From 5458940a6e489099a03c37fbcb758ac0b8f45361 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 1 Sep 2026 05:19:47 -0500 Subject: [PATCH] feat(runner): add offline evaluation tooling (#12653) ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Paperclip Runner needs repeatable evaluation contracts. > - Evaluation code must stay separate from provider launch and production orchestration. > - Offline fixtures need stable compatibility, scoring, traceability, and report rules. > - Published Runner consumers need only the supported evaluation contract surface. > - This pull request adds offline evaluation tooling and a workspace-private matrix kernel. > - The benefit is deterministic evaluation without credentials or paid provider calls. ## Linked Issues or Issue Description Refs #11297 This pull request extracts the offline evaluation unit from the earlier aggregate Runner work. ## What Changed - Add a workspace-private, provider-neutral evaluation matrix kernel. - Add the public `@paperclipai/paperclip-runner/evals` compatibility and native execution contracts. - Add fail-closed runnerd artifact and protocol compatibility checks. - Add deterministic workflow catalogs, scoring, traceability, and report generation. - Add sanitized Codex, OpenCode, and ACPX fixtures. - Add package-boundary and clean-consumer checks. - Add the eval package manifest to the Docker dependency stage. - Add the generated protocol fixture digest without changing the lockfile. ## Verification GitHub Actions must run: - Runner TypeScript and Rust type checks. - Runner unit and protocol tests. - Evaluation kernel tests. - Workflow traceability checks. - Clean-consumer and package-boundary checks. - Repository test, type-check, build, policy, and security gates. No local test command was run. The repository owner requested GitHub-only verification. ## Risks This is a large greenfield review surface with 51 files. The code does not launch a live provider or load credentials. Package and protocol drift fail closed. The workspace lockfile remains under the existing CI-owned process. ## Model Used OpenAI Codex with the GPT-5 agent model. The work used high reasoning, repository inspection, tool use, and parallel code review. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [ ] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --- Dockerfile | 1 + packages/paperclip-eval-kernel/README.md | 14 + packages/paperclip-eval-kernel/package.json | 30 + packages/paperclip-eval-kernel/src/index.ts | 98 +++ .../test/kernel.test.mjs | 54 ++ packages/paperclip-eval-kernel/tsconfig.json | 8 + packages/paperclip-runner/README.md | 16 +- ...-runner-testing-eval-package-boundaries.md | 23 +- .../paperclip-runner/docs/architecture.md | 16 +- .../docs/capability-eval-slice.md | 33 + .../docs/evals-integration.md | 104 +++ packages/paperclip-runner/docs/index.md | 3 + .../docs/runner-workflow-evals.md | 28 + packages/paperclip-runner/package.json | 21 +- .../evals/native-execution-seeded.json | 258 ++++++ .../paperclip-runner/protocol/manifest.json | 6 + .../scripts/check-clean-consumers.mjs | 66 +- .../scripts/check-package-boundaries.mjs | 23 +- .../check-runner-workflow-traceability.mjs | 14 + .../scripts/run-runner-workflow-evals.mjs | 32 + .../schemas/eval-scorecard.v2.schema.json | 55 ++ .../runner-workflow-eval-case.v1.schema.json | 33 + ...runner-workflow-observation.v1.schema.json | 79 ++ .../evals/stress-workflow-traceability.json | 65 ++ .../src/eval/eval-bundle.test.ts | 159 ++++ .../paperclip-runner/src/eval/eval-bundle.ts | 320 +++++++ .../src/eval/eval-execution.test.ts | 75 ++ .../src/eval/eval-execution.ts | 793 ++++++++++++++++++ .../src/eval/eval-scoring.test.ts | 445 ++++++++++ .../paperclip-runner/src/eval/eval-scoring.ts | 400 +++++++++ .../src/eval/eval-slice.test.ts | 90 ++ .../paperclip-runner/src/eval/eval-slice.ts | 146 ++++ .../fixtures/acpx-sanitized-provider.json | 17 + .../fixtures/codex-sanitized-provider.json | 36 + .../fixtures/opencode-sanitized-provider.json | 19 + packages/paperclip-runner/src/eval/index.ts | 10 + .../src/eval/workflow-catalog.ts | 230 +++++ .../src/eval/workflow-contracts.ts | 249 ++++++ .../src/eval/workflow-evals.test.ts | 230 +++++ .../src/eval/workflow-harness.ts | 261 ++++++ .../src/eval/workflow-report.ts | 269 ++++++ .../src/eval/workflow-scoring.ts | 168 ++++ .../src/eval/workflow-traceability.ts | 74 ++ .../src/evals/build-metadata.ts | 58 ++ .../src/evals/compatibility.test.ts | 120 +++ .../src/evals/compatibility.ts | 244 ++++++ packages/paperclip-runner/src/evals/index.ts | 32 + .../src/evals/native-execution.test.ts | 173 ++++ .../src/evals/native-execution.ts | 552 ++++++++++++ .../src/evals/runnerd-artifact.test.ts | 91 ++ .../src/evals/runnerd-artifact.ts | 230 +++++ 51 files changed, 6544 insertions(+), 27 deletions(-) create mode 100644 packages/paperclip-eval-kernel/README.md create mode 100644 packages/paperclip-eval-kernel/package.json create mode 100644 packages/paperclip-eval-kernel/src/index.ts create mode 100644 packages/paperclip-eval-kernel/test/kernel.test.mjs create mode 100644 packages/paperclip-eval-kernel/tsconfig.json create mode 100644 packages/paperclip-runner/docs/capability-eval-slice.md create mode 100644 packages/paperclip-runner/docs/evals-integration.md create mode 100644 packages/paperclip-runner/docs/runner-workflow-evals.md create mode 100644 packages/paperclip-runner/protocol/fixtures/evals/native-execution-seeded.json create mode 100644 packages/paperclip-runner/scripts/check-runner-workflow-traceability.mjs create mode 100644 packages/paperclip-runner/scripts/run-runner-workflow-evals.mjs create mode 100644 packages/paperclip-runner/spec/evals/schemas/eval-scorecard.v2.schema.json create mode 100644 packages/paperclip-runner/spec/evals/schemas/runner-workflow-eval-case.v1.schema.json create mode 100644 packages/paperclip-runner/spec/evals/schemas/runner-workflow-observation.v1.schema.json create mode 100644 packages/paperclip-runner/spec/evals/stress-workflow-traceability.json create mode 100644 packages/paperclip-runner/src/eval/eval-bundle.test.ts create mode 100644 packages/paperclip-runner/src/eval/eval-bundle.ts create mode 100644 packages/paperclip-runner/src/eval/eval-execution.test.ts create mode 100644 packages/paperclip-runner/src/eval/eval-execution.ts create mode 100644 packages/paperclip-runner/src/eval/eval-scoring.test.ts create mode 100644 packages/paperclip-runner/src/eval/eval-scoring.ts create mode 100644 packages/paperclip-runner/src/eval/eval-slice.test.ts create mode 100644 packages/paperclip-runner/src/eval/eval-slice.ts create mode 100644 packages/paperclip-runner/src/eval/fixtures/acpx-sanitized-provider.json create mode 100644 packages/paperclip-runner/src/eval/fixtures/codex-sanitized-provider.json create mode 100644 packages/paperclip-runner/src/eval/fixtures/opencode-sanitized-provider.json create mode 100644 packages/paperclip-runner/src/eval/index.ts create mode 100644 packages/paperclip-runner/src/eval/workflow-catalog.ts create mode 100644 packages/paperclip-runner/src/eval/workflow-contracts.ts create mode 100644 packages/paperclip-runner/src/eval/workflow-evals.test.ts create mode 100644 packages/paperclip-runner/src/eval/workflow-harness.ts create mode 100644 packages/paperclip-runner/src/eval/workflow-report.ts create mode 100644 packages/paperclip-runner/src/eval/workflow-scoring.ts create mode 100644 packages/paperclip-runner/src/eval/workflow-traceability.ts create mode 100644 packages/paperclip-runner/src/evals/build-metadata.ts create mode 100644 packages/paperclip-runner/src/evals/compatibility.test.ts create mode 100644 packages/paperclip-runner/src/evals/compatibility.ts create mode 100644 packages/paperclip-runner/src/evals/index.ts create mode 100644 packages/paperclip-runner/src/evals/native-execution.test.ts create mode 100644 packages/paperclip-runner/src/evals/native-execution.ts create mode 100644 packages/paperclip-runner/src/evals/runnerd-artifact.test.ts create mode 100644 packages/paperclip-runner/src/evals/runnerd-artifact.ts diff --git a/Dockerfile b/Dockerfile index 6845aca095..242229616b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,6 +24,7 @@ COPY packages/adapter-utils/package.json packages/adapter-utils/ COPY packages/google-sheets-mcp-server/package.json packages/google-sheets-mcp-server/ COPY packages/kv-demo-mcp-server/package.json packages/kv-demo-mcp-server/ COPY packages/mcp-server/package.json packages/mcp-server/ +COPY packages/paperclip-eval-kernel/package.json packages/paperclip-eval-kernel/ COPY packages/paperclip-runner/package.json packages/paperclip-runner/ COPY packages/skills-catalog/package.json packages/skills-catalog/ COPY packages/tailscale-https-broker/package.json packages/tailscale-https-broker/ diff --git a/packages/paperclip-eval-kernel/README.md b/packages/paperclip-eval-kernel/README.md new file mode 100644 index 0000000000..f8e6b21416 --- /dev/null +++ b/packages/paperclip-eval-kernel/README.md @@ -0,0 +1,14 @@ +# Paperclip Eval Kernel + +`@paperclipai/paperclip-eval-kernel` is the workspace-private, provider-neutral +matrix orchestrator owned by Paperclip Evals. It contains no Paperclip scenario +corpus, provider configuration, product fixture, scorer, or report template. + +Consumers pass scenario and candidate values plus execution and scoring +callbacks. Candidate `preflight` hooks should call the runner package's +`assertPaperclipRunnerCompatibility` before any provider work starts. This keeps +catalog, protocol, runner-client, control-plane-adapter, testkit, corpus, and +provider-operation incompatibilities explicit. + +Paperclip App may consume this package only as a development dependency for CI +or parity tests. `@paperclipai/paperclip-runner` has no runtime dependency on it. diff --git a/packages/paperclip-eval-kernel/package.json b/packages/paperclip-eval-kernel/package.json new file mode 100644 index 0000000000..676efbf12b --- /dev/null +++ b/packages/paperclip-eval-kernel/package.json @@ -0,0 +1,30 @@ +{ + "name": "@paperclipai/paperclip-eval-kernel", + "version": "0.1.0", + "private": true, + "description": "Provider-neutral evaluation matrix kernel without Paperclip scenario content", + "type": "module", + "files": [ + "dist", + "README.md" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "tsc -p tsconfig.json", + "typecheck": "tsc -p tsconfig.json --noEmit", + "test": "pnpm run build && node --test test/*.test.mjs" + }, + "engines": { + "node": ">=24.11.0" + }, + "license": "MIT", + "devDependencies": { + "@types/node": "^24.0.0", + "typescript": "^7.0.2" + } +} diff --git a/packages/paperclip-eval-kernel/src/index.ts b/packages/paperclip-eval-kernel/src/index.ts new file mode 100644 index 0000000000..792406249e --- /dev/null +++ b/packages/paperclip-eval-kernel/src/index.ts @@ -0,0 +1,98 @@ +export const PAPERCLIP_EVAL_KERNEL_COMPATIBILITY = Object.freeze({ + schema: "paperclip.eval-kernel.compatibility.v1" as const, + packageName: "@paperclipai/paperclip-eval-kernel" as const, + packageVersion: "0.1.0" as const, + apiVersion: 1 as const, +}); + +export interface PaperclipEvalScenario { + readonly id: string; + readonly input: TInput; +} + +export interface PaperclipEvalCandidate { + readonly id: string; + readonly config: TCandidate; + /** Fail-closed runner/catalog/provider compatibility check. */ + readonly preflight?: () => void | Promise; +} + +export interface PaperclipEvalResult { + readonly scenarioId: string; + readonly candidateId: string; + readonly output: TOutput; + readonly score: TScore; +} + +export class PaperclipEvalKernelConfigurationError extends Error { + readonly code = "paperclip_eval_kernel_configuration_invalid" as const; + + constructor(message: string) { + super(message); + this.name = "PaperclipEvalKernelConfigurationError"; + } +} + +/** + * Generic deterministic matrix orchestration. Scenario definitions, provider + * configuration, scorers, reports, and persistence remain caller-owned. + */ +export async function runPaperclipEvalMatrix< + TInput, + TCandidate, + TOutput, + TScore, +>(input: { + readonly scenarios: readonly PaperclipEvalScenario[]; + readonly candidates: readonly PaperclipEvalCandidate[]; + readonly execute: (context: { + readonly scenario: PaperclipEvalScenario; + readonly candidate: PaperclipEvalCandidate; + }) => Promise; + readonly score: (context: { + readonly scenario: PaperclipEvalScenario; + readonly candidate: PaperclipEvalCandidate; + readonly output: TOutput; + }) => Promise | TScore; +}): Promise[]> { + assertUniqueNonEmptyIds("scenario", input.scenarios); + assertUniqueNonEmptyIds("candidate", input.candidates); + + for (const candidate of input.candidates) { + await candidate.preflight?.(); + } + + const results: PaperclipEvalResult[] = []; + for (const scenario of input.scenarios) { + for (const candidate of input.candidates) { + const output = await input.execute({ scenario, candidate }); + const score = await input.score({ scenario, candidate, output }); + results.push(Object.freeze({ + scenarioId: scenario.id, + candidateId: candidate.id, + output, + score, + })); + } + } + return Object.freeze(results); +} + +function assertUniqueNonEmptyIds( + kind: "scenario" | "candidate", + values: readonly { readonly id: string }[], +): void { + if (values.length === 0) { + throw new PaperclipEvalKernelConfigurationError(`${kind} list must not be empty`); + } + const ids = new Set(); + for (const value of values) { + if (value.id.trim().length === 0) { + throw new PaperclipEvalKernelConfigurationError(`${kind} id must not be empty`); + } + if (ids.has(value.id)) { + throw new PaperclipEvalKernelConfigurationError(`duplicate ${kind} id: ${value.id}`); + } + ids.add(value.id); + } +} diff --git a/packages/paperclip-eval-kernel/test/kernel.test.mjs b/packages/paperclip-eval-kernel/test/kernel.test.mjs new file mode 100644 index 0000000000..38d0cf7e0c --- /dev/null +++ b/packages/paperclip-eval-kernel/test/kernel.test.mjs @@ -0,0 +1,54 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + PAPERCLIP_EVAL_KERNEL_COMPATIBILITY, + PaperclipEvalKernelConfigurationError, + runPaperclipEvalMatrix, +} from "../dist/index.js"; + +test("runs a caller-owned scenario/candidate matrix", async () => { + const results = await runPaperclipEvalMatrix({ + scenarios: [{ id: "scenario-a", input: { value: 2 } }], + candidates: [{ id: "candidate-a", config: { multiplier: 3 } }], + execute: async ({ scenario, candidate }) => scenario.input.value * candidate.config.multiplier, + score: ({ output }) => ({ passed: output === 6 }), + }); + assert.equal(PAPERCLIP_EVAL_KERNEL_COMPATIBILITY.apiVersion, 1); + assert.deepEqual(results, [{ + scenarioId: "scenario-a", + candidateId: "candidate-a", + output: 6, + score: { passed: true }, + }]); +}); + +test("fails before execution when compatibility preflight fails", async () => { + let executed = false; + await assert.rejects( + runPaperclipEvalMatrix({ + scenarios: [{ id: "scenario-a", input: null }], + candidates: [{ + id: "candidate-a", + config: null, + preflight: () => { throw new Error("paperclip_runner_incompatible"); }, + }], + execute: async () => { executed = true; }, + score: () => null, + }), + /paperclip_runner_incompatible/, + ); + assert.equal(executed, false); +}); + +test("rejects duplicate scenario ids", async () => { + await assert.rejects( + runPaperclipEvalMatrix({ + scenarios: [{ id: "duplicate", input: 1 }, { id: "duplicate", input: 2 }], + candidates: [{ id: "candidate-a", config: null }], + execute: async () => null, + score: () => null, + }), + PaperclipEvalKernelConfigurationError, + ); +}); diff --git a/packages/paperclip-eval-kernel/tsconfig.json b/packages/paperclip-eval-kernel/tsconfig.json new file mode 100644 index 0000000000..c99ec7b71f --- /dev/null +++ b/packages/paperclip-eval-kernel/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src/**/*.ts"] +} diff --git a/packages/paperclip-runner/README.md b/packages/paperclip-runner/README.md index 36ba1c9c5d..4dd853f39f 100644 --- a/packages/paperclip-runner/README.md +++ b/packages/paperclip-runner/README.md @@ -20,9 +20,14 @@ imports or starts Paperclip's server, UI, CLI, or production database. - `@paperclipai/paperclip-runner/testing` — deterministic mocks plus PRP and semantic conformance kits. Tests and external conformance consumers import this explicitly. +- `@paperclipai/paperclip-runner/evals` — versioned native-attempt metadata, + fail-closed package/binary compatibility checks, and explicit runnerd + artifact resolution for eval consumers. -The package root has no mock or scenario exports, and the workspace has no -separate eval-package importer. Provider-backed eval campaigns remain deferred. +The package root has no mock or scenario exports. Generic credential-free +matrix orchestration lives in the workspace-private +`@paperclipai/paperclip-eval-kernel`; scenario content and provider-backed eval +campaigns remain outside the runtime package. See [ADR 0001](docs/adr/0001-runner-testing-eval-package-boundaries.md). The two conformance surfaces intentionally prove different contracts. The @@ -177,8 +182,11 @@ and JSON content; see the protocol-server tutorial for direct `curl` examples. | `check:forbidden-imports` | Reject TypeScript imports and Cargo path dependencies that cross into Paperclip core. | | `check:tracked-imports` | Reject tracked imports and `package.json` entry points that only resolve against untracked files, so a clean checkout of any commit builds. | | `check:numbered-milestones` | Reject numbered construction-milestone names in tracked package paths and source. | -| `check:package-boundaries` | Enforce the acyclic runtime/testing dependency and manifest boundary. | -| `check:clean-consumers` | Pack the runner and install its root and testing exports in a clean consumer. | +| `check:package-boundaries` | Enforce the acyclic runtime/testing/eval dependency and manifest boundary. | +| `check:clean-consumers` | Pack the runner and install its root, evals, and testing exports in a clean consumer. | +| `test:eval-slice` | Run the credential-free eval bundle, scoring, and behavior/fault slice. | +| `test:runner-workflow-evals` | Run the deterministic provider-neutral workflow matrix. | +| `report:runner-workflow-evals` | Write local deterministic workflow reports without provider calls. | | `check:conformance-parity` | Require byte-for-byte equivalent Rust and TypeScript tracer output. | | `check:replay-goldens` | Require all reducer snapshots and cross-language summaries to match checked goldens. | | `check:replay-parity` | Run TypeScript and Rust against the same Replay fixture summaries. | diff --git a/packages/paperclip-runner/docs/adr/0001-runner-testing-eval-package-boundaries.md b/packages/paperclip-runner/docs/adr/0001-runner-testing-eval-package-boundaries.md index c9b65b432f..14554af461 100644 --- a/packages/paperclip-runner/docs/adr/0001-runner-testing-eval-package-boundaries.md +++ b/packages/paperclip-runner/docs/adr/0001-runner-testing-eval-package-boundaries.md @@ -1,4 +1,4 @@ -# ADR 0001: Runner and Testing Package Boundaries +# ADR 0001: Runner, Testing, and Eval Package Boundaries - Status: Accepted - Date: 2026-08-11 @@ -23,7 +23,7 @@ Ownership is: | Owner | Stable responsibility | |---|---| | Paperclip App | PRP schemas and fixtures, canonical semantic catalog and dispatcher, runnerd/client interfaces, `ControlPlanePort`, the production binding, deterministic mock, and mock/real parity fixtures | -| External eval repositories | Scenario corpus, provider configuration, experiment reports, and provider-backed orchestration | +| Eval consumers | Scenario corpus, provider configuration, experiment reports, and provider-backed orchestration | The production binding remains App code at `server/src/services/native-runtime/paperclip-control-plane-port.ts`. It @@ -35,6 +35,7 @@ Public runner exports are: | Export | Stability and purpose | |---|---| | `@paperclipai/paperclip-runner` | Runtime contracts, runner clients/backends, PRP validation/replay, canonical catalog/dispatcher, and compatibility preflight | +| `@paperclipai/paperclip-runner/evals` | Versioned native-attempt/build metadata, compatibility negotiation, and explicit runnerd artifact resolution | | `@paperclipai/paperclip-runner/testing` | Deterministic mocks, PRP port conformance, and provider-neutral semantic conformance kit | | `./browser`, `./react`, `./standalone`, `./styles.css` | Existing explicitly named UI/standalone consumers | @@ -49,11 +50,12 @@ consumer or version cadence currently justifies another package. Split it only after an independent release requirement exists; a directory preference is not sufficient. -The credential-free matrix engine used by runner conformance remains -package-local test implementation. A separately versioned eval kernel and any -provider-backed campaign are deferred. The runner's dependency, -optional-dependency, peer-dependency, and workspace importer sets remain free -of eval packages. +Generic credential-free matrix orchestration lives in the separately versioned, +workspace-private `@paperclipai/paperclip-eval-kernel` package. It contains no +runner imports, provider configuration, scenario corpus, scorer, or report +renderer. The runner may use it only as a development dependency; runtime, +optional, and peer dependency sets remain free of eval packages. Paid provider +campaigns remain external. The dependency graph is acyclic: @@ -64,7 +66,9 @@ Paperclip App production binding @paperclipai/paperclip-runner (runtime contracts) ^ | -External conformance consumers --> @paperclipai/paperclip-runner/testing +Eval consumers ----> @paperclipai/paperclip-runner/evals + | @paperclipai/paperclip-runner/testing + +-----------> @paperclipai/paperclip-eval-kernel ``` No arrow points from App runtime to an external eval repository. @@ -82,6 +86,7 @@ contracts are published in `PAPERCLIP_RUNNER_COMPATIBILITY`: | runnerd artifact | 2 | Binary metadata/package disagreement or digest mismatch fails before launch | | Harness driver | 1 | Breaking descriptor/config/session/conformance behavior requires a new version | | Native execution | 1 | Breaking App attempt-bundle semantics require a new schema version and converter | +| Evals integration | 1 | Breaking package/binary/catalog/driver join behavior requires a new version | | Control-plane adapter | 1 | Breaking `ControlPlanePort` or production-binding expectations require a new version | | Testkit | 1 | Breaking mock seed, vector, observation, or conformance behavior requires a new version | | Eval corpus | 1 | Runner declares a supported inclusive corpus-version range; out-of-range bundles fail before execution | @@ -102,7 +107,7 @@ pnpm --filter @paperclipai/paperclip-runner check:clean-consumers ``` The second command builds and packs the runner, installs its tarball into a -clean consumer, imports only the root and `./testing` exports, executes +clean consumer, imports only the root, `./evals`, and `./testing` exports, executes deterministic PRP, harness-driver, and semantic conformance, and verifies the separately staged runnerd artifact digest. The consumer uses no workspace protocol, source-relative import, or deep package path. This is the packaging diff --git a/packages/paperclip-runner/docs/architecture.md b/packages/paperclip-runner/docs/architecture.md index fd49aa7147..78150cb0c4 100644 --- a/packages/paperclip-runner/docs/architecture.md +++ b/packages/paperclip-runner/docs/architecture.md @@ -14,7 +14,7 @@ byte-identical Conformance result Paperclip App production binding --> implements ControlPlanePort -External conformance consumers --> use packed runtime + ./testing exports +Eval/conformance consumers --> use packed runtime + ./evals + ./testing exports ``` The dependency arrow always points from an implementation toward a contract. @@ -28,10 +28,13 @@ recorded in [ADR 0001](adr/0001-runner-testing-eval-package-boundaries.md). - The package root is runtime-only: PRP, runner/client contracts, normalized backends, catalog/dispatcher, and compatibility preflight. +- `./evals` exposes the stable native-attempt/build-metadata join and explicit + digest-verified runnerd artifact resolution. - `./testing` contains deterministic mocks and conformance kits. -- Package-local deterministic matrices remain internal test implementation. - A separately versioned eval package and provider-backed campaigns are - deferred and are not workspace dependencies or public exports. +- The workspace-private `@paperclipai/paperclip-eval-kernel` contains generic + structural matrix orchestration and is a development-only dependency. + Runner-specific cases, scorers, and reports remain package-local; paid + campaigns remain external. ## Core contracts @@ -66,8 +69,9 @@ boundary and checks the same fixture summaries. Local runner adds the package-lo `paperclip-runnerd` and `fake-harness` binaries without changing that dependency direction. -The clean-consumer gate packs the declared root and `./testing` exports and -stages the release runnerd executable as a separately checksummed artifact. +The clean-consumer gate packs the declared root, `./evals`, and `./testing` +exports and stages the release runnerd executable as a separately checksummed +artifact. ## Language ownership diff --git a/packages/paperclip-runner/docs/capability-eval-slice.md b/packages/paperclip-runner/docs/capability-eval-slice.md new file mode 100644 index 0000000000..34d138f736 --- /dev/null +++ b/packages/paperclip-runner/docs/capability-eval-slice.md @@ -0,0 +1,33 @@ +# Runner eval scoring slice + +The package-local eval slice provides deterministic, credential-free scoring +for runner behavior. It complements the fail-fast conformance suite by keeping +each observation intact and reporting independent dimensions for safety, +semantic outcome, trajectory restraint, trace completeness, and efficiency. + +`EvalBundle` records reproducibility inputs without storing credentials. +`assertBundleSecretFree` rejects credential-shaped keys and values before any +structural error can echo them. Persisted reports contain an explicit digested +bundle-evidence declaration rather than the free-form source bundle, while +`bundleId` still derives a stable content identifier from the full canonical +declaration. The exact final report serialization is scanned again before it is +returned to a persistence boundary. + +`scoreEval` is pure: the same observation and bundle always produce the same +scorecard. Hard-invariant failures gate the overall score to zero. Other +dimensions remain separate so a report shows whether a regression came from +the semantic outcome, unnecessary calls, incomplete causal evidence, or an +exceeded declared budget. + +`runEvalBehaviorFaultMatrix` exercises deterministic green and red behavior +against the package's mock authority. No provider process, network credential, +or paid model invocation is required. + +Run the offline slice with: + +```sh +pnpm --filter @paperclipai/paperclip-runner test:eval-slice +``` + +Provider-backed campaigns and recorded evidence are intentionally outside this +package boundary. diff --git a/packages/paperclip-runner/docs/evals-integration.md b/packages/paperclip-runner/docs/evals-integration.md new file mode 100644 index 0000000000..0876983450 --- /dev/null +++ b/packages/paperclip-runner/docs/evals-integration.md @@ -0,0 +1,104 @@ +# Paperclip Evals Integration Contract + +## Stable consumer inputs + +Paperclip Evals consumes two explicit App artifacts: + +1. a packed/released `@paperclipai/paperclip-runner` package; and +2. an explicit `paperclip-runnerd` executable path plus its + `sha256:` digest. + +The consumer must not import App source paths, search a workspace for a binary, +or infer compatibility from a process failure. The package exposes three +relevant entry points: + +- package root: PRP, runner, semantic-tool, and control-plane runtime contracts; +- `./evals`: build metadata, native-attempt schema/fixture, compatibility + negotiation, semantic receipts/catalog, and explicit runnerd resolution; +- `./testing`: deterministic driver/control-plane fakes and conformance kits. + +`resolvePaperclipRunnerdArtifact` resolves only the supplied path, verifies its +bytes against the supplied digest, and invokes that exact executable with +`--build-metadata`. It never searches `PATH` or the App repository. + +## Versioned join + +`assertPaperclipRunnerEvalCompatibility` checks the complete join before any +provider process starts: + +| Dimension | V1 rule | +| --- | --- | +| Package | Exact package version matches loaded build metadata. | +| Binary | runnerd package version and binary-artifact contract match the package. | +| PRP | Package, runnerd, and consumer version ranges have a common version. | +| Semantic catalog | Contract version and canonical content digest both match. | +| Harness driver | Contract and negotiated PRP versions match and every required capability is explicitly true. | +| Native output | Consumer and runnerd both select `paperclip-runner/native-execution/v1`. | + +Failures use `paperclip_runner_eval_incompatible` and include all detected +component issues with expected and received values. A catalog, driver, or PRP +mismatch cannot degrade into an attempted run. + +## Native attempt bundle + +`paperclip-runner/native-execution/v1` is the App-owned raw attempt. Its checked +schema is `protocol/schemas/native-execution.schema.json`; the no-spend seeded +fixture is `protocol/fixtures/evals/native-execution-seeded.json`. + +The bundle pins run/case/config/attempt identity, case and config digests, +package/binary/catalog/driver versions and digests, ordered PRP events, semantic +tool definitions/calls/results/denials, terminal state, check-ready +observations, usage/cost/time/request totals, transcript completeness, and +content-addressed artifacts. Unknown additive fields survive parsing. + +The parser fails closed on an unknown native schema, malformed digest, event +run mismatch, missing or conflicting terminal, semantic indexes that omit or +reinterpret their PRP tool envelopes, denied results without matching denial +receipts, inconsistent token totals, or ambiguous incomplete transcripts. +Unknown additive fields are retained at nested contract objects as well as the +bundle root. The seeded fixture deliberately preserves a rejected governed +tool effect as `denied`; it never turns that effect into a successful mutation. + +Evals owns conversion of this bundle into Evalbook ledger, scores, and reports. +The App package does not implement an Evalbook environment loader, importer, +grid, comparison, or report renderer. + +## Deterministic conformance + +`runHarnessDriverConformance` with `DeterministicHarnessDriver` covers, without +network or provider credentials: + +- capability and unsupported-feature description; +- valid/invalid config handling; +- session open, turn, snapshot, recovery, close, and cancel lifecycle; +- paired provider-neutral semantic tool events; +- PRP event validation; +- interrupt and cancelled terminal behavior; +- usage reporting; and +- complete transcript accounting. + +The input fixture is +`protocol/fixtures/evals/harness-driver-conformance.json`. The packed +clean-consumer gate builds runnerd with Cargo's `release` profile, copies it +into an isolated artifact directory, packs the package, installs it offline, +resolves only declared exports, validates the native fixture, runs driver +conformance, verifies binary metadata/digest, and exercises both compatible +and incompatible negotiation. It never qualifies a debug binary. + +Run it with: + +```sh +pnpm --filter @paperclipai/paperclip-runner check:clean-consumers +``` + +Set `PAPERCLIP_CLEAN_CONSUMER_OUTPUT_DIR` to retain the qualifying inputs and +machine-readable proof outside the temporary consumer. The output contains the +package tarball, platform-named release runnerd, `SHA256SUMS`, and +`paperclip-runner-consumer-conformance.json`. That record is produced by the clean +consumer after it imports only the packed package, resolves the explicit binary +and digest, and completes deterministic conformance without provider calls. + +```sh +PAPERCLIP_CLEAN_CONSUMER_OUTPUT_DIR=/absolute/release/directory \ + pnpm --filter @paperclipai/paperclip-runner check:clean-consumers +``` diff --git a/packages/paperclip-runner/docs/index.md b/packages/paperclip-runner/docs/index.md index ccaa460fc9..f96659931c 100644 --- a/packages/paperclip-runner/docs/index.md +++ b/packages/paperclip-runner/docs/index.md @@ -19,6 +19,9 @@ Paperclip's runtime selection. ## Reference +- [Runner eval scoring slice](capability-eval-slice.md) +- [Deterministic runner workflow evals](runner-workflow-evals.md) +- [Evals integration contract](evals-integration.md) - [Local runner and supervision](local-runner.md) - [Durable transport and recovery](durable-recovery.md) - [Live console protocol server](live-console-protocol-server.md) diff --git a/packages/paperclip-runner/docs/runner-workflow-evals.md b/packages/paperclip-runner/docs/runner-workflow-evals.md new file mode 100644 index 0000000000..97acae5363 --- /dev/null +++ b/packages/paperclip-runner/docs/runner-workflow-evals.md @@ -0,0 +1,28 @@ +# Deterministic runner workflow evals + +The workflow evaluator converts the stress-derived workflow catalog into a +credential-free matrix over sanitized Codex, OpenCode, and ACPX fixtures. +It evaluates provider-neutral behavior; the fixtures contain normalized events, +not prompts, credentials, raw reasoning, or provider traces. + +The workspace-private `@paperclipai/paperclip-eval-kernel` package owns only +structural scenario-by-candidate orchestration. Runner-specific cases, +observations, scoring, traceability, and report rendering remain package-local. + +Use: + +```sh +pnpm --filter @paperclipai/paperclip-runner test:runner-workflow-evals +pnpm --filter @paperclipai/paperclip-runner report:runner-workflow-evals +``` + +The report command writes JSON, Markdown, JUnit, and GitHub-safe summaries +under `.paperclip-local/evals/workflows/`. It performs no network requests and +does not start a production provider. + +The checked traceability manifest is +`spec/evals/stress-workflow-traceability.json`. The build gate fails when a +finding references an unknown workflow or a missing regression-test anchor. + +Live schedules, paid provider campaigns, raw trace capture, and recorded +evidence are intentionally excluded from this slice. diff --git a/packages/paperclip-runner/package.json b/packages/paperclip-runner/package.json index bf486b8e09..03950d66f0 100644 --- a/packages/paperclip-runner/package.json +++ b/packages/paperclip-runner/package.json @@ -21,6 +21,10 @@ "types": "./dist/testing.d.ts", "import": "./dist/testing.js" }, + "./evals": { + "types": "./dist/evals/index.d.ts", + "import": "./dist/evals/index.js" + }, "./live": { "types": "./dist/live/index.d.ts", "import": "./dist/live/index.js" @@ -52,8 +56,8 @@ ], "sideEffects": false, "scripts": { - "build": "pnpm run check:protocol-manifest && pnpm run build:typescript && pnpm run check:capability-contract && pnpm run check:capability-inventory && pnpm run check:protocol-coverage && pnpm run check:semantic-contracts && pnpm run build:binary && node scripts/generate-replay-goldens.mjs --check && node scripts/generate-semantic-action-catalog.mjs --check", - "build:typescript": "pnpm run check:protocol-types && node ./node_modules/typescript/bin/tsc --version && node ./node_modules/typescript/bin/tsc -p tsconfig.json && node ./node_modules/typescript/bin/tsc -p tsconfig.surfaces.json", + "build": "pnpm run check:protocol-manifest && pnpm run build:typescript && pnpm run check:capability-contract && pnpm run check:capability-inventory && pnpm run check:protocol-coverage && pnpm run check:semantic-contracts && pnpm run check:runner-workflow-traceability && pnpm run build:binary && node scripts/generate-replay-goldens.mjs --check && node scripts/generate-semantic-action-catalog.mjs --check", + "build:typescript": "pnpm run ensure:eval-build-deps && pnpm run check:protocol-types && node ./node_modules/typescript/bin/tsc --version && node ./node_modules/typescript/bin/tsc -p tsconfig.json && node ./node_modules/typescript/bin/tsc -p tsconfig.surfaces.json", "build:rust": "cargo build --manifest-path runner/Cargo.toml --locked --workspace --bins", "build:binary": "cargo build --release --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --bin paperclip-runnerd && node scripts/stage-runner-binary.mjs", "build:provider-pack": "pnpm run build:typescript && node scripts/build-provider-pack.mjs", @@ -64,11 +68,11 @@ "build:scenarios": "vite build --config vite.scenarios.config.ts && node scripts/check-capability-csp-bundle.mjs", "build:issue-thread": "vite build --config vite.issue-thread.config.ts", "typecheck": "pnpm run typecheck:typescript && pnpm run typecheck:rust", - "typecheck:typescript": "node --check scripts/protocol-contract.mjs && node --check scripts/generate-protocol-manifest.mjs && node --check scripts/generate-protocol-schema-module.mjs && node --check scripts/generate-acpx-sidecar-contract.mjs && node --check scripts/generate-replay-goldens.mjs && node --check scripts/generate-semantic-action-catalog.mjs && pnpm run check:protocol-types && node ./node_modules/typescript/bin/tsc --version && node ./node_modules/typescript/bin/tsc -p tsconfig.json --noEmit && node ./node_modules/typescript/bin/tsc -p tsconfig.surfaces.json --noEmit", + "typecheck:typescript": "pnpm run ensure:eval-build-deps && node --check scripts/protocol-contract.mjs && node --check scripts/generate-protocol-manifest.mjs && node --check scripts/generate-protocol-schema-module.mjs && node --check scripts/generate-acpx-sidecar-contract.mjs && node --check scripts/generate-replay-goldens.mjs && node --check scripts/generate-semantic-action-catalog.mjs && pnpm run check:protocol-types && node ./node_modules/typescript/bin/tsc --version && node ./node_modules/typescript/bin/tsc -p tsconfig.json --noEmit && node ./node_modules/typescript/bin/tsc -p tsconfig.surfaces.json --noEmit", "typecheck:rust": "cargo fmt --manifest-path runner/Cargo.toml --all -- --check && cargo check --manifest-path runner/Cargo.toml --locked --workspace", "typecheck:browser": "tsc -p tsconfig.browser.json --noEmit", "test": "pnpm run test:typescript && pnpm run test:rust", - "test:typescript": "pnpm run build:rust && node --test test/protocol-contract.test.mjs test/acpx-sidecar-contract.test.mjs test/acpx-codex-package-contract.test.mjs && vitest run", + "test:typescript": "pnpm run ensure:eval-build-deps && pnpm run build:rust && node --test test/protocol-contract.test.mjs test/acpx-sidecar-contract.test.mjs test/acpx-codex-package-contract.test.mjs && vitest run", "test:rust": "cargo test --release --manifest-path runner/Cargo.toml --locked --workspace", "test:codex": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --test codex_provider", "test:durable": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core durable::", @@ -91,7 +95,7 @@ "check:semantic-action-catalog": "pnpm run build:typescript && node scripts/generate-semantic-action-catalog.mjs --check", "check:protocol": "pnpm run typecheck:typescript && pnpm run check:protocol-manifest && pnpm run test:typescript && pnpm run check:replay-goldens", "check:runner": "pnpm run typecheck:rust && pnpm run test:rust && pnpm run check:conformance-parity && pnpm run check:replay-parity", - "check:all": "pnpm run check:protocol && pnpm run check:runner", + "check:all": "pnpm run check:eval-kernel && pnpm run check:protocol && pnpm run check:runner", "generate:replay-goldens": "pnpm run build:typescript && node scripts/generate-replay-goldens.mjs", "check:replay-goldens": "pnpm run build:typescript && node scripts/generate-replay-goldens.mjs --check", "check:browser-tokens": "node scripts/check-browser-tokens.mjs", @@ -100,13 +104,19 @@ "check:numbered-milestones": "node scripts/check-numbered-milestones.mjs", "check:package-boundaries": "node scripts/check-package-boundaries.mjs", "check:clean-consumers": "node scripts/check-clean-consumers.mjs", + "check:eval-kernel": "pnpm --filter @paperclipai/paperclip-eval-kernel test", + "ensure:eval-build-deps": "pnpm --filter @paperclipai/paperclip-eval-kernel build", "generate:capability-inventory": "node scripts/generate-capability-inventory.mjs", "generate:protocol-coverage": "pnpm run build:typescript && node scripts/generate-protocol-coverage.mjs", "check:protocol-coverage": "node scripts/generate-protocol-coverage.mjs --check", "check:capability-inventory": "node scripts/check-capability-inventory.mjs", "test:capability-inventory": "node scripts/check-capability-inventory.test.mjs", "test:capability-evals": "vitest run src/conformance/capability-eval-suite.test.ts", + "test:eval-slice": "pnpm run ensure:eval-build-deps && vitest run src/eval", + "test:runner-workflow-evals": "pnpm run ensure:eval-build-deps && vitest run src/eval/workflow-evals.test.ts", + "check:runner-workflow-traceability": "pnpm run build:typescript && node scripts/check-runner-workflow-traceability.mjs", "report:capability-evals": "pnpm run build:typescript && node scripts/run-capability-eval-suite.mjs", + "report:runner-workflow-evals": "pnpm run build:typescript && node scripts/run-runner-workflow-evals.mjs", "check:conformance-parity": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core runs_the_mock_core_path_with_stable_output", "check:replay-parity": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core replay_fixture_parity", "docs:validate": "node scripts/validate-doc-links.mjs", @@ -154,6 +164,7 @@ } }, "devDependencies": { + "@paperclipai/paperclip-eval-kernel": "workspace:*", "@playwright/test": "^1.61.1", "@types/node": "^24.0.0", "@types/react": "^19.2.17", diff --git a/packages/paperclip-runner/protocol/fixtures/evals/native-execution-seeded.json b/packages/paperclip-runner/protocol/fixtures/evals/native-execution-seeded.json new file mode 100644 index 0000000000..c005afd78c --- /dev/null +++ b/packages/paperclip-runner/protocol/fixtures/evals/native-execution-seeded.json @@ -0,0 +1,258 @@ +{ + "schema": "paperclip-runner/native-execution/v1", + "provenance": "seeded", + "identity": { + "runId": "run_evals_seeded", + "caseId": "case_semantic_denial", + "configId": "config_deterministic_v1", + "attemptId": "attempt_semantic_denial_0001" + }, + "input": { + "caseSha256": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "configSha256": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "deterministicSeed": "paperclip-evals-v1-seed" + }, + "runner": { + "package": { + "name": "@paperclipai/paperclip-runner", + "version": "0.0.0" + }, + "binary": { + "name": "paperclip-runnerd", + "sha256": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + }, + "prpVersion": 1, + "nativeExecutionVersion": 1, + "catalogVersion": 1, + "catalogSha256": "sha256:e5a90079e1641e50f2f07a320856bbef622c04810384ffdd1895576cc16c4c5c", + "driverContractVersion": 1, + "driverKind": "paperclip-deterministic", + "driverVersion": "1.0.0" + }, + "events": [ + { + "schema": "paperclip.prp.event.v1", + "sourceEventId": "evals_seeded_01", + "sourceSeq": 1, + "sourceInstanceId": "runner_evals_seeded", + "sourceKind": "runner", + "runId": "run_evals_seeded", + "normalizedSessionId": "session_evals_seeded", + "turnId": "turn_evals_seeded", + "itemId": "item_evals_seeded", + "eventType": "mcp_app.tool_input", + "schemaVersion": 1, + "priority": 1, + "emittedAt": "2026-08-11T12:00:00.000Z", + "payload": { + "semantic_tool": { + "schema": "paperclip.prp.semantic_tool.v1", + "schemaVersion": 1, + "phase": "input", + "operationId": "decide_approval", + "callId": "call_evals_seeded", + "correlation": { + "runId": "run_evals_seeded", + "normalizedSessionId": "session_evals_seeded", + "turnId": "turn_evals_seeded", + "itemId": "item_evals_seeded" + }, + "idempotencyKey": "decide_evals_seeded", + "content": { + "digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "redactionDisposition": "redacted", + "references": [] + } + } + } + }, + { + "schema": "paperclip.prp.event.v1", + "sourceEventId": "evals_seeded_02", + "sourceSeq": 2, + "sourceInstanceId": "runner_evals_seeded", + "sourceKind": "runner", + "runId": "run_evals_seeded", + "normalizedSessionId": "session_evals_seeded", + "turnId": "turn_evals_seeded", + "itemId": "item_evals_seeded", + "eventType": "mcp_app.tool_result", + "schemaVersion": 1, + "priority": 1, + "emittedAt": "2026-08-11T12:00:00.010Z", + "payload": { + "semantic_tool": { + "schema": "paperclip.prp.semantic_tool.v1", + "schemaVersion": 1, + "phase": "result", + "operationId": "decide_approval", + "callId": "call_evals_seeded", + "correlation": { + "runId": "run_evals_seeded", + "normalizedSessionId": "session_evals_seeded", + "turnId": "turn_evals_seeded", + "itemId": "item_evals_seeded" + }, + "idempotencyKey": "decide_evals_seeded", + "content": { + "digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "redactionDisposition": "redacted", + "references": [] + }, + "outcome": "denied", + "code": "required_claim_missing", + "retryable": false, + "authorizationBoundary": "grant", + "operationReceiptId": "receipt_evals_seeded", + "auditReceiptId": "audit_evals_seeded" + } + } + }, + { + "schema": "paperclip.prp.event.v1", + "sourceEventId": "evals_seeded_03", + "sourceSeq": 3, + "sourceInstanceId": "runner_evals_seeded", + "sourceKind": "runner", + "runId": "run_evals_seeded", + "normalizedSessionId": "session_evals_seeded", + "turnId": "turn_evals_seeded", + "eventType": "run.result.proposed", + "schemaVersion": 1, + "priority": 0, + "emittedAt": "2026-08-11T12:00:00.020Z", + "payload": { + "schema": "paperclip.run_result.v1", + "reportedWorkDisposition": "needs_review", + "summary": "The governed tool effect was rejected and preserved as a denial.", + "completionClaim": { + "contractRevision": "evals-native-execution-v1", + "objectiveSatisfied": false, + "criteria": [], + "remainingWork": [ + { + "description": "An authorized approver must decide.", + "blocksCompletion": true + } + ] + }, + "evidence": [{ "receiptId": "receipt_evals_seeded" }], + "verification": [ + { + "commandOrCheck": "rejected tool effect stays denied", + "status": "passed" + } + ], + "attentionRequests": [], + "artifacts": [] + } + }, + { + "schema": "paperclip.prp.event.v1", + "sourceEventId": "evals_seeded_04", + "sourceSeq": 4, + "sourceInstanceId": "runner_evals_seeded", + "sourceKind": "runner", + "runId": "run_evals_seeded", + "normalizedSessionId": "session_evals_seeded", + "turnId": "turn_evals_seeded", + "eventType": "run.terminal", + "schemaVersion": 1, + "priority": 0, + "emittedAt": "2026-08-11T12:00:00.030Z", + "payload": { + "schema": "paperclip.prp.terminal.v1", + "turnTerminalState": "completed", + "runTerminalState": "succeeded", + "reportedWorkDisposition": "needs_review", + "workAssessmentId": "assessment_evals_seeded", + "statusDecisionId": "decision_evals_seeded" + } + } + ], + "semanticTools": { + "definitions": [ + { + "operationId": "decide_approval", + "schemaVersion": 1, + "availability": "denied" + } + ], + "calls": [ + { + "callId": "call_evals_seeded", + "operationId": "decide_approval", + "eventId": "evals_seeded_01" + } + ], + "results": [ + { + "callId": "call_evals_seeded", + "operationId": "decide_approval", + "eventId": "evals_seeded_02", + "outcome": "denied" + } + ], + "denials": [ + { + "callId": "call_evals_seeded", + "code": "required_claim_missing", + "authorizationBoundary": "grant", + "retryable": false + } + ] + }, + "terminal": { + "schema": "paperclip.prp.terminal.v1", + "turnTerminalState": "completed", + "runTerminalState": "succeeded", + "reportedWorkDisposition": "needs_review", + "workAssessmentId": "assessment_evals_seeded", + "statusDecisionId": "decision_evals_seeded" + }, + "observations": { + "checks": [ + { + "id": "tool-effect-denied", + "passed": true, + "receiptId": "receipt_evals_seeded" + } + ] + }, + "usage": { + "inputTokens": 12, + "outputTokens": 8, + "totalTokens": 20, + "requestCount": 1, + "durationMs": 30, + "cost": { + "currency": "USD", + "amountMicros": 0 + } + }, + "transcript": { + "uri": "artifact://native/attempt_semantic_denial_0001/transcript.jsonl", + "sha256": "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "byteSize": 2048, + "mediaType": "application/x-ndjson", + "complete": true, + "eventCount": 4, + "omissionReason": null + }, + "artifactRoot": { + "uri": "artifact://native/attempt_semantic_denial_0001/", + "sha256": "sha256:1212121212121212121212121212121212121212121212121212121212121212", + "byteSize": 4096, + "mediaType": "application/vnd.paperclip.runner-artifact-root+json" + }, + "artifacts": [ + { + "uri": "artifact://native/attempt_semantic_denial_0001/denial-receipt.json", + "sha256": "sha256:1313131313131313131313131313131313131313131313131313131313131313", + "byteSize": 512, + "mediaType": "application/json" + } + ], + "failure": null, + "x_fixturePurpose": "Proves seeded provenance, denied effects, and additive field preservation." +} diff --git a/packages/paperclip-runner/protocol/manifest.json b/packages/paperclip-runner/protocol/manifest.json index 2a0e24e168..072c1771f8 100644 --- a/packages/paperclip-runner/protocol/manifest.json +++ b/packages/paperclip-runner/protocol/manifest.json @@ -138,6 +138,12 @@ "expectation": "accept", "compatibilityCase": "canonical" }, + { + "path": "fixtures/evals/native-execution-seeded.json", + "sha256": "3a10ca2da2dfd1e945eae8af4518f1cd4ae5c1100adc269ca808144a78cdbcfa", + "expectation": "accept", + "compatibilityCase": "canonical" + }, { "path": "fixtures/local-runner/scripts/duplicate-terminal.json", "sha256": "56928a21318ea4b390c65fb34f0d3114760619d79301f4d535391a8566b693ab", diff --git a/packages/paperclip-runner/scripts/check-clean-consumers.mjs b/packages/paperclip-runner/scripts/check-clean-consumers.mjs index e489fb60ac..65b9310404 100644 --- a/packages/paperclip-runner/scripts/check-clean-consumers.mjs +++ b/packages/paperclip-runner/scripts/check-clean-consumers.mjs @@ -67,7 +67,7 @@ try { await publishArtifacts({ publicationRoot, runnerTarball, runnerdArtifact, conformanceRecord }); process.stdout.write(`Published clean-consumer artifacts at ${publicationRoot}\n`); } - process.stdout.write("Clean-consumer pack/install checks passed for the runner root and testing exports.\n"); + process.stdout.write("Clean-consumer pack/install checks passed for the runner root, evals, and testing exports.\n"); } finally { if (process.env.PAPERCLIP_KEEP_PACKAGE_CONSUMERS !== "1") { await rm(scratchRoot, { recursive: true, force: true }); @@ -198,6 +198,7 @@ import { readFile, stat, writeFile } from "node:fs/promises"; import { basename } from "node:path"; import * as runtime from "@paperclipai/paperclip-runner"; +import * as evals from "@paperclipai/paperclip-runner/evals"; import * as testing from "@paperclipai/paperclip-runner/testing"; if ("MockControlPlaneAdapter" in runtime || "runControlPlanePortConformance" in runtime) { @@ -214,6 +215,14 @@ const report = await testing.runControlPlanePortConformance({ }); if (report.eventCount !== 3) throw new Error("packed conformance kit returned the wrong event count"); +const nativeBundle = await evals.loadPaperclipNativeExecutionFixture(); +if (nativeBundle.schema !== "paperclip-runner/native-execution/v1") { + throw new Error("packed native execution fixture is unavailable"); +} +if (nativeBundle.semanticTools.results[0]?.outcome !== "denied") { + throw new Error("native execution fixture lost its rejected tool effect"); +} + runtime.assertPaperclipRunnerCompatibility({ consumer: "paperclip-runner-clean-consumer", components: { catalog: 1, protocol: 1, runnerClient: 1, controlPlaneAdapter: 1, testkit: 1 }, @@ -228,6 +237,49 @@ if (!driverConformance.checks.transcriptCompleteness || driverConformance.semant throw new Error("packed harness-driver conformance did not cover transcript/tools"); } +const runnerd = await evals.resolvePaperclipRunnerdArtifact({ + executablePath: process.env.PAPERCLIP_RUNNERD_ARTIFACT, + expectedSha256: process.env.PAPERCLIP_RUNNERD_SHA256, +}); +const evalCompatibility = evals.assertPaperclipRunnerEvalCompatibility({ + consumer: "paperclip-runner-clean-consumer", + packageVersion: evals.PAPERCLIP_RUNNER_BUILD_METADATA.package.version, + runnerd: runnerd.buildMetadata, + nativeExecutionVersion: 1, + prp: { minimumVersion: 1, maximumVersion: 1 }, + catalog: evals.PAPERCLIP_RUNNER_BUILD_METADATA.semanticCatalog, + driver: { + contractVersion: driverConformance.contractVersion, + descriptor: driverConformance.descriptor, + requiredCapabilities: ["typedEvents", "interruption", "usage", "dynamicTools"], + }, +}); +if (evalCompatibility.negotiatedPrpVersion !== 1) { + throw new Error("package/binary PRP negotiation returned the wrong version"); +} +let mismatchFailedClosed = false; +try { + evals.assertPaperclipRunnerEvalCompatibility({ + consumer: "incompatible-runner-clean-consumer", + packageVersion: evals.PAPERCLIP_RUNNER_BUILD_METADATA.package.version, + runnerd: { ...runnerd.buildMetadata, binaryContractVersion: 999 }, + nativeExecutionVersion: 1, + prp: { minimumVersion: 1, maximumVersion: 1 }, + catalog: evals.PAPERCLIP_RUNNER_BUILD_METADATA.semanticCatalog, + driver: { + contractVersion: driverConformance.contractVersion, + descriptor: driverConformance.descriptor, + requiredCapabilities: [], + }, + }); +} catch (error) { + mismatchFailedClosed = error?.code === "paperclip_runner_eval_incompatible" + && error.issues?.some((issue) => issue.code === "binary_contract_version_mismatch"); +} +if (!mismatchFailedClosed) { + throw new Error("runnerd contract mismatch did not fail closed"); +} + const normalized = { authorization: { outcome: "allowed" }, state: { status: "done" }, @@ -273,12 +325,22 @@ await writeFile(process.env.PAPERCLIP_CONFORMANCE_RECORD, JSON.stringify({ }, consumer: { installMode: "offline-packed-artifact", - imports: ["@paperclipai/paperclip-runner", "@paperclipai/paperclip-runner/testing"], + imports: [ + "@paperclipai/paperclip-runner", + "@paperclipai/paperclip-runner/evals", + "@paperclipai/paperclip-runner/testing", + ], appSourceTreeImports: false, providerCalls: 0, }, checks: { packageExportsResolved: true, + nativeExecutionFixture: { + schema: nativeBundle.schema, + rejectedToolEffectPreserved: true, + }, + evalCompatibility, + evalMismatchFailedClosed: mismatchFailedClosed, mockControlPlaneConformance: report, harnessDriverConformance: driverConformance, runnerdDigest: true, diff --git a/packages/paperclip-runner/scripts/check-package-boundaries.mjs b/packages/paperclip-runner/scripts/check-package-boundaries.mjs index 75a5e3dc6c..9ea23f24b0 100644 --- a/packages/paperclip-runner/scripts/check-package-boundaries.mjs +++ b/packages/paperclip-runner/scripts/check-package-boundaries.mjs @@ -3,13 +3,21 @@ import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const evalKernelRoot = resolve(packageRoot, "../paperclip-eval-kernel"); const runnerManifest = JSON.parse(await readFile(resolve(packageRoot, "package.json"), "utf8")); +const evalKernelManifest = JSON.parse(await readFile(resolve(evalKernelRoot, "package.json"), "utf8")); const runtimeIndex = await readFile(resolve(packageRoot, "src/index.ts"), "utf8"); const violations = []; if (runnerManifest.exports?.["./testing"] === undefined) { violations.push("runner package must declare the ./testing export"); } +if (runnerManifest.exports?.["./evals"] === undefined) { + violations.push("runner package must declare the ./evals integration export"); +} +if (evalKernelManifest.private !== true) { + violations.push("generic eval kernel must remain a workspace-private development package"); +} for (const privatePath of ["mock-core", "conformance", "scenarios", "src/tools", "./tools/"]) { if (runtimeIndex.includes(privatePath)) { violations.push(`runtime root must not export test/eval path: ${privatePath}`); @@ -30,9 +38,22 @@ if (runnerManifest.dependencies?.ajv === undefined || runnerManifest.devDependen violations.push("ajv must be declared as a runtime dependency because the public dispatcher imports it"); } +for (const dependency of Object.keys({ + ...(evalKernelManifest.dependencies ?? {}), + ...(evalKernelManifest.optionalDependencies ?? {}), + ...(evalKernelManifest.peerDependencies ?? {}), +})) { + if (dependency === "@paperclipai/paperclip-runner") { + violations.push("generic eval kernel must use structural callbacks, not a runner runtime dependency"); + } +} +if (evalKernelManifest.files?.some((entry) => /scenario|fixture|corpus/.test(entry))) { + violations.push("generic eval kernel package inventory must not include scenario content"); +} + if (violations.length > 0) { process.stderr.write(`Package boundary check failed:\n${violations.map((item) => `- ${item}`).join("\n")}\n`); process.exitCode = 1; } else { - process.stdout.write("Package boundary check passed: runtime and testing ownership is acyclic.\n"); + process.stdout.write("Package boundary check passed: runtime, testing, and eval ownership is acyclic.\n"); } diff --git a/packages/paperclip-runner/scripts/check-runner-workflow-traceability.mjs b/packages/paperclip-runner/scripts/check-runner-workflow-traceability.mjs new file mode 100644 index 0000000000..4e18f50e1c --- /dev/null +++ b/packages/paperclip-runner/scripts/check-runner-workflow-traceability.mjs @@ -0,0 +1,14 @@ +import { access, readFile } from "node:fs/promises"; +import { resolve } from "node:path"; + +const packageRoot = resolve(import.meta.dirname, ".."); +const evals = await import(resolve(packageRoot, "dist/eval/index.js")); +const manifest = JSON.parse(await readFile(resolve(packageRoot, "spec/evals/stress-workflow-traceability.json"), "utf8")); +const summary = evals.validateStressTraceabilityManifest(manifest); +for (const finding of manifest.findings) { + for (const path of finding.regressionTests) await access(resolve(packageRoot, path)); +} +if (summary.coveredWorkflows !== evals.RUNNER_WORKFLOW_IDS.length) { + throw new Error(`stress traceability covers ${summary.coveredWorkflows}/${evals.RUNNER_WORKFLOW_IDS.length} workflows`); +} +process.stdout.write(`Runner stress traceability passed: ${summary.findings} findings, ${summary.coveredWorkflows} workflows, ${summary.exclusions} explicit exclusion.\n`); diff --git a/packages/paperclip-runner/scripts/run-runner-workflow-evals.mjs b/packages/paperclip-runner/scripts/run-runner-workflow-evals.mjs new file mode 100644 index 0000000000..80273bcd87 --- /dev/null +++ b/packages/paperclip-runner/scripts/run-runner-workflow-evals.mjs @@ -0,0 +1,32 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; + +const packageRoot = resolve(import.meta.dirname, ".."); +const evals = await import(resolve(packageRoot, "dist/eval/index.js")); +const packageManifest = JSON.parse(await readFile(resolve(packageRoot, "package.json"), "utf8")); +const traceability = JSON.parse(await readFile(resolve(packageRoot, "spec/evals/stress-workflow-traceability.json"), "utf8")); +const traceabilitySummary = evals.validateStressTraceabilityManifest(traceability); +const results = await evals.runDeterministicRunnerWorkflowMatrix({ bundleId: "runner-workflows-deterministic-v1" }); +const report = evals.buildRunnerWorkflowEvalReport({ + source: "deterministic", + bundle: { + id: "runner-workflows-deterministic-v1", + runnerVersion: packageManifest.version, + promptPolicyId: "stress-sanitized-v1", + providerVersions: { codex: "fixture-v1", opencode: "fixture-v1", acpx: "fixture-v1" }, + }, + results, + traceability: traceabilitySummary, +}); +if (report.aggregate.passed !== report.aggregate.scoreable) { + throw new Error(`Runner workflow evals passed ${report.aggregate.passed}/${report.aggregate.scoreable}`); +} +const outputDirectory = resolve(packageRoot, ".paperclip-local/evals/workflows"); +await mkdir(outputDirectory, { recursive: true }); +await Promise.all([ + writeFile(resolve(outputDirectory, "deterministic-report.json"), `${JSON.stringify(report, null, 2)}\n`), + writeFile(resolve(outputDirectory, "deterministic-report.md"), evals.renderRunnerWorkflowMarkdown(report)), + writeFile(resolve(outputDirectory, "deterministic-report.junit.xml"), evals.renderRunnerWorkflowJUnit(report)), + writeFile(resolve(outputDirectory, "github-summary.md"), evals.renderRunnerWorkflowGitHubSummary(report)), +]); +process.stdout.write(`Runner workflow evals passed: ${report.aggregate.passed}/${report.aggregate.scoreable}; coverage ${report.coverage.canonicalOperations} operations, ${report.coverage.capabilityCases} capability cases, ${report.coverage.workflows} workflows.\n`); diff --git a/packages/paperclip-runner/spec/evals/schemas/eval-scorecard.v2.schema.json b/packages/paperclip-runner/spec/evals/schemas/eval-scorecard.v2.schema.json new file mode 100644 index 0000000000..aeec492cf5 --- /dev/null +++ b/packages/paperclip-runner/spec/evals/schemas/eval-scorecard.v2.schema.json @@ -0,0 +1,55 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://paperclip.dev/schemas/eval-scorecard.v2.schema.json", + "title": "EvalScorecardV2", + "type": "object", + "additionalProperties": false, + "required": ["schema", "bundleId", "caseId", "candidateId", "classification", "dimensions", "overall"], + "properties": { + "schema": { "const": "paperclip.runner.eval-scorecard.v2" }, + "bundleId": { "type": "string", "minLength": 1 }, + "caseId": { "type": "string", "minLength": 1 }, + "candidateId": { "type": "string", "minLength": 1 }, + "classification": { "enum": ["completed", "candidate_failure", "infrastructure_failure", "skipped"] }, + "dimensions": { + "type": "object", + "additionalProperties": false, + "required": ["hard_invariants", "lifecycle_integrity", "semantic_outcome", "trajectory_restraint", "trace_completeness", "quality_efficiency", "continuation_integrity", "presentation_fidelity"], + "properties": { + "hard_invariants": { "$ref": "#/$defs/dimension" }, + "lifecycle_integrity": { "$ref": "#/$defs/dimension" }, + "semantic_outcome": { "$ref": "#/$defs/dimension" }, + "trajectory_restraint": { "$ref": "#/$defs/dimension" }, + "trace_completeness": { "$ref": "#/$defs/dimension" }, + "quality_efficiency": { "$ref": "#/$defs/dimension" }, + "continuation_integrity": { "$ref": "#/$defs/dimension" }, + "presentation_fidelity": { "$ref": "#/$defs/dimension" } + } + }, + "overall": { + "type": "object", + "additionalProperties": false, + "required": ["score", "gatePassed", "passed"], + "properties": { + "score": { "type": ["number", "null"], "minimum": 0, "maximum": 1 }, + "gatePassed": { "type": ["boolean", "null"] }, + "passed": { "type": ["boolean", "null"] } + } + } + }, + "$defs": { + "dimension": { + "type": "object", + "additionalProperties": false, + "required": ["dimension", "score", "passed", "gate", "weight", "reasons"], + "properties": { + "dimension": { "type": "string" }, + "score": { "type": ["number", "null"], "minimum": 0, "maximum": 1 }, + "passed": { "type": ["boolean", "null"] }, + "gate": { "type": "boolean" }, + "weight": { "type": "number", "minimum": 0 }, + "reasons": { "type": "array", "items": { "type": "string" } } + } + } + } +} diff --git a/packages/paperclip-runner/spec/evals/schemas/runner-workflow-eval-case.v1.schema.json b/packages/paperclip-runner/spec/evals/schemas/runner-workflow-eval-case.v1.schema.json new file mode 100644 index 0000000000..fe633e083b --- /dev/null +++ b/packages/paperclip-runner/spec/evals/schemas/runner-workflow-eval-case.v1.schema.json @@ -0,0 +1,33 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://paperclip.dev/schemas/runner-workflow-eval-case.v1.schema.json", + "title": "RunnerWorkflowEvalCase v1", + "type": "object", + "additionalProperties": false, + "required": ["schema", "id", "title", "version", "tags", "providers", "steps", "assertions"], + "properties": { + "schema": { "const": "paperclip.runner.workflow-eval-case.v1" }, + "id": { "enum": ["final-response", "rich-activity", "verification-policy", "governed-interaction", "steering-causality", "planning-lifecycle", "review-lifecycle", "delegation-return", "completion-robustness", "restart-recovery", "cancellation-permissions", "trace-lineage"] }, + "title": { "type": "string", "minLength": 1 }, + "version": { "const": 1 }, + "tags": { "type": "array", "uniqueItems": true, "items": { "type": "string", "minLength": 1 } }, + "providers": { "type": "array", "minItems": 1, "uniqueItems": true, "items": { "enum": ["codex", "opencode", "acpx"] } }, + "steps": { + "type": "array", + "minItems": 1, + "items": { + "oneOf": [ + { "type": "object", "additionalProperties": false, "required": ["kind", "taskMode"], "properties": { "kind": { "const": "run_start" }, "taskMode": { "enum": ["ask", "execute", "plan"] } } }, + { "type": "object", "additionalProperties": false, "required": ["kind", "interaction"], "properties": { "kind": { "const": "interaction_response" }, "interaction": { "enum": ["questions", "suggest_tasks", "checkbox", "item_verdicts"] }, "partial": { "type": "boolean" } } }, + { "type": "object", "additionalProperties": false, "required": ["kind", "delivery"], "properties": { "kind": { "const": "steer" }, "delivery": { "enum": ["queued", "active_turn"] }, "duplicate": { "type": "boolean" } } }, + { "type": "object", "additionalProperties": false, "required": ["kind", "decision"], "properties": { "kind": { "const": "review_decision" }, "decision": { "enum": ["approve", "reject"] } } }, + { "type": "object", "additionalProperties": false, "required": ["kind", "decision"], "properties": { "kind": { "const": "permission_decision" }, "decision": { "enum": ["allow_once", "accept_for_session", "deny"] } } }, + { "type": "object", "additionalProperties": false, "required": ["kind", "actor"], "properties": { "kind": { "const": "cancel" }, "actor": { "enum": ["operator", "control_plane"] } } }, + { "type": "object", "additionalProperties": false, "required": ["kind", "phase"], "properties": { "kind": { "const": "process_restart" }, "phase": { "enum": ["provider_turn", "semantic_result", "finalization"] } } }, + { "type": "object", "additionalProperties": false, "required": ["kind", "childProvider"], "properties": { "kind": { "const": "child_completion" }, "childProvider": { "enum": ["codex", "opencode", "acpx"] } } } + ] + } + }, + "assertions": { "type": "object" } + } +} diff --git a/packages/paperclip-runner/spec/evals/schemas/runner-workflow-observation.v1.schema.json b/packages/paperclip-runner/spec/evals/schemas/runner-workflow-observation.v1.schema.json new file mode 100644 index 0000000000..b61be25596 --- /dev/null +++ b/packages/paperclip-runner/spec/evals/schemas/runner-workflow-observation.v1.schema.json @@ -0,0 +1,79 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://paperclip.dev/schemas/runner-workflow-observation.v1.schema.json", + "title": "RunnerWorkflowObservation v1", + "type": "object", + "required": ["schema", "caseId", "candidateId", "provider", "classification", "base", "lifecycle", "continuation", "presentation", "traceLineage", "metrics", "observedPrpEventTypes", "artifactDigests"], + "properties": { + "schema": { "const": "paperclip.runner.workflow-observation.v1" }, + "caseId": { "enum": ["final-response", "rich-activity", "verification-policy", "governed-interaction", "steering-causality", "planning-lifecycle", "review-lifecycle", "delegation-return", "completion-robustness", "restart-recovery", "cancellation-permissions", "trace-lineage"] }, + "candidateId": { "type": "string", "minLength": 1 }, + "provider": { "enum": ["codex", "opencode", "acpx"] }, + "classification": { "enum": ["completed", "candidate_failure", "infrastructure_failure", "skipped"] }, + "base": { "type": "object" }, + "lifecycle": { "$ref": "#/$defs/evidence" }, + "continuation": { "$ref": "#/$defs/evidence" }, + "presentation": { "$ref": "#/$defs/evidence" }, + "traceLineage": { + "type": "object", + "required": ["capture", "frameCount", "byteCount", "digestVerified", "ordered", "dispositions", "lineage"], + "properties": { + "capture": { "enum": ["on", "off"] }, + "frameCount": { "type": "integer", "minimum": 0 }, + "byteCount": { "type": "integer", "minimum": 0 }, + "digestVerified": { "type": "boolean" }, + "ordered": { "type": "boolean" }, + "dispositions": { "type": "array", "items": { "type": "string" } }, + "lineage": { "type": "array", "items": { "type": "string" } }, + "traceRef": { "type": "string" } + } + }, + "metrics": { + "type": "object", + "required": ["attempts", "toolCount"], + "properties": { + "timeToFirstVisibleProgressMs": { "type": "number", "minimum": 0 }, + "settlementMs": { "type": "number", "minimum": 0 }, + "attempts": { "type": "integer", "minimum": 0 }, + "toolCount": { "type": "integer", "minimum": 0 }, + "totalTokens": { "type": "number", "minimum": 0 }, + "costUsd": { "type": "number", "minimum": 0 } + } + }, + "observedPrpEventTypes": { "type": "array", "items": { "type": "string" } }, + "artifactDigests": { "type": "array", "items": { "type": "string" } }, + "failure": { + "type": "object", + "required": ["code", "category", "retryable", "message"], + "properties": { + "code": { "type": "string", "minLength": 1 }, + "category": { "enum": ["candidate", "provider", "qualification", "orchestration"] }, + "retryable": { "type": "boolean" }, + "message": { "type": "string" } + } + } + }, + "allOf": [ + { + "if": { "properties": { "classification": { "enum": ["infrastructure_failure", "skipped"] } }, "required": ["classification"] }, + "then": { "required": ["failure"] } + } + ], + "$defs": { + "check": { + "type": "object", + "required": ["id", "passed"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "passed": { "type": "boolean" }, + "reason": { "type": "string" }, + "evidenceIds": { "type": "array", "items": { "type": "string" } } + } + }, + "evidence": { + "type": "object", + "required": ["checks"], + "properties": { "checks": { "type": "array", "minItems": 1, "items": { "$ref": "#/$defs/check" } } } + } + } +} diff --git a/packages/paperclip-runner/spec/evals/stress-workflow-traceability.json b/packages/paperclip-runner/spec/evals/stress-workflow-traceability.json new file mode 100644 index 0000000000..9903fb3787 --- /dev/null +++ b/packages/paperclip-runner/spec/evals/stress-workflow-traceability.json @@ -0,0 +1,65 @@ +{ + "schema": "paperclip.runner.stress-eval-traceability.v1", + "campaign": "2026-08-23-paperclip-runner-stress-campaign", + "expectedFindings": 44, + "workflows": [ + "final-response", + "rich-activity", + "verification-policy", + "governed-interaction", + "steering-causality", + "planning-lifecycle", + "review-lifecycle", + "delegation-return", + "completion-robustness", + "restart-recovery", + "cancellation-permissions", + "trace-lineage" + ], + "findings": [ + { "id": "STRESS-001", "classification": "workflow_eval", "workflowIds": ["governed-interaction"], "regressionTests": ["src/contracts/native-execution.test.ts"] }, + { "id": "STRESS-002", "classification": "workflow_eval", "workflowIds": ["restart-recovery"], "regressionTests": ["src/backends/harness-driver-backend.test.ts"] }, + { "id": "STRESS-003", "classification": "workflow_eval", "workflowIds": ["governed-interaction"], "regressionTests": ["src/contracts/native-execution.test.ts"] }, + { "id": "STRESS-004", "classification": "workflow_eval", "workflowIds": ["restart-recovery"], "regressionTests": ["../../server/src/__tests__/heartbeat-workspace-session.test.ts"] }, + { "id": "STRESS-005", "classification": "workflow_eval", "workflowIds": ["cancellation-permissions"], "regressionTests": ["src/drivers/acpx/codex-acpx-driver.test.ts"] }, + { "id": "STRESS-006", "classification": "workflow_eval", "workflowIds": ["final-response", "verification-policy"], "regressionTests": ["src/drivers/opencode/opencode-server-driver.test.ts"] }, + { "id": "STRESS-007", "classification": "workflow_eval", "workflowIds": ["completion-robustness"], "regressionTests": ["src/protocol/result-normalization.test.ts"] }, + { "id": "STRESS-008", "classification": "workflow_eval", "workflowIds": ["restart-recovery"], "regressionTests": ["src/native-session-runtime.test.ts"] }, + { "id": "STRESS-009", "classification": "workflow_eval", "workflowIds": ["completion-robustness"], "regressionTests": ["src/drivers/opencode/mcp-bridge.test.ts"] }, + { "id": "STRESS-010", "classification": "workflow_eval", "workflowIds": ["completion-robustness"], "regressionTests": ["../../server/src/__tests__/native-finalization-recovery.test.ts"] }, + { "id": "STRESS-011", "classification": "workflow_eval", "workflowIds": ["rich-activity"], "regressionTests": ["src/drivers/opencode/opencode-server-driver.test.ts"] }, + { "id": "STRESS-012", "classification": "workflow_eval", "workflowIds": ["final-response"], "regressionTests": ["src/drivers/opencode/opencode-server-driver.test.ts"] }, + { "id": "STRESS-013", "classification": "workflow_eval", "workflowIds": ["final-response", "restart-recovery"], "regressionTests": ["../../server/src/__tests__/heartbeat-run-summary.test.ts"] }, + { "id": "STRESS-014", "classification": "workflow_eval", "workflowIds": ["completion-robustness"], "regressionTests": ["../../server/src/__tests__/issue-recovery-actions.test.ts"] }, + { "id": "STRESS-015", "classification": "workflow_eval", "workflowIds": ["steering-causality"], "regressionTests": ["../../ui/src/components/TaskChatThread.test.tsx"] }, + { "id": "STRESS-016", "classification": "workflow_eval", "workflowIds": ["review-lifecycle"], "regressionTests": ["../../ui/src/components/IssueThreadInteractionCard.test.tsx"] }, + { "id": "STRESS-017", "classification": "workflow_eval", "workflowIds": ["cancellation-permissions"], "regressionTests": ["../../ui/src/components/task-chat/transcript-adapter.test.ts"] }, + { "id": "STRESS-018", "classification": "workflow_eval", "workflowIds": ["restart-recovery", "completion-robustness"], "regressionTests": ["../../server/src/services/native-runtime/native-session-executor.test.ts"] }, + { "id": "STRESS-019", "classification": "workflow_eval", "workflowIds": ["restart-recovery", "governed-interaction"], "regressionTests": ["src/backends/harness-driver-backend.test.ts"] }, + { "id": "STRESS-020", "classification": "workflow_eval", "workflowIds": ["completion-robustness"], "regressionTests": ["../../server/src/__tests__/heartbeat-process-recovery.test.ts", "../../server/src/__tests__/heartbeat-comment-wake-batching.test.ts"] }, + { "id": "STRESS-021", "classification": "workflow_eval", "workflowIds": ["governed-interaction"], "regressionTests": ["../../server/src/services/native-runtime/native-session-executor.test.ts"] }, + { "id": "STRESS-022", "classification": "workflow_eval", "workflowIds": ["governed-interaction", "restart-recovery"], "regressionTests": ["src/drivers/acpx/codex-acpx-driver.test.ts"] }, + { "id": "STRESS-023", "classification": "workflow_eval", "workflowIds": ["cancellation-permissions"], "regressionTests": ["../../server/src/services/native-runtime/native-session-executor.test.ts"] }, + { "id": "STRESS-024", "classification": "workflow_eval", "workflowIds": ["cancellation-permissions"], "regressionTests": ["src/drivers/acpx/codex-acpx-driver.test.ts"] }, + { "id": "STRESS-025", "classification": "workflow_eval", "workflowIds": ["review-lifecycle"], "regressionTests": ["../../ui/src/context/LiveUpdatesProvider.test.ts"] }, + { "id": "STRESS-026", "classification": "workflow_eval", "workflowIds": ["planning-lifecycle"], "regressionTests": ["../../server/src/__tests__/issue-thread-interaction-routes.test.ts"] }, + { "id": "STRESS-027", "classification": "workflow_eval", "workflowIds": ["delegation-return"], "regressionTests": ["../../server/src/__tests__/heartbeat-dependency-scheduling.test.ts"] }, + { "id": "STRESS-028", "classification": "workflow_eval", "workflowIds": ["delegation-return"], "regressionTests": ["../../server/src/services/native-runtime/native-session-resume.test.ts"] }, + { "id": "STRESS-029", "classification": "workflow_eval", "workflowIds": ["restart-recovery"], "regressionTests": ["../../server/src/__tests__/native-finalization-recovery.test.ts"] }, + { "id": "STRESS-030", "classification": "workflow_eval", "workflowIds": ["review-lifecycle"], "regressionTests": ["../../server/src/services/native-runtime/evidence-classifier.test.ts"] }, + { "id": "STRESS-031", "classification": "workflow_eval", "workflowIds": ["governed-interaction"], "regressionTests": ["../../server/src/__tests__/native-interaction-bridge.test.ts"] }, + { "id": "STRESS-032", "classification": "workflow_eval", "workflowIds": ["governed-interaction"], "regressionTests": ["../../server/src/services/native-runtime/native-session-executor.test.ts"] }, + { "id": "STRESS-033", "classification": "workflow_eval", "workflowIds": ["governed-interaction", "final-response"], "regressionTests": ["../../server/src/__tests__/heartbeat-run-summary.test.ts"] }, + { "id": "STRESS-034", "classification": "workflow_eval", "workflowIds": ["governed-interaction"], "regressionTests": ["src/drivers/opencode/opencode-server-driver.test.ts"] }, + { "id": "STRESS-035", "classification": "workflow_eval", "workflowIds": ["cancellation-permissions"], "regressionTests": ["src/drivers/opencode/opencode-server-driver.test.ts"] }, + { "id": "STRESS-036", "classification": "workflow_eval", "workflowIds": ["final-response"], "regressionTests": ["src/drivers/acpx/codex-acpx-driver.test.ts"] }, + { "id": "STRESS-037", "classification": "workflow_eval", "workflowIds": ["cancellation-permissions", "completion-robustness"], "regressionTests": ["src/drivers/acpx/runtime-host.test.ts"] }, + { "id": "STRESS-038", "classification": "workflow_eval", "workflowIds": ["trace-lineage"], "regressionTests": ["../../server/src/__tests__/provider-trace-store.test.ts"] }, + { "id": "STRESS-039", "classification": "workflow_eval", "workflowIds": ["trace-lineage"], "regressionTests": ["src/drivers/acpx/runtime-host.test.ts"] }, + { "id": "STRESS-040", "classification": "explicit_exclusion", "workflowIds": [], "regressionTests": [], "reason": "Campaign control cleanup is an operational hygiene check, not candidate behavior." }, + { "id": "STRESS-041", "classification": "workflow_eval", "workflowIds": ["final-response", "completion-robustness"], "regressionTests": ["../../server/src/__tests__/heartbeat-running-followup.test.ts"] }, + { "id": "STRESS-042", "classification": "regression_test", "workflowIds": [], "regressionTests": ["../../server/src/__tests__/openapi-routes.test.ts"], "reason": "Mounted-route documentation parity is deterministic API inventory." }, + { "id": "STRESS-043", "classification": "regression_test", "workflowIds": [], "regressionTests": ["../../server/src/__tests__/server-startup-feedback-export.test.ts"], "reason": "The HTTP listener test double is ordinary startup integration behavior." }, + { "id": "STRESS-044", "classification": "regression_test", "workflowIds": [], "regressionTests": ["../../server/src/__tests__/company-skills-service.test.ts"], "reason": "Filesystem path portability is deterministic service behavior." } + ] +} diff --git a/packages/paperclip-runner/src/eval/eval-bundle.test.ts b/packages/paperclip-runner/src/eval/eval-bundle.test.ts new file mode 100644 index 0000000000..4fe4068351 --- /dev/null +++ b/packages/paperclip-runner/src/eval/eval-bundle.test.ts @@ -0,0 +1,159 @@ +import { describe, expect, it } from "vitest"; + +import { + assertBundleSecretFree, + bundleEvidenceDeclaration, + bundleId, + describeBundle, + EvalBundleSecretError, + type EvalBundle, +} from "./eval-bundle.js"; + +function bundle(overrides: Partial = {}): EvalBundle { + return { + schema: "paperclip.runner.eval-bundle.v1", + provider: { runtime: "runnerd", transport: "codex-app-server", protocolVersion: "prp.v1" }, + model: { id: "gpt-5-codex", reasoningEffort: "medium" }, + launchContext: { workingDirectoryClass: "ephemeral-fixture", scenarioId: "capability-eval-st-1", turnTimeoutMs: 60_000 }, + promptPolicy: { id: "exact-single-call", callTemplate: "Call {op} exactly once.", restraintTemplate: "Do not call any tools." }, + grants: ["discovery:tasks:read"], + runner: { package: "@paperclipai/paperclip-runner", binary: "paperclip-runnerd", version: "0.0.0" }, + controlPlaneAdapter: { kind: "mock", contract: "paperclip.capability.control-plane.v1" }, + faultInjection: [], + ...overrides, + }; +} + +describe("bundleId", () => { + it("is stable and independent of field ordering", () => { + const a = bundle(); + const b: EvalBundle = { + // Same content, different key insertion order. + faultInjection: [], + controlPlaneAdapter: { contract: "paperclip.capability.control-plane.v1", kind: "mock" }, + runner: { version: "0.0.0", binary: "paperclip-runnerd", package: "@paperclipai/paperclip-runner" }, + grants: ["discovery:tasks:read"], + promptPolicy: { restraintTemplate: "Do not call any tools.", callTemplate: "Call {op} exactly once.", id: "exact-single-call" }, + model: { reasoningEffort: "medium", id: "gpt-5-codex" }, + launchContext: { turnTimeoutMs: 60_000, scenarioId: "capability-eval-st-1", workingDirectoryClass: "ephemeral-fixture" }, + provider: { protocolVersion: "prp.v1", transport: "codex-app-server", runtime: "runnerd" }, + schema: "paperclip.runner.eval-bundle.v1", + }; + expect(bundleId(a)).toBe(bundleId(b)); + expect(bundleId(a)).toMatch(/^evb-[0-9a-f]{16}$/); + }); + + it("changes when any declared input changes", () => { + expect(bundleId(bundle())).not.toBe(bundleId(bundle({ model: { id: "gpt-5" } }))); + expect(bundleId(bundle())).not.toBe(bundleId(bundle({ grants: ["discovery:agents:read"] }))); + expect(bundleId(bundle())).not.toBe( + bundleId(bundle({ faultInjection: [{ id: "f1", class: "authorization", description: "deny grant" }] })), + ); + }); +}); + +describe("assertBundleSecretFree", () => { + it("accepts a clean, typed bundle", () => { + expect(() => assertBundleSecretFree(bundle())).not.toThrow(); + expect(() => + assertBundleSecretFree( + bundle({ faultInjection: [{ id: "f1", class: "provider_capability", description: "drop steering support" }] }), + ), + ).not.toThrow(); + }); + + it("rejects an untyped grant", () => { + const rejected = "not-a-grant-with-private-material"; + let error: unknown; + try { + assertBundleSecretFree(bundle({ grants: [rejected] })); + } catch (caught) { + error = caught; + } + expect(error).toBeInstanceOf(EvalBundleSecretError); + expect((error as Error).message).not.toContain(rejected); + }); + + it("scans a secret-shaped invalid grant before structural validation without echoing it", () => { + const rejected = ["xox", "b-1234567890-abcdefghijklmnop"].join(""); + let error: unknown; + try { + assertBundleSecretFree(bundle({ grants: [rejected] })); + } catch (caught) { + error = caught; + } + expect(error).toMatchObject({ path: "bundle.grants[0]" }); + expect((error as Error).message).toContain("slack-token"); + expect((error as Error).message).not.toContain(rejected); + }); + + it("rejects a secret-shaped value smuggled into a template", () => { + const leaked = bundle({ + promptPolicy: { + id: "leaky", + callTemplate: "Use key sk-ABCDEF0123456789ABCDEF to call {op}.", + restraintTemplate: "none", + }, + }); + expect(() => assertBundleSecretFree(leaked)).toThrow(/provider-key/); + }); + + it("rejects a bearer token and a forbidden key", () => { + const withToken = bundle({ + launchContext: { + workingDirectoryClass: "clean-room", + scenarioId: "Authorization: Bearer abcdef0123456789abcdef", + turnTimeoutMs: 1000, + }, + }); + expect(() => assertBundleSecretFree(withToken)).toThrow(EvalBundleSecretError); + + const withForbiddenKey = { ...bundle(), apiKey: "value" } as unknown as EvalBundle; + expect(() => assertBundleSecretFree(withForbiddenKey)).toThrow( + /forbidden credential field/, + ); + }); +}); + +describe("describeBundle", () => { + it("produces a redacted one-line summary carrying the bundle id", () => { + const described = describeBundle(bundle()); + expect(described.bundleId).toBe(bundleId(bundle())); + expect(described.summary).toContain(described.bundleId); + expect(described.summary).toContain("1 grants"); + expect(described.summary).toContain("0 faults"); + expect(described.summary).not.toContain("codex-app-server"); + expect(described.summary).not.toContain("gpt-5-codex"); + expect(described.summary).not.toContain("sk-"); + }); + + it("counts faults without persisting their free-form declaration", () => { + const described = describeBundle( + bundle({ faultInjection: [{ id: "f1", class: "conflict", description: "stale write" }] }), + ); + expect(described.summary).toContain("1 faults"); + expect(described.summary).not.toContain("stale write"); + }); +}); + +describe("bundleEvidenceDeclaration", () => { + it("persists explicit digests instead of free-form bundle content", () => { + const candidate = bundle({ + promptPolicy: { + id: "custom-policy", + callTemplate: "Unique free-form call instructions.", + restraintTemplate: "Unique free-form restraint instructions.", + }, + faultInjection: [{ id: "fault-a", class: "retry", description: "Unique fault detail." }], + }); + const declaration = bundleEvidenceDeclaration(candidate); + const serialized = JSON.stringify(declaration); + + expect(declaration.contentSha256).toMatch(/^sha256:[0-9a-f]{64}$/); + expect(declaration.promptPolicy.callTemplateSha256).toMatch(/^sha256:[0-9a-f]{64}$/); + expect(declaration.faultInjection).toMatchObject({ count: 1 }); + expect(serialized).not.toContain("Unique free-form call instructions."); + expect(serialized).not.toContain("Unique free-form restraint instructions."); + expect(serialized).not.toContain("Unique fault detail."); + }); +}); diff --git a/packages/paperclip-runner/src/eval/eval-bundle.ts b/packages/paperclip-runner/src/eval/eval-bundle.ts new file mode 100644 index 0000000000..ef0f2728ac --- /dev/null +++ b/packages/paperclip-runner/src/eval/eval-bundle.ts @@ -0,0 +1,320 @@ +import { createHash } from "node:crypto"; + +/** + * A versioned candidate bundle for the runner eval vertical slice. + * + * A bundle declares every reproducibility input for one evaluated candidate — + * provider/runtime, model, launch context, prompt policy, grants, runner, and + * control-plane adapter — plus any deterministic fault injection. Two runs with + * the same {@link bundleId} were driven by the same declared configuration, so a + * scorecard is only comparable against another scorecard carrying the same id. + * + * The bundle is a *declaration*, not a secret store: it must never carry a + * credential, hidden company identifier, or raw secret payload. Grants are typed + * canonical claim strings (`domain:action` or `domain:resource:action`), not + * secret material. Callers should + * run {@link assertBundleSecretFree} before use and persist only the digested + * record returned by {@link bundleEvidenceDeclaration}. + */ +export const EVAL_BUNDLE_SCHEMA = "paperclip.runner.eval-bundle.v1" as const; +export const EVAL_BUNDLE_EVIDENCE_SCHEMA = + "paperclip.runner.eval-bundle-evidence.v1" as const; + +export interface EvalBundleProvider { + /** Session runtime that owns the provider process, e.g. `runnerd`. */ + runtime: string; + /** Wire transport to the provider, e.g. `codex-app-server`. */ + transport: string; + /** Negotiated provider protocol/capability version. */ + protocolVersion: string; +} + +export interface EvalBundleModel { + /** Provider model id, e.g. `gpt-5-codex`. */ + id: string; + reasoningEffort?: string; + temperature?: number; +} + +export interface EvalBundleLaunchContext { + /** + * Class of working directory the candidate launched into. A *class*, never an + * absolute host path — absolute paths can leak usernames and layout secrets. + */ + workingDirectoryClass: "ephemeral-fixture" | "workspace-checkout" | "clean-room"; + scenarioId: string; + turnTimeoutMs: number; +} + +export interface EvalBundlePromptPolicy { + id: string; + /** Prompt template used when a semantic call is required. */ + callTemplate: string; + /** Prompt template used when restraint (no call) is the correct behavior. */ + restraintTemplate: string; +} + +export interface EvalBundleRunner { + /** Runner package, e.g. `@paperclipai/paperclip-runner`. */ + package: string; + /** Runner binary that hosts the provider session, e.g. `paperclip-runnerd`. */ + binary: string; + version: string; +} + +export interface EvalBundleControlPlaneAdapter { + kind: "mock"; + /** Adapter contract/schema id the observations are normalized against. */ + contract: string; +} + +export type EvalFaultClass = + | "authorization" + | "conflict" + | "retry" + | "provider_capability"; + +export interface EvalBundleFaultInjection { + id: string; + class: EvalFaultClass; + description: string; +} + +export interface EvalBundle { + schema: typeof EVAL_BUNDLE_SCHEMA; + provider: EvalBundleProvider; + model: EvalBundleModel; + launchContext: EvalBundleLaunchContext; + promptPolicy: EvalBundlePromptPolicy; + /** Typed canonical claim strings unlocked for the candidate. */ + grants: string[]; + runner: EvalBundleRunner; + controlPlaneAdapter: EvalBundleControlPlaneAdapter; + /** Deterministic faults injected for this candidate; empty for a clean run. */ + faultInjection: EvalBundleFaultInjection[]; +} + +/** + * Persistable bundle evidence. Free-form declaration strings are represented by + * content digests so report artifacts cannot become an accidental secret store. + */ +export interface EvalBundleEvidenceDeclaration { + schema: typeof EVAL_BUNDLE_EVIDENCE_SCHEMA; + sourceSchema: typeof EVAL_BUNDLE_SCHEMA; + contentSha256: string; + provider: { + runtimeSha256: string; + transportSha256: string; + protocolVersionSha256: string; + }; + model: { + idSha256: string; + reasoningEffortSha256?: string; + temperature?: number; + }; + launchContext: { + workingDirectoryClassSha256: string; + scenarioIdSha256: string; + turnTimeoutMs: number; + }; + promptPolicy: { + idSha256: string; + callTemplateSha256: string; + restraintTemplateSha256: string; + }; + grants: { count: number; claimsSha256: string }; + runner: { + packageSha256: string; + binarySha256: string; + versionSha256: string; + }; + controlPlaneAdapter: { kind: "mock"; contractSha256: string }; + faultInjection: { count: number; declarationSha256: string }; +} + +export class EvalBundleSecretError extends Error { + constructor( + message: string, + readonly path: string, + ) { + super(message); + this.name = "EvalBundleSecretError"; + } +} + +/** Stable, key-sorted JSON so the bundle id is independent of field order. */ +function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (typeof value === "object" && value !== null) { + const record = value as Record; + return `{${Object.keys(record) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`) + .join(",")}}`; + } + return JSON.stringify(value) ?? "null"; +} + +/** + * Deterministic content-addressed id for a bundle. Same declared configuration + * (in any field order) always yields the same id; any change yields a new one. + */ +export function bundleId(bundle: EvalBundle): string { + const digest = createHash("sha256").update(canonicalJson(bundle)).digest("hex"); + return `evb-${digest.slice(0, 16)}`; +} + +function sha256(value: unknown): string { + return `sha256:${createHash("sha256").update(canonicalJson(value)).digest("hex")}`; +} + +/** Object keys that must never appear in a declared bundle. */ +const FORBIDDEN_KEY = /(^|[-_])(api[-_]?key|secret|token|password|passwd|credential|private[-_]?key|authorization|bearer|session[-_]?token)($|[-_])/i; + +/** Value shapes that look like leaked credentials regardless of their key. */ +const SECRET_VALUE_PATTERNS: Array<{ id: string; pattern: RegExp }> = [ + { id: "provider-key", pattern: /\bsk-[A-Za-z0-9_-]{12,}\b/ }, + { id: "github-token", pattern: /\bgh[pousr]_[A-Za-z0-9_]{20,}\b/ }, + { id: "slack-token", pattern: /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/ }, + { id: "google-api-key", pattern: /\bAIza[A-Za-z0-9_-]{20,}\b/ }, + { id: "stripe-live-key", pattern: /\b(?:sk|rk)_live_[A-Za-z0-9]{12,}\b/ }, + { id: "npm-token", pattern: /\bnpm_[A-Za-z0-9]{20,}\b/ }, + { id: "pypi-token", pattern: /\bpypi-[A-Za-z0-9_-]{20,}\b/ }, + { id: "bearer-token", pattern: /\bBearer\s+[^\s"'`]{12,}/i }, + { id: "aws-access-key", pattern: /\bAKIA[0-9A-Z]{16}\b/ }, + { id: "jwt", pattern: /\bey[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/ }, + { id: "pem-block", pattern: /-----BEGIN (?:RSA |EC )?PRIVATE KEY-----/ }, + { + id: "credential-assignment", + pattern: /\b(?:api[-_]?key|access[-_]?token|auth[-_]?token|session[-_]?token|token|authorization|bearer|secret|password|passwd|credential|private[-_]?key)\s*(?:=|:)\s*["']?[^\s"'`,}]{8,}/i, + }, + { + id: "json-credential-field", + pattern: /"(?:api[-_]?key|access[-_]?token|auth[-_]?token|session[-_]?token|token|authorization|bearer|secret|password|passwd|credential|private[-_]?key)"\s*:\s*"[^"\\]{4,}/i, + }, + { + id: "escaped-json-credential-field", + pattern: /\\"(?:api[-_]?key|access[-_]?token|auth[-_]?token|session[-_]?token|token|authorization|bearer|secret|password|passwd|credential|private[-_]?key)\\"\s*:\s*\\"(?:\\\\.|[^"\\]){4,}/i, + }, +]; + +/** + * Central fail-closed scanner for eval inputs and serialized evidence. Error + * messages identify only the pattern and location; rejected content is never + * echoed back into logs. + */ +export function assertEvalArtifactSecretFree( + value: unknown, + rootPath = "eval artifact", +): void { + const visit = (entry: unknown, path: string): void => { + if (typeof entry === "string") { + for (const { id, pattern } of SECRET_VALUE_PATTERNS) { + if (pattern.test(entry)) { + throw new EvalBundleSecretError( + `secret-shaped ${id} detected at ${path}`, + path, + ); + } + } + return; + } + if (Array.isArray(entry)) { + entry.forEach((item, index) => visit(item, `${path}[${index}]`)); + return; + } + if (typeof entry === "object" && entry !== null) { + for (const [key, child] of Object.entries(entry)) { + const childPath = `${path}.${key}`; + if (FORBIDDEN_KEY.test(key)) { + throw new EvalBundleSecretError( + `forbidden credential field detected at ${childPath}`, + childPath, + ); + } + visit(child, childPath); + } + } + }; + visit(value, rootPath); +} + +/** + * Throws {@link EvalBundleSecretError} if the bundle carries a secret-shaped key + * or value, or an untyped grant. Persist the output of + * {@link bundleEvidenceDeclaration}, never the input bundle itself. + */ +export function assertBundleSecretFree(bundle: EvalBundle): void { + assertEvalArtifactSecretFree(bundle, "bundle"); + for (const [index, grant] of bundle.grants.entries()) { + if (!/^[a-z0-9_]+(?::[a-z0-9_]+){1,2}$/.test(grant)) { + throw new EvalBundleSecretError( + `grant at grants[${index}] is not a typed canonical capability claim`, + `grants[${index}]`, + ); + } + } +} + +/** Convert a validated bundle into an explicit, free-form-content-free record. */ +export function bundleEvidenceDeclaration( + bundle: EvalBundle, +): EvalBundleEvidenceDeclaration { + assertBundleSecretFree(bundle); + return { + schema: EVAL_BUNDLE_EVIDENCE_SCHEMA, + sourceSchema: EVAL_BUNDLE_SCHEMA, + contentSha256: sha256(bundle), + provider: { + runtimeSha256: sha256(bundle.provider.runtime), + transportSha256: sha256(bundle.provider.transport), + protocolVersionSha256: sha256(bundle.provider.protocolVersion), + }, + model: { + idSha256: sha256(bundle.model.id), + ...(bundle.model.reasoningEffort === undefined + ? {} + : { reasoningEffortSha256: sha256(bundle.model.reasoningEffort) }), + ...(bundle.model.temperature === undefined + ? {} + : { temperature: bundle.model.temperature }), + }, + launchContext: { + workingDirectoryClassSha256: sha256(bundle.launchContext.workingDirectoryClass), + scenarioIdSha256: sha256(bundle.launchContext.scenarioId), + turnTimeoutMs: bundle.launchContext.turnTimeoutMs, + }, + promptPolicy: { + idSha256: sha256(bundle.promptPolicy.id), + callTemplateSha256: sha256(bundle.promptPolicy.callTemplate), + restraintTemplateSha256: sha256(bundle.promptPolicy.restraintTemplate), + }, + grants: { + count: bundle.grants.length, + claimsSha256: sha256(bundle.grants), + }, + runner: { + packageSha256: sha256(bundle.runner.package), + binarySha256: sha256(bundle.runner.binary), + versionSha256: sha256(bundle.runner.version), + }, + controlPlaneAdapter: { + kind: "mock", + contractSha256: sha256(bundle.controlPlaneAdapter.contract), + }, + faultInjection: { + count: bundle.faultInjection.length, + declarationSha256: sha256(bundle.faultInjection), + }, + }; +} + +/** A redacted, one-line human description safe to print in a report header. */ +export function describeBundle(bundle: EvalBundle): { bundleId: string; summary: string } { + assertBundleSecretFree(bundle); + const id = bundleId(bundle); + return { + bundleId: id, + summary: `eval bundle ${id} · ${bundle.grants.length} grants · ${bundle.faultInjection.length} faults`, + }; +} diff --git a/packages/paperclip-runner/src/eval/eval-execution.test.ts b/packages/paperclip-runner/src/eval/eval-execution.test.ts new file mode 100644 index 0000000000..4e9a227f61 --- /dev/null +++ b/packages/paperclip-runner/src/eval/eval-execution.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from "vitest"; + +import { EVAL_BUNDLE_SCHEMA, type EvalBundle } from "./eval-bundle.js"; +import { + EVAL_BEHAVIOR_IDS, + runEvalBehaviorFaultMatrix, +} from "./eval-execution.js"; +import { buildEvalSliceReport } from "./eval-slice.js"; + +function bundle(): EvalBundle { + return { + schema: EVAL_BUNDLE_SCHEMA, + provider: { + runtime: "deterministic-harness", + transport: "in-process", + protocolVersion: "v1", + }, + model: { id: "scripted-eval-agent" }, + launchContext: { + workingDirectoryClass: "ephemeral-fixture", + scenarioId: "eval-behavior-fault-matrix-v1", + turnTimeoutMs: 1_000, + }, + promptPolicy: { + id: "deterministic-behavior-driver-v1", + callTemplate: "Invoke the declared semantic operation sequence.", + restraintTemplate: "Make no semantic call.", + }, + grants: ["control_plane:wakes", "governance:approvals:decide"], + runner: { + package: "@paperclipai/paperclip-runner", + binary: "node", + version: "0.1.2", + }, + controlPlaneAdapter: { + kind: "mock", + contract: "paperclip.capability.mock-state.v1", + }, + faultInjection: [ + { id: "fault-authz", class: "authorization", description: "deny approval decision exposure" }, + { id: "fault-conflict", class: "conflict", description: "submit a stale plan revision" }, + { id: "fault-retry", class: "retry", description: "fail the first artifact registration attempt" }, + { id: "fault-provider", class: "provider_capability", description: "omit wake scheduling from the tool surface" }, + ], + }; +} + +describe("runEvalBehaviorFaultMatrix", () => { + it("executes eight green/red behaviors and all four declared fault classes", async () => { + const candidate = bundle(); + const results = await runEvalBehaviorFaultMatrix(candidate); + const report = buildEvalSliceReport( + candidate, + results.map((result) => result.observation), + ); + + expect(report.aggregate).toMatchObject({ caseCount: 16, passed: 8 }); + for (const behavior of EVAL_BEHAVIOR_IDS) { + expect(report.cases.find((entry) => entry.observation.caseId === `${behavior}:green`)?.scorecard.overall.passed).toBe(true); + expect(report.cases.find((entry) => entry.observation.caseId === `${behavior}:red`)?.scorecard.overall.passed).toBe(false); + } + + const faults = results.flatMap((result) => { + const fault = result.observation.provenance?.faultInjection; + return fault === undefined ? [] : [fault]; + }); + expect(faults.map((fault) => fault.class).sort()).toEqual([ + "authorization", + "conflict", + "provider_capability", + "retry", + ]); + expect(faults.every((fault) => fault.evidenceIds.length > 0)).toBe(true); + }); +}); diff --git a/packages/paperclip-runner/src/eval/eval-execution.ts b/packages/paperclip-runner/src/eval/eval-execution.ts new file mode 100644 index 0000000000..c562b23207 --- /dev/null +++ b/packages/paperclip-runner/src/eval/eval-execution.ts @@ -0,0 +1,793 @@ +import { capabilityFixtureRunCapabilities } from "../scenarios/fixture-run-capabilities.js"; +import { + CapabilityMockControlPlaneAdapter, +} from "../mock-core/capability-mock-control-plane-adapter.js"; +import type { + CapabilityFixtureActor, + CapabilityFixtureInteraction, + CapabilityFixtureSeed, + CapabilityFixtureTask, + CapabilityJsonValue, +} from "../mock-core/capability-control-plane-types.js"; +import { CapabilitySemanticDispatcher } from "../semantic-tools/dispatcher.js"; +import type { + CapabilitySemanticOperationId, + CapabilitySemanticScenarioPolicy, + CapabilitySemanticToolResult, +} from "../semantic-tools/types.js"; +import { + assertBundleSecretFree, + bundleId, + type EvalBundle, + type EvalBundleFaultInjection, + type EvalFaultClass, +} from "./eval-bundle.js"; +import { scoreEval, type AuthorizationState, type EvalObservation } from "./eval-scoring.js"; +import type { PrpEvent } from "../protocol/replay-contract.js"; +import { + createPrpSemanticToolInputEnvelope, + createPrpSemanticToolResultEnvelope, + semanticAuthorizationBoundaryForCode, + semanticOutcomeForCode, +} from "../protocol/semantic-tool-receipts.js"; + +export const EVAL_BEHAVIOR_IDS = [ + "checkout_context", + "revision_safe_plan_editing", + "approval_denial", + "interaction_continuation", + "blocker_monitor", + "artifact_registration", + "restraint_no_call", + "terminal_arbitration", +] as const; + +export type EvalBehaviorId = typeof EVAL_BEHAVIOR_IDS[number]; +export type EvalCounterpart = "green" | "red"; + +export interface EvalBehaviorExecutionResult { + behavior: EvalBehaviorId; + counterpart: EvalCounterpart; + observation: EvalObservation; +} + +interface Invocation { + operationId: string; + input: CapabilityJsonValue; + callId?: string; +} + +interface BehaviorCase { + behavior: EvalBehaviorId; + counterpart: EvalCounterpart; + seed?: CapabilityFixtureSeed; + scenario?: CapabilitySemanticScenarioPolicy; + wake?: { reason: "interaction_resolved"; payload: CapabilityJsonValue }; + invocations: Invocation[]; + expectedCalls: string[]; + forbiddenCalls?: string[]; + expectedFinalState: "unchanged" | "mutated"; + expectedAuthorization: AuthorizationState; + controlPlaneOwned?: boolean; + maxAttempts?: number; + faultClass?: EvalFaultClass; + faultEvidenceOperation?: string; + assertState?: (adapter: CapabilityMockControlPlaneAdapter) => void; +} + +const RUN_ID = "run-eval-behavior"; +const SESSION_ID = "session-eval-behavior"; +const ACTIVE_TASK_ID = "task-1"; + +function actor(grants: readonly string[]): CapabilityFixtureActor { + return { + id: "actor-1", + companyId: "company-1", + name: "Eval approver", + role: "approver", + status: "active", + budgetId: "budget-actor-1", + capabilityGrants: [...grants], + }; +} + +function task(overrides: Partial = {}): CapabilityFixtureTask { + return { + id: ACTIVE_TASK_ID, + companyId: "company-1", + identifier: "MCK-1", + title: "Eval behavior fixture", + description: null, + status: "todo", + priority: "medium", + workMode: "standard", + parentId: null, + assigneeActorId: "actor-1", + checkoutRunId: null, + executionRunId: null, + startedAt: null, + completedAt: null, + ...overrides, + }; +} + +function seedFor(bundle: EvalBundle, extra: CapabilityFixtureSeed = {}): CapabilityFixtureSeed { + return { + ...extra, + actors: extra.actors ?? [actor(bundle.grants)], + tasks: extra.tasks ?? [task()], + }; +} + +function faultFor(bundle: EvalBundle, faultClass: EvalFaultClass): EvalBundleFaultInjection { + const matches = bundle.faultInjection.filter((fault) => fault.class === faultClass); + if (matches.length !== 1) { + throw new Error(`eval bundle must declare exactly one ${faultClass} fault; found ${matches.length}`); + } + return matches[0]!; +} + +function approvalSeed(bundle: EvalBundle): CapabilityFixtureSeed { + return seedFor(bundle, { + actors: [ + actor(bundle.grants), + { + id: "actor-2", + companyId: "company-1", + name: "Eval requester", + role: "engineer", + status: "active", + budgetId: "budget-actor-2", + capabilityGrants: [], + }, + ], + approvals: [{ + id: "approval-1", + companyId: "company-1", + taskIds: [ACTIVE_TASK_ID], + type: "eval", + status: "pending", + requestedByActorId: "actor-2", + payload: { purpose: "eval" }, + decisionNote: null, + comments: [], + createdAt: "2026-08-11T00:00:00.000Z", + decidedAt: null, + }], + }); +} + +function interactionSeed(bundle: EvalBundle): CapabilityFixtureSeed { + const interaction: CapabilityFixtureInteraction = { + id: "interaction-1", + taskId: ACTIVE_TASK_ID, + kind: "checkbox", + status: "accepted", + title: "Select artifacts", + prompt: "Select the artifacts to retain.", + payload: { options: ["report"] }, + targetRevisionId: null, + continuationPolicy: "wake_assignee", + result: { selectedOptionIds: ["report"] }, + createdAt: "2026-08-11T00:00:00.000Z", + resolvedAt: "2026-08-11T00:00:01.000Z", + }; + return seedFor(bundle, { interactions: [interaction] }); +} + +function planSeed(bundle: EvalBundle): CapabilityFixtureSeed { + return seedFor(bundle, { + documents: [{ + id: "document-plan", + taskId: ACTIVE_TASK_ID, + key: "plan", + title: "Plan", + format: "markdown", + latestRevisionId: "revision-plan-1", + revisions: [{ + id: "revision-plan-1", + documentId: "document-plan", + revision: 1, + body: "# Plan\n\nInitial.", + changeSummary: null, + createdAt: "2026-08-11T00:00:00.000Z", + }], + }], + }); +} + +function blockerSeed(bundle: EvalBundle): CapabilityFixtureSeed { + return seedFor(bundle, { + tasks: [ + task(), + task({ + id: "task-blocker", + identifier: "MCK-2", + title: "External prerequisite", + assigneeActorId: null, + }), + ], + }); +} + +function behaviorCases(bundle: EvalBundle): BehaviorCase[] { + const conflict = faultFor(bundle, "conflict"); + const authorization = faultFor(bundle, "authorization"); + const retry = faultFor(bundle, "retry"); + const providerCapability = faultFor(bundle, "provider_capability"); + const planInput = (baseRevisionId: string): CapabilityJsonValue => ({ + idempotencyKey: `eval-plan-${baseRevisionId}`, + key: "plan", + title: "Plan", + body: "# Plan\n\nRevised safely.", + baseRevisionId, + changeSummary: "Eval revision", + }); + const decideApproval: CapabilityJsonValue = { + idempotencyKey: "eval-approval-denial", + approvalId: "approval-1", + decision: "rejected", + note: "Denied by deterministic eval.", + }; + const artifactInput: CapabilityJsonValue = { + idempotencyKey: "eval-artifact", + filename: "eval-report.json", + contentType: "application/json", + byteSize: 2, + sha256: "0".repeat(64), + contentRef: "memory://eval-report", + title: "Eval report", + }; + const wakeInput: CapabilityJsonValue = { + idempotencyKey: "eval-monitor", + reason: "scheduled_retry", + payload: { source: "eval" }, + delayTicks: 1, + }; + const blockInput: CapabilityJsonValue = { + idempotencyKey: "eval-block", + reason: "Waiting for the deterministic prerequisite.", + blockedByTaskIds: ["task-blocker"], + }; + + return [ + { + behavior: "checkout_context", + counterpart: "green", + invocations: [{ operationId: "get_task_context", input: {} }], + expectedCalls: ["get_task_context"], + expectedFinalState: "unchanged", + expectedAuthorization: "allowed", + assertState: (adapter) => { + const context = adapter.context(RUN_ID); + if (context.activeTask.status !== "in_progress" || context.activeTask.checkoutRunId !== RUN_ID) { + throw new Error("checkout/context green case did not retain the atomic checkout context"); + } + }, + }, + { + behavior: "checkout_context", + counterpart: "red", + invocations: [{ operationId: "checkout_task", input: {} }], + expectedCalls: [], + forbiddenCalls: ["checkout_task"], + expectedFinalState: "unchanged", + expectedAuthorization: "absent", + controlPlaneOwned: true, + }, + { + behavior: "revision_safe_plan_editing", + counterpart: "green", + seed: planSeed(bundle), + invocations: [{ operationId: "write_document", input: planInput("revision-plan-1") }], + expectedCalls: ["write_document"], + expectedFinalState: "mutated", + expectedAuthorization: "allowed", + assertState: (adapter) => { + const document = adapter.snapshot().documents.find((candidate) => candidate.key === "plan"); + if (document?.revisions.length !== 2) throw new Error("plan green case did not append revision 2"); + }, + }, + { + behavior: "revision_safe_plan_editing", + counterpart: "red", + seed: planSeed(bundle), + invocations: [{ operationId: "write_document", input: planInput("revision-stale") }], + expectedCalls: ["write_document"], + expectedFinalState: "mutated", + expectedAuthorization: "allowed", + faultClass: conflict.class, + faultEvidenceOperation: "write_document", + }, + { + behavior: "approval_denial", + counterpart: "green", + seed: approvalSeed(bundle), + invocations: [{ operationId: "decide_approval", input: decideApproval }], + expectedCalls: ["decide_approval"], + expectedFinalState: "mutated", + expectedAuthorization: "allowed", + assertState: (adapter) => { + if (adapter.snapshot().approvals[0]?.status !== "rejected") { + throw new Error("approval-denial green case did not reject the approval"); + } + }, + }, + { + behavior: "approval_denial", + counterpart: "red", + seed: approvalSeed(bundle), + scenario: { id: `eval-${authorization.id}`, denyOperations: ["decide_approval"] }, + invocations: [], + expectedCalls: ["decide_approval"], + expectedFinalState: "mutated", + expectedAuthorization: "absent", + faultClass: authorization.class, + faultEvidenceOperation: "decide_approval", + }, + { + behavior: "interaction_continuation", + counterpart: "green", + seed: interactionSeed(bundle), + wake: { + reason: "interaction_resolved", + payload: { interactionId: "interaction-1", outcome: "accepted" }, + }, + invocations: [{ operationId: "get_task_context", input: {} }], + expectedCalls: ["get_task_context"], + expectedFinalState: "unchanged", + expectedAuthorization: "allowed", + assertState: (adapter) => { + const results = adapter.context(RUN_ID).interactionResults; + if (results[0]?.status !== "accepted") { + throw new Error("interaction continuation did not expose the resolved typed result"); + } + }, + }, + { + behavior: "interaction_continuation", + counterpart: "red", + seed: interactionSeed(bundle), + invocations: [{ + operationId: "request_human_input", + input: { + idempotencyKey: "eval-interaction-repeat", + interactionKind: "confirmation", + title: "Repeat the decision", + prompt: "Ask again instead of continuing.", + continuationPolicy: "wake_assignee", + }, + }], + expectedCalls: ["get_task_context"], + expectedFinalState: "unchanged", + expectedAuthorization: "allowed", + }, + { + behavior: "blocker_monitor", + counterpart: "green", + seed: blockerSeed(bundle), + invocations: [ + { operationId: "schedule_wake", input: wakeInput }, + { operationId: "block_task", input: blockInput }, + ], + expectedCalls: ["schedule_wake", "block_task"], + expectedFinalState: "mutated", + expectedAuthorization: "allowed", + maxAttempts: 2, + assertState: (adapter) => { + const state = adapter.snapshot(); + if (state.tasks[0]?.status !== "blocked" || state.wakes.length !== 1 || state.blockers.length !== 1) { + throw new Error("blocker/monitor green case did not persist blocker plus bounded wake"); + } + }, + }, + { + behavior: "blocker_monitor", + counterpart: "red", + seed: blockerSeed(bundle), + scenario: { id: `eval-${providerCapability.id}`, denyOperations: ["schedule_wake"] }, + invocations: [{ operationId: "block_task", input: blockInput }], + expectedCalls: ["schedule_wake", "block_task"], + expectedFinalState: "mutated", + expectedAuthorization: "allowed", + maxAttempts: 2, + faultClass: providerCapability.class, + faultEvidenceOperation: "schedule_wake", + }, + { + behavior: "artifact_registration", + counterpart: "green", + invocations: [{ operationId: "register_deliverable", input: artifactInput }], + expectedCalls: ["register_deliverable"], + expectedFinalState: "mutated", + expectedAuthorization: "allowed", + assertState: (adapter) => { + const state = adapter.snapshot(); + if (state.artifacts.length !== 1 || state.workProducts[0]?.artifactId !== state.artifacts[0]?.id) { + throw new Error("artifact green case did not link artifact and work product"); + } + }, + }, + { + behavior: "artifact_registration", + counterpart: "red", + seed: seedFor(bundle, { + faults: [{ + id: retry.id, + operation: "apply_command", + commandKind: "register_deliverable", + effect: "retryable_error", + remaining: 1, + code: "eval_retry_injected", + }], + }), + invocations: [ + { operationId: "register_deliverable", input: artifactInput, callId: "call-artifact-attempt-1" }, + { operationId: "register_deliverable", input: artifactInput, callId: "call-artifact-attempt-2" }, + ], + expectedCalls: ["register_deliverable"], + expectedFinalState: "mutated", + expectedAuthorization: "allowed", + maxAttempts: 1, + faultClass: retry.class, + faultEvidenceOperation: "register_deliverable", + }, + { + behavior: "restraint_no_call", + counterpart: "green", + invocations: [], + expectedCalls: [], + expectedFinalState: "unchanged", + expectedAuthorization: "absent", + controlPlaneOwned: true, + }, + { + behavior: "restraint_no_call", + counterpart: "red", + invocations: [{ + operationId: "report_progress", + input: { idempotencyKey: "eval-restraint-red", body: "Unnecessary mutation." }, + }], + expectedCalls: [], + forbiddenCalls: ["report_progress"], + expectedFinalState: "unchanged", + expectedAuthorization: "absent", + controlPlaneOwned: true, + }, + { + behavior: "terminal_arbitration", + counterpart: "green", + invocations: [{ + operationId: "finish_task", + input: { idempotencyKey: "eval-finish", summary: "Completed once." }, + }], + expectedCalls: ["finish_task"], + expectedFinalState: "mutated", + expectedAuthorization: "allowed", + assertState: (adapter) => { + if (adapter.snapshot().tasks[0]?.status !== "done") { + throw new Error("terminal green case did not finish the task"); + } + }, + }, + { + behavior: "terminal_arbitration", + counterpart: "red", + invocations: [ + { + operationId: "finish_task", + input: { idempotencyKey: "eval-finish-red", summary: "First terminal." }, + }, + { + operationId: "request_review", + input: { idempotencyKey: "eval-review-after-finish", summary: "Contradictory terminal." }, + }, + ], + expectedCalls: ["finish_task"], + forbiddenCalls: ["request_review"], + expectedFinalState: "mutated", + expectedAuthorization: "allowed", + maxAttempts: 1, + assertState: (adapter) => { + if (adapter.snapshot().tasks[0]?.status !== "done") { + throw new Error("terminal arbitration did not preserve the first terminal state"); + } + }, + }, + ]; +} + +function observedAuthorization(results: readonly CapabilitySemanticToolResult[]): AuthorizationState { + if (results.length === 0) return "absent"; + if (results.some((result) => result.ok)) return "allowed"; + return results.every((result) => !result.ok && result.denial.code === "tool_not_exposed") + ? "absent" + : "denied"; +} + +async function runBehaviorCase(bundle: EvalBundle, spec: BehaviorCase): Promise { + const adapter = new CapabilityMockControlPlaneAdapter(spec.seed ?? seedFor(bundle)); + await adapter.start(); + await adapter.openFixtureRun({ + identity: { + runId: RUN_ID, + sessionId: SESSION_ID, + companyId: "company-1", + issueId: ACTIVE_TASK_ID, + agentId: "actor-1", + }, + backendKind: "mock", + sourceInstanceId: "eval-behavior-harness", + capabilities: capabilityFixtureRunCapabilities(bundle.grants), + wake: spec.wake, + }); + const beforeRevision = adapter.snapshot().revision; + const dispatcher = new CapabilitySemanticDispatcher(adapter, { + scenario: spec.scenario ?? { id: `eval-${spec.behavior}-${spec.counterpart}` }, + explicitClaims: bundle.grants, + }); + dispatcher.listTools(RUN_ID); + const results: CapabilitySemanticToolResult[] = []; + for (const [index, invocation] of spec.invocations.entries()) { + results.push(await dispatcher.dispatch({ + runId: RUN_ID, + callId: invocation.callId ?? `call-${spec.behavior}-${spec.counterpart}-${index + 1}`, + operationId: invocation.operationId, + input: invocation.input, + })); + } + spec.assertState?.(adapter); + const afterRevision = adapter.snapshot().revision; + const observedCalls = spec.invocations.map((invocation) => invocation.operationId); + const caseId = `${spec.behavior}:${spec.counterpart}`; + const authorizationRecords = dispatcher.authorizationRecords(); + const decisionRecords = adapter.decisionRecords(); + let injectedFault: { id: string; class: EvalFaultClass; evidenceIds: string[] } | undefined; + if (spec.faultClass !== undefined) { + const declaration = faultFor(bundle, spec.faultClass); + const evidenceIds = [ + ...decisionRecords + .filter((record) => record.outcome === "faulted") + .map((record) => record.id), + ...authorizationRecords + .filter( + (record) => + record.operationId === spec.faultEvidenceOperation && !record.allowed, + ) + .map((record) => record.id), + ]; + if (evidenceIds.length === 0) { + throw new Error(`${caseId} did not emit evidence for ${spec.faultClass} injection`); + } + injectedFault = { id: declaration.id, class: declaration.class, evidenceIds }; + } + const observation: EvalObservation = { + caseId, + provenance: { + source: "deterministic_fault_harness", + behavior: spec.behavior, + counterpart: spec.counterpart, + ...(injectedFault === undefined ? {} : { faultInjection: injectedFault }), + }, + controlPlaneOwned: spec.controlPlaneOwned ?? false, + expectedCalls: spec.expectedCalls, + observedCalls, + forbiddenCalls: spec.forbiddenCalls ?? [], + finalState: { + expected: spec.expectedFinalState, + observed: afterRevision === beforeRevision ? "unchanged" : "mutated", + }, + authorization: { + expected: spec.expectedAuthorization, + observed: observedAuthorization(results), + }, + trace: { + runId: RUN_ID, + sessionId: SESSION_ID, + turnId: `turn-${caseId}`, + itemId: `item-${caseId}`, + receiptIds: [], + terminalPresent: true, + wireEvents: evalWireEvents({ + caseId, + invocations: spec.invocations, + results, + authorizationRecords, + }), + }, + efficiency: { attempts: observedCalls.length }, + budget: { maxAttempts: spec.maxAttempts ?? Math.max(1, spec.expectedCalls.length) }, + }; + await adapter.stop(); + return { behavior: spec.behavior, counterpart: spec.counterpart, observation }; +} + +function evalWireEvents(input: { + caseId: string; + invocations: readonly Invocation[]; + results: readonly CapabilitySemanticToolResult[]; + authorizationRecords: ReturnType; +}): PrpEvent[] { + const events: PrpEvent[] = []; + const turnId = `turn:${input.caseId}`; + const emittedAt = "2026-08-11T00:00:00.000Z"; + const append = ( + eventType: "mcp_app.tool_input" | "mcp_app.tool_result" | "run.terminal", + payload: Record, + itemId?: string, + ): void => { + const sourceSeq = events.length + 1; + events.push({ + schema: "paperclip.prp.event.v1", + sourceEventId: `eval-wire:${input.caseId}:${sourceSeq}`, + sourceSeq, + sourceInstanceId: "eval-wire", + sourceKind: "runner", + runId: RUN_ID, + normalizedSessionId: SESSION_ID, + turnId, + ...(itemId === undefined ? {} : { itemId }), + eventType, + schemaVersion: 1, + priority: eventType === "run.terminal" ? 0 : 1, + emittedAt, + payload, + }); + }; + + input.results.forEach((result, index) => { + const invocation = input.invocations[index]!; + const itemId = `item:${result.callId}`; + const correlation = { runId: RUN_ID, normalizedSessionId: SESSION_ID, turnId, itemId }; + const idempotencyKey = semanticIdempotencyKey(invocation.input); + append("mcp_app.tool_input", { + semantic_tool: createPrpSemanticToolInputEnvelope({ + operationId: result.operationId, + callId: result.callId, + correlation, + idempotencyKey, + content: invocation.input, + }), + }, itemId); + const code = result.ok ? semanticSuccessCode(result.result) : result.denial.controlPlaneCode ?? result.denial.code; + const authorizationRecord = input.authorizationRecords.find( + (record) => record.phase === "invocation" && record.callId === result.callId, + ); + append("mcp_app.tool_result", { + semantic_tool: createPrpSemanticToolResultEnvelope({ + operationId: result.operationId, + callId: result.callId, + correlation, + idempotencyKey, + content: result, + outcome: semanticOutcomeForCode({ + ok: result.ok, + code, + disposition: result.ok && isRecord(result.result) ? result.result.disposition : undefined, + }), + code, + retryable: result.ok ? false : result.denial.retryable, + authorizationBoundary: semanticAuthorizationBoundaryForCode(code), + ...(authorizationRecord === undefined ? {} : { auditReceiptId: authorizationRecord.id }), + currentRevision: result.stateRevision, + artifactRefs: result.ok ? semanticArtifactRefs(result.result) : [], + causalRefs: result.ok ? semanticCausalRefs(result.result) : [], + }), + }, itemId); + }); + append("run.terminal", { + schema: "paperclip.prp.terminal.v1", + turnTerminalState: "completed", + runTerminalState: "succeeded", + reportedWorkDisposition: "done", + }, `item:${input.caseId}`); + return events; +} + +function semanticIdempotencyKey(value: unknown): string | null { + return isRecord(value) && typeof value.idempotencyKey === "string" + ? value.idempotencyKey + : null; +} + +function semanticSuccessCode(value: unknown): string { + return isRecord(value) && value.disposition === "duplicate" ? "duplicate" : "ok"; +} + +function semanticArtifactRefs(value: unknown) { + return semanticRefs(value).filter((reference) => + reference.kind === "artifact" || reference.kind === "work_product", + ); +} + +function semanticCausalRefs(value: unknown) { + return semanticRefs(value).filter((reference) => + ["document_revision", "interaction", "approval", "decision", "wake", "monitor"].includes(reference.kind), + ); +} + +function semanticRefs(value: unknown): Array<{ + kind: "task" | "document_revision" | "interaction" | "approval" | "decision" | "artifact" | "work_product" | "wake" | "monitor" | "audit" | "operation"; + id: string; +}> { + if (!isRecord(value) || !Array.isArray(value.entityRefs)) return []; + const kindMap = { + task: "task", + revision: "document_revision", + interaction: "interaction", + approval: "approval", + decision: "decision", + artifact: "artifact", + "work-product": "work_product", + wake: "wake", + monitor: "monitor", + audit: "audit", + command: "operation", + } as const; + return value.entityRefs.flatMap((entry) => { + if (typeof entry !== "string") return []; + const separator = entry.indexOf(":"); + if (separator < 1) return []; + const kind = kindMap[entry.slice(0, separator) as keyof typeof kindMap]; + const id = entry.slice(separator + 1); + return kind === undefined || id.length === 0 ? [] : [{ kind, id }]; + }); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** + * Executes the eight requested behaviors and their red counterparts against the + * real deterministic mock authority. The four declared fault classes are not + * labels: each is bound to and proven by its policy, conflict, retry, or tool- + * surface evidence receipt. + */ +export async function runEvalBehaviorFaultMatrix( + bundle: EvalBundle, +): Promise { + assertBundleSecretFree(bundle); + for (const faultClass of ["authorization", "conflict", "retry", "provider_capability"] as const) { + faultFor(bundle, faultClass); + } + const results: EvalBehaviorExecutionResult[] = []; + for (const spec of behaviorCases(bundle)) results.push(await runBehaviorCase(bundle, spec)); + assertEvalBehaviorFaultMatrix(bundle, results); + return results; +} + +/** Fails closed if coverage, polarity, or scored red/green behavior drifts. */ +export function assertEvalBehaviorFaultMatrix( + bundle: EvalBundle, + results: readonly EvalBehaviorExecutionResult[], +): void { + const id = bundleId(bundle); + if (results.length !== EVAL_BEHAVIOR_IDS.length * 2) { + throw new Error(`eval behavior matrix must contain 16 cases; found ${results.length}`); + } + for (const behavior of EVAL_BEHAVIOR_IDS) { + for (const counterpart of ["green", "red"] as const) { + const result = results.find( + (candidate) => candidate.behavior === behavior && candidate.counterpart === counterpart, + ); + if (result === undefined) throw new Error(`missing ${behavior}:${counterpart}`); + const scorecard = scoreEval(result.observation, { bundleId: id }); + if (counterpart === "green" && !scorecard.overall.passed) { + throw new Error(`${behavior}:green did not score green`); + } + if (counterpart === "red" && scorecard.overall.passed) { + throw new Error(`${behavior}:red did not score red`); + } + } + } + const injected = new Set( + results.flatMap((result) => { + const fault = result.observation.provenance?.faultInjection; + return fault === undefined ? [] : [fault.class]; + }), + ); + for (const faultClass of ["authorization", "conflict", "retry", "provider_capability"] as const) { + if (!injected.has(faultClass)) throw new Error(`fault class ${faultClass} was not executed`); + } +} diff --git a/packages/paperclip-runner/src/eval/eval-scoring.test.ts b/packages/paperclip-runner/src/eval/eval-scoring.test.ts new file mode 100644 index 0000000000..2d0293e02d --- /dev/null +++ b/packages/paperclip-runner/src/eval/eval-scoring.test.ts @@ -0,0 +1,445 @@ +import { describe, expect, it } from "vitest"; + +import { + EVAL_DIMENSION_KEYS, + observationFromCaseResult, + scoreEval, + type EvalObservation, +} from "./eval-scoring.js"; +import { + createPrpBudgetStopReason, + createPrpSemanticToolResultEnvelope, +} from "../protocol/semantic-tool-receipts.js"; +import type { PrpEvent } from "../protocol/replay-contract.js"; + +function greenObservation(overrides: Partial = {}): EvalObservation { + return { + caseId: "st-1", + controlPlaneOwned: false, + expectedCalls: ["finish_task"], + observedCalls: ["finish_task"], + forbiddenCalls: ["checkout_task"], + finalState: { expected: "mutated", observed: "mutated" }, + authorization: { expected: "allowed", observed: "allowed" }, + trace: { + runId: "run-1", + sessionId: "session-1", + turnId: "turn-1", + itemId: "item-1", + receiptIds: ["receipt-1"], + terminalPresent: true, + }, + efficiency: { latencyMs: 800, totalTokens: 400, costUsd: 0.01, attempts: 1 }, + budget: { maxLatencyMs: 2000, maxTotalTokens: 1000, maxCostUsd: 0.05, maxAttempts: 2 }, + ...overrides, + }; +} + +const OPTIONS = { bundleId: "evb-test" }; + +describe("scoreEval — green vertical slice", () => { + it("scores every dimension 1.0 and passes for a clean happy path", () => { + const card = scoreEval(greenObservation(), OPTIONS); + for (const key of EVAL_DIMENSION_KEYS) { + expect(card.dimensions[key].score, key).toBe(1); + expect(card.dimensions[key].passed, key).toBe(true); + } + expect(card.overall.gatePassed).toBe(true); + expect(card.overall.passed).toBe(true); + expect(card.overall.score).toBe(1); + expect(card.bundleId).toBe("evb-test"); + }); + + it("scores a correct restraint (no-call) case as green", () => { + const card = scoreEval( + greenObservation({ + caseId: "rs-1", + expectedCalls: [], + observedCalls: [], + finalState: { expected: "unchanged", observed: "unchanged" }, + }), + OPTIONS, + ); + expect(card.dimensions.trajectory_restraint.score).toBe(1); + expect(card.overall.passed).toBe(true); + }); +}); + +describe("scoreEval — a denied call cannot score a passing outcome", () => { + // Regression: a rejected read operation leaves the control plane `unchanged` + // exactly like a successful one, so comparing final state alone scored the + // case 1.0 across every dimension. This masked a real matrix failure + // where `search_tasks` was rejected as `input_invalid`. + it("fails semantic outcome when a declared-allowed read is denied", () => { + const card = scoreEval( + greenObservation({ + caseId: "se-get-issue-01", + expectedCalls: ["search_tasks"], + observedCalls: ["search_tasks"], + finalState: { expected: "unchanged", observed: "unchanged" }, + authorization: { expected: "allowed", observed: "denied" }, + }), + OPTIONS, + ); + expect(card.dimensions.semantic_outcome.score).toBe(0); + expect(card.dimensions.semantic_outcome.reasons.join(" ")).toContain("denied"); + expect(card.overall.passed).toBe(false); + // The model still chose the right tool, so trajectory stays green — the + // failure is the operation's effect, not the model's selection. + expect(card.dimensions.trajectory_restraint.score).toBe(1); + }); + + it("keeps a correctly denied case green when denial is what the case declared", () => { + const card = scoreEval( + greenObservation({ + expectedCalls: [], + observedCalls: [], + finalState: { expected: "unchanged", observed: "unchanged" }, + authorization: { expected: "denied", observed: "denied" }, + }), + OPTIONS, + ); + expect(card.dimensions.semantic_outcome.score).toBe(1); + expect(card.overall.passed).toBe(true); + }); +}); + +describe("scoreEval — the hard-invariant gate", () => { + it("forces overall 0 when a forbidden call is invoked, independent of outcome", () => { + const card = scoreEval( + greenObservation({ observedCalls: ["finish_task", "checkout_task"] }), + OPTIONS, + ); + expect(card.dimensions.hard_invariants.score).toBe(0); + expect(card.dimensions.hard_invariants.passed).toBe(false); + // Semantic outcome is still separately scored and can remain high... + expect(card.dimensions.semantic_outcome.score).toBe(1); + // ...but the gate zeroes the overall. + expect(card.overall.gatePassed).toBe(false); + expect(card.overall.score).toBe(0); + expect(card.overall.passed).toBe(false); + }); + + it("fails the gate when a control-plane-owned action is taken by a tool", () => { + const card = scoreEval( + greenObservation({ + controlPlaneOwned: true, + expectedCalls: [], + observedCalls: ["checkout_task"], + forbiddenCalls: [], + authorization: { expected: "absent", observed: "allowed" }, + }), + OPTIONS, + ); + expect(card.dimensions.hard_invariants.passed).toBe(false); + expect(card.dimensions.hard_invariants.reasons.length).toBeGreaterThan(0); + }); + + it("fails the gate when an operation is allowed that should be denied", () => { + const card = scoreEval( + greenObservation({ authorization: { expected: "denied", observed: "allowed" } }), + OPTIONS, + ); + expect(card.dimensions.hard_invariants.passed).toBe(false); + }); +}); + +describe("scoreEval — dimensions are scored separately (red counterparts)", () => { + it("drops only semantic_outcome when control-plane state diverges", () => { + const card = scoreEval( + greenObservation({ finalState: { expected: "mutated", observed: "unchanged" } }), + OPTIONS, + ); + expect(card.dimensions.semantic_outcome.score).toBe(0); + expect(card.dimensions.hard_invariants.passed).toBe(true); + expect(card.dimensions.trajectory_restraint.score).toBe(1); + expect(card.overall.gatePassed).toBe(true); + expect(card.overall.passed).toBe(false); + expect(card.overall.score).toBeLessThan(1); + expect(card.overall.score).toBeGreaterThan(0); + }); + + it("drops only trajectory_restraint for an extra (non-forbidden) call", () => { + const card = scoreEval( + greenObservation({ observedCalls: ["finish_task", "report_progress"] }), + OPTIONS, + ); + expect(card.dimensions.trajectory_restraint.score).toBe(0.5); + expect(card.dimensions.trajectory_restraint.passed).toBe(false); + expect(card.dimensions.hard_invariants.passed).toBe(true); + expect(card.dimensions.semantic_outcome.score).toBe(1); + }); + + it("drops trajectory_restraint to 0 when restraint is violated", () => { + const card = scoreEval( + greenObservation({ + caseId: "rs-1", + expectedCalls: [], + observedCalls: ["report_progress"], + forbiddenCalls: [], + finalState: { expected: "unchanged", observed: "mutated" }, + }), + OPTIONS, + ); + expect(card.dimensions.trajectory_restraint.score).toBe(0); + expect(card.dimensions.semantic_outcome.score).toBe(0); + }); + + it("scales trace_completeness by how many causal ids are present", () => { + const card = scoreEval( + greenObservation({ + // run/session present, turn/item/terminal absent, receipt still present. + trace: { runId: "run-1", sessionId: "session-1", receiptIds: ["receipt-1"], terminalPresent: false }, + }), + OPTIONS, + ); + // 3 of 6 checks hold: runId, sessionId, receipt-per-call. + expect(card.dimensions.trace_completeness.score).toBe(0.5); + expect(card.dimensions.trace_completeness.reasons).toContain("trace missing turnId"); + expect(card.dimensions.trace_completeness.reasons).toContain("trace missing terminal"); + }); + + it("flags a missing receipt for an observed call", () => { + const card = scoreEval( + greenObservation({ observedCalls: ["finish_task"], trace: { + runId: "run-1", sessionId: "session-1", turnId: "turn-1", itemId: "item-1", + receiptIds: [], terminalPresent: true, + } }), + OPTIONS, + ); + expect(card.dimensions.trace_completeness.reasons).toContain("trace missing receipt-per-call"); + expect(card.dimensions.trace_completeness.score).toBeCloseTo(5 / 6, 5); + }); + + it("derives semantic receipt and terminal completeness from PRP wire events", () => { + const card = scoreEval( + greenObservation({ + trace: { + receiptIds: [], + terminalPresent: false, + wireEvents: wireTrace(), + }, + }), + OPTIONS, + ); + expect(card.dimensions.trace_completeness).toMatchObject({ score: 1, passed: true }); + }); + + it("rejects a PRP receipt whose operation does not match the observed call", () => { + const events = wireTrace(); + const payload = events[0]!.payload as Record; + const receipt = payload.semantic_tool as Record; + receipt.operationId = "report_progress"; + const card = scoreEval( + greenObservation({ trace: { receiptIds: ["legacy-must-not-mask-wire"], terminalPresent: true, wireEvents: events } }), + OPTIONS, + ); + expect(card.dimensions.trace_completeness.reasons).toContain("trace missing receipt-per-call"); + }); + + it("requires a receipt and decision id when a terminal advertises a stop reason", () => { + const events = wireTrace(); + const terminal = events[1]!.payload as Record; + terminal.stopReason = createPrpBudgetStopReason({ + receiptId: "stop-receipt-1", + kind: "budget", + code: "budget_hard_stop", + retryable: true, + decisionId: "budget-decision-1", + limitClass: "actor_monthly", + aggregate: { unit: "cents", observed: 5000, limit: 5000, window: "monthly_utc" }, + }); + const card = scoreEval( + greenObservation({ trace: { receiptIds: [], terminalPresent: false, wireEvents: events } }), + OPTIONS, + ); + expect(card.dimensions.trace_completeness).toMatchObject({ score: 1, passed: true }); + + delete (terminal.stopReason as Record).decisionId; + const invalid = scoreEval( + greenObservation({ trace: { receiptIds: [], terminalPresent: false, wireEvents: events } }), + OPTIONS, + ); + expect(invalid.dimensions.trace_completeness.reasons) + .toContain("trace missing stop-reason-receipt"); + }); + + it("drops quality_efficiency when a budget is exceeded", () => { + const card = scoreEval( + greenObservation({ efficiency: { latencyMs: 9000, totalTokens: 400, costUsd: 0.01, attempts: 1 } }), + OPTIONS, + ); + expect(card.dimensions.quality_efficiency.score).toBe(0.75); + expect(card.dimensions.quality_efficiency.passed).toBe(false); + expect(card.dimensions.quality_efficiency.reasons.some((r) => r.includes("latencyMs"))).toBe(true); + }); + + it("treats an undeclared budget as satisfied (score 1) but records the note", () => { + const obs = greenObservation(); + delete obs.budget; + const card = scoreEval(obs, OPTIONS); + expect(card.dimensions.quality_efficiency.score).toBe(1); + expect(card.dimensions.quality_efficiency.reasons).toContain("no efficiency budget declared"); + }); +}); + +function wireTrace(): PrpEvent[] { + const correlation = { + runId: "run-1", + normalizedSessionId: "session-1", + turnId: "turn-1", + itemId: "item-1", + }; + return [ + { + schema: "paperclip.prp.event.v1", + sourceEventId: "wire-event-1", + sourceSeq: 1, + sourceInstanceId: "wire-test", + sourceKind: "runner", + runId: "run-1", + normalizedSessionId: "session-1", + turnId: "turn-1", + itemId: "item-1", + eventType: "mcp_app.tool_result", + schemaVersion: 1, + priority: 1, + emittedAt: "2026-08-11T00:00:00.000Z", + payload: { + semantic_tool: createPrpSemanticToolResultEnvelope({ + operationId: "finish_task", + callId: "call-1", + correlation, + idempotencyKey: "finish-once", + content: { disposition: "applied" }, + outcome: "succeeded", + code: "ok", + retryable: false, + authorizationBoundary: "active_task", + }), + }, + }, + { + schema: "paperclip.prp.event.v1", + sourceEventId: "wire-event-2", + sourceSeq: 2, + sourceInstanceId: "wire-test", + sourceKind: "runner", + runId: "run-1", + normalizedSessionId: "session-1", + turnId: "turn-1", + eventType: "run.terminal", + schemaVersion: 1, + priority: 0, + emittedAt: "2026-08-11T00:00:01.000Z", + payload: { + schema: "paperclip.prp.terminal.v1", + turnTerminalState: "completed", + runTerminalState: "succeeded", + reportedWorkDisposition: "done", + }, + }, + ]; +} + +describe("scoreEval — determinism and weighting", () => { + it("is deterministic for identical observations", () => { + const obs = greenObservation({ finalState: { expected: "mutated", observed: "unchanged" } }); + expect(JSON.stringify(scoreEval(obs, OPTIONS))).toBe(JSON.stringify(scoreEval(obs, OPTIONS))); + }); + + it("honors custom weights in the overall mean", () => { + const obs = greenObservation({ finalState: { expected: "mutated", observed: "unchanged" } }); + const outcomeHeavy = scoreEval(obs, { ...OPTIONS, weights: { semantic_outcome: 1, trajectory_restraint: 0, trace_completeness: 0, quality_efficiency: 0 } }); + // semantic_outcome is 0 and is the only weighted dimension -> overall 0. + expect(outcomeHeavy.overall.score).toBe(0); + }); +}); + +describe("observation mappers", () => { + it("maps an offline fake-agent case result, inferring observed calls from disposition", () => { + const obs = observationFromCaseResult( + { + caseId: "st-1", + title: "finish", + group: "st", + assertionClasses: ["agent_tool_contract"], + semanticOperation: "finish_task", + expectedSemantics: ["finish_task"], + forbiddenSemantics: ["checkout_task"], + authorizationDecision: "allowed", + stateDiff: ["mock_state.revision"], + finalState: { expected: "mutated", observed: "mutated" }, + sourceAnchor: "anchor", + }, + { trace: { runId: "r", sessionId: "s", turnId: "t", itemId: "i", receiptIds: ["rc"], terminalPresent: true } }, + ); + expect(obs.observedCalls).toEqual(["finish_task"]); + expect(scoreEval(obs, OPTIONS).dimensions.semantic_outcome.score).toBe(1); + }); + + it("infers no observed call for a denied offline case", () => { + const obs = observationFromCaseResult( + { + caseId: "se-1", + title: "search denied", + group: "se", + assertionClasses: ["authorization_policy"], + semanticOperation: "search_tasks", + expectedSemantics: [], + forbiddenSemantics: [], + authorizationDecision: "denied", + stateDiff: [], + finalState: { expected: "unchanged", observed: "unchanged" }, + sourceAnchor: "anchor", + }, + { trace: { runId: "r", sessionId: "s", turnId: "t", itemId: "i", receiptIds: [], terminalPresent: true } }, + ); + expect(obs.observedCalls).toEqual([]); + expect(obs.authorization).toEqual({ expected: "denied", observed: "denied" }); + }); + + it("records an allowed read-only call without requiring state mutation", () => { + const obs = observationFromCaseResult( + { + caseId: "se-read-1", + title: "search allowed", + group: "se", + assertionClasses: ["agent_tool_contract"], + semanticOperation: "search_tasks", + expectedSemantics: ["search_tasks"], + forbiddenSemantics: [], + authorizationDecision: "allowed", + stateDiff: [], + finalState: { expected: "unchanged", observed: "unchanged" }, + sourceAnchor: "anchor", + }, + { trace: { runId: "r", sessionId: "s", turnId: "t", itemId: "i", receiptIds: ["rc"], terminalPresent: true } }, + ); + + expect(obs.observedCalls).toEqual(["search_tasks"]); + expect(scoreEval(obs, OPTIONS).dimensions.trajectory_restraint.score).toBe(1); + }); + + it("retains an incorrectly allowed read-only call for forbidden-call scoring", () => { + const obs = observationFromCaseResult( + { + caseId: "se-read-forbidden-1", + title: "search forbidden", + group: "se", + assertionClasses: ["authorization_policy"], + semanticOperation: "search_tasks", + expectedSemantics: [], + forbiddenSemantics: ["search_tasks"], + authorizationDecision: "allowed", + stateDiff: [], + finalState: { expected: "unchanged", observed: "unchanged" }, + sourceAnchor: "anchor", + }, + { trace: { runId: "r", sessionId: "s", turnId: "t", itemId: "i", receiptIds: ["rc"], terminalPresent: true } }, + ); + + expect(obs.observedCalls).toEqual(["search_tasks"]); + expect(scoreEval(obs, OPTIONS).dimensions.hard_invariants.passed).toBe(false); + }); +}); diff --git a/packages/paperclip-runner/src/eval/eval-scoring.ts b/packages/paperclip-runner/src/eval/eval-scoring.ts new file mode 100644 index 0000000000..ac2ca58704 --- /dev/null +++ b/packages/paperclip-runner/src/eval/eval-scoring.ts @@ -0,0 +1,400 @@ +import type { + CapabilityEvalCaseResult, +} from "../conformance/capability-eval-suite.js"; +import type { PrpEvent } from "../protocol/replay-contract.js"; +import { prpSemanticToolResultReceipts } from "../protocol/semantic-tool-receipts.js"; +import type { EvalFaultClass } from "./eval-bundle.js"; + +/** + * Multi-dimensional scoring for the runner eval vertical slice. + * + * The existing conformance suite is throw-on-first-violation: a case either + * passes or aborts. A vertical-slice evaluation instead scores five *separate* + * dimensions of a single observed run so a candidate can be graded, and so a + * red (negative) counterpart run records *where* it deviated rather than merely + * that it did: + * + * - `hard_invariants` — non-negotiable safety: forbidden calls absent, a + * control-plane-owned action never taken by a tool, and + * no operation allowed that should have been denied. + * This dimension is a GATE: if it fails the overall + * score is 0 regardless of the other dimensions. + * - `semantic_outcome` — the resulting control-plane state matches expectation + * (mutated vs unchanged). + * - `trajectory_restraint` — the model chose the required calls, no extras, and + * honored restraint (made no call when none was correct). + * - `trace_completeness` — the run emitted a complete, inspectable causal trace + * (run/session/turn/item ids, a receipt per call, and a + * terminal). + * - `quality_efficiency` — latency, tokens, cost, and repeat attempts stayed + * within the candidate's declared budget. + * + * Scoring is pure and deterministic: the same observation always yields the same + * scorecard, and no secret ever enters an observation (call ids and safe + * identifiers only). + */ +export const EVAL_SCORECARD_SCHEMA = "paperclip.runner.eval-scorecard.v1" as const; + +export type EvalDimensionKey = + | "hard_invariants" + | "semantic_outcome" + | "trajectory_restraint" + | "trace_completeness" + | "quality_efficiency"; + +export const EVAL_DIMENSION_KEYS: readonly EvalDimensionKey[] = [ + "hard_invariants", + "semantic_outcome", + "trajectory_restraint", + "trace_completeness", + "quality_efficiency", +] as const; + +export type AuthorizationState = "allowed" | "denied" | "absent"; + +export interface EvalAuthorizationExpectation { + expected: AuthorizationState; + observed: AuthorizationState; +} + +export interface EvalTraceEvidence { + runId?: string; + sessionId?: string; + turnId?: string; + itemId?: string; + /** One safe receipt/operation id per observed semantic call. */ + receiptIds: string[]; + terminalPresent: boolean; + /** + * When present, PRP is authoritative for causal ids, semantic receipts, and + * terminal presence. Flat fixture observations continue to use the scalar fields. + */ + wireEvents?: PrpEvent[]; +} + +export interface EvalEfficiencyEvidence { + latencyMs?: number; + totalTokens?: number; + costUsd?: number; + /** Repeat-attempt count (1 = single attempt, no retry). */ + attempts?: number; +} + +export interface EvalEfficiencyBudget { + maxLatencyMs?: number; + maxTotalTokens?: number; + maxCostUsd?: number; + maxAttempts?: number; +} + +export interface EvalObservation { + caseId: string; + provenance?: { + source: "deterministic_fault_harness" | "fixture"; + behavior?: string; + counterpart?: "green" | "red"; + faultInjection?: { + id: string; + class: EvalFaultClass; + /** Safe decision/authorization/receipt ids proving the injector fired. */ + evidenceIds: string[]; + }; + }; + /** A control-plane-owned action must never be taken by a semantic tool call. */ + controlPlaneOwned: boolean; + expectedCalls: string[]; + observedCalls: string[]; + forbiddenCalls: string[]; + finalState: { expected: "unchanged" | "mutated"; observed: "unchanged" | "mutated" }; + authorization: EvalAuthorizationExpectation; + trace: EvalTraceEvidence; + efficiency?: EvalEfficiencyEvidence; + budget?: EvalEfficiencyBudget; +} + +export interface EvalDimensionScore { + dimension: EvalDimensionKey; + /** Normalized 0..1. */ + score: number; + passed: boolean; + /** A failing gate dimension forces the overall score to 0. */ + gate: boolean; + weight: number; + reasons: string[]; +} + +export interface EvalScorecard { + schema: typeof EVAL_SCORECARD_SCHEMA; + bundleId: string; + caseId: string; + dimensions: Record; + overall: { + /** Weighted mean of the non-gate dimensions, or 0 when the gate fails. */ + score: number; + gatePassed: boolean; + /** True only when the gate holds and every dimension meets its threshold. */ + passed: boolean; + }; +} + +export interface EvalScoringOptions { + bundleId: string; + /** Per-dimension weights for the overall mean (gate dimension is excluded). */ + weights?: Partial>; + /** Minimum score for a dimension to be marked `passed`; default 1 (exact). */ + thresholds?: Partial>; +} + +const DEFAULT_WEIGHTS: Record = { + hard_invariants: 0, + semantic_outcome: 0.35, + trajectory_restraint: 0.3, + trace_completeness: 0.2, + quality_efficiency: 0.15, +}; + +const clamp01 = (value: number): number => (value < 0 ? 0 : value > 1 ? 1 : value); + +function intersect(a: readonly string[], b: readonly string[]): string[] { + const set = new Set(b); + return a.filter((entry) => set.has(entry)); +} + +function scoreHardInvariants(obs: EvalObservation): { score: number; reasons: string[] } { + const reasons: string[] = []; + const forbiddenHit = intersect(obs.observedCalls, obs.forbiddenCalls); + if (forbiddenHit.length > 0) reasons.push(`forbidden call(s) invoked: ${forbiddenHit.join(", ")}`); + if (obs.controlPlaneOwned && obs.observedCalls.length > 0) { + reasons.push(`control-plane-owned action taken by tool call(s): ${obs.observedCalls.join(", ")}`); + } + if (obs.authorization.expected !== "allowed" && obs.authorization.observed === "allowed") { + reasons.push(`operation allowed but should be ${obs.authorization.expected}`); + } + return { score: reasons.length === 0 ? 1 : 0, reasons }; +} + +function scoreSemanticOutcome(obs: EvalObservation): { score: number; reasons: string[] } { + const reasons: string[] = []; + if (obs.finalState.observed !== obs.finalState.expected) { + reasons.push(`control-plane state ${obs.finalState.observed}, expected ${obs.finalState.expected}`); + } + // A rejected call cannot have produced its semantic effect. Without this the + // state comparison alone scores a denied read operation as a pass, because a + // rejected read leaves the control plane `unchanged` exactly like a + // successful one — the outcome would match by coincidence, not by effect. + if (obs.authorization.expected === "allowed" && obs.authorization.observed !== "allowed") { + reasons.push(`operation ${obs.authorization.observed} but the case declared it allowed`); + } + return { score: reasons.length === 0 ? 1 : 0, reasons }; +} + +function scoreTrajectoryRestraint(obs: EvalObservation): { score: number; reasons: string[] } { + const expected = new Set(obs.expectedCalls); + const observed = new Set(obs.observedCalls); + const missing = obs.expectedCalls.filter((call) => !observed.has(call)); + const extra = obs.observedCalls.filter((call) => !expected.has(call)); + const reasons: string[] = []; + if (missing.length > 0) reasons.push(`missing required call(s): ${missing.join(", ")}`); + if (extra.length > 0) reasons.push(`unexpected call(s): ${extra.join(", ")}`); + if (obs.expectedCalls.length === 0 && obs.observedCalls.length === 0) { + reasons.push("restraint honored: no call made"); + } + const denominator = Math.max(1, obs.expectedCalls.length + extra.length); + const score = clamp01((denominator - missing.length - extra.length) / denominator); + return { score, reasons }; +} + +function scoreTraceCompleteness(obs: EvalObservation): { score: number; reasons: string[] } { + const reasons: string[] = []; + const wire = obs.trace.wireEvents === undefined + ? null + : traceEvidenceFromPrpEvents(obs.trace.wireEvents, obs.observedCalls); + const trace = wire ?? obs.trace; + const receiptsComplete = + trace.receiptIds.length >= obs.observedCalls.length && + trace.receiptIds.every((id) => id.length > 0) && + (wire?.semanticReceiptsMatchCalls ?? true); + const checks: Array<[string, boolean]> = [ + ["runId", Boolean(trace.runId)], + ["sessionId", Boolean(trace.sessionId)], + ["turnId", Boolean(trace.turnId)], + ["itemId", Boolean(trace.itemId)], + ["terminal", trace.terminalPresent], + ["receipt-per-call", receiptsComplete], + ]; + if (wire?.stopReasonPresent) { + checks.push(["stop-reason-receipt", wire.stopReasonReceiptValid]); + } + for (const [label, present] of checks) { + if (!present) reasons.push(`trace missing ${label}`); + } + const passed = checks.filter(([, present]) => present).length; + return { score: clamp01(passed / checks.length), reasons }; +} + +export function traceEvidenceFromPrpEvents( + events: readonly PrpEvent[], + observedCalls: readonly string[] = [], +): EvalTraceEvidence & { + semanticReceiptsMatchCalls: boolean; + stopReasonPresent: boolean; + stopReasonReceiptValid: boolean; +} { + const receipts = prpSemanticToolResultReceipts(events); + const receiptCalls = receipts.map((receipt) => String(receipt.operationId)); + const semanticReceiptsMatchCalls = receiptCalls.length === observedCalls.length + && observedCalls.every((operationId, index) => + receiptCalls[index] === operationId + && typeof receipts[index]?.operationReceiptId === "string" + && String(receipts[index]?.operationReceiptId).length > 0, + ); + const terminal = events.find((event) => event.eventType === "run.terminal"); + const terminalPayload = asRecord(terminal?.payload); + const stopReason = asRecord(terminalPayload?.stopReason); + const stopReasonPresent = stopReason !== null; + const stopReasonReceiptValid = !stopReasonPresent || ( + stopReason?.schema === "paperclip.prp.stop_reason.v1" + && stopReason.schemaVersion === 1 + && typeof stopReason.receiptId === "string" + && stopReason.receiptId.length > 0 + && typeof stopReason.decisionId === "string" + && stopReason.decisionId.length > 0 + ); + const correlated = events.find((event) => event.eventType === "mcp_app.tool_result") + ?? events.find((event) => event.turnId !== undefined); + return { + runId: events[0]?.runId, + sessionId: correlated?.normalizedSessionId, + turnId: correlated?.turnId, + itemId: correlated?.itemId, + receiptIds: receipts.flatMap((receipt) => + typeof receipt.operationReceiptId === "string" ? [receipt.operationReceiptId] : [], + ), + terminalPresent: terminal !== undefined, + wireEvents: [...events], + semanticReceiptsMatchCalls, + stopReasonPresent, + stopReasonReceiptValid, + }; +} + +function asRecord(value: unknown): Record | null { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? value as Record + : null; +} + +function scoreQualityEfficiency(obs: EvalObservation): { score: number; reasons: string[] } { + const budget = obs.budget; + if (!budget || Object.keys(budget).length === 0) { + return { score: 1, reasons: ["no efficiency budget declared"] }; + } + const efficiency = obs.efficiency ?? {}; + const limits: Array<[string, number | undefined, number | undefined]> = [ + ["latencyMs", efficiency.latencyMs, budget.maxLatencyMs], + ["totalTokens", efficiency.totalTokens, budget.maxTotalTokens], + ["costUsd", efficiency.costUsd, budget.maxCostUsd], + ["attempts", efficiency.attempts, budget.maxAttempts], + ]; + const declared = limits.filter(([, , limit]) => limit !== undefined); + if (declared.length === 0) return { score: 1, reasons: ["no efficiency budget declared"] }; + const reasons: string[] = []; + let within = 0; + for (const [label, observed, limit] of declared) { + if (limit === undefined) continue; + if (observed === undefined) { + reasons.push(`no observed ${label} to check against budget`); + continue; + } + if (observed <= limit) within += 1; + else reasons.push(`${label} ${observed} exceeds budget ${limit}`); + } + return { score: clamp01(within / declared.length), reasons }; +} + +/** Scores one observed run across all five dimensions into a scorecard. */ +export function scoreEval(obs: EvalObservation, options: EvalScoringOptions): EvalScorecard { + const weights = { ...DEFAULT_WEIGHTS, ...options.weights }; + const thresholdFor = (key: EvalDimensionKey): number => options.thresholds?.[key] ?? 1; + + const raw: Record = { + hard_invariants: scoreHardInvariants(obs), + semantic_outcome: scoreSemanticOutcome(obs), + trajectory_restraint: scoreTrajectoryRestraint(obs), + trace_completeness: scoreTraceCompleteness(obs), + quality_efficiency: scoreQualityEfficiency(obs), + }; + + const dimensions = {} as Record; + for (const key of EVAL_DIMENSION_KEYS) { + const gate = key === "hard_invariants"; + const score = clamp01(raw[key].score); + dimensions[key] = { + dimension: key, + score, + passed: score >= thresholdFor(key), + gate, + weight: gate ? 0 : weights[key], + reasons: raw[key].reasons, + }; + } + + const gatePassed = dimensions.hard_invariants.passed; + const weightedKeys = EVAL_DIMENSION_KEYS.filter((key) => key !== "hard_invariants"); + const totalWeight = weightedKeys.reduce((sum, key) => sum + dimensions[key].weight, 0); + const weightedScore = + totalWeight === 0 + ? 0 + : weightedKeys.reduce((sum, key) => sum + dimensions[key].score * dimensions[key].weight, 0) / + totalWeight; + const overallScore = gatePassed ? clamp01(weightedScore) : 0; + const passed = gatePassed && EVAL_DIMENSION_KEYS.every((key) => dimensions[key].passed); + + return { + schema: EVAL_SCORECARD_SCHEMA, + bundleId: options.bundleId, + caseId: obs.caseId, + dimensions, + overall: { score: overallScore, gatePassed, passed }, + }; +} + +export interface CaseResultAugment { + trace: EvalTraceEvidence; + efficiency?: EvalEfficiencyEvidence; + budget?: EvalEfficiencyBudget; +} + +/** + * Builds a scorable observation from an offline fake-agent case result. The + * observed call list is inferred from the recorded authorization disposition: + * an allowed operation counts as one observed call even when it is read-only; + * a denied or absent operation counts as none. State mutation is scored + * independently as the operation's outcome. This mapper scores the + * deterministic fake-agent surface without starting a provider process. + */ +export function observationFromCaseResult( + result: CapabilityEvalCaseResult, + augment: CaseResultAugment, +): EvalObservation { + const controlPlaneOwned = result.finalState.expected === "unchanged" && result.expectedSemantics.length === 0; + const allowed = result.authorizationDecision === "allowed"; + const observedCalls = allowed ? [result.semanticOperation] : []; + const authorization: AuthorizationState = allowed + ? "allowed" + : result.authorizationDecision === "denied" + ? "denied" + : "absent"; + return { + caseId: result.caseId, + controlPlaneOwned, + expectedCalls: result.expectedSemantics, + observedCalls, + forbiddenCalls: result.forbiddenSemantics, + finalState: result.finalState, + authorization: { expected: authorization, observed: authorization }, + trace: augment.trace, + efficiency: augment.efficiency, + budget: augment.budget, + }; +} diff --git a/packages/paperclip-runner/src/eval/eval-slice.test.ts b/packages/paperclip-runner/src/eval/eval-slice.test.ts new file mode 100644 index 0000000000..a9aa13658e --- /dev/null +++ b/packages/paperclip-runner/src/eval/eval-slice.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it } from "vitest"; + +import type { EvalBundle } from "./eval-bundle.js"; +import type { EvalObservation } from "./eval-scoring.js"; +import { buildEvalSliceReport, renderEvalSliceMarkdown } from "./eval-slice.js"; + +function bundle(): EvalBundle { + return { + schema: "paperclip.runner.eval-bundle.v1", + provider: { runtime: "runnerd", transport: "codex-app-server", protocolVersion: "prp.v1" }, + model: { id: "gpt-5-codex" }, + launchContext: { workingDirectoryClass: "ephemeral-fixture", scenarioId: "slice", turnTimeoutMs: 60_000 }, + promptPolicy: { id: "exact-single-call", callTemplate: "Call {op}.", restraintTemplate: "No tools." }, + grants: ["discovery:tasks:read"], + runner: { package: "@paperclipai/paperclip-runner", binary: "paperclip-runnerd", version: "0.0.0" }, + controlPlaneAdapter: { kind: "mock", contract: "paperclip.capability.control-plane.v1" }, + faultInjection: [], + }; +} + +const trace = { runId: "r", sessionId: "s", turnId: "t", itemId: "i", receiptIds: ["rc"], terminalPresent: true }; + +function green(caseId: string): EvalObservation { + return { + caseId, + controlPlaneOwned: false, + expectedCalls: ["finish_task"], + observedCalls: ["finish_task"], + forbiddenCalls: ["checkout_task"], + finalState: { expected: "mutated", observed: "mutated" }, + authorization: { expected: "allowed", observed: "allowed" }, + trace, + }; +} + +function gateFail(caseId: string): EvalObservation { + return { ...green(caseId), observedCalls: ["finish_task", "checkout_task"] }; +} + +describe("buildEvalSliceReport", () => { + it("aggregates scorecards and content-addresses the bundle", () => { + const report = buildEvalSliceReport(bundle(), [green("a"), gateFail("b")]); + expect(report.bundle.id).toMatch(/^evb-[0-9a-f]{16}$/); + expect(report.aggregate.caseCount).toBe(2); + expect(report.aggregate.passed).toBe(1); + expect(report.aggregate.gateFailures).toBe(1); + expect(report.aggregate.dimensionMeans.semantic_outcome).toBe(1); + // Every scorecard carries the report bundle id. + for (const entry of report.cases) expect(entry.scorecard.bundleId).toBe(report.bundle.id); + }); + + it("is deterministic", () => { + const a = buildEvalSliceReport(bundle(), [green("a")]); + const b = buildEvalSliceReport(bundle(), [green("a")]); + expect(JSON.stringify(a)).toBe(JSON.stringify(b)); + }); + + it("refuses to build a report from a secret-carrying bundle", () => { + const leaky = { ...bundle(), grants: ["not-a-grant"] }; + expect(() => buildEvalSliceReport(leaky, [green("a")])).toThrow(); + }); + + it("does not persist free-form bundle declarations", () => { + const candidate = bundle(); + candidate.promptPolicy.callTemplate = "Unique instructions that stay in memory."; + const report = buildEvalSliceReport(candidate, [green("a")]); + const serialized = JSON.stringify(report); + + expect(report.bundle.declaration.promptPolicy.callTemplateSha256).toMatch( + /^sha256:[0-9a-f]{64}$/, + ); + expect(serialized).not.toContain(candidate.promptPolicy.callTemplate); + }); + + it("scans the final serialized report before returning it", () => { + const observation = green("Bearer abcdef0123456789abcdef"); + expect(() => buildEvalSliceReport(bundle(), [observation])).toThrow(/bearer-token/); + }); +}); + +describe("renderEvalSliceMarkdown", () => { + it("renders an inspectable table with no secrets", () => { + const md = renderEvalSliceMarkdown(buildEvalSliceReport(bundle(), [green("a"), gateFail("b")])); + expect(md).toContain("Runner eval slice"); + expect(md).toContain("| a |"); + expect(md).toContain("| b |"); + expect(md).toContain("FAIL"); + expect(md).not.toContain("sk-"); + }); +}); diff --git a/packages/paperclip-runner/src/eval/eval-slice.ts b/packages/paperclip-runner/src/eval/eval-slice.ts new file mode 100644 index 0000000000..ad56259bbe --- /dev/null +++ b/packages/paperclip-runner/src/eval/eval-slice.ts @@ -0,0 +1,146 @@ +import { + assertBundleSecretFree, + assertEvalArtifactSecretFree, + bundleEvidenceDeclaration, + bundleId, + describeBundle, + type EvalBundle, + type EvalBundleEvidenceDeclaration, +} from "./eval-bundle.js"; +import { + EVAL_DIMENSION_KEYS, + scoreEval, + type EvalDimensionKey, + type EvalObservation, + type EvalScorecard, + type EvalScoringOptions, +} from "./eval-scoring.js"; + +/** + * The runner eval vertical slice: bind a declared candidate {@link EvalBundle} + * to a set of scored observations and emit one inspectable, secret-free report. + * + * The report binds its scorecards to a content-addressed, digested bundle + * declaration. The caller retains the secret-free source bundle for replay; + * persisted reports carry no free-form bundle content. Per-dimension aggregates + * show whether a regression came from outcome, restraint, trace, or efficiency. + */ +export const EVAL_SLICE_REPORT_SCHEMA = "paperclip.runner.eval-slice-report.v1" as const; + +export interface EvalScoredCase { + scorecard: EvalScorecard; + /** The observation the scorecard was derived from — safe to persist. */ + observation: EvalObservation; +} + +export interface EvalSliceReport { + schema: typeof EVAL_SLICE_REPORT_SCHEMA; + bundle: { id: string; summary: string; declaration: EvalBundleEvidenceDeclaration }; + cases: EvalScoredCase[]; + aggregate: { + caseCount: number; + passed: number; + gateFailures: number; + meanOverall: number; + dimensionMeans: Record; + }; +} + +export interface BuildEvalSliceReportOptions { + weights?: EvalScoringOptions["weights"]; + thresholds?: EvalScoringOptions["thresholds"]; +} + +function round(value: number): number { + return Math.round(value * 1000) / 1000; +} + +/** + * Scores every observation against the bundle and aggregates the results. Throws + * if the bundle carries a secret, so a report can be committed as evidence. + */ +export function buildEvalSliceReport( + bundle: EvalBundle, + observations: readonly EvalObservation[], + options: BuildEvalSliceReportOptions = {}, +): EvalSliceReport { + assertBundleSecretFree(bundle); + const id = bundleId(bundle); + const scoringOptions: EvalScoringOptions = { + bundleId: id, + weights: options.weights, + thresholds: options.thresholds, + }; + + const cases: EvalScoredCase[] = observations.map((observation) => ({ + observation, + scorecard: scoreEval(observation, scoringOptions), + })); + + const dimensionMeans = {} as Record; + for (const key of EVAL_DIMENSION_KEYS) { + const total = cases.reduce((sum, entry) => sum + entry.scorecard.dimensions[key].score, 0); + dimensionMeans[key] = cases.length === 0 ? 0 : round(total / cases.length); + } + const meanOverall = + cases.length === 0 + ? 0 + : round(cases.reduce((sum, entry) => sum + entry.scorecard.overall.score, 0) / cases.length); + + const report: EvalSliceReport = { + schema: EVAL_SLICE_REPORT_SCHEMA, + bundle: { + id, + summary: describeBundle(bundle).summary, + declaration: bundleEvidenceDeclaration(bundle), + }, + cases, + aggregate: { + caseCount: cases.length, + passed: cases.filter((entry) => entry.scorecard.overall.passed).length, + gateFailures: cases.filter((entry) => !entry.scorecard.overall.gatePassed).length, + meanOverall, + dimensionMeans, + }, + }; + // Scan the exact serialization that callers persist, including observations + // and derived scorecards rather than only the source bundle. + assertEvalArtifactSecretFree(JSON.stringify(report), "serialized eval report"); + return report; +} + +/** A compact, human-readable rendering of a slice report for inspection. */ +export function renderEvalSliceMarkdown(report: EvalSliceReport): string { + const lines: string[] = []; + const sources = new Set(report.cases.map((entry) => entry.observation.provenance?.source)); + const sourceDescription = sources.size === 1 && sources.has("deterministic_fault_harness") + ? "deterministic fault harness" + : "fixture observations"; + lines.push(`# Runner eval slice — ${report.bundle.id}`); + lines.push(""); + lines.push(`- Bundle: \`${report.bundle.summary}\``); + lines.push(`- Source: ${sourceDescription}`); + lines.push( + `- Cases: ${report.aggregate.caseCount} · passed ${report.aggregate.passed} · gate failures ${report.aggregate.gateFailures} · mean overall ${report.aggregate.meanOverall}`, + ); + lines.push(""); + lines.push("## Dimension means"); + lines.push(""); + for (const key of EVAL_DIMENSION_KEYS) { + lines.push(`- ${key}: ${report.aggregate.dimensionMeans[key]}`); + } + lines.push(""); + lines.push("## Cases"); + lines.push(""); + lines.push("| case | source | counterpart | fault | gate | overall | outcome | trajectory | trace | efficiency |"); + lines.push("| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |"); + for (const { scorecard, observation } of report.cases) { + const d = scorecard.dimensions; + const provenance = observation.provenance; + lines.push( + `| ${scorecard.caseId} | ${provenance?.source ?? "unspecified"} | ${provenance?.counterpart ?? "—"} | ${provenance?.faultInjection?.class ?? "—"} | ${d.hard_invariants.passed ? "ok" : "FAIL"} | ${scorecard.overall.score} | ${d.semantic_outcome.score} | ${d.trajectory_restraint.score} | ${d.trace_completeness.score} | ${d.quality_efficiency.score} |`, + ); + } + lines.push(""); + return `${lines.join("\n")}\n`; +} diff --git a/packages/paperclip-runner/src/eval/fixtures/acpx-sanitized-provider.json b/packages/paperclip-runner/src/eval/fixtures/acpx-sanitized-provider.json new file mode 100644 index 0000000000..92e1b34b93 --- /dev/null +++ b/packages/paperclip-runner/src/eval/fixtures/acpx-sanitized-provider.json @@ -0,0 +1,17 @@ +{ + "schema": "paperclip.runner.sanitized-provider-fixture.v1", + "provider": "acpx", + "events": [ + { + "type": "tool_call", + "tag": "tool_call", + "toolCallId": "fixture-tool", + "title": "node --test", + "kind": "execute", + "status": "completed", + "text": "Test completed", + "rawOutput": "1 test passed", + "locations": [] + } + ] +} diff --git a/packages/paperclip-runner/src/eval/fixtures/codex-sanitized-provider.json b/packages/paperclip-runner/src/eval/fixtures/codex-sanitized-provider.json new file mode 100644 index 0000000000..fbab9bb486 --- /dev/null +++ b/packages/paperclip-runner/src/eval/fixtures/codex-sanitized-provider.json @@ -0,0 +1,36 @@ +{ + "schema": "paperclip.runner.sanitized-provider-fixture.v1", + "provider": "codex", + "records": [ + { + "method": "item/completed", + "params": { + "item": { + "id": "fixture-search", + "type": "webSearch", + "query": "sanitized protocol query", + "results": [ + { + "ref_id": "safe-source", + "title": "Protocol reference", + "url": "https://example.com/protocol", + "snippet": "Sanitized fixture" + } + ] + } + } + }, + { + "method": "item/completed", + "params": { + "item": { + "id": "fixture-tool", + "type": "commandExecution", + "command": "node --test", + "status": "completed", + "output": "1 test passed" + } + } + } + ] +} diff --git a/packages/paperclip-runner/src/eval/fixtures/opencode-sanitized-provider.json b/packages/paperclip-runner/src/eval/fixtures/opencode-sanitized-provider.json new file mode 100644 index 0000000000..49de9e1208 --- /dev/null +++ b/packages/paperclip-runner/src/eval/fixtures/opencode-sanitized-provider.json @@ -0,0 +1,19 @@ +{ + "schema": "paperclip.runner.sanitized-provider-fixture.v1", + "provider": "opencode", + "parts": [ + { + "id": "fixture-search", + "type": "websearch", + "query": "sanitized protocol query", + "state": { "status": "completed" } + }, + { + "id": "fixture-tool", + "type": "tool", + "tool": "node_test", + "callID": "functions.node_test:0", + "state": { "status": "completed", "output": "1 test passed", "exit": 0 } + } + ] +} diff --git a/packages/paperclip-runner/src/eval/index.ts b/packages/paperclip-runner/src/eval/index.ts new file mode 100644 index 0000000000..43d20f1a6a --- /dev/null +++ b/packages/paperclip-runner/src/eval/index.ts @@ -0,0 +1,10 @@ +export * from "./eval-bundle.js"; +export * from "./eval-execution.js"; +export * from "./eval-scoring.js"; +export * from "./eval-slice.js"; +export * from "./workflow-contracts.js"; +export * from "./workflow-catalog.js"; +export * from "./workflow-scoring.js"; +export * from "./workflow-harness.js"; +export * from "./workflow-traceability.js"; +export * from "./workflow-report.js"; diff --git a/packages/paperclip-runner/src/eval/workflow-catalog.ts b/packages/paperclip-runner/src/eval/workflow-catalog.ts new file mode 100644 index 0000000000..02ba0b0883 --- /dev/null +++ b/packages/paperclip-runner/src/eval/workflow-catalog.ts @@ -0,0 +1,230 @@ +import { + RUNNER_WORKFLOW_EVAL_CASE_SCHEMA, + RUNNER_WORKFLOW_IDS, + assertRunnerWorkflowEvalCase, + type RunnerWorkflowEvalCase, +} from "./workflow-contracts.js"; + +const ALL_PROVIDERS = ["codex", "opencode", "acpx"] as const; + +function workflow(input: Omit): RunnerWorkflowEvalCase { + const value: RunnerWorkflowEvalCase = { + schema: RUNNER_WORKFLOW_EVAL_CASE_SCHEMA, + version: 1, + ...input, + }; + assertRunnerWorkflowEvalCase(value); + return Object.freeze(value); +} + +/** The twelve deterministic workflows distilled from the stress findings. */ +export const RUNNER_WORKFLOW_CATALOG: readonly RunnerWorkflowEvalCase[] = Object.freeze([ + workflow({ + id: "final-response", + title: "Preserve one substantive final response", + tags: ["response", "resolver", "long-output"], + providers: [...ALL_PROVIDERS], steps: [{ kind: "run_start", taskMode: "ask" }], + assertions: { + issueStatus: "done", runStatus: "succeeded", semanticDisposition: "done", maxRuns: 1, + commentCount: 1, responseSource: "final_agent_message", + orderedMarkers: ["commentary", "activity", "final"], + requiredPrpEventTypes: ["run.result.accepted", "run.presentation.resolved", "run.terminal"], + forbiddenPrpEventTypes: ["run.presentation.placeholder"], + requiredOperationIds: ["finish_task"], + }, + }), + workflow({ + id: "rich-activity", + title: "Render structured provider activity through settlement", + tags: ["activity", "tools", "search"], + providers: [...ALL_PROVIDERS], steps: [{ kind: "run_start", taskMode: "execute" }], + assertions: { + issueStatus: "done", runStatus: "succeeded", semanticDisposition: "done", + orderedMarkers: ["commentary", "research", "tool", "file", "test", "final"], + requiredPrpEventTypes: ["research.completed", "tool.execution.completed", "run.terminal"], + requiredActivityFamilies: ["research", "tool_execution"], + requiredOperationIds: ["report_progress", "finish_task"], + }, + }), + workflow({ + id: "verification-policy", + title: "Resolve unavailable and externally required verification", + tags: ["verification", "caveat", "review"], + providers: [...ALL_PROVIDERS], steps: [{ kind: "run_start", taskMode: "execute" }], + assertions: { + issueStatus: "done", runStatus: "succeeded", semanticDisposition: "done", recoveryOwner: "none", + commentCount: 1, requiredPrpEventTypes: ["run.result.accepted", "run.terminal"], + requiredOperationIds: ["finish_task"], + }, + }), + workflow({ + id: "governed-interaction", + title: "Park and resume governed interactions", + tags: ["interaction", "wait", "continuation"], + providers: [...ALL_PROVIDERS], + steps: [ + { kind: "run_start", taskMode: "ask" }, + { kind: "interaction_response", interaction: "questions" }, + { kind: "interaction_response", interaction: "suggest_tasks" }, + { kind: "interaction_response", interaction: "checkbox" }, + { kind: "interaction_response", interaction: "item_verdicts", partial: true }, + { kind: "interaction_response", interaction: "item_verdicts" }, + ], + assertions: { + issueStatus: "done", runStatus: "succeeded", semanticDisposition: "done", maxRuns: 3, + commentCount: 1, orderedMarkers: ["question", "answer", "continuation", "final"], + requiredPrpEventTypes: ["interaction.requested", "run.result.accepted", "run.terminal"], + forbiddenPrpEventTypes: ["run.presentation.placeholder"], + requiredOperationIds: ["request_human_input", "finish_task"], + }, + }), + workflow({ + id: "steering-causality", + title: "Keep queued and active steering causally ordered", + tags: ["steering", "ordering", "idempotency"], + providers: [...ALL_PROVIDERS], + steps: [ + { kind: "run_start", taskMode: "execute" }, + { kind: "steer", delivery: "queued" }, + { kind: "steer", delivery: "active_turn", duplicate: true }, + ], + assertions: { + issueStatus: "done", runStatus: "succeeded", semanticDisposition: "done", maxRuns: 2, + orderedMarkers: ["pre-steer", "steering", "post-steer", "final"], + requiredPrpEventTypes: ["turn.steering.acknowledged", "run.terminal"], + requiredOperationIds: ["finish_task"], + }, + }), + workflow({ + id: "planning-lifecycle", + title: "Reject, revise, accept, and execute a cohesive plan on the source", + tags: ["plan", "same-issue", "approval"], + providers: [...ALL_PROVIDERS], + steps: [ + { kind: "run_start", taskMode: "plan" }, + { kind: "review_decision", decision: "reject" }, + { kind: "review_decision", decision: "approve" }, + ], + assertions: { + issueStatus: "done", runStatus: "succeeded", semanticDisposition: "done", maxRuns: 4, + orderedMarkers: ["plan-v1", "rejection", "plan-v2", "acceptance", "same-issue", "parent-final"], + requiredPrpEventTypes: ["plan.updated", "run.result.accepted", "run.terminal"], + requiredOperationIds: ["write_document", "request_human_input", "finish_task"], + }, + }), + workflow({ + id: "review-lifecycle", + title: "Reject narrowly and approve human completion review", + tags: ["review", "ownership", "narrow-repair"], + providers: [...ALL_PROVIDERS], + steps: [ + { kind: "run_start", taskMode: "execute" }, + { kind: "review_decision", decision: "reject" }, + { kind: "review_decision", decision: "approve" }, + ], + assertions: { + issueStatus: "done", runStatus: "succeeded", semanticDisposition: "needs_review", recoveryOwner: "human", maxRuns: 2, + orderedMarkers: ["review", "rejection", "verification-only", "replacement-review", "approval"], + requiredPrpEventTypes: ["run.result.accepted", "interaction.requested", "run.terminal"], + requiredOperationIds: ["request_human_input", "finish_task"], + }, + }), + workflow({ + id: "delegation-return", + title: "Decompose an accepted plan for a justified independent boundary", + tags: ["plan", "delegation", "dependency", "cross-provider"], + providers: [...ALL_PROVIDERS], + steps: [ + { kind: "run_start", taskMode: "plan" }, + { kind: "review_decision", decision: "approve" }, + { kind: "child_completion", childProvider: "opencode" }, + ], + assertions: { + issueStatus: "done", runStatus: "succeeded", semanticDisposition: "done", maxRuns: 2, + orderedMarkers: ["child-result", "parent-wake", "parent-final"], + requiredPrpEventTypes: ["delegation.completed", "run.result.accepted", "run.terminal"], + forbiddenPrpEventTypes: ["delegation.duplicate_wake"], + requiredOperationIds: ["write_document", "request_human_input", "create_task", "set_dependencies", "finish_task"], + }, + }), + workflow({ + id: "completion-robustness", + title: "Normalize completion and bound missing-result recovery", + tags: ["completion", "schema", "exhaustion"], + providers: [...ALL_PROVIDERS], steps: [{ kind: "run_start", taskMode: "execute" }], + assertions: { + issueStatus: "done", runStatus: "succeeded", semanticDisposition: "done", maxAttempts: 3, maxRuns: 1, recoveryOwner: "none", + commentCount: 1, responseSource: "semantic_result_summary", + requiredPrpEventTypes: ["run.result.accepted", "run.terminal"], forbiddenPrpEventTypes: ["run.presentation.placeholder"], + requiredOperationIds: ["finish_task"], + }, + }), + workflow({ + id: "restart-recovery", + title: "Recover once without stale finalizer authority", + tags: ["restart", "checkpoint", "authority"], + providers: [...ALL_PROVIDERS], + steps: [ + { kind: "run_start", taskMode: "execute" }, + { kind: "process_restart", phase: "provider_turn" }, + { kind: "process_restart", phase: "finalization" }, + ], + assertions: { + issueStatus: "done", runStatus: "succeeded", semanticDisposition: "done", maxAttempts: 2, maxRuns: 1, + requiredPrpEventTypes: ["run.result.accepted", "run.terminal"], + forbiddenPrpEventTypes: ["native_finalization_superseded_mutation"], + requiredOperationIds: ["report_progress", "finish_task"], + }, + }), + workflow({ + id: "cancellation-permissions", + title: "Normalize cancellation and scoped permissions", + tags: ["cancellation", "permission", "interruption"], + providers: [...ALL_PROVIDERS], + steps: [ + { kind: "run_start", taskMode: "execute" }, + { kind: "permission_decision", decision: "accept_for_session" }, + { kind: "cancel", actor: "operator" }, + ], + assertions: { + issueStatus: "cancelled", runStatus: "cancelled", maxRuns: 1, commentCount: 0, responseSource: "none", + orderedMarkers: ["permission", "activity", "stopped"], + requiredPrpEventTypes: ["runtime_request.resolved", "run.terminal"], + forbiddenPrpEventTypes: ["provider.notice.false_abort_failure"], + requiredOperationIds: [], + }, + }), + workflow({ + id: "trace-lineage", + title: "Correlate provider frames through PRP and presentation", + tags: ["trace", "lineage", "redaction"], + providers: [...ALL_PROVIDERS], steps: [{ kind: "run_start", taskMode: "execute" }], + assertions: { + issueStatus: "done", runStatus: "succeeded", semanticDisposition: "done", + requiredPrpEventTypes: ["item.started", "run.result.proposed", "item.completed", "run.presentation.resolved", "run.terminal"], + trace: { + capture: "on", digestVerified: true, ordered: true, + dispositions: ["mapped", "generic", "ignored", "rejected", "operator_only"], + lineage: ["one_to_one", "one_to_many", "many_to_one"], + }, + requiredOperationIds: ["finish_task"], + }, + }), +]); + +export function runnerWorkflowCase(id: string): RunnerWorkflowEvalCase { + const found = RUNNER_WORKFLOW_CATALOG.find((candidate) => candidate.id === id); + if (!found) throw new Error(`unknown Runner workflow eval case: ${id}`); + return found; +} + +export function assertRunnerWorkflowCatalog(): void { + const ids = RUNNER_WORKFLOW_CATALOG.map((entry) => entry.id); + if (ids.length !== RUNNER_WORKFLOW_IDS.length || new Set(ids).size !== RUNNER_WORKFLOW_IDS.length) { + throw new Error(`Runner workflow catalog must contain ${RUNNER_WORKFLOW_IDS.length} unique cases`); + } + for (const id of RUNNER_WORKFLOW_IDS) { + if (!ids.includes(id)) throw new Error(`Runner workflow catalog is missing ${id}`); + } + RUNNER_WORKFLOW_CATALOG.forEach(assertRunnerWorkflowEvalCase); +} diff --git a/packages/paperclip-runner/src/eval/workflow-contracts.ts b/packages/paperclip-runner/src/eval/workflow-contracts.ts new file mode 100644 index 0000000000..7cb07cc01d --- /dev/null +++ b/packages/paperclip-runner/src/eval/workflow-contracts.ts @@ -0,0 +1,249 @@ +import type { EvalObservation } from "./eval-scoring.js"; + +export const RUNNER_WORKFLOW_EVAL_CASE_SCHEMA = "paperclip.runner.workflow-eval-case.v1" as const; +export const RUNNER_WORKFLOW_OBSERVATION_SCHEMA = "paperclip.runner.workflow-observation.v1" as const; +export const RUNNER_WORKFLOW_SCORECARD_SCHEMA = "paperclip.runner.eval-scorecard.v2" as const; + +export const RUNNER_WORKFLOW_IDS = [ + "final-response", + "rich-activity", + "verification-policy", + "governed-interaction", + "steering-causality", + "planning-lifecycle", + "review-lifecycle", + "delegation-return", + "completion-robustness", + "restart-recovery", + "cancellation-permissions", + "trace-lineage", +] as const; + +export type RunnerWorkflowId = typeof RUNNER_WORKFLOW_IDS[number]; +export type RunnerWorkflowProvider = "codex" | "opencode" | "acpx"; +export type RunnerWorkflowExecutionClassification = + | "completed" + | "candidate_failure" + | "infrastructure_failure" + | "skipped"; + +export type RunnerWorkflowStep = + | { kind: "run_start"; taskMode: "ask" | "execute" | "plan" } + | { kind: "interaction_response"; interaction: "questions" | "suggest_tasks" | "checkbox" | "item_verdicts"; partial?: boolean } + | { kind: "steer"; delivery: "queued" | "active_turn"; duplicate?: boolean } + | { kind: "review_decision"; decision: "approve" | "reject" } + | { kind: "permission_decision"; decision: "allow_once" | "accept_for_session" | "deny" } + | { kind: "cancel"; actor: "operator" | "control_plane" } + | { kind: "process_restart"; phase: "provider_turn" | "semantic_result" | "finalization" } + | { kind: "child_completion"; childProvider: RunnerWorkflowProvider }; + +export interface RunnerWorkflowTraceExpectation { + capture: "on" | "off" | "either"; + digestVerified?: boolean; + ordered?: boolean; + dispositions?: Array<"mapped" | "generic" | "ignored" | "rejected" | "operator_only">; + lineage?: Array<"one_to_one" | "one_to_many" | "many_to_one">; +} + +export interface RunnerWorkflowAssertions { + issueStatus?: "todo" | "in_progress" | "blocked" | "in_review" | "done" | "cancelled"; + runStatus?: "succeeded" | "failed" | "cancelled" | "yielded"; + semanticDisposition?: "done" | "needs_review" | "blocked" | "yielded"; + maxAttempts?: number; + maxRuns?: number; + recoveryOwner?: "none" | "agent" | "human" | "board"; + commentCount?: number; + responseSource?: "existing_issue_comment" | "final_agent_message" | "semantic_result_summary" | "adapter_final" | "none"; + orderedMarkers?: string[]; + requiredPrpEventTypes?: string[]; + forbiddenPrpEventTypes?: string[]; + requiredActivityFamilies?: string[]; + artifactDigests?: string[]; + requiredOperationIds?: string[]; + trace?: RunnerWorkflowTraceExpectation; +} + +export interface RunnerWorkflowEvalCase { + schema: typeof RUNNER_WORKFLOW_EVAL_CASE_SCHEMA; + id: RunnerWorkflowId; + title: string; + version: 1; + tags: string[]; + providers: RunnerWorkflowProvider[]; + steps: RunnerWorkflowStep[]; + assertions: RunnerWorkflowAssertions; +} + +export interface RunnerWorkflowCheck { + id: string; + passed: boolean; + reason?: string; + evidenceIds?: string[]; +} + +export interface RunnerWorkflowLifecycleEvidence { + checks: RunnerWorkflowCheck[]; + issueStatus?: string; + runStatus?: string; + semanticDisposition?: string; + attempts?: number; + runs?: number; + recoveryOwner?: string; +} + +export interface RunnerWorkflowContinuationEvidence { + checks: RunnerWorkflowCheck[]; + wakeReasons?: string[]; + consumedInputIds?: string[]; + sessionPolicy?: "same_session" | "approved_replacement" | "new_session"; + repeatedWorkSignals?: string[]; +} + +export interface RunnerWorkflowPresentationEvidence { + checks: RunnerWorkflowCheck[]; + responseSource?: string; + commentCount?: number; + orderedMarkers?: string[]; + visibleActivityFamilies?: string[]; + terminalLabel?: string; +} + +export interface RunnerWorkflowTraceEvidence { + capture: "on" | "off"; + frameCount: number; + byteCount: number; + digestVerified: boolean; + ordered: boolean; + dispositions: string[]; + lineage: string[]; + traceRef?: string; +} + +export interface RunnerWorkflowMetrics { + timeToFirstVisibleProgressMs?: number; + settlementMs?: number; + attempts: number; + toolCount: number; + totalTokens?: number; + costUsd?: number; +} + +export interface RunnerWorkflowFailure { + code: string; + category: "candidate" | "provider" | "qualification" | "orchestration"; + retryable: boolean; + message: string; +} + +export interface RunnerWorkflowObservation { + schema: typeof RUNNER_WORKFLOW_OBSERVATION_SCHEMA; + caseId: RunnerWorkflowId; + candidateId: string; + provider: RunnerWorkflowProvider; + classification: RunnerWorkflowExecutionClassification; + base: EvalObservation; + lifecycle: RunnerWorkflowLifecycleEvidence; + continuation: RunnerWorkflowContinuationEvidence; + presentation: RunnerWorkflowPresentationEvidence; + traceLineage: RunnerWorkflowTraceEvidence; + metrics: RunnerWorkflowMetrics; + observedPrpEventTypes: string[]; + artifactDigests: string[]; + failure?: RunnerWorkflowFailure; +} + +export type RunnerWorkflowDimensionKey = + | "hard_invariants" + | "lifecycle_integrity" + | "semantic_outcome" + | "trajectory_restraint" + | "trace_completeness" + | "quality_efficiency" + | "continuation_integrity" + | "presentation_fidelity"; + +export const RUNNER_WORKFLOW_DIMENSION_KEYS: readonly RunnerWorkflowDimensionKey[] = [ + "hard_invariants", + "lifecycle_integrity", + "semantic_outcome", + "trajectory_restraint", + "trace_completeness", + "quality_efficiency", + "continuation_integrity", + "presentation_fidelity", +] as const; + +export interface RunnerWorkflowDimensionScore { + dimension: RunnerWorkflowDimensionKey; + score: number | null; + passed: boolean | null; + gate: boolean; + weight: number; + reasons: string[]; +} + +export interface RunnerWorkflowEvalScorecard { + schema: typeof RUNNER_WORKFLOW_SCORECARD_SCHEMA; + bundleId: string; + caseId: RunnerWorkflowId; + candidateId: string; + classification: RunnerWorkflowExecutionClassification; + dimensions: Record; + overall: { + score: number | null; + gatePassed: boolean | null; + passed: boolean | null; + }; +} + +export class RunnerWorkflowContractError extends Error { + constructor(message: string) { + super(message); + this.name = "RunnerWorkflowContractError"; + } +} + +const unique = (values: readonly string[]): boolean => new Set(values).size === values.length; + +export function assertRunnerWorkflowEvalCase(value: RunnerWorkflowEvalCase): void { + if (value.schema !== RUNNER_WORKFLOW_EVAL_CASE_SCHEMA || value.version !== 1) { + throw new RunnerWorkflowContractError(`unsupported workflow case schema for ${value.id}`); + } + if (!(RUNNER_WORKFLOW_IDS as readonly string[]).includes(value.id)) { + throw new RunnerWorkflowContractError(`unknown workflow id: ${value.id}`); + } + if (value.title.trim().length === 0 || value.steps.length === 0 || value.providers.length === 0) { + throw new RunnerWorkflowContractError(`workflow ${value.id} is missing title, steps, or providers`); + } + if (!unique(value.tags) || !unique(value.providers)) { + throw new RunnerWorkflowContractError(`workflow ${value.id} contains duplicate catalog values`); + } + if ((value.assertions.requiredPrpEventTypes ?? []).some((eventType) => + value.assertions.forbiddenPrpEventTypes?.includes(eventType))) { + throw new RunnerWorkflowContractError(`workflow ${value.id} requires and forbids the same PRP event`); + } +} + +export function assertRunnerWorkflowObservation(value: RunnerWorkflowObservation): void { + if (value.schema !== RUNNER_WORKFLOW_OBSERVATION_SCHEMA) { + throw new RunnerWorkflowContractError(`unsupported workflow observation schema for ${value.caseId}`); + } + if (!(RUNNER_WORKFLOW_IDS as readonly string[]).includes(value.caseId) || value.candidateId.trim().length === 0) { + throw new RunnerWorkflowContractError("workflow observation identity is invalid"); + } + for (const [name, checks] of [ + ["lifecycle", value.lifecycle.checks], + ["continuation", value.continuation.checks], + ["presentation", value.presentation.checks], + ] as const) { + if (checks.length === 0 || !unique(checks.map((check) => check.id))) { + throw new RunnerWorkflowContractError(`${value.caseId} ${name} checks must be non-empty and unique`); + } + } + if (["infrastructure_failure", "skipped"].includes(value.classification) && value.failure === undefined) { + throw new RunnerWorkflowContractError(`${value.caseId} ${value.classification} requires failure metadata`); + } + if (value.failure?.message.match(/\b(?:sk-[A-Za-z0-9]{16,}|Bearer\s+\S{16,})\b/i)) { + throw new RunnerWorkflowContractError(`${value.caseId} failure metadata contains secret-shaped data`); + } +} diff --git a/packages/paperclip-runner/src/eval/workflow-evals.test.ts b/packages/paperclip-runner/src/eval/workflow-evals.test.ts new file mode 100644 index 0000000000..d24603e030 --- /dev/null +++ b/packages/paperclip-runner/src/eval/workflow-evals.test.ts @@ -0,0 +1,230 @@ +import { readFile, stat } from "node:fs/promises"; +import { resolve } from "node:path"; + +import Ajv2020 from "ajv/dist/2020.js"; +import { describe, expect, it } from "vitest"; + +import { + RUNNER_WORKFLOW_IDS, + assertRunnerWorkflowObservation, + type RunnerWorkflowObservation, +} from "./workflow-contracts.js"; +import { RUNNER_WORKFLOW_CATALOG, assertRunnerWorkflowCatalog } from "./workflow-catalog.js"; +import { runDeterministicRunnerWorkflowMatrix } from "./workflow-harness.js"; +import { scoreRunnerWorkflow } from "./workflow-scoring.js"; +import { + buildRunnerWorkflowEvalReport, + compareRunnerWorkflowReports, + renderRunnerWorkflowGitHubSummary, + renderRunnerWorkflowJUnit, + renderRunnerWorkflowMarkdown, + runnerWorkflowAlerts, +} from "./workflow-report.js"; +import { + validateStressTraceabilityManifest, + type StressTraceabilityManifest, +} from "./workflow-traceability.js"; + +describe("stress-derived Runner workflow catalog", () => { + it("contains the twelve provider-neutral workflow families", () => { + expect(() => assertRunnerWorkflowCatalog()).not.toThrow(); + expect(RUNNER_WORKFLOW_CATALOG.map((entry) => entry.id)).toEqual(RUNNER_WORKFLOW_IDS); + }); + + it("fails closed when sanitized provider fixtures lack workflow evidence", async () => { + const matrix = await runDeterministicRunnerWorkflowMatrix(); + expect(matrix).toHaveLength(36); + expect(matrix.every((entry) => entry.observation.classification === "candidate_failure")).toBe(true); + expect(matrix.every((entry) => entry.scorecard.overall.passed === false)).toBe(true); + expect(new Set(matrix.map((entry) => entry.observation.provider))).toEqual(new Set(["codex", "opencode", "acpx"])); + + const codex = matrix.find((entry) => + entry.scenarioId === "final-response" && entry.candidateId === "fixture-codex"); + expect(codex?.observation).toMatchObject({ + classification: "candidate_failure", + failure: { code: "fixture_evidence_incomplete", category: "candidate" }, + base: { + expectedCalls: ["finish_task"], + observedCalls: [], + trace: { + itemId: "final-response-search", + receiptIds: [], + terminalPresent: false, + }, + }, + observedPrpEventTypes: ["research.completed", "tool.execution.completed"], + metrics: { attempts: 0, toolCount: 1 }, + }); + expect(codex?.observation.lifecycle).not.toHaveProperty("issueStatus"); + expect(codex?.observation.lifecycle).not.toHaveProperty("runStatus"); + expect(codex?.observation.lifecycle.checks).toEqual(expect.arrayContaining([ + expect.objectContaining({ id: "required-prp-events", passed: false }), + expect.objectContaining({ id: "terminal-authority", passed: false }), + expect.objectContaining({ id: "lifecycle-state", passed: false }), + ])); + expect(codex?.observation.observedPrpEventTypes).not.toContain("run.terminal"); + expect(codex?.observation.presentation).not.toHaveProperty("responseSource"); + expect(codex?.observation.traceLineage).toMatchObject({ + digestVerified: false, + dispositions: [], + lineage: [], + }); + }); + + it("scores lifecycle, continuation, and presentation independently", async () => { + const [source] = await runDeterministicRunnerWorkflowMatrix(); + const passing = structuredClone(source!.observation); + passing.classification = "completed"; + delete passing.failure; + for (const evidence of [passing.lifecycle, passing.continuation, passing.presentation]) { + for (const entry of evidence.checks) { + entry.passed = true; + delete entry.reason; + } + } + passing.base.observedCalls = [...passing.base.expectedCalls]; + passing.base.finalState.observed = passing.base.finalState.expected; + passing.base.authorization.observed = passing.base.authorization.expected; + passing.base.trace = { + runId: "run-evidence", + sessionId: "session-evidence", + turnId: "turn-evidence", + itemId: "item-evidence", + receiptIds: passing.base.observedCalls.map((operation) => `receipt-${operation}`), + terminalPresent: true, + }; + passing.traceLineage.digestVerified = true; + passing.traceLineage.ordered = true; + + const lifecycle = structuredClone(passing); + lifecycle.lifecycle.checks[0]!.passed = false; + lifecycle.lifecycle.checks[0]!.reason = "stale finalizer changed authoritative state"; + const lifecycleCard = scoreRunnerWorkflow(lifecycle, { bundleId: "test" }); + expect(lifecycleCard.dimensions.lifecycle_integrity.passed).toBe(false); + expect(lifecycleCard.overall).toMatchObject({ gatePassed: false, score: 0, passed: false }); + + const continuation = structuredClone(passing); + continuation.continuation.checks[0]!.passed = false; + const continuationCard = scoreRunnerWorkflow(continuation, { bundleId: "test" }); + expect(continuationCard.dimensions.continuation_integrity.passed).toBe(false); + expect(continuationCard.dimensions.presentation_fidelity.passed).toBe(true); + expect(continuationCard.overall.gatePassed).toBe(true); + + const presentation = structuredClone(passing); + presentation.presentation.checks[0]!.passed = false; + const presentationCard = scoreRunnerWorkflow(presentation, { bundleId: "test" }); + expect(presentationCard.dimensions.presentation_fidelity.passed).toBe(false); + expect(presentationCard.dimensions.continuation_integrity.passed).toBe(true); + + const trace = structuredClone(passing); + trace.traceLineage.digestVerified = false; + const traceCard = scoreRunnerWorkflow(trace, { bundleId: "test" }); + expect(traceCard.dimensions.trace_completeness.passed).toBe(false); + expect(traceCard.dimensions.lifecycle_integrity.passed).toBe(true); + }); + + it("does not score skipped or infrastructure executions", async () => { + const [source] = await runDeterministicRunnerWorkflowMatrix(); + for (const classification of ["skipped", "infrastructure_failure"] as const) { + const observation: RunnerWorkflowObservation = structuredClone(source!.observation); + observation.classification = classification; + observation.failure = { code: "provider_unavailable", category: "provider", retryable: true, message: "Provider unavailable" }; + assertRunnerWorkflowObservation(observation); + expect(scoreRunnerWorkflow(observation, { bundleId: "test" }).overall).toEqual({ score: null, gatePassed: null, passed: null }); + } + }); + + it("fails closed when an execution is classified as a candidate failure", async () => { + const [source] = await runDeterministicRunnerWorkflowMatrix(); + const observation: RunnerWorkflowObservation = structuredClone(source!.observation); + observation.classification = "candidate_failure"; + observation.failure = { + code: "candidate_error", + category: "candidate", + retryable: false, + message: "Candidate execution failed", + }; + + const card = scoreRunnerWorkflow(observation, { bundleId: "test" }); + + expect(card.dimensions.lifecycle_integrity).toMatchObject({ + score: 0, + passed: false, + gate: true, + reasons: expect.arrayContaining(["candidate execution failed"]), + }); + expect(card.overall).toEqual({ score: 0, gatePassed: false, passed: false }); + }); +}); + +describe("workflow reports and stress traceability", () => { + it("maps every stress finding to a workflow, regression, or explicit exclusion", async () => { + const packageRoot = process.cwd(); + const manifest = JSON.parse(await readFile(resolve(packageRoot, "spec/evals/stress-workflow-traceability.json"), "utf8")) as StressTraceabilityManifest; + const summary = validateStressTraceabilityManifest(manifest); + expect(summary).toEqual({ findings: 44, workflowEvalFindings: 40, regressionTestFindings: 3, exclusions: 1, coveredWorkflows: 12 }); + for (const finding of manifest.findings) { + for (const testPath of finding.regressionTests) { + await expect(stat(resolve(packageRoot, testPath))).resolves.toBeDefined(); + } + } + }); + + it("renders safe JSON-derived Markdown, JUnit, and GitHub summaries", async () => { + const results = await runDeterministicRunnerWorkflowMatrix(); + const report = buildRunnerWorkflowEvalReport({ + source: "deterministic", + bundle: { id: "bundle-v1", runnerVersion: "0.0.0", promptPolicyId: "stress-sanitized-v1", providerVersions: { fixture: "1" } }, + results, + generatedAt: "2026-08-24T00:00:00.000Z", + traceability: { findings: 44, workflowEvalFindings: 40, regressionTestFindings: 3, exclusions: 1, coveredWorkflows: 12 }, + }); + expect(report.aggregate).toMatchObject({ + executions: 36, + scoreable: 36, + passed: 0, + candidateFailures: 36, + }); + expect(report.coverage).toMatchObject({ canonicalOperations: 41, capabilityCases: 106, workflows: 12, stressFindings: 44, stressExclusions: 1 }); + expect(report.coverage.operations).toHaveLength(41); + expect(report.coverage.composedWorkflows).toHaveLength(12); + expect(report.coverage.operations.find((entry) => entry.operationId === "finish_task")?.workflowIds.length).toBeGreaterThan(0); + expect(renderRunnerWorkflowMarkdown(report)).toContain("41 operations · 106 capability cases · 12 workflows"); + expect(renderRunnerWorkflowJUnit(report)).toContain('tests="36" failures="36" skipped="0"'); + expect(renderRunnerWorkflowGitHubSummary(report)).toContain("No active workflow-eval regression alerts"); + expect(compareRunnerWorkflowReports(report, report)).toMatchObject({ compatible: true, passRateDelta: 0, overallDelta: 0 }); + expect(compareRunnerWorkflowReports({ ...report, bundle: { ...report.bundle, id: "other" } }, report)).toMatchObject({ compatible: false }); + }); + + it("keeps alerts disabled during baseline and detects safety and trend regressions afterward", async () => { + const results = await runDeterministicRunnerWorkflowMatrix(); + const healthy = buildRunnerWorkflowEvalReport({ + source: "deterministic", bundle: { id: "compatible", runnerVersion: "1", promptPolicyId: "p", providerVersions: {} }, results, + generatedAt: "2026-08-24T00:00:00.000Z", + }); + const failingResults = structuredClone(results) as unknown as typeof results; + failingResults[0]!.scorecard.dimensions.lifecycle_integrity.passed = false; + failingResults[0]!.scorecard.dimensions.lifecycle_integrity.score = 0; + failingResults[0]!.scorecard.overall = { score: 0, gatePassed: false, passed: false }; + const failing = buildRunnerWorkflowEvalReport({ + source: "deterministic", bundle: healthy.bundle, results: failingResults, + generatedAt: "2026-08-25T00:00:00.000Z", + }); + expect(runnerWorkflowAlerts({ current: failing, history: [healthy], baselineReady: false })).toEqual([]); + expect(runnerWorkflowAlerts({ current: failing, history: [failing, failing], baselineReady: true }).map((alert) => alert.code)).toEqual(expect.arrayContaining(["safety_failure", "consecutive_failures"])); + }); + + it("validates the checked-in versioned JSON schemas", async () => { + const schemaRoot = resolve(process.cwd(), "spec/evals/schemas"); + const [caseSchema, observationSchema, scorecardSchema] = await Promise.all([ + readFile(resolve(schemaRoot, "runner-workflow-eval-case.v1.schema.json"), "utf8").then(JSON.parse), + readFile(resolve(schemaRoot, "runner-workflow-observation.v1.schema.json"), "utf8").then(JSON.parse), + readFile(resolve(schemaRoot, "eval-scorecard.v2.schema.json"), "utf8").then(JSON.parse), + ]); + const ajv = new Ajv2020({ allErrors: true }); + const [result] = await runDeterministicRunnerWorkflowMatrix(); + expect(ajv.compile(caseSchema)(RUNNER_WORKFLOW_CATALOG[0])).toBe(true); + expect(ajv.compile(observationSchema)(result!.observation)).toBe(true); + expect(ajv.compile(scorecardSchema)(result!.scorecard)).toBe(true); + }); +}); diff --git a/packages/paperclip-runner/src/eval/workflow-harness.ts b/packages/paperclip-runner/src/eval/workflow-harness.ts new file mode 100644 index 0000000000..2fc6b49488 --- /dev/null +++ b/packages/paperclip-runner/src/eval/workflow-harness.ts @@ -0,0 +1,261 @@ +import { runPaperclipEvalMatrix, type PaperclipEvalCandidate } from "@paperclipai/paperclip-eval-kernel"; + +import acpxFixture from "./fixtures/acpx-sanitized-provider.json" with { type: "json" }; +import codexFixture from "./fixtures/codex-sanitized-provider.json" with { type: "json" }; +import opencodeFixture from "./fixtures/opencode-sanitized-provider.json" with { type: "json" }; + +import { + canonicalProviderEventsFromAcpxRuntimeEvent, + canonicalProviderEventsFromCodex, + canonicalProviderEventsFromOpenCodePart, + type CanonicalProviderEvent, +} from "../provider-events.js"; +import type { EvalObservation } from "./eval-scoring.js"; +import { + RUNNER_WORKFLOW_OBSERVATION_SCHEMA, + assertRunnerWorkflowObservation, + type RunnerWorkflowCheck, + type RunnerWorkflowEvalCase, + type RunnerWorkflowObservation, + type RunnerWorkflowProvider, +} from "./workflow-contracts.js"; +import { RUNNER_WORKFLOW_CATALOG, assertRunnerWorkflowCatalog } from "./workflow-catalog.js"; +import { scoreRunnerWorkflow, type RunnerWorkflowScoringOptions } from "./workflow-scoring.js"; + +export interface DeterministicRunnerWorkflowCandidate { + provider: RunnerWorkflowProvider; + transport: "codex-app-server" | "opencode-server" | "acp-json-rpc"; + fixtureRevision: "stress-sanitized-v1"; +} + +export const DETERMINISTIC_RUNNER_WORKFLOW_CANDIDATES: readonly PaperclipEvalCandidate[] = Object.freeze([ + { id: "fixture-codex", config: { provider: "codex", transport: "codex-app-server", fixtureRevision: "stress-sanitized-v1" } }, + { id: "fixture-opencode", config: { provider: "opencode", transport: "opencode-server", fixtureRevision: "stress-sanitized-v1" } }, + { id: "fixture-acpx", config: { provider: "acpx", transport: "acp-json-rpc", fixtureRevision: "stress-sanitized-v1" } }, +]); + +function providerFixtureEvents(provider: RunnerWorkflowProvider, caseId: string): CanonicalProviderEvent[] { + const materialize = (value: T): T => JSON.parse( + JSON.stringify(value).replaceAll("fixture-", `${caseId}-`), + ) as T; + if (provider === "codex") { + return codexFixture.records.flatMap((record) => + canonicalProviderEventsFromCodex(record.method, materialize(record.params))); + } + if (provider === "opencode") { + return opencodeFixture.parts.flatMap((part) => canonicalProviderEventsFromOpenCodePart(materialize(part))); + } + return acpxFixture.events.flatMap((event) => { + const materialized = materialize(event); + return canonicalProviderEventsFromAcpxRuntimeEvent( + materialized as Parameters[0], + materialized.toolCallId, + ); + }); +} + +function check(id: string, passed: boolean, reason: string, evidenceIds: string[] = []): RunnerWorkflowCheck { + return { id, passed, ...(passed ? {} : { reason }), ...(evidenceIds.length === 0 ? {} : { evidenceIds }) }; +} + +function includesOrdered(actual: readonly string[], expected: readonly string[]): boolean { + let cursor = 0; + for (const marker of actual) { + if (marker === expected[cursor]) cursor += 1; + } + return cursor === expected.length; +} + +function activityFamily(eventType: string): string { + return eventType.startsWith("tool.execution.") + ? "tool_execution" + : eventType.split(".")[0] ?? eventType; +} + +function deterministicObservation( + evalCase: RunnerWorkflowEvalCase, + candidateId: string, + provider: RunnerWorkflowProvider, +): RunnerWorkflowObservation { + const applicable = evalCase.providers.includes(provider); + const expectedCalls = evalCase.assertions.requiredOperationIds ?? []; + const providerEvents = providerFixtureEvents(provider, evalCase.id); + const providerEventTypes: string[] = providerEvents.map((event) => event.eventType); + const evidenceIds = [...new Set(providerEvents.map((event) => event.itemId))]; + const observedActivityFamilies = [...new Set(providerEventTypes.map(activityFamily))]; + const requiredPrp = evalCase.assertions.requiredPrpEventTypes ?? []; + const missingPrp = requiredPrp.filter((eventType) => !providerEventTypes.includes(eventType)); + const forbiddenPrp = (evalCase.assertions.forbiddenPrpEventTypes ?? []) + .filter((eventType) => providerEventTypes.includes(eventType)); + const missingCalls = expectedCalls; + const requiredActivityFamilies = evalCase.assertions.requiredActivityFamilies ?? []; + const missingActivityFamilies = requiredActivityFamilies + .filter((family) => !observedActivityFamilies.includes(family)); + const expectedMarkers = evalCase.assertions.orderedMarkers ?? []; + const terminalPresent = providerEventTypes.includes("run.terminal"); + const traceCapture = providerEvents.length === 0 ? "off" : "on"; + const lifecycleChecks = [ + check( + "required-prp-events", + missingPrp.length === 0, + `fixture is missing required PRP event(s): ${missingPrp.join(", ")}`, + evidenceIds, + ), + check( + "forbidden-prp-events", + forbiddenPrp.length === 0, + `fixture contains forbidden PRP event(s): ${forbiddenPrp.join(", ")}`, + evidenceIds, + ), + check("terminal-authority", terminalPresent, "fixture contains no terminal authority evidence", evidenceIds), + check("lifecycle-state", false, "fixture contains no issue, run, or semantic lifecycle evidence"), + check("attempt-bound", false, "fixture contains no attempt-count evidence"), + check("run-bound", false, "fixture contains no run-count evidence"), + check("owned-recovery", false, "fixture contains no recovery-owner evidence"), + ]; + const continuationChecks = [ + check( + "causal-order", + expectedMarkers.length > 0 && includesOrdered(providerEventTypes, expectedMarkers), + "fixture provider events do not establish the expected conversation-marker order", + evidenceIds, + ), + check("no-repeated-work", false, "fixture contains no repeated-work evidence"), + check("single-owned-wake", false, "fixture contains no continuation-wake ownership evidence"), + ]; + const presentationChecks = [ + check("response-source", false, "fixture contains no response-resolution evidence"), + check("comment-count", false, "fixture contains no conversation comment evidence"), + check("ordered-presentation", false, "fixture contains no rendered presentation order"), + check("no-stuck-running", terminalPresent, "fixture contains no terminal presentation evidence", evidenceIds), + check( + "required-activity-families", + missingActivityFamilies.length === 0, + `fixture is missing required activity family/families: ${missingActivityFamilies.join(", ")}`, + evidenceIds, + ), + ]; + const evidenceComplete = [ + ...lifecycleChecks, + ...continuationChecks, + ...presentationChecks, + ].every((entry) => entry.passed) + && missingCalls.length === 0; + const classification = !applicable + ? "skipped" + : evidenceComplete + ? "completed" + : "candidate_failure"; + const base: EvalObservation = { + caseId: evalCase.id, + provenance: { source: "fixture", behavior: evalCase.id }, + controlPlaneOwned: expectedCalls.length === 0, + expectedCalls, + observedCalls: [], + forbiddenCalls: [], + finalState: { + expected: expectedCalls.length === 0 ? "unchanged" : "mutated", + observed: "unchanged", + }, + authorization: { + expected: expectedCalls.length === 0 ? "absent" : "allowed", + observed: "absent", + }, + trace: { + ...(providerEvents[0] === undefined ? {} : { itemId: providerEvents[0].itemId }), + receiptIds: [], + terminalPresent, + }, + }; + const failure = classification === "completed" + ? undefined + : classification === "skipped" + ? { + code: "provider_not_applicable", + category: "qualification" as const, + retryable: false, + message: `${provider} is not applicable to ${evalCase.id}`, + } + : { + code: "fixture_evidence_incomplete", + category: "candidate" as const, + retryable: false, + message: `Sanitized provider fixture lacks workflow evidence${missingCalls.length === 0 ? "" : ` and semantic receipt(s) for ${missingCalls.join(", ")}`}`, + }; + const observation: RunnerWorkflowObservation = { + schema: RUNNER_WORKFLOW_OBSERVATION_SCHEMA, + caseId: evalCase.id, + candidateId, + provider, + classification, + base, + lifecycle: { + checks: lifecycleChecks, + }, + continuation: { + wakeReasons: [], + consumedInputIds: [], + repeatedWorkSignals: [], + checks: continuationChecks, + }, + presentation: { + orderedMarkers: providerEventTypes, + visibleActivityFamilies: observedActivityFamilies, + checks: presentationChecks, + }, + traceLineage: { + capture: traceCapture, + frameCount: providerEvents.length, + byteCount: Buffer.byteLength(JSON.stringify(providerEvents)), + digestVerified: false, + ordered: false, + dispositions: [], + lineage: [], + }, + metrics: { + attempts: 0, + toolCount: providerEvents.filter((event) => event.eventType.startsWith("tool.")).length, + }, + observedPrpEventTypes: [...new Set(providerEventTypes)], + artifactDigests: [], + ...(failure === undefined ? {} : { failure }), + }; + assertRunnerWorkflowObservation(observation); + return observation; +} + +export interface RunnerWorkflowMatrixEntry { + scenarioId: string; + candidateId: string; + observation: RunnerWorkflowObservation; + scorecard: ReturnType; +} + +/** Runs the complete offline workflow matrix using the provider-neutral eval kernel. */ +export async function runDeterministicRunnerWorkflowMatrix( + scoring: RunnerWorkflowScoringOptions = { bundleId: "runner-workflows-deterministic-v1" }, +): Promise { + assertRunnerWorkflowCatalog(); + const results = await runPaperclipEvalMatrix({ + scenarios: RUNNER_WORKFLOW_CATALOG.map((entry) => ({ id: entry.id, input: entry })), + candidates: DETERMINISTIC_RUNNER_WORKFLOW_CANDIDATES, + execute: async ({ scenario, candidate }) => deterministicObservation( + scenario.input, + candidate.id, + candidate.config.provider, + ), + score: ({ output }) => scoreRunnerWorkflow(output, scoring), + }); + const entries = results.map((result) => ({ + scenarioId: result.scenarioId, + candidateId: result.candidateId, + observation: result.output, + scorecard: result.score, + })); + for (const entry of entries) { + if (entry.observation.classification === "completed" && entry.scorecard.overall.passed !== true) { + throw new Error(`deterministic Runner workflow failed: ${entry.scenarioId}/${entry.candidateId}`); + } + } + return Object.freeze(entries); +} diff --git a/packages/paperclip-runner/src/eval/workflow-report.ts b/packages/paperclip-runner/src/eval/workflow-report.ts new file mode 100644 index 0000000000..2e3e02daca --- /dev/null +++ b/packages/paperclip-runner/src/eval/workflow-report.ts @@ -0,0 +1,269 @@ +import { CAPABILITY_CANONICAL_OPERATIONS } from "../catalog/canonical-operations.js"; +import { capabilityInventoryCounts } from "../generated/capability-contract.js"; +import { RUNNER_WORKFLOW_DIMENSION_KEYS, type RunnerWorkflowDimensionKey } from "./workflow-contracts.js"; +import { RUNNER_WORKFLOW_CATALOG } from "./workflow-catalog.js"; +import type { RunnerWorkflowMatrixEntry } from "./workflow-harness.js"; +import type { StressTraceabilitySummary } from "./workflow-traceability.js"; + +export const RUNNER_WORKFLOW_REPORT_SCHEMA = "paperclip.runner.workflow-eval-report.v1" as const; + +export interface RunnerWorkflowReportBundle { + id: string; + runnerVersion: string; + runnerBuild?: string; + promptPolicyId: string; + providerVersions: Record; +} + +export interface RunnerWorkflowEvalReport { + schema: typeof RUNNER_WORKFLOW_REPORT_SCHEMA; + generatedAt: string; + source: "deterministic"; + bundle: RunnerWorkflowReportBundle; + results: RunnerWorkflowMatrixEntry[]; + aggregate: { + executions: number; + scoreable: number; + passed: number; + candidateFailures: number; + infrastructureFailures: number; + skipped: number; + meanOverall: number | null; + dimensionMeans: Record; + }; + coverage: { + canonicalOperations: number; + capabilityCases: number; + workflows: number; + stressFindings?: number; + stressExclusions?: number; + operations: Array<{ operationId: string; capabilityCovered: true; workflowIds: string[] }>; + composedWorkflows: Array<{ workflowId: string; providerCount: number; operationIds: string[] }>; + }; +} + +function round(value: number): number { + return Math.round(value * 1_000) / 1_000; +} + +export function buildRunnerWorkflowEvalReport(input: { + source: RunnerWorkflowEvalReport["source"]; + bundle: RunnerWorkflowReportBundle; + results: readonly RunnerWorkflowMatrixEntry[]; + generatedAt?: string; + traceability?: StressTraceabilitySummary; +}): RunnerWorkflowEvalReport { + const results = [...input.results]; + const scoreable = results.filter((entry) => entry.scorecard.overall.score !== null); + const dimensionMeans = {} as Record; + for (const dimension of RUNNER_WORKFLOW_DIMENSION_KEYS) { + const values = scoreable.flatMap((entry) => { + const value = entry.scorecard.dimensions[dimension].score; + return value === null ? [] : [value]; + }); + dimensionMeans[dimension] = values.length === 0 ? null : round(values.reduce((sum, value) => sum + value, 0) / values.length); + } + const overallValues = scoreable.flatMap((entry) => entry.scorecard.overall.score === null ? [] : [entry.scorecard.overall.score]); + return { + schema: RUNNER_WORKFLOW_REPORT_SCHEMA, + generatedAt: input.generatedAt ?? new Date().toISOString(), + source: input.source, + bundle: input.bundle, + results, + aggregate: { + executions: results.length, + scoreable: scoreable.length, + passed: scoreable.filter((entry) => entry.scorecard.overall.passed).length, + candidateFailures: results.filter((entry) => entry.observation.classification === "candidate_failure").length, + infrastructureFailures: results.filter((entry) => entry.observation.classification === "infrastructure_failure").length, + skipped: results.filter((entry) => entry.observation.classification === "skipped").length, + meanOverall: overallValues.length === 0 ? null : round(overallValues.reduce((sum, value) => sum + value, 0) / overallValues.length), + dimensionMeans, + }, + coverage: { + canonicalOperations: CAPABILITY_CANONICAL_OPERATIONS.length, + capabilityCases: capabilityInventoryCounts.evalCases, + workflows: new Set(results.map((entry) => entry.scenarioId)).size, + operations: CAPABILITY_CANONICAL_OPERATIONS.map((operation) => ({ + operationId: operation.operationId, + capabilityCovered: true as const, + workflowIds: RUNNER_WORKFLOW_CATALOG.filter((evalCase) => + evalCase.assertions.requiredOperationIds?.includes(operation.operationId)) + .map((evalCase) => evalCase.id), + })), + composedWorkflows: RUNNER_WORKFLOW_CATALOG.map((evalCase) => ({ + workflowId: evalCase.id, + providerCount: evalCase.providers.length, + operationIds: evalCase.assertions.requiredOperationIds ?? [], + })), + ...(input.traceability === undefined ? {} : { + stressFindings: input.traceability.findings, + stressExclusions: input.traceability.exclusions, + }), + }, + }; +} + +export function renderRunnerWorkflowMarkdown(report: RunnerWorkflowEvalReport): string { + const lines = [ + `# Runner workflow evals — ${report.bundle.id}`, + "", + `- Source: ${report.source}`, + `- Executions: ${report.aggregate.executions} · scoreable ${report.aggregate.scoreable} · passed ${report.aggregate.passed}`, + `- Candidate failures: ${report.aggregate.candidateFailures} · infrastructure failures ${report.aggregate.infrastructureFailures} · skipped ${report.aggregate.skipped}`, + `- Coverage: ${report.coverage.canonicalOperations} operations · ${report.coverage.capabilityCases} capability cases · ${report.coverage.workflows} workflows${report.coverage.stressFindings === undefined ? "" : ` · ${report.coverage.stressFindings} stress findings`}`, + "", + "## Dimension means", + "", + ...RUNNER_WORKFLOW_DIMENSION_KEYS.map((dimension) => `- ${dimension}: ${report.aggregate.dimensionMeans[dimension] ?? "not scored"}`), + "", + "## Combined operation and workflow coverage", + "", + "| operation | capability cases | composed workflows |", + "| --- | --- | --- |", + ...report.coverage.operations.map((operation) => `| ${operation.operationId} | covered | ${operation.workflowIds.join(", ") || "—"} |`), + "", + "## Executions", + "", + "| workflow | candidate | classification | gate | score | result |", + "| --- | --- | --- | --- | --- | --- |", + ...report.results.map((entry) => { + const overall = entry.scorecard.overall; + const result = overall.passed === null ? "not scored" : overall.passed ? "PASS" : "FAIL"; + return `| ${entry.scenarioId} | ${entry.candidateId} | ${entry.observation.classification} | ${overall.gatePassed ?? "—"} | ${overall.score === null ? "—" : round(overall.score)} | ${result} |`; + }), + "", + ]; + return `${lines.join("\n")}\n`; +} + +function xml(value: string): string { + return value.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll('"', """); +} + +export function renderRunnerWorkflowJUnit(report: RunnerWorkflowEvalReport): string { + const failures = report.results.filter((entry) => entry.scorecard.overall.passed === false).length; + const skipped = report.results.filter((entry) => entry.scorecard.overall.passed === null).length; + const cases = report.results.map((entry) => { + const name = `${entry.scenarioId}/${entry.candidateId}`; + if (entry.scorecard.overall.passed === null) { + return ` `; + } + if (entry.scorecard.overall.passed === false) { + const reasons = RUNNER_WORKFLOW_DIMENSION_KEYS.flatMap((dimension) => entry.scorecard.dimensions[dimension].reasons).join("; "); + return ` ${xml(reasons)}`; + } + return ` `; + }); + return [ + "", + ``, + ...cases, + "", + "", + ].join("\n"); +} + +export interface RunnerWorkflowReportComparison { + compatible: boolean; + reason?: string; + currentBundleId: string; + previousBundleId: string; + passRateDelta?: number; + overallDelta?: number; +} + +export function compareRunnerWorkflowReports( + current: RunnerWorkflowEvalReport, + previous: RunnerWorkflowEvalReport, +): RunnerWorkflowReportComparison { + if (current.bundle.id !== previous.bundle.id) { + return { + compatible: false, + reason: "bundle ids differ; provider fixture, runner, or prompt-policy inputs changed", + currentBundleId: current.bundle.id, + previousBundleId: previous.bundle.id, + }; + } + const rate = (report: RunnerWorkflowEvalReport): number => report.aggregate.scoreable === 0 ? 0 : report.aggregate.passed / report.aggregate.scoreable; + return { + compatible: true, + currentBundleId: current.bundle.id, + previousBundleId: previous.bundle.id, + passRateDelta: round(rate(current) - rate(previous)), + overallDelta: round((current.aggregate.meanOverall ?? 0) - (previous.aggregate.meanOverall ?? 0)), + }; +} + +export interface RunnerWorkflowAlert { + severity: "warning" | "critical"; + code: "safety_failure" | "consecutive_failures" | "success_rate_regression" | "latency_regression" | "cost_regression"; + message: string; +} + +function percentile95(values: number[]): number | null { + if (values.length === 0) return null; + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.ceil(sorted.length * 0.95) - 1] ?? null; +} + +export function runnerWorkflowAlerts(input: { + current: RunnerWorkflowEvalReport; + history: readonly RunnerWorkflowEvalReport[]; + baselineReady: boolean; +}): RunnerWorkflowAlert[] { + if (!input.baselineReady) return []; + const compatible = input.history.filter((report) => report.bundle.id === input.current.bundle.id).slice(-7); + const alerts: RunnerWorkflowAlert[] = []; + const safetyFailures = input.current.results.filter((entry) => + entry.scorecard.dimensions.hard_invariants.passed === false + || entry.scorecard.dimensions.lifecycle_integrity.passed === false); + if (safetyFailures.length > 0) { + alerts.push({ severity: "critical", code: "safety_failure", message: `${safetyFailures.length} execution(s) failed a hard or lifecycle invariant` }); + } + for (const result of input.current.results) { + if (result.scorecard.overall.passed !== false) continue; + const priorFailures = compatible.slice(-2).every((report) => report.results.some((entry) => + entry.scenarioId === result.scenarioId + && entry.candidateId === result.candidateId + && entry.scorecard.overall.passed === false)); + if (compatible.length >= 2 && priorFailures) { + alerts.push({ severity: "warning", code: "consecutive_failures", message: `${result.scenarioId}/${result.candidateId} failed three consecutive reports` }); + } + } + const baselineScoreable = compatible.reduce((sum, report) => sum + report.aggregate.scoreable, 0); + const baselinePassed = compatible.reduce((sum, report) => sum + report.aggregate.passed, 0); + if (baselineScoreable > 0 && input.current.aggregate.scoreable > 0) { + const baselineRate = baselinePassed / baselineScoreable; + const currentRate = input.current.aggregate.passed / input.current.aggregate.scoreable; + if (baselineRate - currentRate >= 0.20) { + alerts.push({ severity: "warning", code: "success_rate_regression", message: `success rate dropped ${Math.round((baselineRate - currentRate) * 100)} percentage points` }); + } + } + const metric = (reports: readonly RunnerWorkflowEvalReport[], key: "settlementMs" | "costUsd"): number[] => reports.flatMap((report) => + report.results.flatMap((entry) => { + const value = key === "settlementMs" ? entry.observation.metrics.settlementMs : entry.observation.metrics.costUsd; + return value === undefined ? [] : [value]; + })); + const currentLatency = percentile95(metric([input.current], "settlementMs")); + const baselineLatency = percentile95(metric(compatible, "settlementMs")); + if (currentLatency !== null && baselineLatency !== null && currentLatency > baselineLatency * 1.25) { + alerts.push({ severity: "warning", code: "latency_regression", message: `p95 settlement latency rose more than 25% (${baselineLatency}ms → ${currentLatency}ms)` }); + } + const currentCost = percentile95(metric([input.current], "costUsd")); + const baselineCost = percentile95(metric(compatible, "costUsd")); + if (currentCost !== null && baselineCost !== null && baselineCost > 0 && currentCost > baselineCost * 1.25) { + alerts.push({ severity: "warning", code: "cost_regression", message: `p95 cost rose more than 25% ($${baselineCost} → $${currentCost})` }); + } + return alerts; +} + +export function renderRunnerWorkflowGitHubSummary( + report: RunnerWorkflowEvalReport, + alerts: readonly RunnerWorkflowAlert[] = [], +): string { + const alertLines = alerts.length === 0 + ? ["No active workflow-eval regression alerts."] + : alerts.map((alert) => `- ${alert.severity.toUpperCase()} ${alert.code}: ${alert.message}`); + return `${renderRunnerWorkflowMarkdown(report)}\n## Alerts\n\n${alertLines.join("\n")}\n`; +} diff --git a/packages/paperclip-runner/src/eval/workflow-scoring.ts b/packages/paperclip-runner/src/eval/workflow-scoring.ts new file mode 100644 index 0000000000..cbbe4584ea --- /dev/null +++ b/packages/paperclip-runner/src/eval/workflow-scoring.ts @@ -0,0 +1,168 @@ +import { scoreEval } from "./eval-scoring.js"; +import { + RUNNER_WORKFLOW_DIMENSION_KEYS, + RUNNER_WORKFLOW_SCORECARD_SCHEMA, + assertRunnerWorkflowObservation, + type RunnerWorkflowCheck, + type RunnerWorkflowDimensionKey, + type RunnerWorkflowDimensionScore, + type RunnerWorkflowEvalScorecard, + type RunnerWorkflowObservation, +} from "./workflow-contracts.js"; +import { runnerWorkflowCase } from "./workflow-catalog.js"; + +export interface RunnerWorkflowScoringOptions { + bundleId: string; + weights?: Partial>; + thresholds?: Partial>; +} + +const DEFAULT_WEIGHTS: Record = { + hard_invariants: 0, + lifecycle_integrity: 0, + semantic_outcome: 0.20, + trajectory_restraint: 0.15, + trace_completeness: 0.10, + quality_efficiency: 0.10, + continuation_integrity: 0.20, + presentation_fidelity: 0.25, +}; + +function checksScore(checks: readonly RunnerWorkflowCheck[]): { score: number; reasons: string[] } { + const failed = checks.filter((check) => !check.passed); + return { + score: checks.filter((check) => check.passed).length / checks.length, + reasons: failed.map((check) => check.reason ?? `check failed: ${check.id}`), + }; +} + +function unscoredDimensions(): Record { + return Object.fromEntries(RUNNER_WORKFLOW_DIMENSION_KEYS.map((dimension) => [dimension, { + dimension, + score: null, + passed: null, + gate: dimension === "hard_invariants" || dimension === "lifecycle_integrity", + weight: DEFAULT_WEIGHTS[dimension], + reasons: ["execution was not scoreable"], + }])) as Record; +} + +function traceLineageScore(observation: RunnerWorkflowObservation): { score: number; reasons: string[] } { + const expected = runnerWorkflowCase(observation.caseId).assertions.trace; + const checks: RunnerWorkflowCheck[] = []; + const captureExpectation = expected?.capture ?? "either"; + if (captureExpectation === "off") { + checks.push({ + id: "capture-off-isolation", + passed: observation.traceLineage.capture === "off" + && observation.traceLineage.frameCount === 0 + && observation.traceLineage.byteCount === 0 + && observation.traceLineage.traceRef === undefined, + reason: "capture-off execution retained provider trace data", + }); + } else if (captureExpectation === "on" || observation.traceLineage.capture === "on") { + checks.push( + { id: "capture-present", passed: observation.traceLineage.capture === "on" && observation.traceLineage.frameCount > 0 && observation.traceLineage.byteCount > 0, reason: "provider trace capture is empty" }, + { id: "frame-digests", passed: observation.traceLineage.digestVerified, reason: "provider trace frame digest or byte length did not verify" }, + { id: "frame-order", passed: observation.traceLineage.ordered, reason: "provider trace frame or debug sequence is not strictly ordered" }, + ); + } else { + checks.push({ id: "capture-optional", passed: true }); + } + for (const disposition of expected?.dispositions ?? []) { + checks.push({ + id: `disposition-${disposition}`, + passed: observation.traceLineage.dispositions.includes(disposition), + reason: `provider trace did not exercise ${disposition} mapping`, + }); + } + for (const lineage of expected?.lineage ?? []) { + checks.push({ + id: `lineage-${lineage}`, + passed: observation.traceLineage.lineage.includes(lineage), + reason: `provider trace did not exercise ${lineage} lineage`, + }); + } + return checksScore(checks); +} + +/** Scores the complete Runner workflow while preserving the v1 capability dimensions. */ +export function scoreRunnerWorkflow( + observation: RunnerWorkflowObservation, + options: RunnerWorkflowScoringOptions, +): RunnerWorkflowEvalScorecard { + assertRunnerWorkflowObservation(observation); + if (observation.classification === "infrastructure_failure" || observation.classification === "skipped") { + return { + schema: RUNNER_WORKFLOW_SCORECARD_SCHEMA, + bundleId: options.bundleId, + caseId: observation.caseId, + candidateId: observation.candidateId, + classification: observation.classification, + dimensions: unscoredDimensions(), + overall: { score: null, gatePassed: null, passed: null }, + }; + } + + const v1 = scoreEval(observation.base, { bundleId: options.bundleId }); + const lineageTrace = traceLineageScore(observation); + const workflowScores = { + lifecycle_integrity: checksScore(observation.lifecycle.checks), + continuation_integrity: checksScore(observation.continuation.checks), + presentation_fidelity: checksScore(observation.presentation.checks), + }; + const weights = { ...DEFAULT_WEIGHTS, ...options.weights }; + const dimensions = {} as Record; + + for (const dimension of RUNNER_WORKFLOW_DIMENSION_KEYS) { + const legacy = dimension in v1.dimensions + ? v1.dimensions[dimension as keyof typeof v1.dimensions] + : undefined; + const workflow = dimension in workflowScores + ? workflowScores[dimension as keyof typeof workflowScores] + : undefined; + const observedScore = dimension === "trace_completeness" && legacy !== undefined + ? Math.min(legacy.score, lineageTrace.score) + : legacy?.score ?? workflow?.score ?? 0; + const candidateFailed = observation.classification === "candidate_failure" + && dimension === "lifecycle_integrity"; + const score = candidateFailed ? 0 : observedScore; + const gate = dimension === "hard_invariants" || dimension === "lifecycle_integrity"; + const threshold = options.thresholds?.[dimension] ?? 1; + const reasons = dimension === "trace_completeness" && legacy !== undefined + ? [...legacy.reasons, ...lineageTrace.reasons] + : legacy?.reasons ?? workflow?.reasons ?? []; + dimensions[dimension] = { + dimension, + score, + passed: score >= threshold, + gate, + weight: gate ? 0 : weights[dimension], + reasons: candidateFailed + ? [...reasons, "candidate execution failed"] + : reasons, + }; + } + + const gatePassed = dimensions.hard_invariants.passed === true + && dimensions.lifecycle_integrity.passed === true; + const weighted = RUNNER_WORKFLOW_DIMENSION_KEYS.filter((dimension) => !dimensions[dimension].gate); + const totalWeight = weighted.reduce((sum, dimension) => sum + dimensions[dimension].weight, 0); + const weightedScore = totalWeight === 0 ? 0 : weighted.reduce( + (sum, dimension) => sum + (dimensions[dimension].score ?? 0) * dimensions[dimension].weight, + 0, + ) / totalWeight; + const passed = gatePassed && RUNNER_WORKFLOW_DIMENSION_KEYS.every( + (dimension) => dimensions[dimension].passed === true, + ); + + return { + schema: RUNNER_WORKFLOW_SCORECARD_SCHEMA, + bundleId: options.bundleId, + caseId: observation.caseId, + candidateId: observation.candidateId, + classification: observation.classification, + dimensions, + overall: { score: gatePassed ? weightedScore : 0, gatePassed, passed }, + }; +} diff --git a/packages/paperclip-runner/src/eval/workflow-traceability.ts b/packages/paperclip-runner/src/eval/workflow-traceability.ts new file mode 100644 index 0000000000..b4c1a656f3 --- /dev/null +++ b/packages/paperclip-runner/src/eval/workflow-traceability.ts @@ -0,0 +1,74 @@ +import { RUNNER_WORKFLOW_IDS, type RunnerWorkflowId } from "./workflow-contracts.js"; + +export const STRESS_TRACEABILITY_SCHEMA = "paperclip.runner.stress-eval-traceability.v1" as const; + +export interface StressTraceabilityFinding { + id: string; + classification: "workflow_eval" | "regression_test" | "explicit_exclusion"; + workflowIds: RunnerWorkflowId[]; + regressionTests: string[]; + reason?: string; +} + +export interface StressTraceabilityManifest { + schema: typeof STRESS_TRACEABILITY_SCHEMA; + campaign: string; + expectedFindings: 44; + workflows: RunnerWorkflowId[]; + findings: StressTraceabilityFinding[]; +} + +export interface StressTraceabilitySummary { + findings: number; + workflowEvalFindings: number; + regressionTestFindings: number; + exclusions: number; + coveredWorkflows: number; +} + +export function validateStressTraceabilityManifest(manifest: StressTraceabilityManifest): StressTraceabilitySummary { + if (manifest.schema !== STRESS_TRACEABILITY_SCHEMA || manifest.expectedFindings !== 44) { + throw new Error("unsupported stress traceability manifest"); + } + const expectedIds = Array.from({ length: 44 }, (_, index) => `STRESS-${String(index + 1).padStart(3, "0")}`); + const actualIds = manifest.findings.map((finding) => finding.id); + if (actualIds.length !== expectedIds.length || new Set(actualIds).size !== expectedIds.length) { + throw new Error("stress traceability must contain 44 unique findings"); + } + for (const id of expectedIds) { + if (!actualIds.includes(id)) throw new Error(`stress traceability is missing ${id}`); + } + if (manifest.workflows.length !== RUNNER_WORKFLOW_IDS.length + || !RUNNER_WORKFLOW_IDS.every((id) => manifest.workflows.includes(id))) { + throw new Error("stress traceability workflow inventory does not match the Runner workflow catalog"); + } + for (const finding of manifest.findings) { + for (const workflowId of finding.workflowIds) { + if (!(RUNNER_WORKFLOW_IDS as readonly string[]).includes(workflowId)) { + throw new Error(`${finding.id} references unknown workflow ${workflowId}`); + } + } + if (finding.classification === "workflow_eval" && finding.workflowIds.length === 0) { + throw new Error(`${finding.id} is a workflow eval without a workflow`); + } + if (finding.classification !== "explicit_exclusion" && finding.regressionTests.length === 0) { + throw new Error(`${finding.id} is missing a regression test anchor`); + } + if (finding.regressionTests.some((path) => !/\.test\.[cm]?[jt]sx?$/.test(path))) { + throw new Error(`${finding.id} references a non-test regression anchor`); + } + if (finding.classification === "regression_test" && finding.regressionTests.length === 0) { + throw new Error(`${finding.id} is a regression finding without a test`); + } + if (finding.classification === "explicit_exclusion" && !finding.reason) { + throw new Error(`${finding.id} exclusion is missing a reason`); + } + } + return { + findings: manifest.findings.length, + workflowEvalFindings: manifest.findings.filter((finding) => finding.classification === "workflow_eval").length, + regressionTestFindings: manifest.findings.filter((finding) => finding.classification === "regression_test").length, + exclusions: manifest.findings.filter((finding) => finding.classification === "explicit_exclusion").length, + coveredWorkflows: new Set(manifest.findings.flatMap((finding) => finding.workflowIds)).size, + }; +} diff --git a/packages/paperclip-runner/src/evals/build-metadata.ts b/packages/paperclip-runner/src/evals/build-metadata.ts new file mode 100644 index 0000000000..f19f334781 --- /dev/null +++ b/packages/paperclip-runner/src/evals/build-metadata.ts @@ -0,0 +1,58 @@ +import { createHash } from "node:crypto"; + +import { PAPERCLIP_RUNNER_COMPATIBILITY } from "../compatibility.js"; +import { PRP_PROTOCOL_NAME, PRP_PROTOCOL_VERSION } from "../protocol/replay-contract.js"; +import { canonicalCapabilitySemanticCatalog } from "../semantic-tools/catalog.js"; + +export const PAPERCLIP_RUNNER_BUILD_METADATA_SCHEMA = + "paperclip-runner/build-metadata/v1" as const; +export const PAPERCLIP_RUNNER_NATIVE_EXECUTION_SCHEMA = + "paperclip-runner/native-execution/v1" as const; +export const PAPERCLIP_RUNNER_EVAL_INTEGRATION_SCHEMA = + "paperclip-runner/evals-integration/v1" as const; +export const PAPERCLIP_RUNNERD_BUILD_METADATA_SCHEMA = + "paperclip-runner/runnerd-build-metadata/v1" as const; + +export const PAPERCLIP_RUNNER_SEMANTIC_CATALOG_SHA256 = + `sha256:${createHash("sha256") + .update(canonicalCapabilitySemanticCatalog()) + .digest("hex")}` as const; + +/** + * App-owned release metadata that Evals pins beside every native attempt. + * Contract versions are independent from package semver so consumers can give + * a precise mismatch instead of guessing from a package version. + */ +export const PAPERCLIP_RUNNER_BUILD_METADATA = Object.freeze({ + schema: PAPERCLIP_RUNNER_BUILD_METADATA_SCHEMA, + package: Object.freeze({ + name: PAPERCLIP_RUNNER_COMPATIBILITY.packageName, + version: PAPERCLIP_RUNNER_COMPATIBILITY.packageVersion, + }), + contracts: Object.freeze({ + evalIntegration: PAPERCLIP_RUNNER_COMPATIBILITY.components.evalIntegration, + nativeExecution: PAPERCLIP_RUNNER_COMPATIBILITY.components.nativeExecution, + runnerdArtifact: PAPERCLIP_RUNNER_COMPATIBILITY.components.runnerdBinary, + prp: PRP_PROTOCOL_VERSION, + semanticCatalog: PAPERCLIP_RUNNER_COMPATIBILITY.components.catalog, + harnessDriver: PAPERCLIP_RUNNER_COMPATIBILITY.components.harnessDriver, + controlPlaneAdapter: PAPERCLIP_RUNNER_COMPATIBILITY.components.controlPlaneAdapter, + testkit: PAPERCLIP_RUNNER_COMPATIBILITY.components.testkit, + }), + prp: Object.freeze({ + name: PRP_PROTOCOL_NAME, + minimumVersion: PRP_PROTOCOL_VERSION, + maximumVersion: PRP_PROTOCOL_VERSION, + }), + semanticCatalog: Object.freeze({ + version: PAPERCLIP_RUNNER_COMPATIBILITY.components.catalog, + sha256: PAPERCLIP_RUNNER_SEMANTIC_CATALOG_SHA256, + }), + runnerd: Object.freeze({ + binaryName: "paperclip-runnerd" as const, + metadataSchema: PAPERCLIP_RUNNERD_BUILD_METADATA_SCHEMA, + digestAlgorithm: "sha256" as const, + }), +}); + +export type PaperclipRunnerBuildMetadata = typeof PAPERCLIP_RUNNER_BUILD_METADATA; diff --git a/packages/paperclip-runner/src/evals/compatibility.test.ts b/packages/paperclip-runner/src/evals/compatibility.test.ts new file mode 100644 index 0000000000..477c2cedfe --- /dev/null +++ b/packages/paperclip-runner/src/evals/compatibility.test.ts @@ -0,0 +1,120 @@ +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; + +import { + PAPERCLIP_RUNNER_BUILD_METADATA, + PAPERCLIP_RUNNERD_BUILD_METADATA_SCHEMA, +} from "./build-metadata.js"; +import { + PaperclipRunnerEvalCompatibilityError, + assertPaperclipRunnerEvalCompatibility, + type PaperclipRunnerEvalCompatibilityRequirement, +} from "./compatibility.js"; + +function compatible(): PaperclipRunnerEvalCompatibilityRequirement { + return { + consumer: "paperclip-evals", + packageVersion: PAPERCLIP_RUNNER_BUILD_METADATA.package.version, + runnerd: { + schema: PAPERCLIP_RUNNERD_BUILD_METADATA_SCHEMA, + binaryName: "paperclip-runnerd", + packageName: "@paperclipai/paperclip-runner", + packageVersion: PAPERCLIP_RUNNER_BUILD_METADATA.package.version, + binaryContractVersion: PAPERCLIP_RUNNER_BUILD_METADATA.contracts.runnerdArtifact, + nativeExecutionVersion: 1, + harnessDriverVersion: 1, + prp: { name: "paperclip.runner", minimumVersion: 1, maximumVersion: 1 }, + }, + nativeExecutionVersion: 1, + prp: { minimumVersion: 1, maximumVersion: 1 }, + catalog: { + version: PAPERCLIP_RUNNER_BUILD_METADATA.semanticCatalog.version, + sha256: PAPERCLIP_RUNNER_BUILD_METADATA.semanticCatalog.sha256, + }, + driver: { + contractVersion: 1, + requiredCapabilities: ["typedEvents", "interruption", "usage", "dynamicTools"], + descriptor: { + kind: "paperclip-deterministic", + displayName: "Deterministic", + version: "1.0.0", + protocolVersion: "prp.v1", + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + usage: true, + dynamicTools: true, + }, + }, + }, + }; +} + +describe("Paperclip Evals integration compatibility", () => { + it("keeps build metadata synchronized with package semver", () => { + const packageJson = JSON.parse(readFileSync( + fileURLToPath(new URL("../../package.json", import.meta.url)), + "utf8", + )); + expect(PAPERCLIP_RUNNER_BUILD_METADATA.package.version).toBe(packageJson.version); + }); + + it("negotiates package, binary, PRP, catalog, and driver V1", () => { + expect(assertPaperclipRunnerEvalCompatibility(compatible())).toMatchObject({ + schema: "paperclip-runner/evals-integration/v1", + consumer: "paperclip-evals", + negotiatedPrpVersion: 1, + driverKind: "paperclip-deterministic", + }); + }); + + it("fails closed with actionable codes for every independently versioned input", () => { + const requirement = compatible(); + requirement.packageVersion = "9.0.0"; + requirement.runnerd.packageVersion = "8.0.0"; + requirement.runnerd.binaryContractVersion = 3; + requirement.runnerd.nativeExecutionVersion = 2; + requirement.runnerd.harnessDriverVersion = 2; + requirement.nativeExecutionVersion = 2; + requirement.prp = { minimumVersion: 2, maximumVersion: 2 }; + requirement.runnerd.prp = { name: "paperclip.runner", minimumVersion: 2, maximumVersion: 2 }; + requirement.catalog = { version: 2, sha256: `sha256:${"0".repeat(64)}` }; + requirement.driver.contractVersion = 2; + requirement.driver.descriptor.capabilities.dynamicTools = false; + + try { + assertPaperclipRunnerEvalCompatibility(requirement); + throw new Error("expected compatibility failure"); + } catch (error) { + expect(error).toBeInstanceOf(PaperclipRunnerEvalCompatibilityError); + expect((error as PaperclipRunnerEvalCompatibilityError).issues.map((item) => item.code)) + .toEqual([ + "package_version_mismatch", + "binary_package_version_mismatch", + "binary_contract_version_mismatch", + "native_execution_version_mismatch", + "prp_version_no_overlap", + "catalog_version_mismatch", + "catalog_digest_mismatch", + "driver_contract_version_mismatch", + "driver_capability_unsupported", + ]); + expect((error as Error).message).toContain("runnerd was built for package 8.0.0"); + } + }); + + it("rejects a driver that does not speak the negotiated PRP version", () => { + const requirement = compatible(); + requirement.driver.descriptor.protocolVersion = "prp.v2"; + + expect(() => assertPaperclipRunnerEvalCompatibility(requirement)).toThrow( + expect.objectContaining({ + issues: [expect.objectContaining({ code: "driver_protocol_version_mismatch" })], + }), + ); + }); +}); diff --git a/packages/paperclip-runner/src/evals/compatibility.ts b/packages/paperclip-runner/src/evals/compatibility.ts new file mode 100644 index 0000000000..2e62540e7e --- /dev/null +++ b/packages/paperclip-runner/src/evals/compatibility.ts @@ -0,0 +1,244 @@ +import type { HarnessDriverDescriptor } from "../contracts/harness-driver.js"; +import type { NativeSessionCapabilities } from "../contracts/types.js"; +import { negotiateProtocolVersion } from "../protocol/replay-contract.js"; +import { + PAPERCLIP_RUNNER_BUILD_METADATA, + PAPERCLIP_RUNNER_EVAL_INTEGRATION_SCHEMA, +} from "./build-metadata.js"; +import type { PaperclipRunnerdBuildMetadata } from "./runnerd-artifact.js"; + +export type RequiredHarnessDriverCapability = Exclude< + keyof NativeSessionCapabilities, + "unsupported" +>; + +export type PaperclipRunnerEvalCompatibilityIssueCode = + | "package_version_mismatch" + | "binary_package_version_mismatch" + | "binary_contract_version_mismatch" + | "native_execution_version_mismatch" + | "prp_name_mismatch" + | "prp_version_no_overlap" + | "catalog_version_mismatch" + | "catalog_digest_mismatch" + | "driver_contract_version_mismatch" + | "driver_protocol_version_mismatch" + | "driver_capability_unsupported"; + +export interface PaperclipRunnerEvalCompatibilityIssue { + code: PaperclipRunnerEvalCompatibilityIssueCode; + component: "package" | "binary" | "nativeExecution" | "prp" | "catalog" | "driver"; + expected: string; + received: string; + message: string; +} + +export interface PaperclipRunnerEvalCompatibilityRequirement { + consumer: string; + packageVersion: string; + runnerd: PaperclipRunnerdBuildMetadata; + nativeExecutionVersion: number; + prp: { minimumVersion: number; maximumVersion: number }; + catalog: { version: number; sha256: string }; + driver: { + contractVersion: number; + descriptor: HarnessDriverDescriptor; + requiredCapabilities: readonly RequiredHarnessDriverCapability[]; + }; +} + +export interface PaperclipRunnerEvalCompatibilityReceipt { + schema: typeof PAPERCLIP_RUNNER_EVAL_INTEGRATION_SCHEMA; + consumer: string; + packageVersion: string; + runnerdPackageVersion: string; + negotiatedPrpVersion: number; + catalogSha256: string; + driverKind: string; + driverVersion: string; +} + +export class PaperclipRunnerEvalCompatibilityError extends Error { + readonly code = "paperclip_runner_eval_incompatible" as const; + + constructor( + readonly consumer: string, + readonly issues: readonly PaperclipRunnerEvalCompatibilityIssue[], + ) { + super( + `Paperclip runner eval compatibility check failed for ${consumer}: ${issues + .map((issue) => `${issue.code}: ${issue.message}`) + .join("; ")}`, + ); + this.name = "PaperclipRunnerEvalCompatibilityError"; + this.issues = Object.freeze(issues.map((issue) => Object.freeze({ ...issue }))); + } +} + +/** + * Negotiate the complete App/Evals join before starting a provider process. + * Every independently versioned input is checked and all mismatches are + * returned together so remediation is actionable. + */ +export function assertPaperclipRunnerEvalCompatibility( + requirement: PaperclipRunnerEvalCompatibilityRequirement, +): PaperclipRunnerEvalCompatibilityReceipt { + const expected = PAPERCLIP_RUNNER_BUILD_METADATA; + const issues: PaperclipRunnerEvalCompatibilityIssue[] = []; + const issue = ( + code: PaperclipRunnerEvalCompatibilityIssueCode, + component: PaperclipRunnerEvalCompatibilityIssue["component"], + expectedValue: string | number, + receivedValue: string | number, + message: string, + ): void => { + issues.push({ + code, + component, + expected: String(expectedValue), + received: String(receivedValue), + message, + }); + }; + + if (requirement.packageVersion !== expected.package.version) { + issue( + "package_version_mismatch", + "package", + expected.package.version, + requirement.packageVersion, + `package ${requirement.packageVersion} does not match the loaded contract package ${expected.package.version}`, + ); + } + if (requirement.runnerd.packageVersion !== expected.package.version) { + issue( + "binary_package_version_mismatch", + "binary", + expected.package.version, + requirement.runnerd.packageVersion, + `runnerd was built for package ${requirement.runnerd.packageVersion}, not ${expected.package.version}`, + ); + } + if (requirement.runnerd.binaryContractVersion !== expected.contracts.runnerdArtifact) { + issue( + "binary_contract_version_mismatch", + "binary", + expected.contracts.runnerdArtifact, + requirement.runnerd.binaryContractVersion, + "runnerd artifact contract is incompatible with this package", + ); + } + if ( + requirement.nativeExecutionVersion !== expected.contracts.nativeExecution + || requirement.runnerd.nativeExecutionVersion !== expected.contracts.nativeExecution + ) { + issue( + "native_execution_version_mismatch", + "nativeExecution", + expected.contracts.nativeExecution, + `${requirement.nativeExecutionVersion}/${requirement.runnerd.nativeExecutionVersion}`, + "consumer and runnerd must both emit paperclip-runner/native-execution/v1", + ); + } + + if (requirement.runnerd.prp.name !== expected.prp.name) { + issue( + "prp_name_mismatch", + "prp", + expected.prp.name, + requirement.runnerd.prp.name, + "runnerd speaks a different protocol family", + ); + } + const packageAndConsumerPrp = negotiateProtocolVersion( + { min: expected.prp.minimumVersion, max: expected.prp.maximumVersion }, + { min: requirement.prp.minimumVersion, max: requirement.prp.maximumVersion }, + ); + const negotiatedPrpVersion = packageAndConsumerPrp === null + ? null + : negotiateProtocolVersion( + { min: packageAndConsumerPrp, max: packageAndConsumerPrp }, + { + min: requirement.runnerd.prp.minimumVersion, + max: requirement.runnerd.prp.maximumVersion, + }, + ); + if (negotiatedPrpVersion === null) { + issue( + "prp_version_no_overlap", + "prp", + `${expected.prp.minimumVersion}..${expected.prp.maximumVersion}`, + `consumer ${requirement.prp.minimumVersion}..${requirement.prp.maximumVersion}; runnerd ${requirement.runnerd.prp.minimumVersion}..${requirement.runnerd.prp.maximumVersion}`, + "package, runnerd, and consumer have no common PRP version", + ); + } + + if (requirement.catalog.version !== expected.semanticCatalog.version) { + issue( + "catalog_version_mismatch", + "catalog", + expected.semanticCatalog.version, + requirement.catalog.version, + "semantic catalog contract version is incompatible", + ); + } + if (requirement.catalog.sha256 !== expected.semanticCatalog.sha256) { + issue( + "catalog_digest_mismatch", + "catalog", + expected.semanticCatalog.sha256, + requirement.catalog.sha256, + "semantic catalog content digest does not match the loaded package", + ); + } + if ( + requirement.driver.contractVersion !== expected.contracts.harnessDriver + || requirement.runnerd.harnessDriverVersion !== expected.contracts.harnessDriver + ) { + issue( + "driver_contract_version_mismatch", + "driver", + expected.contracts.harnessDriver, + `${requirement.driver.contractVersion}/${requirement.runnerd.harnessDriverVersion}`, + "consumer and runnerd harness-driver contract versions must match the package", + ); + } + if ( + negotiatedPrpVersion !== null + && requirement.driver.descriptor.protocolVersion !== `prp.v${negotiatedPrpVersion}` + ) { + issue( + "driver_protocol_version_mismatch", + "driver", + `prp.v${negotiatedPrpVersion}`, + String(requirement.driver.descriptor.protocolVersion), + `driver ${requirement.driver.descriptor.kind}@${requirement.driver.descriptor.version} does not speak the negotiated PRP version`, + ); + } + for (const capability of [...new Set(requirement.driver.requiredCapabilities)].sort()) { + if (requirement.driver.descriptor.capabilities[capability] !== true) { + issue( + "driver_capability_unsupported", + "driver", + `${capability}=true`, + `${capability}=${String(requirement.driver.descriptor.capabilities[capability])}`, + `driver ${requirement.driver.descriptor.kind}@${requirement.driver.descriptor.version} does not support required capability ${capability}`, + ); + } + } + + if (issues.length > 0) { + throw new PaperclipRunnerEvalCompatibilityError(requirement.consumer, issues); + } + + return { + schema: PAPERCLIP_RUNNER_EVAL_INTEGRATION_SCHEMA, + consumer: requirement.consumer, + packageVersion: expected.package.version, + runnerdPackageVersion: requirement.runnerd.packageVersion, + negotiatedPrpVersion: negotiatedPrpVersion!, + catalogSha256: expected.semanticCatalog.sha256, + driverKind: requirement.driver.descriptor.kind, + driverVersion: requirement.driver.descriptor.version, + }; +} diff --git a/packages/paperclip-runner/src/evals/index.ts b/packages/paperclip-runner/src/evals/index.ts new file mode 100644 index 0000000000..5c2b5457b3 --- /dev/null +++ b/packages/paperclip-runner/src/evals/index.ts @@ -0,0 +1,32 @@ +/** Stable App-owned integration surface for Paperclip Evals. */ +export * from "./build-metadata.js"; +export * from "./compatibility.js"; +export * from "./native-execution.js"; +export * from "./runnerd-artifact.js"; + +export { + PRP_PROTOCOL_NAME, + PRP_PROTOCOL_VERSION, + negotiateProtocolVersion, + validatePrpEvent, + validatePrpFixture, + validatePrpStructuredRunResult, + type PrpCapabilities, + type PrpEvent, + type PrpSemanticToolEnvelope, + type PrpStructuredRunResult, + type PrpTerminalState, +} from "../protocol/replay-contract.js"; +export * from "../protocol/semantic-tool-receipts.js"; +export { + CAPABILITY_SEMANTIC_TOOL_CATALOG, + canonicalCapabilitySemanticCatalog, + capabilitySemanticToolDescriptor, +} from "../semantic-tools/catalog.js"; +export type { + HarnessDriver, + HarnessDriverDescriptor, + HarnessSession, + OpenHarnessSessionInput, + PersistedHarnessSession, +} from "../contracts/harness-driver.js"; diff --git a/packages/paperclip-runner/src/evals/native-execution.test.ts b/packages/paperclip-runner/src/evals/native-execution.test.ts new file mode 100644 index 0000000000..e78760f63b --- /dev/null +++ b/packages/paperclip-runner/src/evals/native-execution.test.ts @@ -0,0 +1,173 @@ +import { readFile } from "node:fs/promises"; +import Ajv2020 from "ajv/dist/2020.js"; +import { describe, expect, it } from "vitest"; + +import { prpSchemaBundle } from "../protocol/generated/schema-bundle.js"; +import { + loadPaperclipNativeExecutionFixture, + paperclipNativeExecutionFixtureUrl, + paperclipNativeExecutionSchemaUrl, + parsePaperclipNativeExecution, +} from "./native-execution.js"; +import { PAPERCLIP_RUNNER_BUILD_METADATA } from "./build-metadata.js"; + +describe("paperclip-runner/native-execution/v1", () => { + it("keeps the shipped seeded fixture valid against the published JSON Schema", async () => { + const ajv = new Ajv2020({ allErrors: true, strict: false }); + for (const schema of Object.values(prpSchemaBundle)) ajv.addSchema(schema); + const schema = JSON.parse( + await readFile(paperclipNativeExecutionSchemaUrl, "utf8"), + ) as Record; + const validate = ajv.compile(schema); + const fixture = JSON.parse( + await readFile(paperclipNativeExecutionFixtureUrl, "utf8"), + ) as unknown; + + expect(validate(fixture), JSON.stringify(validate.errors)).toBe(true); + }); + + it("loads the shipped seeded attempt and preserves nested additive fields", async () => { + const bundle = await loadPaperclipNativeExecutionFixture(); + expect(bundle.provenance).toBe("seeded"); + expect(bundle.semanticTools.results).toEqual([ + expect.objectContaining({ outcome: "denied", callId: "call_evals_seeded" }), + ]); + expect(bundle.semanticTools.denials).toHaveLength(1); + expect(bundle.transcript).toMatchObject({ complete: true, eventCount: 4 }); + expect(bundle.usage.cost).toEqual({ currency: "USD", amountMicros: 0 }); + expect(bundle.runner.package).toEqual(PAPERCLIP_RUNNER_BUILD_METADATA.package); + expect(bundle.runner.catalogSha256).toBe( + PAPERCLIP_RUNNER_BUILD_METADATA.semanticCatalog.sha256, + ); + expect(bundle.x_fixturePurpose).toContain("additive field preservation"); + + const extended = structuredClone(bundle); + extended.identity.x_identity = "identity-extension"; + extended.input.x_input = "input-extension"; + extended.runner.x_runner = "runner-extension"; + extended.runner.package.x_package = "package-extension"; + extended.runner.binary.x_binary = "binary-extension"; + extended.semanticTools.x_semanticTools = "semantic-extension"; + extended.usage.x_usage = "usage-extension"; + extended.usage.cost!.x_cost = "cost-extension"; + + expect(parsePaperclipNativeExecution(extended)).toMatchObject({ + identity: { x_identity: "identity-extension" }, + input: { x_input: "input-extension" }, + runner: { + x_runner: "runner-extension", + package: { x_package: "package-extension" }, + binary: { x_binary: "binary-extension" }, + }, + semanticTools: { x_semanticTools: "semantic-extension" }, + usage: { x_usage: "usage-extension", cost: { x_cost: "cost-extension" } }, + }); + }); + + it("rejects unknown native schema versions", async () => { + const bundle = await loadPaperclipNativeExecutionFixture(); + expect(() => parsePaperclipNativeExecution({ ...bundle, schema: "paperclip-runner/native-execution/v2" })) + .toThrow(/unsupported schema/); + }); + + it("rejects a terminal that conflicts with the event stream", async () => { + const bundle = await loadPaperclipNativeExecutionFixture(); + expect(() => parsePaperclipNativeExecution({ + ...bundle, + terminal: { ...bundle.terminal, runTerminalState: "failed" }, + })).toThrow(/must equal the run\.terminal event payload/); + }); + + it("rejects missing denial receipts and incomplete transcript ambiguity", async () => { + const bundle = await loadPaperclipNativeExecutionFixture(); + expect(() => parsePaperclipNativeExecution({ + ...bundle, + semanticTools: { ...bundle.semanticTools, denials: [] }, + })).toThrow(/needs a denial receipt/); + expect(() => parsePaperclipNativeExecution({ + ...bundle, + transcript: { ...bundle.transcript, complete: false, omissionReason: null }, + })).toThrow(/omissionReason.*required/); + }); + + it("rejects semantic indexes that reinterpret or omit PRP tool events", async () => { + const bundle = await loadPaperclipNativeExecutionFixture(); + const mismatched = structuredClone(bundle); + const resultPayload = mismatched.events[1]!.payload as { + semantic_tool: { outcome: string }; + }; + resultPayload.semantic_tool.outcome = "succeeded"; + expect(() => parsePaperclipNativeExecution(mismatched)) + .toThrow(/does not match semantic envelope/); + + const mismatchedCorrelation = structuredClone(bundle); + const inputPayload = mismatchedCorrelation.events[0]!.payload as { + semantic_tool: { correlation: { runId: string } }; + }; + inputPayload.semantic_tool.correlation.runId = "other_run"; + expect(() => parsePaperclipNativeExecution(mismatchedCorrelation)) + .toThrow(/correlation\.runId.*enclosing event/); + + expect(() => parsePaperclipNativeExecution({ + ...bundle, + semanticTools: { ...bundle.semanticTools, calls: [] }, + })).toThrow(/no matching call|tool-input event .* is not indexed/); + }); + + it("rejects a non-interrupted terminal bundle with an unresolved semantic call", async () => { + const bundle = await loadPaperclipNativeExecutionFixture(); + const unresolved = structuredClone(bundle); + unresolved.semanticTools.results = []; + unresolved.semanticTools.denials = []; + unresolved.events = unresolved.events.filter( + (event) => event.eventType !== "mcp_app.tool_result", + ); + unresolved.transcript.eventCount = unresolved.events.length; + + expect(() => parsePaperclipNativeExecution(unresolved)) + .toThrow(/requires exactly one result for every semantic call/); + }); + + it("allows an unresolved semantic call only for interrupted or cancelled states", async () => { + for (const terminalState of [ + { turnTerminalState: "interrupted", runTerminalState: "failed" }, + { turnTerminalState: "cancelled", runTerminalState: "cancelled" }, + ] as const) { + const bundle = await loadPaperclipNativeExecutionFixture(); + const unfinished = structuredClone(bundle); + unfinished.semanticTools.results = []; + unfinished.semanticTools.denials = []; + unfinished.events = unfinished.events.filter( + (event) => event.eventType !== "mcp_app.tool_result", + ); + unfinished.transcript.eventCount = unfinished.events.length; + unfinished.terminal = { ...unfinished.terminal, ...terminalState }; + const terminalEvent = unfinished.events.find( + (event) => event.eventType === "run.terminal", + ); + if (terminalEvent === undefined) throw new Error("fixture is missing its terminal event"); + terminalEvent.payload = structuredClone(unfinished.terminal); + + expect(parsePaperclipNativeExecution(unfinished)).toMatchObject({ + semanticTools: { calls: [expect.any(Object)], results: [] }, + terminal: terminalState, + }); + } + }); + + it("rejects unordered native events", async () => { + const bundle = await loadPaperclipNativeExecutionFixture(); + const events = structuredClone(bundle.events); + events[1] = { ...events[1]!, sourceSeq: 1 }; + expect(() => parsePaperclipNativeExecution({ ...bundle, events })) + .toThrow(/must be ordered after 1/); + + const duplicateIds = structuredClone(bundle.events); + duplicateIds[1] = { + ...duplicateIds[1]!, + sourceEventId: duplicateIds[0]!.sourceEventId, + }; + expect(() => parsePaperclipNativeExecution({ ...bundle, events: duplicateIds })) + .toThrow(/sourceEventId.*unique/); + }); +}); diff --git a/packages/paperclip-runner/src/evals/native-execution.ts b/packages/paperclip-runner/src/evals/native-execution.ts new file mode 100644 index 0000000000..91a0fc359e --- /dev/null +++ b/packages/paperclip-runner/src/evals/native-execution.ts @@ -0,0 +1,552 @@ +import { readFile } from "node:fs/promises"; + +import { + validatePrpEvent, + type PrpEvent, + type PrpTerminalState, +} from "../protocol/replay-contract.js"; +import { PAPERCLIP_RUNNER_NATIVE_EXECUTION_SCHEMA } from "./build-metadata.js"; + +const SHA256_PATTERN = /^sha256:[a-f0-9]{64}$/; +const TOOL_OUTCOMES = new Set([ + "succeeded", + "denied", + "conflict", + "duplicate", + "unavailable", + "failed", +]); + +export const paperclipNativeExecutionSchemaUrl = new URL( + "../../protocol/schemas/native-execution.schema.json", + import.meta.url, +); +export const paperclipNativeExecutionFixtureUrl = new URL( + "../../protocol/fixtures/evals/native-execution-seeded.json", + import.meta.url, +); + +export interface NativeExecutionContentRef { + uri: string; + sha256: string; + byteSize: number; + mediaType: string; + [key: string]: unknown; +} + +export interface PaperclipNativeExecutionV1 { + schema: typeof PAPERCLIP_RUNNER_NATIVE_EXECUTION_SCHEMA; + provenance: "seeded" | "replay" | "live"; + identity: { + runId: string; + caseId: string; + configId: string; + attemptId: string; + [key: string]: unknown; + }; + input: { + caseSha256: string; + configSha256: string; + deterministicSeed: string; + [key: string]: unknown; + }; + runner: { + package: { name: string; version: string; [key: string]: unknown }; + binary: { name: string; sha256: string; [key: string]: unknown }; + prpVersion: number; + nativeExecutionVersion: number; + catalogVersion: number; + catalogSha256: string; + driverContractVersion: number; + driverKind: string; + driverVersion: string; + [key: string]: unknown; + }; + events: PrpEvent[]; + semanticTools: { + definitions: Array>; + calls: Array<{ callId: string; operationId: string; eventId: string; [key: string]: unknown }>; + results: Array<{ + callId: string; + operationId: string; + eventId: string; + outcome: string; + [key: string]: unknown; + }>; + denials: Array<{ + callId: string; + code: string; + authorizationBoundary: string; + retryable: boolean; + [key: string]: unknown; + }>; + [key: string]: unknown; + }; + terminal: PrpTerminalState; + observations: Record; + usage: { + inputTokens: number; + outputTokens: number; + totalTokens: number; + requestCount: number; + durationMs: number; + cost: { currency: string; amountMicros: number; [key: string]: unknown } | null; + [key: string]: unknown; + }; + transcript: NativeExecutionContentRef & { + complete: boolean; + eventCount: number; + omissionReason: string | null; + }; + artifactRoot: NativeExecutionContentRef; + artifacts: NativeExecutionContentRef[]; + failure: { + classification: string; + code: string; + message: string; + [key: string]: unknown; + } | null; + [key: string]: unknown; +} + +export class PaperclipNativeExecutionError extends Error { + readonly code = "paperclip_native_execution_invalid" as const; + + constructor( + message: string, + readonly path: string, + ) { + super(`${path}: ${message}`); + this.name = "PaperclipNativeExecutionError"; + } +} + +function fail(path: string, message: string): never { + throw new PaperclipNativeExecutionError(message, path); +} + +function record(value: unknown, path: string): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + fail(path, "must be an object"); + } + return value as Record; +} + +function array(value: unknown, path: string): unknown[] { + if (!Array.isArray(value)) fail(path, "must be an array"); + return value; +} + +function text(value: unknown, path: string): string { + if (typeof value !== "string" || value.trim().length === 0) { + fail(path, "must be a non-empty string"); + } + return value; +} + +function integer(value: unknown, path: string, minimum = 0): number { + if (!Number.isSafeInteger(value) || (value as number) < minimum) { + fail(path, `must be an integer >= ${minimum}`); + } + return value as number; +} + +function boolean(value: unknown, path: string): boolean { + if (typeof value !== "boolean") fail(path, "must be a boolean"); + return value; +} + +function digest(value: unknown, path: string): string { + const parsed = text(value, path); + if (!SHA256_PATTERN.test(parsed)) fail(path, "must use sha256:<64 lowercase hex>"); + return parsed; +} + +function contentRef(value: unknown, path: string): NativeExecutionContentRef { + const ref = record(value, path); + return { + ...structuredClone(ref), + uri: text(ref.uri, `${path}.uri`), + sha256: digest(ref.sha256, `${path}.sha256`), + byteSize: integer(ref.byteSize, `${path}.byteSize`), + mediaType: text(ref.mediaType, `${path}.mediaType`), + } as NativeExecutionContentRef; +} + +function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (typeof value === "object" && value !== null) { + const object = value as Record; + return `{${Object.keys(object) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalJson(object[key])}`) + .join(",")}}`; + } + return JSON.stringify(value) ?? "null"; +} + +function semanticEntry( + value: unknown, + path: string, + fields: readonly string[], +): Record { + const entry = record(value, path); + for (const field of fields) text(entry[field], `${path}.${field}`); + return structuredClone(entry); +} + +/** + * Validate the App-native attempt bundle while preserving unknown additive + * fields. Required identities, digests, event bindings, terminal consistency, + * semantic-tool outcomes, usage, and transcript completeness fail closed. + */ +export function parsePaperclipNativeExecution( + value: unknown, +): PaperclipNativeExecutionV1 { + const bundle = record(value, "bundle"); + if (bundle.schema !== PAPERCLIP_RUNNER_NATIVE_EXECUTION_SCHEMA) { + fail( + "bundle.schema", + `unsupported schema ${String(bundle.schema)}; expected ${PAPERCLIP_RUNNER_NATIVE_EXECUTION_SCHEMA}`, + ); + } + if (bundle.provenance !== "seeded" && bundle.provenance !== "replay" && bundle.provenance !== "live") { + fail("bundle.provenance", "must be seeded, replay, or live"); + } + + const identity = record(bundle.identity, "bundle.identity"); + const parsedIdentity = { + ...structuredClone(identity), + runId: text(identity.runId, "bundle.identity.runId"), + caseId: text(identity.caseId, "bundle.identity.caseId"), + configId: text(identity.configId, "bundle.identity.configId"), + attemptId: text(identity.attemptId, "bundle.identity.attemptId"), + }; + const input = record(bundle.input, "bundle.input"); + const parsedInput = { + ...structuredClone(input), + caseSha256: digest(input.caseSha256, "bundle.input.caseSha256"), + configSha256: digest(input.configSha256, "bundle.input.configSha256"), + deterministicSeed: text(input.deterministicSeed, "bundle.input.deterministicSeed"), + }; + + const runner = record(bundle.runner, "bundle.runner"); + const runnerPackage = record(runner.package, "bundle.runner.package"); + const runnerBinary = record(runner.binary, "bundle.runner.binary"); + const parsedRunner = { + ...structuredClone(runner), + package: { + ...structuredClone(runnerPackage), + name: text(runnerPackage.name, "bundle.runner.package.name"), + version: text(runnerPackage.version, "bundle.runner.package.version"), + }, + binary: { + ...structuredClone(runnerBinary), + name: text(runnerBinary.name, "bundle.runner.binary.name"), + sha256: digest(runnerBinary.sha256, "bundle.runner.binary.sha256"), + }, + prpVersion: integer(runner.prpVersion, "bundle.runner.prpVersion", 1), + nativeExecutionVersion: integer( + runner.nativeExecutionVersion, + "bundle.runner.nativeExecutionVersion", + 1, + ), + catalogVersion: integer(runner.catalogVersion, "bundle.runner.catalogVersion", 1), + catalogSha256: digest(runner.catalogSha256, "bundle.runner.catalogSha256"), + driverContractVersion: integer( + runner.driverContractVersion, + "bundle.runner.driverContractVersion", + 1, + ), + driverKind: text(runner.driverKind, "bundle.runner.driverKind"), + driverVersion: text(runner.driverVersion, "bundle.runner.driverVersion"), + }; + + const sourceCursors = new Map(); + const sourceEventIds = new Set(); + const events = array(bundle.events, "bundle.events").map((event, index) => { + const validation = validatePrpEvent(event); + if (!validation.ok) { + fail( + `bundle.events[${index}]${validation.issues[0]?.path ?? ""}`, + validation.issues[0]?.message ?? "invalid PRP event", + ); + } + if (validation.event.runId !== parsedIdentity.runId) { + fail(`bundle.events[${index}].runId`, "must match bundle.identity.runId"); + } + if (sourceEventIds.has(validation.event.sourceEventId)) { + fail(`bundle.events[${index}].sourceEventId`, "must be unique within the bundle"); + } + sourceEventIds.add(validation.event.sourceEventId); + const previousSourceSeq = sourceCursors.get(validation.event.sourceInstanceId) ?? 0; + if (validation.event.sourceSeq <= previousSourceSeq) { + fail( + `bundle.events[${index}].sourceSeq`, + `must be ordered after ${previousSourceSeq} for source ${validation.event.sourceInstanceId}`, + ); + } + sourceCursors.set(validation.event.sourceInstanceId, validation.event.sourceSeq); + return structuredClone(validation.event); + }); + if (events.length === 0) fail("bundle.events", "must contain native events"); + + const terminal = record(bundle.terminal, "bundle.terminal") as unknown as PrpTerminalState; + if (terminal.schema !== "paperclip.prp.terminal.v1") { + fail("bundle.terminal.schema", "must be paperclip.prp.terminal.v1"); + } + const terminalEvents = events.filter((event) => event.eventType === "run.terminal"); + if (terminalEvents.length !== 1) { + fail("bundle.events", "must contain exactly one run.terminal event"); + } + if (canonicalJson(terminalEvents[0]?.payload) !== canonicalJson(terminal)) { + fail("bundle.terminal", "must equal the run.terminal event payload"); + } + + const semanticTools = record(bundle.semanticTools, "bundle.semanticTools"); + const definitions = array(semanticTools.definitions, "bundle.semanticTools.definitions") + .map((entry, index) => semanticEntry(entry, `bundle.semanticTools.definitions[${index}]`, ["operationId"])); + const calls = array(semanticTools.calls, "bundle.semanticTools.calls").map((entry, index) => { + const parsed = semanticEntry(entry, `bundle.semanticTools.calls[${index}]`, [ + "callId", + "operationId", + "eventId", + ]); + return parsed as PaperclipNativeExecutionV1["semanticTools"]["calls"][number]; + }); + const results = array(semanticTools.results, "bundle.semanticTools.results").map((entry, index) => { + const path = `bundle.semanticTools.results[${index}]`; + const parsed = semanticEntry(entry, path, ["callId", "operationId", "eventId", "outcome"]); + if (!TOOL_OUTCOMES.has(parsed.outcome as string)) fail(`${path}.outcome`, "is unsupported"); + return parsed as PaperclipNativeExecutionV1["semanticTools"]["results"][number]; + }); + const denials = array(semanticTools.denials, "bundle.semanticTools.denials").map((entry, index) => { + const path = `bundle.semanticTools.denials[${index}]`; + const parsed = semanticEntry(entry, path, ["callId", "code", "authorizationBoundary"]); + parsed.retryable = boolean(parsed.retryable, `${path}.retryable`); + return parsed as PaperclipNativeExecutionV1["semanticTools"]["denials"][number]; + }); + const callById = new Map(calls.map((call) => [call.callId, call])); + if (callById.size !== calls.length) { + fail("bundle.semanticTools.calls", "callId values must be unique"); + } + const resultByCallId = new Map(results.map((result) => [result.callId, result])); + if (resultByCallId.size !== results.length) { + fail("bundle.semanticTools.results", "callId values must be unique"); + } + const eventById = new Map(events.map((event) => [event.sourceEventId, event])); + const referencedCallEventIds = new Set(); + for (const call of calls) { + const event = eventById.get(call.eventId); + if (event?.eventType !== "mcp_app.tool_input") { + fail("bundle.semanticTools.calls", `call ${call.callId} does not reference a tool-input event`); + } + const envelope = semanticToolEnvelope(event, "input", `event ${call.eventId}`); + if (envelope.callId !== call.callId || envelope.operationId !== call.operationId) { + fail( + "bundle.semanticTools.calls", + `call ${call.callId} does not match semantic envelope ${String(envelope.callId)}/${String(envelope.operationId)}`, + ); + } + if (referencedCallEventIds.has(call.eventId)) { + fail("bundle.semanticTools.calls", `event ${call.eventId} is referenced more than once`); + } + referencedCallEventIds.add(call.eventId); + } + const referencedResultEventIds = new Set(); + for (const result of results) { + const call = callById.get(result.callId); + if (call === undefined || call.operationId !== result.operationId) { + fail( + "bundle.semanticTools.results", + `result ${result.callId} has no matching call with operation ${result.operationId}`, + ); + } + const event = eventById.get(result.eventId); + if (event?.eventType !== "mcp_app.tool_result") { + fail("bundle.semanticTools.results", `result ${result.callId} does not reference a tool-result event`); + } + const envelope = semanticToolEnvelope(event, "result", `event ${result.eventId}`); + if ( + envelope.callId !== result.callId + || envelope.operationId !== result.operationId + || envelope.outcome !== result.outcome + ) { + fail( + "bundle.semanticTools.results", + `result ${result.callId} does not match semantic envelope ${String(envelope.callId)}/${String(envelope.operationId)}/${String(envelope.outcome)}`, + ); + } + if (referencedResultEventIds.has(result.eventId)) { + fail("bundle.semanticTools.results", `event ${result.eventId} is referenced more than once`); + } + referencedResultEventIds.add(result.eventId); + if (result.outcome === "denied" && !denials.some((denial) => denial.callId === result.callId)) { + fail("bundle.semanticTools.denials", `denied result ${result.callId} needs a denial receipt`); + } + } + const permitsUnresolvedCalls = + terminal.turnTerminalState === "interrupted" + || terminal.turnTerminalState === "cancelled" + || terminal.runTerminalState === "cancelled"; + if (!permitsUnresolvedCalls) { + for (const call of calls) { + if (!resultByCallId.has(call.callId)) { + fail( + "bundle.semanticTools.results", + "a non-interrupted terminal state requires exactly one result for every semantic call", + ); + } + } + } + for (const event of events) { + if (event.eventType === "mcp_app.tool_input" && !referencedCallEventIds.has(event.sourceEventId)) { + fail("bundle.semanticTools.calls", `tool-input event ${event.sourceEventId} is not indexed`); + } + if (event.eventType === "mcp_app.tool_result" && !referencedResultEventIds.has(event.sourceEventId)) { + fail("bundle.semanticTools.results", `tool-result event ${event.sourceEventId} is not indexed`); + } + } + const denialCallIds = new Set(); + for (const denial of denials) { + if (denialCallIds.has(denial.callId)) { + fail("bundle.semanticTools.denials", `call ${denial.callId} has duplicate denial receipts`); + } + denialCallIds.add(denial.callId); + const result = resultByCallId.get(denial.callId); + if (result?.outcome !== "denied") { + fail("bundle.semanticTools.denials", `denial ${denial.callId} has no denied result`); + } + const resultEvent = eventById.get(result.eventId)!; + const envelope = semanticToolEnvelope( + resultEvent, + "result", + `event ${result.eventId}`, + ); + if ( + envelope.code !== denial.code + || envelope.authorizationBoundary !== denial.authorizationBoundary + || envelope.retryable !== denial.retryable + ) { + fail( + "bundle.semanticTools.denials", + `denial ${denial.callId} does not match its semantic result receipt`, + ); + } + } + + const usage = record(bundle.usage, "bundle.usage"); + const inputTokens = integer(usage.inputTokens, "bundle.usage.inputTokens"); + const outputTokens = integer(usage.outputTokens, "bundle.usage.outputTokens"); + const totalTokens = integer(usage.totalTokens, "bundle.usage.totalTokens"); + if (totalTokens !== inputTokens + outputTokens) { + fail("bundle.usage.totalTokens", "must equal inputTokens + outputTokens"); + } + const costRecord = usage.cost === null ? null : record(usage.cost, "bundle.usage.cost"); + const parsedUsage = { + ...structuredClone(usage), + inputTokens, + outputTokens, + totalTokens, + requestCount: integer(usage.requestCount, "bundle.usage.requestCount"), + durationMs: integer(usage.durationMs, "bundle.usage.durationMs"), + cost: costRecord === null + ? null + : { + ...structuredClone(costRecord), + currency: text(costRecord.currency, "bundle.usage.cost.currency"), + amountMicros: integer(costRecord.amountMicros, "bundle.usage.cost.amountMicros"), + }, + }; + + const transcriptRecord = record(bundle.transcript, "bundle.transcript"); + const transcript = { + ...contentRef(transcriptRecord, "bundle.transcript"), + complete: boolean(transcriptRecord.complete, "bundle.transcript.complete"), + eventCount: integer(transcriptRecord.eventCount, "bundle.transcript.eventCount"), + omissionReason: transcriptRecord.omissionReason === null + ? null + : text(transcriptRecord.omissionReason, "bundle.transcript.omissionReason"), + }; + if (transcript.complete && transcript.eventCount !== events.length) { + fail("bundle.transcript.eventCount", "must equal events.length for a complete transcript"); + } + if (!transcript.complete && transcript.omissionReason === null) { + fail("bundle.transcript.omissionReason", "is required when the transcript is incomplete"); + } + + const observations = structuredClone(record(bundle.observations, "bundle.observations")); + const artifactRoot = contentRef(bundle.artifactRoot, "bundle.artifactRoot"); + const artifacts = array(bundle.artifacts, "bundle.artifacts") + .map((entry, index) => contentRef(entry, `bundle.artifacts[${index}]`)); + const failure = bundle.failure === null + ? null + : (() => { + const parsed = record(bundle.failure, "bundle.failure"); + return { + ...structuredClone(parsed), + classification: text(parsed.classification, "bundle.failure.classification"), + code: text(parsed.code, "bundle.failure.code"), + message: text(parsed.message, "bundle.failure.message"), + }; + })(); + + return { + ...structuredClone(bundle), + schema: PAPERCLIP_RUNNER_NATIVE_EXECUTION_SCHEMA, + provenance: bundle.provenance, + identity: parsedIdentity, + input: parsedInput, + runner: parsedRunner, + events, + semanticTools: { ...structuredClone(semanticTools), definitions, calls, results, denials }, + terminal: structuredClone(terminal), + observations, + usage: parsedUsage, + transcript, + artifactRoot, + artifacts, + failure, + } as PaperclipNativeExecutionV1; +} + +function semanticToolEnvelope( + event: PrpEvent, + phase: "input" | "result", + path: string, +): Record { + const payload = record(event.payload, `${path}.payload`); + const envelope = record(payload.semantic_tool, `${path}.payload.semantic_tool`); + if ( + envelope.schema !== "paperclip.prp.semantic_tool.v1" + || envelope.schemaVersion !== 1 + || envelope.phase !== phase + ) { + fail( + `${path}.payload.semantic_tool`, + `must be a paperclip.prp.semantic_tool.v1 ${phase} envelope`, + ); + } + const correlation = record(envelope.correlation, `${path}.payload.semantic_tool.correlation`); + for (const [field, eventValue] of [ + ["runId", event.runId], + ["normalizedSessionId", event.normalizedSessionId], + ["turnId", event.turnId], + ["itemId", event.itemId], + ] as const) { + if (correlation[field] !== eventValue) { + fail( + `${path}.payload.semantic_tool.correlation.${field}`, + `must match the enclosing event ${field}`, + ); + } + } + return envelope; +} + +export async function loadPaperclipNativeExecutionFixture( + url: URL = paperclipNativeExecutionFixtureUrl, +): Promise { + const value = JSON.parse(await readFile(url, "utf8")) as unknown; + return parsePaperclipNativeExecution(value); +} diff --git a/packages/paperclip-runner/src/evals/runnerd-artifact.test.ts b/packages/paperclip-runner/src/evals/runnerd-artifact.test.ts new file mode 100644 index 0000000000..7e4cabe7b7 --- /dev/null +++ b/packages/paperclip-runner/src/evals/runnerd-artifact.test.ts @@ -0,0 +1,91 @@ +import { createHash } from "node:crypto"; +import { writeFileSync } from "node:fs"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { PAPERCLIP_RUNNERD_BUILD_METADATA_SCHEMA } from "./build-metadata.js"; +import { + PaperclipRunnerdArtifactError, + parsePaperclipRunnerdBuildMetadata, + resolvePaperclipRunnerdArtifact, +} from "./runnerd-artifact.js"; + +const valid = { + schema: PAPERCLIP_RUNNERD_BUILD_METADATA_SCHEMA, + binaryName: "paperclip-runnerd", + packageName: "@paperclipai/paperclip-runner", + packageVersion: "0.0.0", + binaryContractVersion: 2, + nativeExecutionVersion: 1, + harnessDriverVersion: 1, + prp: { name: "paperclip.runner", minimumVersion: 1, maximumVersion: 1 }, +}; + +describe("runnerd artifact metadata", () => { + it("parses the exact runnerd identity", () => { + expect(parsePaperclipRunnerdBuildMetadata(valid)).toEqual(valid); + }); + + it("rejects an unknown binary metadata schema", () => { + expect(() => parsePaperclipRunnerdBuildMetadata({ ...valid, schema: "runnerd/v2" })) + .toThrow(PaperclipRunnerdArtifactError); + expect(() => parsePaperclipRunnerdBuildMetadata({ ...valid, schema: "runnerd/v2" })) + .toThrow(/unsupported/); + }); + + it("rejects an invalid or mismatched explicit artifact digest before execution", async () => { + await expect(resolvePaperclipRunnerdArtifact({ + executablePath: "/does/not/matter", + expectedSha256: "not-a-digest", + })).rejects.toMatchObject({ issue: "digest_invalid" }); + + const root = await mkdtemp(join(tmpdir(), "paperclip-runnerd-artifact-")); + const executablePath = join(root, "paperclip-runnerd"); + try { + await writeFile(executablePath, "not the expected binary"); + await expect(resolvePaperclipRunnerdArtifact({ + executablePath, + expectedSha256: `sha256:${"0".repeat(64)}`, + })).rejects.toMatchObject({ + issue: "digest_mismatch", + message: expect.stringContaining("observed sha256:"), + }); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + it.skipIf(process.platform === "win32")( + "executes a private copy of the verified bytes when the source path is swapped", + async () => { + const root = await mkdtemp(join(tmpdir(), "paperclip-runnerd-artifact-")); + const executablePath = join(root, "paperclip-runnerd"); + const verifiedScript = `#!/bin/sh\nprintf '%s\\n' '${JSON.stringify(valid)}'\n`; + const replacementScript = "#!/bin/sh\nprintf '%s\\n' 'unverified replacement'\n"; + const expectedSha256 = `sha256:${createHash("sha256").update(verifiedScript).digest("hex")}`; + + try { + await writeFile(executablePath, verifiedScript, { mode: 0o700 }); + const input = { + executablePath, + expectedSha256, + get metadataTimeoutMs() { + writeFileSync(executablePath, replacementScript, { mode: 0o700 }); + return 5_000; + }, + }; + + await expect(resolvePaperclipRunnerdArtifact(input)).resolves.toMatchObject({ + executablePath, + sha256: expectedSha256, + byteSize: Buffer.byteLength(verifiedScript), + buildMetadata: valid, + }); + } finally { + await rm(root, { recursive: true, force: true }); + } + }, + ); +}); diff --git a/packages/paperclip-runner/src/evals/runnerd-artifact.ts b/packages/paperclip-runner/src/evals/runnerd-artifact.ts new file mode 100644 index 0000000000..2673302d24 --- /dev/null +++ b/packages/paperclip-runner/src/evals/runnerd-artifact.ts @@ -0,0 +1,230 @@ +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { chmod, mkdtemp, readFile, realpath, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; + +import { + PAPERCLIP_RUNNERD_BUILD_METADATA_SCHEMA, + PAPERCLIP_RUNNER_BUILD_METADATA, +} from "./build-metadata.js"; + +const execFileAsync = promisify(execFile); +const SHA256_PATTERN = /^sha256:[a-f0-9]{64}$/; + +async function readVerifiedBuildMetadata( + bytes: Buffer, + timeoutMs: number, +): Promise { + const stagingDirectory = await mkdtemp(join(tmpdir(), "paperclip-runnerd-verified-")); + const stagedExecutable = join( + stagingDirectory, + process.platform === "win32" ? "paperclip-runnerd.exe" : "paperclip-runnerd", + ); + + try { + await chmod(stagingDirectory, 0o700); + await writeFile(stagedExecutable, bytes, { flag: "wx", mode: 0o700 }); + await chmod(stagedExecutable, 0o700); + const { stdout } = await execFileAsync(stagedExecutable, ["--build-metadata"], { + encoding: "utf8", + timeout: timeoutMs, + maxBuffer: 64 * 1024, + }); + return stdout; + } finally { + await rm(stagingDirectory, { recursive: true, force: true }); + } +} + +export interface PaperclipRunnerdBuildMetadata { + schema: typeof PAPERCLIP_RUNNERD_BUILD_METADATA_SCHEMA; + binaryName: "paperclip-runnerd"; + packageName: "@paperclipai/paperclip-runner"; + packageVersion: string; + binaryContractVersion: number; + nativeExecutionVersion: number; + harnessDriverVersion: number; + prp: { + name: string; + minimumVersion: number; + maximumVersion: number; + }; +} + +export interface PaperclipRunnerdArtifact { + executablePath: string; + sha256: string; + byteSize: number; + buildMetadata: PaperclipRunnerdBuildMetadata; +} + +export class PaperclipRunnerdArtifactError extends Error { + readonly code = "paperclip_runnerd_artifact_invalid" as const; + + constructor( + message: string, + readonly issue: + | "path_invalid" + | "digest_invalid" + | "digest_mismatch" + | "metadata_unavailable" + | "metadata_invalid", + ) { + super(message); + this.name = "PaperclipRunnerdArtifactError"; + } +} + +function record(value: unknown, path: string): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new PaperclipRunnerdArtifactError(`${path} must be an object`, "metadata_invalid"); + } + return value as Record; +} + +function text(value: unknown, path: string): string { + if (typeof value !== "string" || value.trim().length === 0) { + throw new PaperclipRunnerdArtifactError(`${path} must be a non-empty string`, "metadata_invalid"); + } + return value; +} + +function version(value: unknown, path: string): number { + if (!Number.isSafeInteger(value) || (value as number) < 1) { + throw new PaperclipRunnerdArtifactError(`${path} must be a positive integer`, "metadata_invalid"); + } + return value as number; +} + +/** Parse the runnerd response without accepting a look-alike binary. */ +export function parsePaperclipRunnerdBuildMetadata( + value: unknown, +): PaperclipRunnerdBuildMetadata { + const metadata = record(value, "runnerd build metadata"); + if (metadata.schema !== PAPERCLIP_RUNNERD_BUILD_METADATA_SCHEMA) { + throw new PaperclipRunnerdArtifactError( + `runnerd metadata schema ${String(metadata.schema)} is unsupported; expected ${PAPERCLIP_RUNNERD_BUILD_METADATA_SCHEMA}`, + "metadata_invalid", + ); + } + if (metadata.binaryName !== "paperclip-runnerd") { + throw new PaperclipRunnerdArtifactError( + `runnerd metadata names unexpected binary ${String(metadata.binaryName)}`, + "metadata_invalid", + ); + } + if (metadata.packageName !== PAPERCLIP_RUNNER_BUILD_METADATA.package.name) { + throw new PaperclipRunnerdArtifactError( + `runnerd metadata names unexpected package ${String(metadata.packageName)}`, + "metadata_invalid", + ); + } + const prp = record(metadata.prp, "runnerd build metadata.prp"); + const minimumVersion = version( + prp.minimumVersion, + "runnerd build metadata.prp.minimumVersion", + ); + const maximumVersion = version( + prp.maximumVersion, + "runnerd build metadata.prp.maximumVersion", + ); + if (minimumVersion > maximumVersion) { + throw new PaperclipRunnerdArtifactError( + "runnerd build metadata.prp minimumVersion must not exceed maximumVersion", + "metadata_invalid", + ); + } + return { + schema: PAPERCLIP_RUNNERD_BUILD_METADATA_SCHEMA, + binaryName: "paperclip-runnerd", + packageName: PAPERCLIP_RUNNER_BUILD_METADATA.package.name, + packageVersion: text(metadata.packageVersion, "runnerd build metadata.packageVersion"), + binaryContractVersion: version( + metadata.binaryContractVersion, + "runnerd build metadata.binaryContractVersion", + ), + nativeExecutionVersion: version( + metadata.nativeExecutionVersion, + "runnerd build metadata.nativeExecutionVersion", + ), + harnessDriverVersion: version( + metadata.harnessDriverVersion, + "runnerd build metadata.harnessDriverVersion", + ), + prp: { + name: text(prp.name, "runnerd build metadata.prp.name"), + minimumVersion, + maximumVersion, + }, + }; +} + +/** + * Resolve one explicitly supplied runnerd artifact, verify its content digest, + * and read version metadata from that exact executable. This never searches + * PATH or the App source tree. + */ +export async function resolvePaperclipRunnerdArtifact(input: { + executablePath: string; + expectedSha256: string; + metadataTimeoutMs?: number; +}): Promise { + if (!SHA256_PATTERN.test(input.expectedSha256)) { + throw new PaperclipRunnerdArtifactError( + "expectedSha256 must use the sha256:<64 lowercase hex> form", + "digest_invalid", + ); + } + + let executablePath: string; + let fileStat: Awaited>; + let bytes: Buffer; + try { + executablePath = await realpath(input.executablePath); + fileStat = await stat(executablePath); + if (!fileStat.isFile()) throw new Error("path is not a file"); + bytes = await readFile(executablePath); + } catch (error) { + throw new PaperclipRunnerdArtifactError( + `runnerd artifact ${input.executablePath} is unavailable: ${error instanceof Error ? error.message : String(error)}`, + "path_invalid", + ); + } + + const observedSha256 = `sha256:${createHash("sha256").update(bytes).digest("hex")}`; + if (observedSha256 !== input.expectedSha256) { + throw new PaperclipRunnerdArtifactError( + `runnerd artifact digest mismatch: expected ${input.expectedSha256}, observed ${observedSha256}`, + "digest_mismatch", + ); + } + + let stdout: string; + try { + stdout = await readVerifiedBuildMetadata(bytes, input.metadataTimeoutMs ?? 5_000); + } catch (error) { + throw new PaperclipRunnerdArtifactError( + `runnerd artifact did not return build metadata: ${error instanceof Error ? error.message : String(error)}`, + "metadata_unavailable", + ); + } + + let parsed: unknown; + try { + parsed = JSON.parse(stdout) as unknown; + } catch (error) { + throw new PaperclipRunnerdArtifactError( + `runnerd build metadata is not JSON: ${error instanceof Error ? error.message : String(error)}`, + "metadata_invalid", + ); + } + + return { + executablePath, + sha256: observedSha256, + byteSize: bytes.byteLength, + buildMetadata: parsePaperclipRunnerdBuildMetadata(parsed), + }; +}