From 75289b06b1f12df1a951cd8bf6f876113ecb71f5 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 6 Sep 2026 10:56:27 -0500 Subject: [PATCH] fix(runner): mirror production context in direct evals --- .../docs/runner-protocol-live-evals.md | 6 ++ .../src/cli/eval-session-contract.test.ts | 30 ++++++ .../paperclip-runner/src/cli/eval-session.ts | 94 ++++++++++++++++++- .../src/live/runnerd-codex-transport.test.ts | 16 +++- .../src/live/runnerd-codex-transport.ts | 19 ++-- 5 files changed, 150 insertions(+), 15 deletions(-) diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index 8c4761e980..62092030af 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -50,6 +50,12 @@ versions and patched ACP server bytes. Do not replace this step with a fresh `patchedDependencies`, so the resulting ACPX executables no longer match their qualified digests. +The direct eval CLI also materializes a minimal immutable native runtime +context in each isolated attempt workspace. This keeps the direct layer on the +same `paperclip.native-execution-input.v3` contract as production, including +the AgentCore HarnessSkill upload path, without borrowing any browser E2E +setup. + `all` is intentionally literal. A disabled driver, missing remote profile, or unavailable provider is retained as an infrastructure result; it is not silently omitted. In particular, the ACPX Pi roster remains visible while Pi diff --git a/packages/paperclip-runner/src/cli/eval-session-contract.test.ts b/packages/paperclip-runner/src/cli/eval-session-contract.test.ts index e2765f0307..9f5857cc1f 100644 --- a/packages/paperclip-runner/src/cli/eval-session-contract.test.ts +++ b/packages/paperclip-runner/src/cli/eval-session-contract.test.ts @@ -1,5 +1,10 @@ +import { chmod, mkdtemp, readFile, rm, stat } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + import { describe, expect, it } from "vitest"; +import { parseNativeRuntimeContext } from "../contracts/runtime-context.js"; import type { CapabilityLiveSessionSnapshot } from "../live/live-session.js"; import { evalSessionUsage, @@ -8,6 +13,7 @@ import { import { boundedEvalSessionUsage, evalSessionProviderVersion, + prepareEvalRuntimeContext, } from "./eval-session.js"; function request(overrides: Record = {}): unknown { @@ -55,6 +61,30 @@ function agentCoreProfile(overrides: Record = {}) { } describe("eval-session request contract", () => { + it("materializes a production-v3 runtime context for direct live providers", async () => { + const workspace = await mkdtemp(join(tmpdir(), "paperclip-eval-context-")); + let instructionRoot: string | null = null; + try { + const context = await prepareEvalRuntimeContext(workspace); + instructionRoot = context.instructions.bundle.rootPath; + expect(parseNativeRuntimeContext(context)).toEqual(context); + expect(context.skills).toEqual([]); + expect(context.mcp.assignmentSetId).toBe("paperclip-runner-direct-eval-v1"); + expect(context.instructions.entryPath).toBe("AGENTS.md"); + expect(await readFile( + join(context.instructions.bundle.rootPath, "AGENTS.md"), + "utf8", + )).toContain("Paperclip direct live evaluation"); + expect((await stat(context.instructions.bundle.rootPath)).mode & 0o777) + .toBe(0o555); + } finally { + if (instructionRoot !== null) { + await chmod(instructionRoot, 0o700).catch(() => undefined); + } + await rm(workspace, { recursive: true, force: true }); + } + }); + it("normalizes the current local live-session provider contract", () => { expect(parseEvalSessionRequest(request())).toMatchObject({ provider: "codex", diff --git a/packages/paperclip-runner/src/cli/eval-session.ts b/packages/paperclip-runner/src/cli/eval-session.ts index 609ec73090..a53fa3f7d4 100644 --- a/packages/paperclip-runner/src/cli/eval-session.ts +++ b/packages/paperclip-runner/src/cli/eval-session.ts @@ -1,9 +1,18 @@ #!/usr/bin/env node import { createHash } from "node:crypto"; -import { readFile, writeFile } from "node:fs/promises"; +import { chmod, mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { + NATIVE_RUNTIME_ASSET_SCHEMA, + PAPERCLIP_EXECUTION_PROMPT, + PAPERCLIP_EXECUTION_PROMPT_REVISION, + canonicalNativeRuntimeContextDigest, + nativeRuntimePromptDigest, + parseNativeRuntimeContext, + type NativeRuntimeContextSnapshot, +} from "../contracts/runtime-context.js"; import { projectCapabilityDevtools } from "../devtools/index.js"; import { resolveQualifiedAcpxProfile } from "../drivers/acpx/qualified-profiles.js"; import { PAPERCLIP_RUNNER_BUILD_METADATA } from "../evals/build-metadata.js"; @@ -53,6 +62,85 @@ async function sha256(path: string): Promise { return createHash("sha256").update(await readFile(path)).digest("hex"); } +const EVAL_RUNTIME_INSTRUCTIONS = [ + "# Paperclip direct live evaluation", + "", + "Use the provided Paperclip semantic tools to inspect and act on the assigned task.", + "Treat the seeded control-plane state as authoritative and keep every action within the requested scope.", + "", +].join("\n"); + +/** Materializes the minimal immutable v3 context used by production-native providers. */ +export async function prepareEvalRuntimeContext( + workingDirectory: string, +): Promise { + const contextRoot = await mkdtemp( + resolve(workingDirectory, ".paperclip-eval-runtime-context-"), + ); + const instructionRoot = resolve(contextRoot, "instructions"); + const entryPath = "AGENTS.md"; + const entry = Buffer.from(EVAL_RUNTIME_INSTRUCTIONS); + const entryDigest = createHash("sha256").update(entry).digest("hex"); + const manifestFiles = [{ + path: entryPath, + sha256: entryDigest, + mode: 0o444, + size: entry.byteLength, + }]; + const assetDigest = createHash("sha256") + .update(JSON.stringify(manifestFiles)) + .digest("hex"); + const manifestText = `${JSON.stringify({ + schema: "paperclip.runtime-asset-manifest.v1", + digest: assetDigest, + fileCount: manifestFiles.length, + totalBytes: entry.byteLength, + files: manifestFiles, + })}\n`; + const manifestDigest = createHash("sha256") + .update(manifestText) + .digest("hex"); + await mkdir(instructionRoot, { recursive: true, mode: 0o700 }); + const entryFile = resolve(instructionRoot, entryPath); + await writeFile(entryFile, entry, { flag: "wx", mode: 0o444 }); + await chmod(entryFile, 0o444); + await chmod(instructionRoot, 0o555); + + const semanticCatalogDigest = + PAPERCLIP_RUNNER_BUILD_METADATA.semanticCatalog.sha256.replace( + /^sha256:/, + "", + ); + const context = { + prompt: { + revision: PAPERCLIP_EXECUTION_PROMPT_REVISION, + text: PAPERCLIP_EXECUTION_PROMPT, + digest: nativeRuntimePromptDigest(), + }, + instructions: { + entryPath, + bundle: { + schema: NATIVE_RUNTIME_ASSET_SCHEMA, + digest: assetDigest, + manifestDigest, + rootPath: instructionRoot, + fileCount: 1, + totalBytes: entry.byteLength, + }, + }, + skills: [], + mcp: { + assignmentSetId: "paperclip-runner-direct-eval-v1", + digest: semanticCatalogDigest, + bindingId: null, + }, + } satisfies Omit; + return parseNativeRuntimeContext({ + ...context, + aggregateDigest: canonicalNativeRuntimeContextDigest(context), + }); +} + export function evalSessionProviderVersion( request: EvalSessionRequest, ): string | null { @@ -189,10 +277,14 @@ export async function runEvalSessionCli( const requestedDriver = request.driver ?? expectedEvalSessionDriver(requestedProvider); const requestedProviderVersion = evalSessionProviderVersion(request); + const runtimeContext = await prepareEvalRuntimeContext( + resolve(request.session.workingDirectory ?? process.cwd()), + ); const service = options.serviceFactory?.(runnerdPath) ?? new CapabilityLiveSessionService({ transportOptions: { runnerBinary: runnerdPath, + runtimeContext, // The transport performs the provider-specific allowlisting. Supplying // the source environment here is still required: without it the // isolated Codex home has no credential source and runnerd receives no diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index c0d62a561d..88916ffbf0 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -635,11 +635,17 @@ it("derives the ACPX package authority only from the verified dist/cli layout", }); it("keeps a deployed ACPX package inside its pnpm-owned dependency root", async () => { - const deployedPackageRoot = await mkdtemp( - join(tmpdir(), "paperclip-deployed-provider-package-"), + const deploymentRoot = await mkdtemp( + join(tmpdir(), "paperclip-deployed-provider-root-"), + ); + const deployedPackageRoot = join( + deploymentRoot, + "node_modules", + "@paperclipai", + "paperclip-runner", ); await mkdir(join(deployedPackageRoot, "dist", "cli"), { recursive: true }); - await mkdir(join(deployedPackageRoot, "node_modules", ".pnpm"), { + await mkdir(join(deploymentRoot, "node_modules", ".pnpm"), { recursive: true, }); try { @@ -654,11 +660,11 @@ it("keeps a deployed ACPX package inside its pnpm-owned dependency root", async deployedPackageRoot, ), ).toEqual({ - root: deployedPackageRoot, + root: deploymentRoot, manifest: join(deployedPackageRoot, "package.json"), }); } finally { - await rm(deployedPackageRoot, { recursive: true, force: true }); + await rm(deploymentRoot, { recursive: true, force: true }); } }); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index ec86cb6166..4bf3965bfe 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -1567,17 +1567,18 @@ function acpxProviderPackageAuthority( ); } const sidecarPackageRoot = resolve(cliDirectory, "../.."); - // A local source build consumes pnpm's workspace-owned node_modules tree. - // A deployed provider pack owns a closed node_modules tree at its own root. - // Source builds resolve provider dependencies from the monorepo root. A - // `pnpm deploy`, however, owns a complete virtual store below the deployed - // package root and must not escape to its caller's filesystem. This marker - // is generated by pnpm itself and keeps the two layouts unambiguous. + // A local source build lives at /packages/paperclip-runner and + // resolves dependencies from /node_modules. A `pnpm deploy` + // package lives at /node_modules/@paperclipai/paperclip-runner, + // beside /node_modules/.pnpm. The provider verifier receives the + // directory that owns node_modules, so step out one more level only for the + // deployed shape. + const sourceDependencyRoot = resolve(ownerPackageRoot, "../.."); const localDependencyRoot = existsSync( - resolve(ownerPackageRoot, "node_modules", ".pnpm"), + resolve(sourceDependencyRoot, ".pnpm"), ) - ? ownerPackageRoot - : resolve(ownerPackageRoot, "../.."); + ? resolve(sourceDependencyRoot, "..") + : sourceDependencyRoot; return sidecarPackageRoot === ownerPackageRoot ? { root: localDependencyRoot,