fix: reuse verified provider packs across equivalent builds

Normalize the provider-pack layout independently of runnerd build order and compare verified content rather than revision provenance, avoiding unnecessary multi-gigabyte uploads.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Dotta 2026-09-09 11:29:58 -05:00
parent a31eba2381
commit d3257a3340
8 changed files with 235 additions and 29 deletions

View File

@ -333,13 +333,26 @@ All routes start at
The Cloud app image includes a build-owned remote provider pack at
`/opt/paperclip-runner/provider-pack` and configures
`PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH` to that directory. Native OpenCode
and ACPX runs verify the sandbox's installed pack against this manifest; if it
differs, the host stages its complete pack before launch. The pack is built
and ACPX runs verify the sandbox's installed pack against this manifest. Reuse
requires a valid full manifest digest and matching content, including artifact
hashes, the distribution tree, dependency pins, platform, and Node requirements.
The source revision remains provenance; a revision-only difference does not
require retransferring identical contents. App and sandbox builds omit the
redundant `dist/bin/paperclip-runnerd` from the pack because that executable is
shipped and verified separately. If content differs, the host stages its
complete pack before launch. The pack is built
from the app revision, includes the production lockfile and artifact hashes,
and must pass its provider-launch checks during the image build. It belongs to
the app image, not the workspace volume or a scoped file collection. Ordinary
local execution is unchanged.
When diagnosing startup delays, distinguish scoped-file hydration from native
runtime preparation. `work_folder.prepared` records the intended layout before
hydration completes. `provider_pack.verify_preinstalled` reports installed-pack
verification; a fallback within `runner.artifact.prepare` can transfer gigabytes
independently of task files. Acceptance must prove that a matching image uses
its installed pack instead of silently relying on that fallback.
Automated tests do not qualify a deployed runner image. Before merging, use a
new pinned staging stack with the branch's Cloud image and matching migrator.
The deployed harness must target that tenant URL without launching a local

View File

@ -73,7 +73,7 @@
"typecheck:rust": "cargo fmt --manifest-path runner/Cargo.toml --all -- --check && cargo check --manifest-path runner/Cargo.toml --locked --workspace",
"typecheck:browser": "tsc -p tsconfig.browser.json --noEmit",
"test": "pnpm run test:typescript && pnpm run test:rust",
"test:typescript": "pnpm run ensure:eval-build-deps && pnpm run build:rust && node --test test/protocol-contract.test.mjs test/acpx-sidecar-contract.test.mjs test/acpx-codex-package-contract.test.mjs scripts/aws-agentcore-provisioning.test.mjs scripts/build-verified-provider-entrypoints.test.mjs scripts/local-provider-smoke-environment.test.mjs scripts/materialize-opencode-binary.test.mjs scripts/materialize-pi-binary.test.mjs scripts/portable-provider-shim.test.mjs && vitest run",
"test:typescript": "pnpm run ensure:eval-build-deps && pnpm run build:rust && node --test test/protocol-contract.test.mjs test/acpx-sidecar-contract.test.mjs test/acpx-codex-package-contract.test.mjs scripts/aws-agentcore-provisioning.test.mjs scripts/build-verified-provider-entrypoints.test.mjs scripts/local-provider-smoke-environment.test.mjs scripts/materialize-opencode-binary.test.mjs scripts/materialize-pi-binary.test.mjs scripts/portable-provider-shim.test.mjs scripts/provider-pack-layout.test.mjs && vitest run",
"test:rust": "cargo test --release --manifest-path runner/Cargo.toml --locked --workspace",
"test:codex": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --test codex_provider",
"test:durable": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core durable::",

View File

@ -1,3 +1,4 @@
import { normalizeProviderPackLayout } from "./provider-pack-layout.mjs";
import { portableProviderShim } from "./portable-provider-shim.mjs";
import { createHash } from "node:crypto";
import { execFileSync, spawnSync } from "node:child_process";
@ -113,6 +114,8 @@ try {
throw new Error(`pnpm deploy failed with exit code ${deployed.status}`);
}
normalizeProviderPackLayout(temporaryRoot);
// Fail the image build if a bridge silently brings back an older/private
// provider CLI. A direct dependency alone does not deduplicate pnpm's graph.
const packRequire = createRequire(join(temporaryRoot, "package.json"));

View File

@ -0,0 +1,19 @@
import { existsSync, lstatSync, readdirSync, rmSync, rmdirSync } from "node:fs";
import { join } from "node:path";
/** runnerd is shipped and verified separately from the JavaScript provider pack. */
export function normalizeProviderPackLayout(packRoot) {
const dist = join(packRoot, "dist");
if (!existsSync(dist)) return;
if (!lstatSync(dist).isDirectory()) {
throw new Error("Provider pack dist must be a directory");
}
const bin = join(dist, "bin");
if (!existsSync(bin)) return;
if (!lstatSync(bin).isDirectory()) {
throw new Error("Provider pack dist/bin must be a directory");
}
rmSync(join(bin, "paperclip-runnerd"), { force: true });
// Keep every other entry, including future provider executables.
if (readdirSync(bin).length === 0) rmdirSync(bin);
}

View File

@ -0,0 +1,43 @@
import assert from "node:assert/strict";
import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, readdirSync, rmSync, symlinkSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import test from "node:test";
import { normalizeProviderPackLayout } from "./provider-pack-layout.mjs";
function fixture(t) {
const root = mkdtempSync(join(tmpdir(), "provider-pack-layout-"));
t.after(() => rmSync(root, { recursive: true, force: true }));
mkdirSync(join(root, "dist", "cli"), { recursive: true });
writeFileSync(join(root, "dist", "cli", "provider.cjs"), "provider bytes\n");
return root;
}
test("app and sandbox image build orders produce the same provider layout", (t) => {
const app = fixture(t), image = fixture(t);
mkdirSync(join(app, "dist", "bin"));
writeFileSync(join(app, "dist", "bin", "paperclip-runnerd"), "separately verified native binary");
normalizeProviderPackLayout(app);
normalizeProviderPackLayout(image);
assert.deepEqual(readdirSync(join(app, "dist")), readdirSync(join(image, "dist")));
assert.deepEqual(readFileSync(join(app, "dist", "cli", "provider.cjs")), readFileSync(join(image, "dist", "cli", "provider.cjs")));
});
test("normalization removes only redundant runnerd and retains other bin entries", (t) => {
const root = fixture(t);
mkdirSync(join(root, "dist", "bin"));
writeFileSync(join(root, "dist", "bin", "paperclip-runnerd"), "runnerd");
writeFileSync(join(root, "dist", "bin", "future-provider"), "keep me");
normalizeProviderPackLayout(root);
normalizeProviderPackLayout(root);
assert.deepEqual(readdirSync(join(root, "dist", "bin")), ["future-provider"]);
assert.equal(readFileSync(join(root, "dist", "bin", "future-provider"), "utf8"), "keep me");
});
test("normalization never follows a substituted bin directory", (t) => {
const root = fixture(t), outside = fixture(t);
writeFileSync(join(outside, "paperclip-runnerd"), "keep outside bytes");
symlinkSync(outside, join(root, "dist", "bin"));
assert.throws(() => normalizeProviderPackLayout(root), /must be a directory/);
assert.equal(readFileSync(join(outside, "paperclip-runnerd"), "utf8"), "keep outside bytes");
});

View File

@ -12,8 +12,9 @@ import {
symlink,
writeFile,
} from "node:fs/promises";
import { execFileSync } from "node:child_process";
import { execFileSync, spawnSync } from "node:child_process";
import { tmpdir } from "node:os";
import { fileURLToPath } from "node:url";
import { join } from "node:path";
import {
heartbeatRuns,
@ -26,7 +27,7 @@ import {
type NativeExecutionInputV1,
type PrpEvent,
} from "@paperclipai/paperclip-runner";
import { createHash } from "node:crypto";
import { createHash, randomUUID } from "node:crypto";
import {
createNativeHarnessBackupStamp,
verifyNativeHarnessBackupStamp,
@ -192,6 +193,7 @@ import {
normalizeNativeUsage,
parseRemoteRunnerProcessIdentity,
readRemoteProviderPackManifest,
buildRemoteProviderPackVerificationScript,
providerSessionIdentityFromDurableProviderState,
providerSessionIdentityTransitionIsAllowed,
providerPlanMarkdown,
@ -589,6 +591,12 @@ describe("native provider usage normalization", () => {
});
describe("remote provider pack manifest", () => {
it("normalizes app and sandbox build layouts in the standard server test lane", () => {
execFileSync(process.execPath, ["--test", fileURLToPath(new URL(
"../../../../packages/paperclip-runner/scripts/provider-pack-layout.test.mjs", import.meta.url,
))], { stdio: "pipe" });
});
const canonical = (value: unknown): string => {
if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`;
if (value && typeof value === "object") {
@ -648,7 +656,7 @@ describe("remote provider pack manifest", () => {
pi: "0.84.2",
piAcp: "0.0.33",
},
target: { platform: "linux", architecture: "x64" },
target: { platform: process.platform, architecture: process.arch },
runnerSourceRevision: "1".repeat(40),
distDigest: sha256DirectoryTree(join(root, "dist")),
bridgeDigest: "",
@ -703,6 +711,116 @@ describe("remote provider pack manifest", () => {
expect(readRemoteProviderPackManifest(root).payload.pins.opencode).toBe(
"1.18.29",
);
// Execute the exact source sent to the sandbox, not a mock of its verdict.
const expectedManifest = readRemoteProviderPackManifest(root);
const expectedArgument = Buffer.from(canonical(expectedManifest)).toString("base64");
const verifyRemote = () => spawnSync(process.execPath, [
"-e", buildRemoteProviderPackVerificationScript(), root, expectedArgument,
], { encoding: "utf8" });
for (const [pkg, version] of Object.entries({
acpx: payload.pins.acpx,
"@agentclientprotocol/claude-agent-acp": payload.pins.claudeAcp,
"@agentclientprotocol/codex-acp": payload.pins.codexAcp,
"opencode-ai": payload.pins.opencode,
})) {
await mkdir(join(root, "node_modules", pkg), { recursive: true });
await writeFile(join(root, "node_modules", pkg, "package.json"), JSON.stringify({ version }));
}
expect(verifyRemote().status).toBe(0);
payload.runnerSourceRevision = "2".repeat(40);
await writeManifest();
expect(verifyRemote().status).toBe(0); // Revision-only provenance allows image reuse.
// Exercise the production selection branch too: a different revision with
// verified identical bytes must link the installed pack without syncIn.
payload.runnerSourceRevision = "1".repeat(40);
await writeManifest();
const syncIn = vi.fn(async () => { throw new Error("unexpected-provider-pack-upload"); });
const logs: string[] = [];
const remoteExecute = vi.fn(async (command: { command: string; args?: string[] }) => {
let stdout = "";
const script = command.args?.[1] ?? "";
if (command.args?.[0] === "--build-metadata") {
stdout = JSON.stringify({
schema: "paperclip-runner/runnerd-build-metadata/v1", binaryName: "paperclip-runnerd",
packageName: "@paperclipai/paperclip-runner", binaryContractVersion: 2,
capabilities: ["codex.warm-attachment.passive-notices.v1"], prpTransportModes: ["listen_ws"],
});
} else if (script.includes("command -v paperclip-runnerd")) {
stdout = "/opt/paperclip-runner/bin/paperclip-runnerd\n";
} else if (script.includes("for candidate in /opt/paperclip-runner/provider-pack")) {
stdout = "/opt/paperclip-runner/provider-pack\n";
} else if (command.args?.[0] === "-e") {
const verified = spawnSync(process.execPath, ["-e", script, root, command.args![3]!], { encoding: "utf8" });
return { exitCode: verified.status, signal: null, timedOut: false, stdout: verified.stdout, stderr: verified.stderr };
} else if (command.command.endsWith("/node_modules/.bin/opencode") && command.args?.[0] === "--version") {
stdout = payload.pins.opencode;
} else if (!script.includes("ln -s")) {
throw new Error("after-provider-pack-verification");
}
return { exitCode: 0, signal: null, timedOut: false, stdout, stderr: "" };
});
const providerExecution = {
...execution,
provider: { kind: "opencode", model: null },
session: { ...execution.session, normalizedSessionId: `revision-only-${randomUUID()}`, driverKind: "opencode_server" },
} as NativeExecutionInputV1;
await createRunnerdBackend({
db: leaseDb(providerExecution), execution: providerExecution,
runnerInstanceId: "runner-revision-only-provider-pack", runnerIngressAuthorized: true,
runnerRemoteProviderPackPath: root,
onLog: async (_stream, chunk) => { logs.push(chunk); },
runnerExecutionTarget: {
kind: "remote", transport: "sandbox", remoteCwd: "/workspace", environmentId: "environment",
leaseId: "lease", providerKey: "daytona", effectiveCapabilities: { runnerWebSocketIngress: true },
runner: { execute: remoteExecute, syncIn },
} as never,
});
payload.runnerSourceRevision = "2".repeat(40);
await writeManifest();
state.createTransport.mockClear();
state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!();
const transport = state.createTransport.mock.calls[0]![0] as RunnerTransportOptions & {
controlPlaneRegistration: (authority: unknown) => Promise<unknown>;
};
await expect(transport.controlPlaneRegistration({})).rejects.toThrow();
expect(logs.join("")).toContain("using content-matched provider pack");
expect(syncIn).not.toHaveBeenCalled();
const revisionManifest = JSON.parse(await readFile(join(root, "provider-pack.json"), "utf8"));
await writeFile(join(root, "provider-pack.json"), JSON.stringify({ ...revisionManifest, digest: `sha256:${"0".repeat(64)}` }));
expect(verifyRemote().stderr).toContain("manifest digest mismatch");
await writeManifest();
for (const [relativePath, bytes] of [
["node_modules/node/bin/node", node],
["pnpm-lock.yaml", lockfile],
["dist/cli/acpx-runtime-sidecar.cjs", sidecar],
]) {
await writeFile(join(root, relativePath), "tampered bytes");
expect(verifyRemote().status).not.toBe(0);
await writeFile(join(root, relativePath), bytes);
}
await writeFile(join(root, "dist", "extra-runtime.js"), "unexpected runtime code");
expect(verifyRemote().stderr).toContain("dist tree digest mismatch");
await rm(join(root, "dist", "extra-runtime.js"));
for (const change of [
(value: typeof payload) => { value.pins.opencode = "0.0.0"; },
(value: typeof payload) => { value.target.architecture = "unexpected" as typeof process.arch; },
(value: typeof payload) => { value.artifacts.nodeCommand = { ...value.artifacts.productionLock }; },
]) {
const changed = structuredClone(payload);
change(changed);
await writeFile(join(root, "provider-pack.json"), JSON.stringify({
schema: expectedManifest.schema,
digest: digest(canonical(changed)), payload: changed,
}));
expect(verifyRemote().stderr).toContain("manifest content mismatch");
}
await writeManifest();
await writeFile(join(root, "node_modules", "acpx", "package.json"), JSON.stringify({ version: "0.0.0" }));
expect(verifyRemote().stderr).toContain("acpx version mismatch");
await writeFile(join(root, "node_modules", "acpx", "package.json"), JSON.stringify({ version: payload.pins.acpx }));
expect(verifyRemote().status).toBe(0);
for (const [artifactName, substituteName] of [
["nodeCommand", "productionLock"],
["opencodeExecutable", "opencodeCommand"],

View File

@ -5341,6 +5341,36 @@ export function readRemoteProviderPackManifest(
return structuredClone(manifest);
}
/** Verify bytes against the host-owned content contract while retaining revision provenance. */
export function buildRemoteProviderPackVerificationScript(): string {
return [
"const fs=require('node:fs')",
"const crypto=require('node:crypto')",
"const path=require('node:path')",
"const root=process.argv[1]",
"const expected=JSON.parse(Buffer.from(process.argv[2],'base64').toString('utf8'))",
"const actual=fs.readFileSync(path.join(root,'provider-pack.json'),'utf8').trim()",
"const canonical=(v)=>Array.isArray(v)?'['+v.map(canonical).join(',')+']':v&&typeof v==='object'?'{'+Object.keys(v).sort().map(k=>JSON.stringify(k)+':'+canonical(v[k])).join(',')+'}':JSON.stringify(v)",
"const manifest=JSON.parse(actual)",
"const digest=(payload)=>'sha256:'+crypto.createHash('sha256').update(canonical(payload)).digest('hex')",
"if(manifest.schema!==expected.schema||!manifest.payload||!(/^[0-9a-f]{40}(?:-dirty)?$/).test(manifest.payload.runnerSourceRevision))throw new Error('manifest schema or revision mismatch')",
"if(digest(manifest.payload)!==manifest.digest)throw new Error('manifest digest mismatch')",
"const content=(value)=>{const {digest:provenanceDigest,...fields}=value;const {runnerSourceRevision,...payload}=fields.payload;return {...fields,payload}}",
"if(canonical(content(manifest))!==canonical(content(expected)))throw new Error('manifest content mismatch')",
"const hash=(p)=>'sha256:'+crypto.createHash('sha256').update(fs.readFileSync(path.join(root,p))).digest('hex')",
"const tree=(treeRoot)=>{const digest=crypto.createHash('sha256');const visit=(directory,prefix='')=>{for(const entry of fs.readdirSync(directory,{withFileTypes:true}).sort((a,b)=>a.name.localeCompare(b.name))){const relative=prefix?prefix+'/'+entry.name:entry.name;const absolute=path.join(directory,entry.name);if(entry.isDirectory()){digest.update('directory\\0'+relative+'\\n');visit(absolute,relative)}else if(entry.isFile()){digest.update('file\\0'+relative+'\\0'+'sha256:'+crypto.createHash('sha256').update(fs.readFileSync(absolute)).digest('hex')+'\\n')}else if(entry.isSymbolicLink()){digest.update('symlink\\0'+relative+'\\0'+fs.readlinkSync(absolute)+'\\n')}else throw new Error('unsupported dist entry '+relative)}};visit(treeRoot);return 'sha256:'+digest.digest('hex')}",
"for(const name of ['nodeCommand','productionLock','opencodeCommand','opencodeExecutable','opencodeProxy','acpxSidecar']){const artifact=manifest.payload.artifacts[name];if(hash(artifact.path)!==artifact.sha256)throw new Error(name+' digest mismatch')}",
"if(tree(path.join(root,'dist'))!==manifest.payload.distDigest)throw new Error('dist tree digest mismatch')",
"const version=process.versions.node.split('.').map(Number)",
"const minimum=manifest.payload.pins.nodeMinimum.split('.').map(Number)",
"if(version[0]<minimum[0]||(version[0]===minimum[0]&&(version[1]<minimum[1]||(version[1]===minimum[1]&&version[2]<minimum[2]))))throw new Error('Node version incompatible')",
"if(process.platform!==manifest.payload.target.platform||process.arch!==manifest.payload.target.architecture)throw new Error('provider pack target mismatch')",
"const packageVersion=(pkg)=>JSON.parse(fs.readFileSync(path.join(root,'node_modules',...pkg.split('/'),'package.json'),'utf8')).version",
"const expectedPackages={acpx:manifest.payload.pins.acpx,'@agentclientprotocol/claude-agent-acp':manifest.payload.pins.claudeAcp,'@agentclientprotocol/codex-acp':manifest.payload.pins.codexAcp,'opencode-ai':manifest.payload.pins.opencode}",
"for(const [pkg,version] of Object.entries(expectedPackages))if(packageVersion(pkg)!==version)throw new Error(pkg+' version mismatch')",
].join(";");
}
export function assertRemoteRunnerBuildMetadata(
value: unknown,
requiredMode: "dial_wss" | "listen_ws",
@ -6544,28 +6574,7 @@ async function createRunnerdBackendWithinSessionClaim(
packRoot,
expectedProviderPackManifest.payload.artifacts.nodeCommand.path,
);
const verifyScript = [
"const fs=require('node:fs')",
"const crypto=require('node:crypto')",
"const path=require('node:path')",
"const root=process.argv[1]",
"const expected=Buffer.from(process.argv[2],'base64').toString('utf8')",
"const actual=fs.readFileSync(path.join(root,'provider-pack.json'),'utf8').trim()",
"const canonical=(v)=>Array.isArray(v)?'['+v.map(canonical).join(',')+']':v&&typeof v==='object'?'{'+Object.keys(v).sort().map(k=>JSON.stringify(k)+':'+canonical(v[k])).join(',')+'}':JSON.stringify(v)",
"const manifest=JSON.parse(actual)",
"if(canonical(manifest)!==expected)throw new Error('manifest mismatch')",
"const hash=(p)=>'sha256:'+crypto.createHash('sha256').update(fs.readFileSync(path.join(root,p))).digest('hex')",
"const tree=(treeRoot)=>{const digest=crypto.createHash('sha256');const visit=(directory,prefix='')=>{for(const entry of fs.readdirSync(directory,{withFileTypes:true}).sort((a,b)=>a.name.localeCompare(b.name))){const relative=prefix?prefix+'/'+entry.name:entry.name;const absolute=path.join(directory,entry.name);if(entry.isDirectory()){digest.update('directory\\0'+relative+'\\n');visit(absolute,relative)}else if(entry.isFile()){digest.update('file\\0'+relative+'\\0'+'sha256:'+crypto.createHash('sha256').update(fs.readFileSync(absolute)).digest('hex')+'\\n')}else if(entry.isSymbolicLink()){digest.update('symlink\\0'+relative+'\\0'+fs.readlinkSync(absolute)+'\\n')}else throw new Error('unsupported dist entry '+relative)}};visit(treeRoot);return 'sha256:'+digest.digest('hex')}",
"for(const name of ['nodeCommand','productionLock','opencodeCommand','opencodeExecutable','opencodeProxy','acpxSidecar']){const artifact=manifest.payload.artifacts[name];if(hash(artifact.path)!==artifact.sha256)throw new Error(name+' digest mismatch')}",
"if(tree(path.join(root,'dist'))!==manifest.payload.distDigest)throw new Error('dist tree digest mismatch')",
"const version=process.versions.node.split('.').map(Number)",
"const minimum=manifest.payload.pins.nodeMinimum.split('.').map(Number)",
"if(version[0]<minimum[0]||(version[0]===minimum[0]&&(version[1]<minimum[1]||(version[1]===minimum[1]&&version[2]<minimum[2]))))throw new Error('Node version incompatible')",
"if(process.platform!==manifest.payload.target.platform||process.arch!==manifest.payload.target.architecture)throw new Error('provider pack target mismatch')",
"const packageVersion=(pkg)=>JSON.parse(fs.readFileSync(path.join(root,'node_modules',...pkg.split('/'),'package.json'),'utf8')).version",
"const expectedPackages={acpx:manifest.payload.pins.acpx,'@agentclientprotocol/claude-agent-acp':manifest.payload.pins.claudeAcp,'@agentclientprotocol/codex-acp':manifest.payload.pins.codexAcp,'opencode-ai':manifest.payload.pins.opencode}",
"for(const [pkg,version] of Object.entries(expectedPackages))if(packageVersion(pkg)!==version)throw new Error(pkg+' version mismatch')",
].join(";");
const verifyScript = buildRemoteProviderPackVerificationScript();
const verified = await remoteCommandRunner.execute({
command: providerNodeCommand,
args: ["-e", verifyScript, packRoot, expected],
@ -6905,7 +6914,7 @@ async function createRunnerdBackendWithinSessionClaim(
activeRemoteProviderPackRoot = stagedRemoteProviderPackRoot;
await input.onLog?.(
"stderr",
"[paperclip-runner] using manifest-matched provider pack from the sandbox image\n",
"[paperclip-runner] using content-matched provider pack from the sandbox image\n",
);
} catch {
preinstalledProviderPack = null;

View File

@ -34,6 +34,7 @@ export const DAYTONA_IMAGE_INPUT_PATHS = [
"packages/paperclip-runner/runner/crates",
"packages/paperclip-runner/scripts/acpx-sidecar-contract.mjs",
"packages/paperclip-runner/scripts/build-provider-pack.mjs",
"packages/paperclip-runner/scripts/provider-pack-layout.mjs",
"packages/paperclip-runner/scripts/materialize-pi-binary.mjs",
"packages/paperclip-runner/scripts/portable-provider-shim.mjs",
"packages/paperclip-runner/scripts/verify-pi-provider-launch.mjs",