RomanNum3ral
/
paperclip
mirror of https://github.com/paperclipai/paperclip.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Make provider pack metadata and nested launchers reproducible

This commit is contained in:
Dotta 2026-09-09 17:17:33 -05:00
parent a1d0ebbb0e
commit bc4e39d2f7
3 changed files with 131 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
packages/paperclip-runner/scripts/assemble-provider-pack.mjs
View File

@ -1,5 +1,5 @@
import { canonicalJson, sha256File, sha256Tree, prepareProviderTree, writeProviderTreeSidecar } from "./provider-pack-integrity.mjs";
import { normalizeProviderPackLayout } from "./provider-pack-layout.mjs";
import { normalizeProviderPackLayout, normalizeProviderPackMetadata } from "./provider-pack-layout.mjs";
import { portableProviderShim } from "./portable-provider-shim.mjs";
import { createHash } from "node:crypto";
import { execFileSync, spawnSync } from "node:child_process";
@ -158,6 +158,8 @@ try {
{ recursive: true, force: true },
);
normalizeProviderPackMetadata(temporaryRoot);
for (const shimName of readdirSync(
join(temporaryRoot, "node_modules", ".bin"),
)) {

61
packages/paperclip-runner/scripts/provider-pack-layout.mjs
View File

@ -1,5 +1,5 @@
import { existsSync, lstatSync, readdirSync, rmSync, rmdirSync } from "node:fs";
import { join } from "node:path";
import { existsSync, lstatSync, readdirSync, readFileSync, realpathSync, rmSync, rmdirSync, writeFileSync } from "node:fs";
import { basename, dirname, join, relative, resolve, sep } from "node:path";
/** runnerd is shipped and verified separately from the JavaScript provider pack. */
export function normalizeProviderPackLayout(packRoot) {
@ -17,3 +17,60 @@ export function normalizeProviderPackLayout(packRoot) {
// Keep every other entry, including future provider executables.
if (readdirSync(bin).length === 0) rmdirSync(bin);
}
/** Remove build-specific bytes before binding the complete immutable pack tree. */
export function normalizeProviderPackMetadata(packRoot) {
const root = realpathSync(packRoot);
const buildRoots = [...new Set([root, resolve(packRoot)])].sort((a, b) => b.length - a.length);
const containsBuildRoot = (text) => buildRoots.some((path) => text.includes(path));
let normalizedShims = 0;
function visit(directory) {
for (const entry of readdirSync(directory, { withFileTypes: true })) {
const file = join(directory, entry.name);
// Walk physical directories only; the integrity verifier checks link confinement.
if (entry.isDirectory()) { visit(file); continue; }
if (!entry.isFile() || basename(directory) !== ".bin") continue;
if (lstatSync(file).size > 64 * 1024) continue; // Preserve non-shim executables.
const body = readFileSync(file, "utf8");
if (!containsBuildRoot(body)) continue;
const lines = body.split("\n"), base = lines.flatMap((line, index) => line.startsWith("basedir=") ? [index] : []);
if (!body.startsWith("#!/bin/sh\n") || base.length !== 1
|| lines.some((line) => containsBuildRoot(line) && !line.startsWith(' export NODE_PATH="'))) {
throw new Error("Unrecognized build-dependent provider shim");
}
// Resolve the physical launcher directory before ascending to the pack.
// Callers can invoke it through pnpm package links or task-local symlinks.
const ascent = relative(dirname(file), root).split(sep).join("/");
const preamble = [
'paperclip_self=$0; paperclip_links=0',
'while [ -L "$paperclip_self" ]; do',
' paperclip_links=$((paperclip_links + 1)); [ "$paperclip_links" -le 40 ] || exit 1',
' paperclip_parent=$(CDPATH= cd -P -- "$(dirname -- "$paperclip_self")" && pwd -P) || exit 1',
' paperclip_self=$(readlink -- "$paperclip_self") || exit 1',
' case "$paperclip_self" in /*) ;; *) paperclip_self=$paperclip_parent/$paperclip_self ;; esac',
'done',
'basedir=$(CDPATH= cd -P -- "$(dirname -- "$paperclip_self")" && pwd -P) || exit 1',
`paperclip_pack_root=$(CDPATH= cd -P -- "$basedir/${ascent}" && pwd -P) || exit 1`,
].join("\n");
lines[base[0]] = preamble;
let normalized = lines.join("\n");
for (const buildRoot of buildRoots) normalized = normalized.split(buildRoot).join("${paperclip_pack_root}");
writeFileSync(file, normalized);
normalizedShims++;
}
}
const modules = join(root, "node_modules");
if (!lstatSync(modules).isDirectory()) throw new Error("Provider node_modules must be a physical directory");
visit(modules);
const metadata = join(modules, ".modules.yaml");
if (existsSync(metadata)) {
const stat = lstatSync(metadata);
if (!stat.isFile() || stat.size > 1024 * 1024) throw new Error("Provider package-manager metadata must be a bounded regular file");
const body = readFileSync(metadata, "utf8");
if ([...body.matchAll(/^prunedAt: .*$/gm)].length !== 1) throw new Error("Unexpected provider package-manager timestamp metadata");
// pnpm's pruning time is bookkeeping, not runtime content. Keep its format
// and all other metadata, with the same value in every isolated build.
writeFileSync(metadata, body.replace(/^prunedAt: .*$/m, "prunedAt: Thu, 01 Jan 1970 00:00:00 GMT"));
}
return { normalizedShims };
}

72
packages/paperclip-runner/scripts/provider-pack-layout.test.mjs
View File

@ -1,9 +1,11 @@
import assert from "node:assert/strict";
import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, readdirSync, rmSync, symlinkSync } from "node:fs";
import { join } from "node:path";
import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, symlinkSync } from "node:fs";
import { execFileSync } from "node:child_process";
import { dirname, join } from "node:path";
import { tmpdir } from "node:os";
import test from "node:test";
import { normalizeProviderPackLayout } from "./provider-pack-layout.mjs";
import { normalizeProviderPackLayout, normalizeProviderPackMetadata } from "./provider-pack-layout.mjs";
import { sha256Tree } from "./provider-pack-integrity.mjs";
function fixture(t) {
const root = mkdtempSync(join(tmpdir(), "provider-pack-layout-"));
@ -41,3 +43,67 @@ test("normalization never follows a substituted bin directory", (t) => {
assert.throws(() => normalizeProviderPackLayout(root), /must be a directory/);
assert.equal(readFileSync(join(outside, "paperclip-runnerd"), "utf8"), "keep outside bytes");
});
function metadataFixture(t, date) {
const root = fixture(t), provider = join(root, "node_modules/.pnpm/tool@1/node_modules/tool");
const shim = join(provider, "node_modules/.bin/tool");
mkdirSync(dirname(shim), { recursive: true });
mkdirSync(join(root, "runtime-extra/only-through-node-path"), { recursive: true });
writeFileSync(join(root, "runtime-extra/only-through-node-path/index.js"), 'module.exports = "dependency resolved";');
writeFileSync(join(provider, "cli.cjs"), 'console.log(JSON.stringify({value:require("only-through-node-path"),args:process.argv.slice(2),nodePath:process.env.NODE_PATH}));');
writeFileSync(shim, ["#!/bin/sh", 'basedir=$(dirname "$0")',
'if [ -z "$NODE_PATH" ]; then', ` export NODE_PATH="${root}/runtime-extra"`,
"else", ` export NODE_PATH="${root}/runtime-extra:$NODE_PATH"`, "fi",
'exec node "$basedir/../../cli.cjs" "$@"', ""].join("\n"), { mode: 0o755 });
symlinkSync(".pnpm/tool@1/node_modules/tool", join(root, "node_modules/tool"));
writeFileSync(join(root, "node_modules/.modules.yaml"), `packageManager: pnpm@9.15.4\nprunedAt: ${date}\nvirtualStoreDir: .pnpm\n`);
return { root, shim };
}
test("independent build paths and pruning times produce identical complete content trees", (t) => {
const a = metadataFixture(t, "Wed, 09 Sep 2026 21:44:13 GMT"), b = metadataFixture(t, "Wed, 09 Sep 2026 21:45:40 GMT");
assert.notEqual(sha256Tree(a.root), sha256Tree(b.root));
for (const pack of [a, b]) assert.equal(normalizeProviderPackMetadata(pack.root).normalizedShims, 1);
assert.equal(sha256Tree(a.root), sha256Tree(b.root));
const before = sha256Tree(a.root);
assert.equal(normalizeProviderPackMetadata(a.root).normalizedShims, 0);
assert.equal(sha256Tree(a.root), before);
});
test("normalized nested launcher resolves dependencies and arguments after relocation and symlink invocation", (t) => {
const pack = metadataFixture(t, "Wed, 09 Sep 2026 21:44:13 GMT");
normalizeProviderPackMetadata(pack.root);
const parent = mkdtempSync(join(tmpdir(), "provider move ' "));
t.after(() => rmSync(parent, { recursive: true, force: true }));
const moved = join(parent, "moved pack"); renameSync(pack.root, moved);
const logical = join(moved, "node_modules/tool/node_modules/.bin/tool");
symlinkSync(logical, join(parent, "absolute")); symlinkSync("absolute", join(parent, "relative"));
for (const existing of [undefined, "/existing caller path"]) {
const env = { ...process.env, PATH: dirname(process.execPath) + ":" + process.env.PATH };
if (existing === undefined) delete env.NODE_PATH; else env.NODE_PATH = existing;
for (const command of [logical, join(parent, "absolute"), join(parent, "relative")]) {
const result = JSON.parse(execFileSync(command, ["argument with spaces", "'quoted'"], { cwd: "/", encoding: "utf8", env }));
assert.equal(result.value, "dependency resolved");
assert.deepEqual(result.args, ["argument with spaces", "'quoted'"]);
assert.equal(result.nodePath, join(realpathSync(moved), "runtime-extra") + (existing ? ":" + existing : ""));
assert(!result.nodePath.includes(pack.root));
}
}
});
test("metadata normalization rejects substituted node_modules and bookkeeping links", (t) => {
const root = fixture(t), outside = metadataFixture(t, "original timestamp");
symlinkSync(join(outside.root, "node_modules"), join(root, "node_modules"));
assert.throws(() => normalizeProviderPackMetadata(root), /physical directory/);
assert(readFileSync(outside.shim, "utf8").includes(outside.root));
const pack = metadataFixture(t, "original timestamp"), metadata = join(pack.root, "node_modules/.modules.yaml");
rmSync(metadata); symlinkSync(join(outside.root, "node_modules/.modules.yaml"), metadata);
assert.throws(() => normalizeProviderPackMetadata(pack.root), /regular file/);
assert(readFileSync(join(outside.root, "node_modules/.modules.yaml"), "utf8").includes("original timestamp"));
});
test("unrecognized build-dependent shims fail instead of hiding unstable executable bytes", (t) => {
const pack = metadataFixture(t, "original timestamp");
writeFileSync(pack.shim, `#!/bin/sh\nexec "${pack.root}/somewhere"\n`);
assert.throws(() => normalizeProviderPackMetadata(pack.root), /Unrecognized/);
});