feat(sandbox-runtime): route all inbound staging through client.syncIn (Codex home -> native uploadFiles; delete usesCustomProvision gate) (#10354)
This commit is contained in:
Nicky Leach
GitHub
parent
7797995038
commit
341993ebae
|
|
@ -599,6 +599,39 @@ describe("command managed runtime", () => {
|
|||
expect(calls[cmd2Idx]?.noProfile).not.toBe(true);
|
||||
});
|
||||
|
||||
it("fallback syncIn runs a post-upload command under its own timeout, not the sync-client default", async () => {
|
||||
// The run-specific timeout (`spec.timeoutMs`, stamped onto each delegated
|
||||
// post-upload command) can differ from the sync client's own default. The
|
||||
// fallback must honor the per-command `timeoutMs` so the delegated
|
||||
// extract/cleanup/merge runs under the run limit — not the sync default.
|
||||
const syncClientTimeoutMs = 30_000;
|
||||
const runTimeoutMs = 7_000;
|
||||
const execTimeouts: Array<number | undefined> = [];
|
||||
const runner: CommandManagedRuntimeRunner = {
|
||||
execute: async (input) => {
|
||||
execTimeouts.push(input.timeoutMs);
|
||||
return { exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "", pid: null, startedAt: "" };
|
||||
},
|
||||
};
|
||||
const client = createCommandManagedRuntimeClient({ runner, commandCwd: "/", timeoutMs: syncClientTimeoutMs });
|
||||
|
||||
await client.syncIn!([
|
||||
{
|
||||
operationId: "op-timeout",
|
||||
files: [],
|
||||
postUploadCommands: [
|
||||
{ command: "echo carries-run-timeout", timeoutMs: runTimeoutMs },
|
||||
{ command: "echo defaults-to-sync-timeout" },
|
||||
],
|
||||
},
|
||||
]);
|
||||
|
||||
// First command carries the run timeout; a command with no explicit timeout
|
||||
// still falls back to the sync-client default (matched by the stamping in
|
||||
// prepareSandboxManagedRuntime, which never leaves a delegated command bare).
|
||||
expect(execTimeouts).toEqual([runTimeoutMs, syncClientTimeoutMs]);
|
||||
});
|
||||
|
||||
it("fallback syncIn stages mode-constrained files before chmod and rename", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-syncin-mode-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
|
|
|
|||
|
|
@ -21,8 +21,10 @@ interface RecordingClient {
|
|||
}
|
||||
|
||||
// A filesystem-backed client that additionally exposes native syncIn/syncOut,
|
||||
// mirroring a provider that opted into the sync verbs. The native transfer is a
|
||||
// faithful destroy-then-replace directory copy honoring followSymlinks.
|
||||
// mirroring a provider that opted into the sync verbs and HONORS an operation's
|
||||
// ordered `postUploadCommands` after its files land (PR-2 contract: execute or
|
||||
// fail-closed, never silently ignore). The native transfer is a faithful copy
|
||||
// honoring followSymlinks; single-file mappings stream verbatim.
|
||||
function makeNativeClient(): RecordingClient {
|
||||
const syncInOps: SandboxSyncOperation[][] = [];
|
||||
const syncOutOps: SandboxSyncOperation[][] = [];
|
||||
|
|
@ -53,6 +55,10 @@ function makeNativeClient(): RecordingClient {
|
|||
filesTransferred += 1;
|
||||
}
|
||||
}
|
||||
// Honor the operation's ordered post-upload commands (PR-2), fail-fast.
|
||||
for (const command of operation.postUploadCommands ?? []) {
|
||||
await execFile("sh", ["-c", command.command], { maxBuffer: 32 * 1024 * 1024 });
|
||||
}
|
||||
return { operationId: operation.operationId, filesTransferred, bytesTransferred: 0 };
|
||||
})),
|
||||
});
|
||||
|
|
@ -106,16 +112,25 @@ describe("sandbox native file sync", () => {
|
|||
assets: [{ key: "skills", localDir: localAssetsDir }],
|
||||
});
|
||||
|
||||
// The default-provision asset was transferred through syncIn as a single
|
||||
// directory mapping with an opaque operationId; the file landed in place.
|
||||
// Both the workspace and the default-provision asset stage through syncIn:
|
||||
// each uploads a single tar as a `file` mapping with an opaque operationId,
|
||||
// and the extract runs as the operation's ordered post-upload command.
|
||||
const inboundOps = syncInOps.flat();
|
||||
expect(inboundOps.length).toBe(1);
|
||||
const assetOp = inboundOps[0];
|
||||
expect(assetOp.operationId).toMatch(/^sync-op-\d+$/);
|
||||
expect(assetOp.operationId).not.toContain("skills");
|
||||
expect(assetOp.files).toEqual([
|
||||
{ sourcePath: localAssetsDir, targetPath: prepared.assetDirs.skills, kind: "directory", exclude: undefined, followSymlinks: undefined },
|
||||
]);
|
||||
expect(inboundOps.length).toBe(2);
|
||||
const assetOp = inboundOps.find((op) =>
|
||||
op.files.some((mapping) => mapping.targetPath.endsWith("skills-upload.tar")),
|
||||
);
|
||||
expect(assetOp).toBeDefined();
|
||||
expect(assetOp!.operationId).toMatch(/^sync-op-\d+$/);
|
||||
expect(assetOp!.operationId).not.toContain("skills");
|
||||
expect(assetOp!.files).toHaveLength(1);
|
||||
expect(assetOp!.files[0]).toMatchObject({
|
||||
targetPath: path.posix.join(prepared.runtimeRootDir, "skills-upload.tar"),
|
||||
kind: "file",
|
||||
});
|
||||
// Default provision → a plain destroy-then-replace tar extract post-command.
|
||||
expect(assetOp!.postUploadCommands).toHaveLength(1);
|
||||
expect(assetOp!.postUploadCommands![0].command).toContain("tar -xf");
|
||||
expect(await readFile(path.join(prepared.assetDirs.skills, "skill.md"), "utf8")).toBe("skill body\n");
|
||||
|
||||
// Mutate the sandbox workspace, then restore through the native outbound path.
|
||||
|
|
@ -130,7 +145,54 @@ describe("sandbox native file sync", () => {
|
|||
expect(await readFile(path.join(localWorkspaceDir, "new.txt"), "utf8")).toBe("added\n");
|
||||
});
|
||||
|
||||
it("keeps a custom-provision asset on the tar fallback even when native sync is available", async () => {
|
||||
it("stamps the run-specific timeout onto every delegated post-upload command", async () => {
|
||||
// The extract/wipe/merge commands are delegated to the provider through
|
||||
// `syncIn` as `postUploadCommands`. They MUST carry the run-specific timeout
|
||||
// (`spec.timeoutMs`) — the same limit the pre-syncIn code passed to
|
||||
// `client.run` — not the provider sync client's own default. When the two
|
||||
// differ, a command left without a `timeoutMs` outlives (or is killed under)
|
||||
// the wrong limit; here a distinctive `spec.timeoutMs` proves propagation.
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-native-timeout-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const localWorkspaceDir = path.join(rootDir, "local-workspace");
|
||||
const remoteWorkspaceDir = path.join(rootDir, "remote-workspace");
|
||||
const defaultAssetDir = path.join(rootDir, "default-asset");
|
||||
const customAssetDir = path.join(rootDir, "custom-asset");
|
||||
await mkdir(localWorkspaceDir, { recursive: true });
|
||||
await mkdir(defaultAssetDir, { recursive: true });
|
||||
await mkdir(customAssetDir, { recursive: true });
|
||||
await writeFile(path.join(localWorkspaceDir, "README.md"), "ws\n", "utf8");
|
||||
await writeFile(path.join(defaultAssetDir, "skill.md"), "skill\n", "utf8");
|
||||
await writeFile(path.join(customAssetDir, "cred.txt"), "secret\n", "utf8");
|
||||
|
||||
const runTimeoutMs = 7_000;
|
||||
const { client, syncInOps } = makeNativeClient();
|
||||
await prepareSandboxManagedRuntime({
|
||||
spec: { transport: "sandbox", provider: "test", sandboxId: "s1", remoteCwd: remoteWorkspaceDir, timeoutMs: runTimeoutMs, apiKey: null },
|
||||
adapterKey: "test-adapter",
|
||||
client,
|
||||
workspaceLocalDir: localWorkspaceDir,
|
||||
assets: [
|
||||
{ key: "skills", localDir: defaultAssetDir },
|
||||
{
|
||||
key: "creds",
|
||||
localDir: customAssetDir,
|
||||
provision: { postUploadCommand: ({ assetTarPath, assetDir }) =>
|
||||
`rm -rf ${assetDir} && mkdir -p ${assetDir} && tar -xf ${assetTarPath} -C ${assetDir} && rm -f ${assetTarPath}` },
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
// Workspace extract + default-asset extract + custom-provision merge — every
|
||||
// delegated command across every operation carries the run timeout.
|
||||
const commands = syncInOps.flat().flatMap((op) => op.postUploadCommands ?? []);
|
||||
expect(commands.length).toBeGreaterThanOrEqual(3);
|
||||
for (const command of commands) {
|
||||
expect(command.timeoutMs).toBe(runTimeoutMs);
|
||||
}
|
||||
});
|
||||
|
||||
it("routes a custom-provision asset through syncIn with its bespoke post-upload command (native)", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-native-custom-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const localWorkspaceDir = path.join(rootDir, "local-workspace");
|
||||
|
|
@ -150,15 +212,22 @@ describe("sandbox native file sync", () => {
|
|||
assets: [{
|
||||
key: "creds",
|
||||
localDir: localAssetsDir,
|
||||
// A bespoke extract command (e.g. a credential merge) cannot be a generic
|
||||
// file mapping, so the orchestrator keeps it on the tar path.
|
||||
provision: { extractCommand: ({ assetTarPath, assetDir }) =>
|
||||
// A bespoke post-upload command (e.g. a credential merge) rides syncIn as
|
||||
// the operation's ordered post-upload command — no native-diversion gate.
|
||||
provision: { postUploadCommand: ({ assetTarPath, assetDir }) =>
|
||||
`rm -rf ${assetDir} && mkdir -p ${assetDir} && tar -xf ${assetTarPath} -C ${assetDir} && rm -f ${assetTarPath}` },
|
||||
}],
|
||||
});
|
||||
|
||||
// No syncIn operation for the custom asset; it still materializes via tar.
|
||||
expect(syncInOps.flat().length).toBe(0);
|
||||
// The custom asset now rides syncIn (native uploadFiles), carrying its
|
||||
// bespoke command as the operation's ordered post-upload command.
|
||||
const credsOp = syncInOps.flat().find((op) =>
|
||||
op.files.some((mapping) => mapping.targetPath.endsWith("creds-upload.tar")),
|
||||
);
|
||||
expect(credsOp).toBeDefined();
|
||||
expect(credsOp!.files.every((mapping) => mapping.kind === "file")).toBe(true);
|
||||
expect(credsOp!.postUploadCommands).toHaveLength(1);
|
||||
expect(credsOp!.postUploadCommands![0].command).toContain("tar -xf");
|
||||
expect(await readFile(path.join(prepared.assetDirs.creds, "cred.txt"), "utf8")).toBe("secret\n");
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -11,8 +11,82 @@ import {
|
|||
mirrorDirectory,
|
||||
prepareSandboxManagedRuntime,
|
||||
type SandboxManagedRuntimeClient,
|
||||
type SandboxSyncOperation,
|
||||
type SandboxSyncResult,
|
||||
} from "./sandbox-managed-runtime.js";
|
||||
|
||||
function toArrayBuffer(bytes: Buffer): ArrayBuffer {
|
||||
return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer;
|
||||
}
|
||||
|
||||
// Give a bare fake client a `syncIn` that reproduces the non-native base64-tar
|
||||
// FALLBACK (place each file mapping via `writeFile`, then run the operation's
|
||||
// ordered `postUploadCommands` fail-fast via `run`) — byte-for-byte the prior
|
||||
// inline `writeFile`+`run` sequence, exercised through the unified seam. Inbound
|
||||
// staging uses only `kind: "file"` mappings.
|
||||
function attachFallbackSyncIn(client: SandboxManagedRuntimeClient, timeoutMs = 30_000): void {
|
||||
client.syncIn = async (operations: SandboxSyncOperation[]): Promise<SandboxSyncResult> => {
|
||||
const resultOperations: SandboxSyncResult["operations"] = [];
|
||||
for (const operation of operations) {
|
||||
let filesTransferred = 0;
|
||||
let bytesTransferred = 0;
|
||||
for (const mapping of operation.files) {
|
||||
const bytes = await readFile(mapping.sourcePath);
|
||||
await client.makeDir(path.posix.dirname(mapping.targetPath));
|
||||
if (mapping.mode != null) {
|
||||
const staged = `${mapping.targetPath}.pcstage`;
|
||||
await client.writeFile(staged, toArrayBuffer(bytes));
|
||||
await client.run(
|
||||
`chmod ${(mapping.mode & 0o7777).toString(8)} '${staged}' && mv -f '${staged}' '${mapping.targetPath}'`,
|
||||
{ timeoutMs },
|
||||
);
|
||||
} else {
|
||||
await client.writeFile(mapping.targetPath, toArrayBuffer(bytes));
|
||||
}
|
||||
filesTransferred += 1;
|
||||
bytesTransferred += bytes.byteLength;
|
||||
}
|
||||
for (const command of operation.postUploadCommands ?? []) {
|
||||
await client.run(command.command, { timeoutMs: command.timeoutMs ?? timeoutMs });
|
||||
}
|
||||
resultOperations.push({ operationId: operation.operationId, filesTransferred, bytesTransferred });
|
||||
}
|
||||
return { operations: resultOperations };
|
||||
};
|
||||
}
|
||||
|
||||
// A NATIVE-simulating `syncIn`: it records the operations for assertion and
|
||||
// materializes files directly (never through the client's `writeFile`/`run`), so
|
||||
// a test can prove the orchestrator delegates entirely to `syncIn` (0 direct
|
||||
// `writeFile`/`run` execs). Post-upload commands still run in-sandbox (via `sh`),
|
||||
// modeling a provider that honors `postUploadCommands` after `uploadFiles`.
|
||||
function attachNativeRecordingSyncIn(
|
||||
client: SandboxManagedRuntimeClient,
|
||||
captured: SandboxSyncOperation[],
|
||||
): void {
|
||||
client.syncIn = async (operations: SandboxSyncOperation[]): Promise<SandboxSyncResult> => {
|
||||
const resultOperations: SandboxSyncResult["operations"] = [];
|
||||
for (const operation of operations) {
|
||||
captured.push(operation);
|
||||
let filesTransferred = 0;
|
||||
let bytesTransferred = 0;
|
||||
for (const mapping of operation.files) {
|
||||
const bytes = await readFile(mapping.sourcePath);
|
||||
await mkdir(path.posix.dirname(mapping.targetPath), { recursive: true });
|
||||
await writeFile(mapping.targetPath, bytes);
|
||||
if (mapping.mode != null) await fsPromises.chmod(mapping.targetPath, mapping.mode);
|
||||
filesTransferred += 1;
|
||||
bytesTransferred += bytes.byteLength;
|
||||
}
|
||||
for (const command of operation.postUploadCommands ?? []) {
|
||||
await execFile("sh", ["-c", command.command], { maxBuffer: 32 * 1024 * 1024 });
|
||||
}
|
||||
resultOperations.push({ operationId: operation.operationId, filesTransferred, bytesTransferred });
|
||||
}
|
||||
return { operations: resultOperations };
|
||||
};
|
||||
}
|
||||
|
||||
vi.mock("node:fs", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("node:fs")>();
|
||||
return {
|
||||
|
|
@ -176,6 +250,7 @@ describe("sandbox managed runtime", () => {
|
|||
};
|
||||
const runtimeStatuses: string[] = [];
|
||||
|
||||
attachFallbackSyncIn(client);
|
||||
const prepared = await prepareSandboxManagedRuntime({
|
||||
spec: {
|
||||
transport: "sandbox",
|
||||
|
|
@ -296,6 +371,7 @@ describe("sandbox managed runtime", () => {
|
|||
};
|
||||
const runtimeStatuses: Array<{ phase: string; message: string }> = [];
|
||||
|
||||
attachFallbackSyncIn(client);
|
||||
const prepared = await prepareSandboxManagedRuntime({
|
||||
spec: {
|
||||
transport: "sandbox",
|
||||
|
|
@ -418,6 +494,7 @@ describe("sandbox managed runtime", () => {
|
|||
},
|
||||
};
|
||||
|
||||
attachFallbackSyncIn(client);
|
||||
const prepared = await prepareSandboxManagedRuntime({
|
||||
spec: {
|
||||
transport: "sandbox",
|
||||
|
|
@ -536,6 +613,7 @@ describe("sandbox managed runtime", () => {
|
|||
},
|
||||
};
|
||||
|
||||
attachFallbackSyncIn(client);
|
||||
const prepared = await prepareSandboxManagedRuntime({
|
||||
spec: {
|
||||
transport: "sandbox",
|
||||
|
|
@ -628,6 +706,7 @@ describe("sandbox managed runtime", () => {
|
|||
},
|
||||
};
|
||||
|
||||
attachFallbackSyncIn(client);
|
||||
const prepared = await prepareSandboxManagedRuntime({
|
||||
spec: {
|
||||
transport: "sandbox",
|
||||
|
|
@ -713,6 +792,7 @@ describe("sandbox managed runtime", () => {
|
|||
},
|
||||
};
|
||||
|
||||
attachFallbackSyncIn(client);
|
||||
const prepared = await prepareSandboxManagedRuntime({
|
||||
spec: {
|
||||
transport: "sandbox",
|
||||
|
|
@ -805,6 +885,7 @@ describe("sandbox managed runtime", () => {
|
|||
};
|
||||
|
||||
const lines: string[] = [];
|
||||
attachFallbackSyncIn(client);
|
||||
const prepared = await prepareSandboxManagedRuntime({
|
||||
spec: {
|
||||
transport: "sandbox",
|
||||
|
|
@ -872,6 +953,7 @@ describe("sandbox managed runtime", () => {
|
|||
},
|
||||
};
|
||||
|
||||
attachFallbackSyncIn(client);
|
||||
const prepared = await prepareSandboxManagedRuntime({
|
||||
spec: {
|
||||
transport: "sandbox",
|
||||
|
|
@ -928,6 +1010,7 @@ describe("sandbox managed runtime", () => {
|
|||
},
|
||||
};
|
||||
|
||||
attachFallbackSyncIn(client);
|
||||
const prepared = await prepareSandboxManagedRuntime({
|
||||
spec: {
|
||||
transport: "sandbox",
|
||||
|
|
@ -994,6 +1077,7 @@ describe("sandbox managed runtime", () => {
|
|||
const restored: string[] = [];
|
||||
const stagedContentSeen: string[] = [];
|
||||
|
||||
attachFallbackSyncIn(client);
|
||||
const prepared = await prepareSandboxManagedRuntime({
|
||||
spec: {
|
||||
transport: "sandbox",
|
||||
|
|
@ -1012,8 +1096,8 @@ describe("sandbox managed runtime", () => {
|
|||
provision: {
|
||||
stageFiles: [{ name: "widget-helper.txt", contents: "helper-bytes\n" }],
|
||||
// Extract the asset AND consume the staged helper file, proving both
|
||||
// stageFiles and extractCommand flow through the core generically.
|
||||
extractCommand: ({ assetTarPath, assetDir, runtimeRootDir }) =>
|
||||
// stageFiles and postUploadCommand flow through the core generically.
|
||||
postUploadCommand: ({ assetTarPath, assetDir, runtimeRootDir }) =>
|
||||
`rm -rf ${q(assetDir)} && mkdir -p ${q(assetDir)} && ` +
|
||||
`tar -xf ${q(assetTarPath)} -C ${q(assetDir)} && rm -f ${q(assetTarPath)} && ` +
|
||||
`cp ${q(path.posix.join(runtimeRootDir, "widget-helper.txt"))} ${q(path.posix.join(assetDir, "helper.copied.txt"))}`,
|
||||
|
|
@ -1070,6 +1154,7 @@ describe("sandbox managed runtime", () => {
|
|||
await execFile("sh", ["-c", command], { maxBuffer: 32 * 1024 * 1024 });
|
||||
},
|
||||
};
|
||||
attachFallbackSyncIn(client);
|
||||
|
||||
// A compromised adapter supplying a traversal name must be rejected before
|
||||
// the core ever writes outside the runtime root.
|
||||
|
|
@ -1103,6 +1188,181 @@ describe("sandbox managed runtime", () => {
|
|||
}
|
||||
});
|
||||
|
||||
it("routes a custom-provisioned asset through a single syncIn operation with its post-upload command (native runner → 0 direct writeFile/run)", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-sandbox-native-asset-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const localWorkspaceDir = path.join(rootDir, "local-workspace");
|
||||
const remoteWorkspaceDir = path.join(rootDir, "remote-workspace");
|
||||
const localAssetsDir = path.join(rootDir, "local-assets");
|
||||
await mkdir(localWorkspaceDir, { recursive: true });
|
||||
await mkdir(localAssetsDir, { recursive: true });
|
||||
await writeFile(path.join(localWorkspaceDir, "README.md"), "workspace\n", "utf8");
|
||||
await writeFile(path.join(localAssetsDir, "seed.txt"), "seed\n", "utf8");
|
||||
|
||||
// A native runner delegates every staging step to `syncIn`; the orchestrator
|
||||
// must make NO direct `writeFile`/`run` exec. These record any leak.
|
||||
const directWrites: string[] = [];
|
||||
const directRuns: string[] = [];
|
||||
const client: SandboxManagedRuntimeClient = {
|
||||
makeDir: async (remotePath) => {
|
||||
await mkdir(remotePath, { recursive: true });
|
||||
},
|
||||
writeFile: async (remotePath, bytes) => {
|
||||
directWrites.push(remotePath);
|
||||
await mkdir(path.dirname(remotePath), { recursive: true });
|
||||
await writeFile(remotePath, Buffer.from(bytes));
|
||||
},
|
||||
readFile: async (remotePath) => await readFile(remotePath),
|
||||
listFiles: async () => [],
|
||||
remove: async (remotePath) => {
|
||||
await rm(remotePath, { recursive: true, force: true });
|
||||
},
|
||||
run: async (command) => {
|
||||
directRuns.push(command);
|
||||
await execFile("sh", ["-c", command], { maxBuffer: 32 * 1024 * 1024 });
|
||||
},
|
||||
};
|
||||
const captured: SandboxSyncOperation[] = [];
|
||||
attachNativeRecordingSyncIn(client, captured);
|
||||
|
||||
const q = (value: string) => `'${value.replace(/'/g, `'\"'\"'`)}'`;
|
||||
const prepared = await prepareSandboxManagedRuntime({
|
||||
spec: {
|
||||
transport: "sandbox",
|
||||
provider: "test",
|
||||
sandboxId: "sandbox-1",
|
||||
remoteCwd: remoteWorkspaceDir,
|
||||
timeoutMs: 30_000,
|
||||
apiKey: null,
|
||||
},
|
||||
adapterKey: "generic-adapter",
|
||||
client,
|
||||
workspaceLocalDir: localWorkspaceDir,
|
||||
assets: [{
|
||||
key: "widget",
|
||||
localDir: localAssetsDir,
|
||||
provision: {
|
||||
stageFiles: [{ name: "widget-helper.sh", contents: "#!/bin/sh\ntar -xf \"$2\" -C \"$1\"\n" }],
|
||||
// A bespoke post-upload command that consumes the staged helper — proves
|
||||
// the custom command (not a plain default `tar -xf`) rides syncIn.
|
||||
postUploadCommand: ({ assetTarPath, assetDir, runtimeRootDir }) =>
|
||||
`rm -rf ${q(assetDir)} && mkdir -p ${q(assetDir)} && ` +
|
||||
`sh ${q(path.posix.join(runtimeRootDir, "widget-helper.sh"))} ${q(assetDir)} ${q(assetTarPath)} && ` +
|
||||
`rm -f ${q(assetTarPath)}`,
|
||||
},
|
||||
}],
|
||||
});
|
||||
|
||||
// The orchestrator delegated everything to syncIn: no direct exec/writeFile.
|
||||
expect(directWrites).toEqual([]);
|
||||
expect(directRuns).toEqual([]);
|
||||
|
||||
// Exactly one operation carries the asset: the asset tar + the staged helper
|
||||
// as `files`, and the bespoke command as the ordered post-upload command.
|
||||
const assetOp = captured.find((op) =>
|
||||
op.files.some((mapping) => mapping.targetPath.endsWith("widget-upload.tar")),
|
||||
);
|
||||
expect(assetOp).toBeDefined();
|
||||
const targets = assetOp!.files.map((mapping) => path.posix.basename(mapping.targetPath)).sort();
|
||||
expect(targets).toEqual(["widget-helper.sh", "widget-upload.tar"]);
|
||||
expect(assetOp!.files.every((mapping) => mapping.kind === "file")).toBe(true);
|
||||
expect(assetOp!.postUploadCommands).toHaveLength(1);
|
||||
expect(assetOp!.postUploadCommands![0].command).toContain("widget-helper.sh");
|
||||
expect(assetOp!.postUploadCommands![0].command).not.toBe(
|
||||
`rm -rf ${q(path.posix.join(prepared.runtimeRootDir, "widget"))} && mkdir -p ${q(path.posix.join(prepared.runtimeRootDir, "widget"))}`,
|
||||
);
|
||||
|
||||
// The asset actually materialized through the native seam.
|
||||
await expect(readFile(path.join(prepared.assetDirs.widget, "seed.txt"), "utf8")).resolves.toBe("seed\n");
|
||||
});
|
||||
|
||||
it("stages git and workspace via syncIn preserving .paperclip-runtime (native runner → 0 direct writeFile/run)", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-sandbox-native-git-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const sourceRepoDir = path.join(rootDir, "source-repo");
|
||||
const localWorkspaceDir = path.join(rootDir, "local-workspace");
|
||||
const remoteWorkspaceDir = path.join(rootDir, "remote-workspace");
|
||||
await mkdir(sourceRepoDir, { recursive: true });
|
||||
await git(sourceRepoDir, ["init"]);
|
||||
await git(sourceRepoDir, ["checkout", "-b", "main"]);
|
||||
await git(sourceRepoDir, ["config", "user.name", "Paperclip Test"]);
|
||||
await git(sourceRepoDir, ["config", "user.email", "test@paperclip.dev"]);
|
||||
await writeFile(path.join(sourceRepoDir, "tracked.txt"), "tracked\n", "utf8");
|
||||
await git(sourceRepoDir, ["add", "tracked.txt"]);
|
||||
await git(sourceRepoDir, ["commit", "-m", "base"]);
|
||||
await git(sourceRepoDir, ["worktree", "add", "-b", "work", localWorkspaceDir, "HEAD"]);
|
||||
// Pre-seed the sandbox with a `.paperclip-runtime` dir that MUST survive.
|
||||
await mkdir(path.join(remoteWorkspaceDir, ".paperclip-runtime"), { recursive: true });
|
||||
await writeFile(path.join(remoteWorkspaceDir, ".paperclip-runtime", "keep.txt"), "keep\n", "utf8");
|
||||
|
||||
const directWrites: string[] = [];
|
||||
const directRuns: string[] = [];
|
||||
const client: SandboxManagedRuntimeClient = {
|
||||
makeDir: async (remotePath) => {
|
||||
await mkdir(remotePath, { recursive: true });
|
||||
},
|
||||
writeFile: async (remotePath, bytes) => {
|
||||
directWrites.push(remotePath);
|
||||
await mkdir(path.dirname(remotePath), { recursive: true });
|
||||
await writeFile(remotePath, Buffer.from(bytes));
|
||||
},
|
||||
readFile: async (remotePath) => await readFile(remotePath),
|
||||
listFiles: async () => [],
|
||||
remove: async (remotePath) => {
|
||||
await rm(remotePath, { recursive: true, force: true });
|
||||
},
|
||||
run: async (command) => {
|
||||
directRuns.push(command);
|
||||
await execFile("sh", ["-c", command], { maxBuffer: 32 * 1024 * 1024 });
|
||||
},
|
||||
};
|
||||
const captured: SandboxSyncOperation[] = [];
|
||||
attachNativeRecordingSyncIn(client, captured);
|
||||
|
||||
const prepared = await prepareSandboxManagedRuntime({
|
||||
spec: {
|
||||
transport: "sandbox",
|
||||
provider: "test",
|
||||
sandboxId: "sandbox-1",
|
||||
remoteCwd: remoteWorkspaceDir,
|
||||
timeoutMs: 30_000,
|
||||
apiKey: null,
|
||||
},
|
||||
adapterKey: "test-adapter",
|
||||
client,
|
||||
workspaceLocalDir: localWorkspaceDir,
|
||||
});
|
||||
|
||||
// Delegated entirely to syncIn.
|
||||
expect(directWrites).toEqual([]);
|
||||
expect(directRuns).toEqual([]);
|
||||
|
||||
// Two operations: git-workspace then workspace overlay. Each uploads a single
|
||||
// tar as a `file` mapping and carries its extract as an ordered post-command.
|
||||
expect(captured.length).toBeGreaterThanOrEqual(2);
|
||||
const gitOp = captured.find((op) =>
|
||||
op.files.some((mapping) => mapping.targetPath.endsWith("git-workspace-upload.tar")),
|
||||
);
|
||||
const workspaceOp = captured.find((op) =>
|
||||
op.files.some((mapping) => mapping.targetPath.endsWith("workspace-upload.tar")),
|
||||
);
|
||||
expect(gitOp).toBeDefined();
|
||||
expect(workspaceOp).toBeDefined();
|
||||
expect(gitOp!.files.every((mapping) => mapping.kind === "file")).toBe(true);
|
||||
// The git operation's post-upload command preserves `.paperclip-runtime` while
|
||||
// replacing the rest of the tree (wipe-except-preserved), then untars.
|
||||
const gitCommand = gitOp!.postUploadCommands![0].command;
|
||||
expect(gitCommand).toContain(".paperclip-runtime");
|
||||
expect(gitCommand).toContain("tar -xf");
|
||||
|
||||
// The pre-seeded runtime dir survived the git+workspace staging.
|
||||
await expect(
|
||||
readFile(path.join(remoteWorkspaceDir, ".paperclip-runtime", "keep.txt"), "utf8"),
|
||||
).resolves.toBe("keep\n");
|
||||
await expect(readFile(path.join(remoteWorkspaceDir, "tracked.txt"), "utf8")).resolves.toBe("tracked\n");
|
||||
expect(prepared.workspaceRemoteDir).toBe(remoteWorkspaceDir);
|
||||
});
|
||||
|
||||
it("keeps the sandbox runtime core free of Codex-specific string literals", async () => {
|
||||
const coreSource = await readFile(new URL("./sandbox-managed-runtime.ts", import.meta.url), "utf8");
|
||||
// The seam must be generic: no adapter (Codex) knowledge may live in the core.
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ export interface SandboxRemoteExecutionSpec {
|
|||
}
|
||||
|
||||
/**
|
||||
* Remote paths handed to an asset's `provision.extractCommand`. All are POSIX
|
||||
* Remote paths handed to an asset's `provision.postUploadCommand`. All are POSIX
|
||||
* paths inside the sandbox: `assetTarPath` is the uploaded asset tarball,
|
||||
* `assetDir` is where the asset should be materialized, and `runtimeRootDir`
|
||||
* is the directory any `stageFiles` were written into.
|
||||
|
|
@ -69,23 +69,32 @@ export interface SandboxManagedRuntimeAssetProvisionContext {
|
|||
|
||||
/**
|
||||
* Per-asset inbound provisioning contribution. The core is adapter-agnostic:
|
||||
* an asset that supplies neither `stageFiles` nor `extractCommand` is extracted
|
||||
* with a plain `tar -xf`. An adapter that needs custom provisioning (e.g. a
|
||||
* credential merge) supplies helper files via `stageFiles` and the shell
|
||||
* command that consumes them via `extractCommand`.
|
||||
* an asset that supplies neither `stageFiles` nor `postUploadCommand` is
|
||||
* materialized with a plain destroy-then-replace `tar -xf`. An adapter that
|
||||
* needs custom provisioning (e.g. a credential merge) supplies helper files via
|
||||
* `stageFiles` and the shell command that consumes them via `postUploadCommand`.
|
||||
*
|
||||
* Both contributions ride the unified {@link SandboxSyncOperation} the core
|
||||
* builds per asset: `stageFiles` become additional `files` mappings placed
|
||||
* alongside the asset tar, and `postUploadCommand` becomes the operation's
|
||||
* ordered `postUploadCommands`. See {@link SandboxPostUploadCommand} for the
|
||||
* command-origin / confinement security contract (C1–C3).
|
||||
*/
|
||||
export interface SandboxManagedRuntimeAssetProvision {
|
||||
/**
|
||||
* Extra files written into `runtimeRootDir` (alongside the asset tar) before
|
||||
* the extract command runs — typically helper scripts the extract command
|
||||
* Extra files placed into `runtimeRootDir` (alongside the asset tar) before
|
||||
* the post-upload command runs — typically helper scripts the command
|
||||
* invokes. Contents may be raw bytes or a UTF-8 string.
|
||||
*/
|
||||
stageFiles?: { name: string; contents: Buffer | string }[];
|
||||
/**
|
||||
* Builds the shell command that materializes the uploaded asset tar into
|
||||
* `assetDir`. Defaults to a plain `tar -xf` extraction when omitted.
|
||||
* Builds the opaque, adapter-authored shell command that materializes the
|
||||
* uploaded asset tar into `assetDir`, run as the operation's ordered
|
||||
* post-upload command after every mapping has landed. Defaults to a plain
|
||||
* destroy-then-replace `tar -xf` extraction when omitted. Any path embedded in
|
||||
* the command MUST be built from already-confined paths and shell-quoted (C3).
|
||||
*/
|
||||
extractCommand?: (ctx: SandboxManagedRuntimeAssetProvisionContext) => string;
|
||||
postUploadCommand?: (ctx: SandboxManagedRuntimeAssetProvisionContext) => string;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -281,6 +290,39 @@ function buildDefaultExtractRuntimeAssetCommand(input: {
|
|||
`rm -f ${shellQuote(input.remoteAssetTar)}`;
|
||||
}
|
||||
|
||||
// Named builder (Security Condition C3): extract an uploaded workspace tarball
|
||||
// into `workspaceRemoteDir`, then remove the tarball. When `wipeExceptNames` is
|
||||
// present the target's direct children (except the preserved names) are removed
|
||||
// before extraction (destroy-then-replace); when null the tarball is overlaid on
|
||||
// top of the existing tree (e.g. a git overlay merge). Every path is
|
||||
// shell-quoted; no untrusted value is concatenated into the shell (C1/C3).
|
||||
function buildWorkspaceTarExtractCommand(input: {
|
||||
workspaceRemoteDir: string;
|
||||
remoteTar: string;
|
||||
wipeExceptNames: string[] | null;
|
||||
}): string {
|
||||
const wipe = input.wipeExceptNames
|
||||
? ` && find ${shellQuote(input.workspaceRemoteDir)} -mindepth 1 -maxdepth 1 ` +
|
||||
`${preserveFindArgs(input.wipeExceptNames)} -exec rm -rf -- {} +`
|
||||
: "";
|
||||
return (
|
||||
`mkdir -p ${shellQuote(input.workspaceRemoteDir)}${wipe} && ` +
|
||||
`tar -xf ${shellQuote(input.remoteTar)} -C ${shellQuote(input.workspaceRemoteDir)} && ` +
|
||||
`rm -f ${shellQuote(input.remoteTar)}`
|
||||
);
|
||||
}
|
||||
|
||||
// Named builder (C3): remove paths deleted in the host git worktree from the
|
||||
// sandbox workspace. Every path is shell-quoted; the caller supplies only
|
||||
// already-confined relative paths from the git snapshot.
|
||||
function buildRemoveDeletedPathsCommand(input: {
|
||||
remoteDir: string;
|
||||
deletedPaths: string[];
|
||||
}): string {
|
||||
const quotedPaths = input.deletedPaths.map((entry) => shellQuote(entry)).join(" ");
|
||||
return `cd ${shellQuote(input.remoteDir)} && rm -rf -- ${quotedPaths}`;
|
||||
}
|
||||
|
||||
function buildUniqueStagingPath(input: { targetPath: string; suffix: string }): string {
|
||||
return `${input.targetPath}${input.suffix}.${randomUUID()}`;
|
||||
}
|
||||
|
|
@ -491,10 +533,6 @@ async function copySelectedWorkspaceEntries(input: {
|
|||
}
|
||||
}
|
||||
|
||||
function toArrayBuffer(bytes: Buffer): ArrayBuffer {
|
||||
return Uint8Array.from(bytes).buffer;
|
||||
}
|
||||
|
||||
function toBuffer(bytes: Buffer | Uint8Array | ArrayBuffer): Buffer {
|
||||
if (Buffer.isBuffer(bytes)) return bytes;
|
||||
if (bytes instanceof ArrayBuffer) return Buffer.from(bytes);
|
||||
|
|
@ -541,20 +579,6 @@ function preserveFindArgs(entries: string[]): string {
|
|||
return entries.map((entry) => `! -name ${shellQuote(entry)}`).join(" ");
|
||||
}
|
||||
|
||||
async function removeDeletedPathsInSandbox(input: {
|
||||
client: SandboxManagedRuntimeClient;
|
||||
spec: SandboxRemoteExecutionSpec;
|
||||
remoteDir: string;
|
||||
deletedPaths: string[];
|
||||
}): Promise<void> {
|
||||
if (input.deletedPaths.length === 0) return;
|
||||
const quotedPaths = input.deletedPaths.map((entry) => shellQuote(entry)).join(" ");
|
||||
await input.client.run(
|
||||
`sh -c ${shellQuote(`cd ${shellQuote(input.remoteDir)} && rm -rf -- ${quotedPaths}`)}`,
|
||||
{ timeoutMs: input.spec.timeoutMs },
|
||||
);
|
||||
}
|
||||
|
||||
// Bridge a single byte-level transfer to the throttled progress reporter. The
|
||||
// transport reports decoded bytes via `options.onProgress`; the reporter turns
|
||||
// them into a throttled, fully-formatted log line. `finish()` emits the terminal
|
||||
|
|
@ -641,60 +665,111 @@ export async function prepareSandboxManagedRuntime(input: {
|
|||
? await captureDirectorySnapshot(input.workspaceLocalDir, { exclude: restoreExclude })
|
||||
: null;
|
||||
|
||||
// Prefer the provider's native file transport when it advertised the sync
|
||||
// verbs; otherwise every branch below falls back to the byte-identical tar +
|
||||
// base64 `writeFile`/`run` path. `nextSyncOperationId` emits opaque, ordered,
|
||||
// non-sensitive tokens — never a caller/asset identifier.
|
||||
const nativeSyncIn = typeof input.client.syncIn === "function";
|
||||
// Every inbound staging step delegates to the provider through `client.syncIn`:
|
||||
// the orchestrator no longer inlines `writeFile`+`run` or chooses a transport,
|
||||
// and there is no `usesCustomProvision` native-diversion gate. `syncIn` is
|
||||
// ALWAYS present in production — the command-managed client exposes a native
|
||||
// transport (Daytona/Kubernetes `uploadFiles` + provider-executed post-upload
|
||||
// commands) or a byte-identical base64-tar fallback that reproduces the prior
|
||||
// `writeFile`+`run` sequence. Require it explicitly so a misconfigured client
|
||||
// fails loud rather than silently skipping staging. `syncOut` stays optional
|
||||
// (native-only) with a tar fallback on the restore path below.
|
||||
const syncIn = input.client.syncIn;
|
||||
if (typeof syncIn !== "function") {
|
||||
throw new Error(
|
||||
"prepareSandboxManagedRuntime requires a client that exposes syncIn " +
|
||||
"(createCommandManagedRuntimeClient provides a native-or-fallback implementation).",
|
||||
);
|
||||
}
|
||||
const nativeSyncOut = typeof input.client.syncOut === "function";
|
||||
let syncOperationSeq = 0;
|
||||
// Opaque, ordered, non-sensitive operation tokens — never a caller/asset id.
|
||||
const nextSyncOperationId = () => `sync-op-${++syncOperationSeq}`;
|
||||
|
||||
// Every delegated post-upload command (extract/wipe/remove-deleted/asset merge)
|
||||
// must run under the run-specific timeout (`spec.timeoutMs`), not the provider
|
||||
// sync client's default timeout — the two can differ, and before staging was
|
||||
// routed through `syncIn` each of these ran via
|
||||
// `client.run(cmd, { timeoutMs: spec.timeoutMs })`. When they mismatch, a
|
||||
// command left without a `timeoutMs` outlives (or is killed under) the wrong
|
||||
// limit. Stamp the run timeout onto every delegated command, preserving any
|
||||
// command that already carries its own explicit timeout.
|
||||
const withRunTimeout = (
|
||||
commands: SandboxPostUploadCommand[],
|
||||
): SandboxPostUploadCommand[] =>
|
||||
commands.map((command) => ({
|
||||
...command,
|
||||
timeoutMs: command.timeoutMs ?? input.spec.timeoutMs,
|
||||
}));
|
||||
|
||||
await withTempDir("paperclip-sandbox-sync-", async (tempDir) => {
|
||||
const preservedNames = new Set([
|
||||
".paperclip-runtime",
|
||||
...(gitSnapshot ? [".git"] : []),
|
||||
...(input.preserveAbsentOnRestore ?? []),
|
||||
]);
|
||||
|
||||
// Build one `SandboxSyncOperation` uploading a host tarball as a single file
|
||||
// mapping (rides native `uploadFiles`, or the base64-tar fallback) with the
|
||||
// extract/wipe/merge steps carried as ordered `postUploadCommands`, confine
|
||||
// it, and delegate to `syncIn`. `finish` emits the terminal progress line.
|
||||
const stageTarball = async (input2: {
|
||||
tarPath: string;
|
||||
remoteTar: string;
|
||||
postUploadCommands: SandboxPostUploadCommand[];
|
||||
progressLabel: string;
|
||||
statusPhase: RuntimeStatusPhase;
|
||||
}): Promise<void> => {
|
||||
const tarSize = (await fs.stat(input2.tarPath)).size;
|
||||
const operations: SandboxSyncOperation[] = [{
|
||||
operationId: nextSyncOperationId(),
|
||||
files: [{ sourcePath: input2.tarPath, targetPath: input2.remoteTar, kind: "file" }],
|
||||
postUploadCommands: withRunTimeout(input2.postUploadCommands),
|
||||
}];
|
||||
assertSyncOperationsConfined(operations, {
|
||||
sourceRoots: [tempDir],
|
||||
targetRoots: [runtimeRootDir],
|
||||
});
|
||||
const upload = makeTransferProgress(
|
||||
input.onProgress,
|
||||
"Syncing",
|
||||
"to",
|
||||
input2.progressLabel,
|
||||
{ sink: input.onRuntimeProgress, phase: input2.statusPhase },
|
||||
);
|
||||
await syncIn(operations);
|
||||
await upload.finish(tarSize, tarSize);
|
||||
};
|
||||
|
||||
if (syncWorkspace && gitSnapshot) {
|
||||
await emitRuntimeStatus(input.onRuntimeProgress, "git_sync", "Syncing git history to sandbox");
|
||||
await withShallowGitWorkspaceClone({
|
||||
localDir: input.workspaceLocalDir,
|
||||
snapshot: gitSnapshot,
|
||||
}, async (cloneDir) => {
|
||||
// The git-workspace and workspace transfers preserve `.paperclip-runtime`
|
||||
// on the target (and the git overlay merges on top rather than replacing),
|
||||
// which the generic destroy-then-replace file mapping cannot express, so
|
||||
// they always take the tar path. Native transfer is used for the clean
|
||||
// destroy-then-replace cases (default-provision assets inbound; the
|
||||
// workspace download into a fresh host dir outbound).
|
||||
// git-workspace preserves `.paperclip-runtime` on the target and the
|
||||
// workspace overlay merges on top rather than replacing — expressed as
|
||||
// the operation's ordered post-upload commands, not a plain replace.
|
||||
const gitTarPath = path.join(tempDir, "git-workspace.tar");
|
||||
await createTarballFromDirectory({
|
||||
localDir: cloneDir,
|
||||
archivePath: gitTarPath,
|
||||
exclude: [".paperclip-runtime"],
|
||||
});
|
||||
const gitTarBytes = await fs.readFile(gitTarPath);
|
||||
const remoteGitTar = path.posix.join(runtimeRootDir, "git-workspace-upload.tar");
|
||||
await input.client.makeDir(runtimeRootDir);
|
||||
const gitUpload = makeTransferProgress(
|
||||
input.onProgress,
|
||||
"Syncing",
|
||||
"to",
|
||||
"git history",
|
||||
{ sink: input.onRuntimeProgress, phase: "git_sync" },
|
||||
);
|
||||
await input.client.writeFile(remoteGitTar, toArrayBuffer(gitTarBytes), gitUpload.options);
|
||||
await gitUpload.finish(gitTarBytes.byteLength, gitTarBytes.byteLength);
|
||||
await input.client.run(
|
||||
`sh -c ${shellQuote(
|
||||
`mkdir -p ${shellQuote(workspaceRemoteDir)} && ` +
|
||||
`find ${shellQuote(workspaceRemoteDir)} -mindepth 1 -maxdepth 1 ${preserveFindArgs([".paperclip-runtime"])} -exec rm -rf -- {} + && ` +
|
||||
`tar -xf ${shellQuote(remoteGitTar)} -C ${shellQuote(workspaceRemoteDir)} && ` +
|
||||
`rm -f ${shellQuote(remoteGitTar)}`,
|
||||
)}`,
|
||||
{ timeoutMs: input.spec.timeoutMs },
|
||||
);
|
||||
await stageTarball({
|
||||
tarPath: gitTarPath,
|
||||
remoteTar: remoteGitTar,
|
||||
postUploadCommands: [{
|
||||
command: buildWorkspaceTarExtractCommand({
|
||||
workspaceRemoteDir,
|
||||
remoteTar: remoteGitTar,
|
||||
wipeExceptNames: [".paperclip-runtime"],
|
||||
}),
|
||||
}],
|
||||
progressLabel: "git history",
|
||||
statusPhase: "git_sync",
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -715,81 +790,43 @@ export async function prepareSandboxManagedRuntime(input: {
|
|||
archivePath: workspaceTarPath,
|
||||
exclude: gitSnapshot ? undefined : workspaceArchiveExclude,
|
||||
});
|
||||
const workspaceTarBytes = await fs.readFile(workspaceTarPath);
|
||||
const remoteWorkspaceTar = path.posix.join(runtimeRootDir, "workspace-upload.tar");
|
||||
await input.client.makeDir(runtimeRootDir);
|
||||
const workspaceUpload = makeTransferProgress(
|
||||
input.onProgress,
|
||||
"Syncing",
|
||||
"to",
|
||||
"workspace",
|
||||
{ sink: input.onRuntimeProgress, phase: "config_sync" },
|
||||
);
|
||||
await input.client.writeFile(
|
||||
remoteWorkspaceTar,
|
||||
toArrayBuffer(workspaceTarBytes),
|
||||
workspaceUpload.options,
|
||||
);
|
||||
await workspaceUpload.finish(workspaceTarBytes.byteLength, workspaceTarBytes.byteLength);
|
||||
const extractWorkspaceTarCommand = gitSnapshot
|
||||
? `mkdir -p ${shellQuote(workspaceRemoteDir)} && ` +
|
||||
`tar -xf ${shellQuote(remoteWorkspaceTar)} -C ${shellQuote(workspaceRemoteDir)} && ` +
|
||||
`rm -f ${shellQuote(remoteWorkspaceTar)}`
|
||||
: `mkdir -p ${shellQuote(workspaceRemoteDir)} && ` +
|
||||
`find ${shellQuote(workspaceRemoteDir)} -mindepth 1 -maxdepth 1 ${preserveFindArgs([...preservedNames])} -exec rm -rf -- {} + && ` +
|
||||
`tar -xf ${shellQuote(remoteWorkspaceTar)} -C ${shellQuote(workspaceRemoteDir)} && ` +
|
||||
`rm -f ${shellQuote(remoteWorkspaceTar)}`;
|
||||
await input.client.run(
|
||||
`sh -c ${shellQuote(extractWorkspaceTarCommand)}`,
|
||||
{ timeoutMs: input.spec.timeoutMs },
|
||||
);
|
||||
if (gitSnapshot) {
|
||||
await removeDeletedPathsInSandbox({
|
||||
client: input.client,
|
||||
spec: input.spec,
|
||||
remoteDir: workspaceRemoteDir,
|
||||
deletedPaths: gitSnapshot.deletedPaths,
|
||||
// git overlay merges on top of the just-extracted git tree (no wipe);
|
||||
// non-git workspace wipes every child except the preserved names first.
|
||||
const workspacePostUploadCommands: SandboxPostUploadCommand[] = [{
|
||||
command: buildWorkspaceTarExtractCommand({
|
||||
workspaceRemoteDir,
|
||||
remoteTar: remoteWorkspaceTar,
|
||||
wipeExceptNames: gitSnapshot ? null : [...preservedNames],
|
||||
}),
|
||||
}];
|
||||
if (gitSnapshot && gitSnapshot.deletedPaths.length > 0) {
|
||||
workspacePostUploadCommands.push({
|
||||
command: buildRemoveDeletedPathsCommand({
|
||||
remoteDir: workspaceRemoteDir,
|
||||
deletedPaths: gitSnapshot.deletedPaths,
|
||||
}),
|
||||
});
|
||||
}
|
||||
await stageTarball({
|
||||
tarPath: workspaceTarPath,
|
||||
remoteTar: remoteWorkspaceTar,
|
||||
postUploadCommands: workspacePostUploadCommands,
|
||||
progressLabel: "workspace",
|
||||
statusPhase: "config_sync",
|
||||
});
|
||||
}
|
||||
|
||||
for (const asset of input.assets ?? []) {
|
||||
await emitRuntimeStatus(input.onRuntimeProgress, "config_sync", "Syncing runtime assets to sandbox");
|
||||
const remoteAssetDir = path.posix.join(runtimeRootDir, asset.key);
|
||||
// Assets with custom provisioning (staged helper files or a bespoke extract
|
||||
// command such as a credential merge) do more than a plain directory
|
||||
// replacement, so they cannot be expressed as a generic file mapping and
|
||||
// always take the tar path. A default-provisioned asset is a clean
|
||||
// destroy-then-replace of its own directory, which the native transport
|
||||
// reproduces exactly.
|
||||
const usesCustomProvision =
|
||||
Boolean(asset.provision?.extractCommand) || (asset.provision?.stageFiles?.length ?? 0) > 0;
|
||||
if (nativeSyncIn && !usesCustomProvision) {
|
||||
const operations: SandboxSyncOperation[] = [{
|
||||
operationId: nextSyncOperationId(),
|
||||
files: [{
|
||||
sourcePath: asset.localDir,
|
||||
targetPath: remoteAssetDir,
|
||||
kind: "directory",
|
||||
exclude: asset.exclude,
|
||||
followSymlinks: asset.followSymlinks,
|
||||
}],
|
||||
}];
|
||||
assertSyncOperationsConfined(operations, {
|
||||
sourceRoots: [asset.localDir],
|
||||
targetRoots: [runtimeRootDir],
|
||||
});
|
||||
const assetUpload = makeTransferProgress(
|
||||
input.onProgress,
|
||||
"Syncing",
|
||||
"to",
|
||||
asset.key,
|
||||
{ sink: input.onRuntimeProgress, phase: "config_sync" },
|
||||
);
|
||||
await input.client.syncIn!(operations);
|
||||
await assetUpload.finish(0, 0);
|
||||
continue;
|
||||
}
|
||||
const remoteAssetTar = path.posix.join(runtimeRootDir, `${asset.key}-upload.tar`);
|
||||
// Every asset — default OR custom-provisioned (e.g. an adapter credential
|
||||
// merge) — rides one `syncIn` operation: the asset tar plus any staged
|
||||
// helper files as `files` mappings, and the extract/merge command as the
|
||||
// ordered post-upload command. There is no native-diversion gate; a
|
||||
// custom-provisioned asset's bytes now ride native `uploadFiles` and its
|
||||
// command runs as a provider-executed post-upload command.
|
||||
const assetTarPath = path.join(tempDir, `${asset.key}.tar`);
|
||||
await createTarballFromDirectory({
|
||||
localDir: asset.localDir,
|
||||
|
|
@ -797,8 +834,42 @@ export async function prepareSandboxManagedRuntime(input: {
|
|||
followSymlinks: asset.followSymlinks,
|
||||
exclude: asset.exclude,
|
||||
});
|
||||
const assetTarBytes = await fs.readFile(assetTarPath);
|
||||
const remoteAssetTar = path.posix.join(runtimeRootDir, `${asset.key}-upload.tar`);
|
||||
const files: SandboxSyncFileMapping[] = [
|
||||
{ sourcePath: assetTarPath, targetPath: remoteAssetTar, kind: "file" },
|
||||
];
|
||||
// Stage provision helper files (e.g. the merge scripts) into the temp dir
|
||||
// and map them alongside the asset tar so they ride the same native upload.
|
||||
for (const stageFile of asset.provision?.stageFiles ?? []) {
|
||||
const safeName = stageFile.name;
|
||||
if (/[\\/]|\.\.(\.|$)/.test(safeName) || safeName === "..") {
|
||||
throw new Error(`provision stageFile.name must be a simple basename, got: ${safeName}`);
|
||||
}
|
||||
const stageBytes = typeof stageFile.contents === "string"
|
||||
? Buffer.from(stageFile.contents)
|
||||
: stageFile.contents;
|
||||
const stageHostPath = path.join(tempDir, `${asset.key}.stage.${safeName}`);
|
||||
await fs.writeFile(stageHostPath, stageBytes);
|
||||
files.push({
|
||||
sourcePath: stageHostPath,
|
||||
targetPath: path.posix.join(runtimeRootDir, safeName),
|
||||
kind: "file",
|
||||
});
|
||||
}
|
||||
const postUploadCommand = asset.provision?.postUploadCommand?.({
|
||||
assetTarPath: remoteAssetTar,
|
||||
assetDir: remoteAssetDir,
|
||||
runtimeRootDir,
|
||||
}) ?? buildDefaultExtractRuntimeAssetCommand({ remoteAssetDir, remoteAssetTar });
|
||||
const operations: SandboxSyncOperation[] = [{
|
||||
operationId: nextSyncOperationId(),
|
||||
files,
|
||||
postUploadCommands: withRunTimeout([{ command: postUploadCommand }]),
|
||||
}];
|
||||
assertSyncOperationsConfined(operations, {
|
||||
sourceRoots: [tempDir],
|
||||
targetRoots: [runtimeRootDir],
|
||||
});
|
||||
const assetTarSize = (await fs.stat(assetTarPath)).size;
|
||||
const assetUpload = makeTransferProgress(
|
||||
input.onProgress,
|
||||
"Syncing",
|
||||
|
|
@ -806,30 +877,8 @@ export async function prepareSandboxManagedRuntime(input: {
|
|||
asset.key,
|
||||
{ sink: input.onRuntimeProgress, phase: "config_sync" },
|
||||
);
|
||||
await input.client.writeFile(remoteAssetTar, toArrayBuffer(assetTarBytes), assetUpload.options);
|
||||
await assetUpload.finish(assetTarBytes.byteLength, assetTarBytes.byteLength);
|
||||
for (const stageFile of asset.provision?.stageFiles ?? []) {
|
||||
const stageBytes = typeof stageFile.contents === "string"
|
||||
? Buffer.from(stageFile.contents)
|
||||
: stageFile.contents;
|
||||
const safeName = stageFile.name;
|
||||
if (/[\\/]|\.\.(\.|$)/.test(safeName) || safeName === "..") {
|
||||
throw new Error(`provision stageFile.name must be a simple basename, got: ${safeName}`);
|
||||
}
|
||||
await input.client.writeFile(
|
||||
path.posix.join(runtimeRootDir, safeName),
|
||||
toArrayBuffer(stageBytes),
|
||||
);
|
||||
}
|
||||
const extractCommand = asset.provision?.extractCommand?.({
|
||||
assetTarPath: remoteAssetTar,
|
||||
assetDir: remoteAssetDir,
|
||||
runtimeRootDir,
|
||||
}) ?? buildDefaultExtractRuntimeAssetCommand({ remoteAssetDir, remoteAssetTar });
|
||||
await input.client.run(
|
||||
`sh -c ${shellQuote(extractCommand)}`,
|
||||
{ timeoutMs: input.spec.timeoutMs },
|
||||
);
|
||||
await syncIn(operations);
|
||||
await assetUpload.finish(assetTarSize, assetTarSize);
|
||||
}
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -22,12 +22,19 @@ const CODEX_AUTH_MERGE_DECISION_SCRIPT_BYTES = readFileSync(
|
|||
|
||||
/**
|
||||
* Builds the inbound (host→sandbox) provisioning contribution for the Codex
|
||||
* managed-home asset: stage the two merge scripts into the runtime root and run
|
||||
* the merge-extract script instead of a plain `tar -xf`, so a sandbox that
|
||||
* already carries a Codex `auth.json` keeps whichever credential is newer.
|
||||
* managed-home asset as **files + an ordered post-upload merge command**: the two
|
||||
* merge scripts ride the sync operation's `files` (staged into the runtime root
|
||||
* alongside the uploaded home tar via native `uploadFiles`), and the merge-extract
|
||||
* script runs as the operation's ordered **post-upload command** instead of a
|
||||
* plain `tar -xf`, so a sandbox that already carries a Codex `auth.json` keeps
|
||||
* whichever credential is newer (newer-`auth.json`-wins, same-identity, atomic
|
||||
* `0600` install — all inside the opaque script, unchanged).
|
||||
*
|
||||
* This is behaviour-identical to the extraction the sandbox core previously
|
||||
* hardcoded for `adapterKey === "codex" && assetKey === "home"`.
|
||||
* The command string handed to the provider is opaque and fully shell-quoted from
|
||||
* already-confined paths (Security Conditions C1/C3): it invokes only the staged
|
||||
* script by path — no `auth.json` bytes, token fields, or workspace content are
|
||||
* interpolated into the shell (C5). This is behaviour-identical to the extraction
|
||||
* the sandbox core previously drove through the custom-provision tar path.
|
||||
*/
|
||||
export function buildCodexAuthInboundProvision(): SandboxManagedRuntimeAssetProvision {
|
||||
return {
|
||||
|
|
@ -35,7 +42,7 @@ export function buildCodexAuthInboundProvision(): SandboxManagedRuntimeAssetProv
|
|||
{ name: CODEX_AUTH_MERGE_EXTRACT_SCRIPT_NAME, contents: CODEX_AUTH_MERGE_EXTRACT_SCRIPT_BYTES },
|
||||
{ name: CODEX_AUTH_MERGE_DECISION_SCRIPT_NAME, contents: CODEX_AUTH_MERGE_DECISION_SCRIPT_BYTES },
|
||||
],
|
||||
extractCommand: ({ assetTarPath, assetDir, runtimeRootDir }) =>
|
||||
postUploadCommand: ({ assetTarPath, assetDir, runtimeRootDir }) =>
|
||||
`sh ${shellQuote(path.posix.join(runtimeRootDir, CODEX_AUTH_MERGE_EXTRACT_SCRIPT_NAME))} ` +
|
||||
`${shellQuote(assetDir)} ${shellQuote(assetTarPath)}`,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import { afterEach, describe, expect, it } from "vitest";
|
|||
import {
|
||||
prepareSandboxManagedRuntime,
|
||||
type SandboxManagedRuntimeClient,
|
||||
type SandboxSyncOperation,
|
||||
} from "@paperclipai/adapter-utils/sandbox-managed-runtime";
|
||||
import { buildCodexAuthInboundProvision } from "./codex-auth-merge-scripts.js";
|
||||
|
||||
|
|
@ -93,6 +94,27 @@ describe("codex home auth merge on sandbox asset extract", () => {
|
|||
outputs.push(result.stdout, result.stderr);
|
||||
},
|
||||
};
|
||||
// Non-native base64-tar fallback `syncIn`: place each file mapping via
|
||||
// `writeFile`, then run the operation's ordered `postUploadCommands` — the
|
||||
// same seam the command-managed client provides in production. The Codex home
|
||||
// asset uploads its tar + the two merge scripts, then runs the auth-merge
|
||||
// command as the operation's post-upload command.
|
||||
client.syncIn = async (operations) => {
|
||||
for (const operation of operations) {
|
||||
for (const mapping of operation.files) {
|
||||
const bytes = await readFile(mapping.sourcePath);
|
||||
await mkdir(path.dirname(mapping.targetPath), { recursive: true });
|
||||
await client.writeFile(mapping.targetPath, bytes.buffer.slice(
|
||||
bytes.byteOffset,
|
||||
bytes.byteOffset + bytes.byteLength,
|
||||
) as ArrayBuffer);
|
||||
}
|
||||
for (const command of operation.postUploadCommands ?? []) {
|
||||
await client.run(command.command, { timeoutMs: 30_000 });
|
||||
}
|
||||
}
|
||||
return { operations: [] };
|
||||
};
|
||||
|
||||
await prepareSandboxManagedRuntime({
|
||||
spec: {
|
||||
|
|
@ -352,6 +374,127 @@ describe("codex home auth merge on sandbox asset extract", () => {
|
|||
expect(result.commandText, entry.name).not.toContain("SENTINEL");
|
||||
}
|
||||
});
|
||||
|
||||
it("routes the Codex home asset through a single native syncIn operation whose post-command is the auth-merge (#4, C5/C6)", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-codex-native-route-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const localWorkspaceDir = path.join(rootDir, "local-workspace");
|
||||
const remoteWorkspaceDir = path.join(rootDir, "remote-workspace");
|
||||
const localHomeDir = path.join(rootDir, "local-codex-home");
|
||||
const remoteHomeDir = path.join(remoteWorkspaceDir, ".paperclip-runtime", "codex", "home");
|
||||
await mkdir(localWorkspaceDir, { recursive: true });
|
||||
await mkdir(localHomeDir, { recursive: true });
|
||||
await mkdir(remoteHomeDir, { recursive: true });
|
||||
await writeFile(path.join(localWorkspaceDir, "README.md"), "workspace\n", "utf8");
|
||||
// Host credential is strictly newer → wins the merge (same identity).
|
||||
const hostAuth = subscriptionAuth({
|
||||
accountId: "acct-native",
|
||||
lastRefresh: "2026-07-10T02:00:00Z",
|
||||
marker: "host-newer-SENTINEL",
|
||||
});
|
||||
const sandboxAuth = subscriptionAuth({
|
||||
accountId: "acct-native",
|
||||
lastRefresh: "2026-07-10T01:00:00Z",
|
||||
marker: "sandbox-older-SENTINEL",
|
||||
});
|
||||
await writeFile(path.join(localHomeDir, "auth.json"), hostAuth, { mode: 0o600 });
|
||||
await writeFile(path.join(localHomeDir, "config.toml"), "model = \"gpt\"\n", "utf8");
|
||||
await writeFile(path.join(remoteHomeDir, "auth.json"), sandboxAuth, { mode: 0o600 });
|
||||
|
||||
// A native runner: the orchestrator must make NO direct writeFile/run — every
|
||||
// byte (incl. auth.json) rides `syncIn` (native uploadFiles), and the merge
|
||||
// runs as the operation's ordered post-upload command.
|
||||
const directWrites: string[] = [];
|
||||
const directRuns: string[] = [];
|
||||
const captured: SandboxSyncOperation[] = [];
|
||||
const client: SandboxManagedRuntimeClient = {
|
||||
makeDir: async (remotePath) => {
|
||||
await mkdir(remotePath, { recursive: true });
|
||||
},
|
||||
writeFile: async (remotePath, bytes) => {
|
||||
directWrites.push(remotePath);
|
||||
await mkdir(path.dirname(remotePath), { recursive: true });
|
||||
await writeFile(remotePath, Buffer.from(bytes));
|
||||
},
|
||||
readFile: async (remotePath) => await readFile(remotePath),
|
||||
listFiles: async () => [],
|
||||
remove: async (remotePath) => {
|
||||
await rm(remotePath, { recursive: true, force: true });
|
||||
},
|
||||
run: async (command) => {
|
||||
directRuns.push(command);
|
||||
await execFile("sh", ["-c", command], { maxBuffer: 32 * 1024 * 1024 });
|
||||
},
|
||||
};
|
||||
client.syncIn = async (operations) => {
|
||||
for (const operation of operations) {
|
||||
captured.push(operation);
|
||||
for (const mapping of operation.files) {
|
||||
const bytes = await readFile(mapping.sourcePath);
|
||||
await mkdir(path.dirname(mapping.targetPath), { recursive: true });
|
||||
await writeFile(mapping.targetPath, bytes);
|
||||
if (mapping.mode != null) await lstat(mapping.targetPath);
|
||||
}
|
||||
for (const command of operation.postUploadCommands ?? []) {
|
||||
await execFile("sh", ["-c", command.command], { maxBuffer: 32 * 1024 * 1024 });
|
||||
}
|
||||
}
|
||||
return { operations: [] };
|
||||
};
|
||||
|
||||
await prepareSandboxManagedRuntime({
|
||||
spec: {
|
||||
transport: "sandbox",
|
||||
provider: "test",
|
||||
sandboxId: "sandbox-1",
|
||||
remoteCwd: remoteWorkspaceDir,
|
||||
timeoutMs: 30_000,
|
||||
apiKey: null,
|
||||
},
|
||||
adapterKey: "codex",
|
||||
client,
|
||||
workspaceLocalDir: localWorkspaceDir,
|
||||
assets: [{
|
||||
key: "home",
|
||||
localDir: localHomeDir,
|
||||
followSymlinks: true,
|
||||
provision: buildCodexAuthInboundProvision(),
|
||||
}],
|
||||
});
|
||||
|
||||
// 0 direct exec/writeFile — pure native delegation.
|
||||
expect(directWrites).toEqual([]);
|
||||
expect(directRuns).toEqual([]);
|
||||
|
||||
// One operation carries the home asset: the home tar + the two merge scripts
|
||||
// as `files` mappings, and the auth-merge as the ordered post-upload command.
|
||||
const homeOp = captured.find((op) =>
|
||||
op.files.some((mapping) => mapping.targetPath.endsWith("home-upload.tar")),
|
||||
);
|
||||
expect(homeOp).toBeDefined();
|
||||
const targets = homeOp!.files.map((mapping) => path.posix.basename(mapping.targetPath)).sort();
|
||||
expect(targets).toEqual([
|
||||
"codex-auth-merge-decision.cjs",
|
||||
"codex-auth-merge-extract.sh",
|
||||
"home-upload.tar",
|
||||
]);
|
||||
expect(homeOp!.files.every((mapping) => mapping.kind === "file")).toBe(true);
|
||||
expect(homeOp!.postUploadCommands).toHaveLength(1);
|
||||
// The post-command is the auth-merge script, NOT a plain `tar -xf` (C6).
|
||||
const mergeCommand = homeOp!.postUploadCommands![0].command;
|
||||
expect(mergeCommand).toContain("codex-auth-merge-extract.sh");
|
||||
expect(mergeCommand).not.toMatch(/^\s*tar -xf/);
|
||||
|
||||
// C5: no token/credential material leaks into the operation metadata.
|
||||
const opJson = JSON.stringify(captured);
|
||||
expect(opJson).not.toContain("SENTINEL");
|
||||
expect(opJson).not.toContain("refresh-token");
|
||||
|
||||
// C6: newer host credential won, installed atomically at mode 0600.
|
||||
const finalAuthPath = path.join(remoteHomeDir, "auth.json");
|
||||
expect(await readFile(finalAuthPath, "utf8")).toBe(hostAuth);
|
||||
expect((await lstat(finalAuthPath)).mode & 0o777).toBe(0o600);
|
||||
});
|
||||
});
|
||||
|
||||
// The extract shell script consumes the decision predicate as a child process
|
||||
|
|
|
|||
Loading…
Reference in New Issue