perf(sandbox): coalesce git-workspace stage-sync into one confined syncIn (#10488)
## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Cloud and sandbox agents need less round-trip overhead at workspace start > - The workspace start path now uploads the git history and the working tree overlay in two separate sync operations > - That adds extra mkdir, guard, upload, and rename work for the same workspace bytes > - This pull request merges those two uploads into one guarded sync operation > - The benefit is one sync round trip for the workspace bytes with the same guard and login-shell contract ## Linked Issues or Issue Description No public GitHub issue exists for this change. ### Problem A git-backed workspace start uploads the git-history clone and the working-tree overlay as two separate host-to-sandbox sync operations. ### Proposed Solution Merge the two uploads into one sync operation. Keep both tar mappings under the same confinement guard. Run the git-history extract first, then the overlay extract, then optional cleanup. ### Alternatives Considered Keep two sync operations. That keeps the current round-trip cost and duplicates the guard, upload, and rename work. ### Roadmap Alignment This change supports the roadmap work on cloud and sandbox agents by reducing workspace start overhead. ## What Changed - Merge the git-history tar and the working-tree overlay tar into one sync operation. - Run the git-history extract first, then the overlay extract, then optional remove-deleted-paths cleanup. - Keep both temporary tar targets under the runtime directory so the first extract cannot delete the second tar before use. - Keep the symlink-escape guard, login-shell contract, and file-mapping checks on the merged file set. ## Verification - `tsc --noEmit` on `@paperclipai/adapter-utils` - `@paperclipai/adapter-utils` full project test suite: 354 pass / 4 skipped - Orchestrator proof: one merged operation with both tars and two ordered extract commands - Orchestrator proof: the confinement guard covers both tar mappings - Daytona plugin `plugin.test.ts`: 94 pass ## Risks - Any regression in the extract order could change workspace start behavior. - Any regression in the merged guard could block valid uploads or miss an escape attempt. ## Model Used OpenAI GPT-5, tool use enabled, current Codex session. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Nicky Leach
GitHub
parent
674d71548a
commit
d5b9f6c8c9
|
|
@ -8,6 +8,7 @@ import { afterEach, describe, expect, it, vi } from "vitest";
|
|||
import { resetLocalGitIndexToHead } from "./git-workspace-sync.js";
|
||||
|
||||
import {
|
||||
assertSyncOperationsConfined,
|
||||
mirrorDirectory,
|
||||
prepareSandboxManagedRuntime,
|
||||
type SandboxManagedRuntimeClient,
|
||||
|
|
@ -443,9 +444,13 @@ describe("sandbox managed runtime", () => {
|
|||
"restore",
|
||||
"finalize",
|
||||
]));
|
||||
// Git history and workspace overlay sync as ONE merged operation, so a single
|
||||
// transfer-progress event rides the config_sync (workspace) phase. The git_sync
|
||||
// phase still emits its plain status message (asserted by the arrayContaining
|
||||
// check above).
|
||||
expect(runtimeStatuses.some((status) => (
|
||||
status.phase === "git_sync" &&
|
||||
/^Syncing git history to sandbox: 100% \(\d+\.\d\/\d+\.\d MB\)$/.test(status.message)
|
||||
status.phase === "config_sync" &&
|
||||
/^Syncing workspace to sandbox: 100% \(\d+\.\d\/\d+\.\d MB\)$/.test(status.message)
|
||||
))).toBe(true);
|
||||
expect(runtimeStatuses.some((status) => (
|
||||
status.phase === "export" &&
|
||||
|
|
@ -1342,21 +1347,18 @@ describe("sandbox managed runtime", () => {
|
|||
expect(directWrites).toEqual([]);
|
||||
expect(directRuns).toEqual([]);
|
||||
|
||||
// Two operations: git-workspace then workspace overlay. Each uploads a single
|
||||
// tar as a `file` mapping and carries its extract as an ordered post-command.
|
||||
expect(captured.length).toBeGreaterThanOrEqual(2);
|
||||
const gitOp = captured.find((op) =>
|
||||
op.files.some((mapping) => mapping.targetPath.endsWith("git-workspace-upload.tar")),
|
||||
);
|
||||
const workspaceOp = captured.find((op) =>
|
||||
op.files.some((mapping) => mapping.targetPath.endsWith("workspace-upload.tar")),
|
||||
);
|
||||
expect(gitOp).toBeDefined();
|
||||
expect(workspaceOp).toBeDefined();
|
||||
expect(gitOp!.files.every((mapping) => mapping.kind === "file")).toBe(true);
|
||||
// The git operation's post-upload command preserves `.paperclip-runtime` while
|
||||
// replacing the rest of the tree (wipe-except-preserved), then untars.
|
||||
const gitCommand = gitOp!.postUploadCommands![0].command;
|
||||
// One merged operation carries BOTH the git-history and workspace-overlay tars
|
||||
// as two `file` mappings, each with its extract as an ordered post-command.
|
||||
expect(captured).toHaveLength(1);
|
||||
const op = captured[0];
|
||||
const byBase = (base: string) =>
|
||||
op.files.find((mapping) => path.posix.basename(mapping.targetPath) === base);
|
||||
expect(byBase("git-workspace-upload.tar")).toBeDefined();
|
||||
expect(byBase("workspace-upload.tar")).toBeDefined();
|
||||
expect(op.files.every((mapping) => mapping.kind === "file")).toBe(true);
|
||||
// The first post-upload command extracts the git history and preserves
|
||||
// `.paperclip-runtime` while replacing the rest of the tree (wipe-except-preserved).
|
||||
const gitCommand = op.postUploadCommands![0].command;
|
||||
expect(gitCommand).toContain(".paperclip-runtime");
|
||||
expect(gitCommand).toContain("tar -xf");
|
||||
|
||||
|
|
@ -1368,16 +1370,168 @@ describe("sandbox managed runtime", () => {
|
|||
expect(prepared.workspaceRemoteDir).toBe(remoteWorkspaceDir);
|
||||
});
|
||||
|
||||
// Regression lock: a representative `codex_local` start stages its inbound
|
||||
// bytes — git history, workspace overlay, and the managed Codex `home` asset
|
||||
// (auth.json merge) — as EXACTLY ONE `syncIn` operation each. Every inbound step
|
||||
// is routed through `client.syncIn` (one native `uploadFiles` round-trip per
|
||||
// operation, with the extract/merge carried as provider-executed
|
||||
// `postUploadCommands`), with no separate custom-provision diversion. Assert
|
||||
// the collapsed round-trip count so a future change that re-inlines a
|
||||
// `writeFile`+`run` sequence — or fans one staging step across multiple
|
||||
// operations — fails loudly here instead of silently regressing the start path.
|
||||
it("collapses a representative codex_local start to one syncIn round-trip per inbound staging step", async () => {
|
||||
it("issues one merged syncIn operation for a git-backed workspace stage-sync with two ordered extract commands", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-sandbox-merged-git-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const sourceRepoDir = path.join(rootDir, "source-repo");
|
||||
const localWorkspaceDir = path.join(rootDir, "local-workspace");
|
||||
const remoteWorkspaceDir = path.join(rootDir, "remote-workspace");
|
||||
await mkdir(sourceRepoDir, { recursive: true });
|
||||
await git(sourceRepoDir, ["init"]);
|
||||
await git(sourceRepoDir, ["checkout", "-b", "main"]);
|
||||
await git(sourceRepoDir, ["config", "user.name", "Paperclip Test"]);
|
||||
await git(sourceRepoDir, ["config", "user.email", "test@paperclip.dev"]);
|
||||
await writeFile(path.join(sourceRepoDir, "tracked.txt"), "tracked\n", "utf8");
|
||||
await git(sourceRepoDir, ["add", "tracked.txt"]);
|
||||
await git(sourceRepoDir, ["commit", "-m", "base"]);
|
||||
await git(sourceRepoDir, ["worktree", "add", "-b", "work", localWorkspaceDir, "HEAD"]);
|
||||
// Pre-seed the sandbox with a `.paperclip-runtime` dir that MUST survive.
|
||||
await mkdir(path.join(remoteWorkspaceDir, ".paperclip-runtime"), { recursive: true });
|
||||
await writeFile(path.join(remoteWorkspaceDir, ".paperclip-runtime", "keep.txt"), "keep\n", "utf8");
|
||||
|
||||
const client: SandboxManagedRuntimeClient = {
|
||||
makeDir: async (remotePath) => {
|
||||
await mkdir(remotePath, { recursive: true });
|
||||
},
|
||||
writeFile: async (remotePath, bytes) => {
|
||||
await mkdir(path.dirname(remotePath), { recursive: true });
|
||||
await writeFile(remotePath, Buffer.from(bytes));
|
||||
},
|
||||
readFile: async (remotePath) => await readFile(remotePath),
|
||||
listFiles: async () => [],
|
||||
remove: async (remotePath) => {
|
||||
await rm(remotePath, { recursive: true, force: true });
|
||||
},
|
||||
run: async (command) => {
|
||||
await execFile("sh", ["-c", command], { maxBuffer: 32 * 1024 * 1024 });
|
||||
},
|
||||
};
|
||||
const captured: SandboxSyncOperation[] = [];
|
||||
attachNativeRecordingSyncIn(client, captured);
|
||||
// Count how many times `syncIn` is invoked so the merge collapses the two
|
||||
// workspace staging steps into a single native round trip.
|
||||
let syncInCallCount = 0;
|
||||
const recordingSyncIn = client.syncIn!;
|
||||
client.syncIn = async (operations) => {
|
||||
syncInCallCount += 1;
|
||||
return recordingSyncIn(operations);
|
||||
};
|
||||
|
||||
await prepareSandboxManagedRuntime({
|
||||
spec: {
|
||||
transport: "sandbox",
|
||||
provider: "test",
|
||||
sandboxId: "sandbox-1",
|
||||
remoteCwd: remoteWorkspaceDir,
|
||||
timeoutMs: 30_000,
|
||||
apiKey: null,
|
||||
},
|
||||
adapterKey: "test-adapter",
|
||||
client,
|
||||
workspaceLocalDir: localWorkspaceDir,
|
||||
});
|
||||
|
||||
// One `syncIn` call carrying exactly one operation for the whole workspace.
|
||||
expect(syncInCallCount).toBe(1);
|
||||
expect(captured).toHaveLength(1);
|
||||
const op = captured[0];
|
||||
|
||||
// Both host tars ride the one operation as two `file` mappings.
|
||||
const byBase = (base: string) =>
|
||||
op.files.find((mapping) => path.posix.basename(mapping.targetPath) === base);
|
||||
const gitMapping = byBase("git-workspace-upload.tar");
|
||||
const overlayMapping = byBase("workspace-upload.tar");
|
||||
expect(gitMapping).toBeDefined();
|
||||
expect(overlayMapping).toBeDefined();
|
||||
expect(op.files).toHaveLength(2);
|
||||
expect(op.files.every((mapping) => mapping.kind === "file")).toBe(true);
|
||||
|
||||
// Both tar targets live under `.paperclip-runtime`, so the git extract's wipe
|
||||
// (which preserves `.paperclip-runtime`) cannot delete the overlay tar before
|
||||
// the overlay extract runs.
|
||||
for (const mapping of op.files) {
|
||||
expect(mapping.targetPath).toContain("/.paperclip-runtime/");
|
||||
}
|
||||
|
||||
// Two ordered extract commands: git history first (wipe-except-preserved),
|
||||
// overlay second (merge, no wipe). No deleted paths in this clean worktree.
|
||||
const commands = op.postUploadCommands ?? [];
|
||||
expect(commands).toHaveLength(2);
|
||||
expect(commands[0].command).toContain("git-workspace-upload.tar");
|
||||
expect(commands[0].command).toContain(".paperclip-runtime");
|
||||
expect(commands[0].command).toContain("find ");
|
||||
expect(commands[1].command).toContain("workspace-upload.tar");
|
||||
expect(commands[1].command).not.toContain("git-workspace-upload.tar");
|
||||
expect(commands[1].command).not.toContain("find ");
|
||||
|
||||
// The pre-seeded runtime dir survived and the workspace overlay applied.
|
||||
await expect(
|
||||
readFile(path.join(remoteWorkspaceDir, ".paperclip-runtime", "keep.txt"), "utf8"),
|
||||
).resolves.toBe("keep\n");
|
||||
await expect(readFile(path.join(remoteWorkspaceDir, "tracked.txt"), "utf8")).resolves.toBe("tracked\n");
|
||||
});
|
||||
|
||||
it("the merged workspace confine guard covers both tar mappings (escape in either trips it)", () => {
|
||||
const runtimeRoot = "/home/daytona/paperclip-workspace/.paperclip-runtime/test-adapter";
|
||||
const tempRoot = "/tmp/paperclip-sandbox-sync-abc";
|
||||
const gitMapping = {
|
||||
sourcePath: `${tempRoot}/git-workspace.tar`,
|
||||
targetPath: `${runtimeRoot}/git-workspace-upload.tar`,
|
||||
kind: "file" as const,
|
||||
};
|
||||
const overlayMapping = {
|
||||
sourcePath: `${tempRoot}/workspace.tar`,
|
||||
targetPath: `${runtimeRoot}/workspace-upload.tar`,
|
||||
kind: "file" as const,
|
||||
};
|
||||
const roots = { sourceRoots: [tempRoot], targetRoots: [runtimeRoot] };
|
||||
|
||||
// A confined merged operation with both tar mappings passes the guard.
|
||||
expect(() =>
|
||||
assertSyncOperationsConfined(
|
||||
[{ operationId: "merged", files: [gitMapping, overlayMapping] }],
|
||||
roots,
|
||||
),
|
||||
).not.toThrow();
|
||||
|
||||
// A `..` target escape in the overlay mapping trips the guard, so the whole
|
||||
// merged operation is rejected before any transfer.
|
||||
expect(() =>
|
||||
assertSyncOperationsConfined(
|
||||
[{
|
||||
operationId: "merged",
|
||||
files: [
|
||||
gitMapping,
|
||||
{ ...overlayMapping, targetPath: `${runtimeRoot}/../../etc/workspace-upload.tar` },
|
||||
],
|
||||
}],
|
||||
roots,
|
||||
),
|
||||
).toThrow(/escapes its confinement root|not a confined absolute path/);
|
||||
|
||||
// An absolute-path source escape in the git mapping trips the guard too.
|
||||
expect(() =>
|
||||
assertSyncOperationsConfined(
|
||||
[{
|
||||
operationId: "merged",
|
||||
files: [{ ...gitMapping, sourcePath: "/etc/passwd" }, overlayMapping],
|
||||
}],
|
||||
roots,
|
||||
),
|
||||
).toThrow(/escapes its confinement root|not a confined absolute path/);
|
||||
});
|
||||
|
||||
// Regression lock: a representative `codex_local` start stages its inbound bytes
|
||||
// as TWO `syncIn` operations. The git-history and workspace-overlay tars share
|
||||
// ONE merged operation (one native `uploadFiles` round-trip that carries both
|
||||
// tars, with the two extract commands as ordered `postUploadCommands`); the
|
||||
// managed Codex `home` asset (auth.json merge) is the second operation. Every
|
||||
// inbound step is routed through `client.syncIn`, with no separate
|
||||
// custom-provision diversion. Assert the collapsed round-trip count so a future
|
||||
// change that re-inlines a `writeFile`+`run` sequence — or splits the merged
|
||||
// workspace operation back into two — fails loudly here instead of silently
|
||||
// regressing the start path.
|
||||
it("collapses a representative codex_local start to two syncIn round-trips: one merged workspace op plus the asset op", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-sandbox-codex-roundtrip-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const sourceRepoDir = path.join(rootDir, "source-repo");
|
||||
|
|
@ -1385,7 +1539,7 @@ describe("sandbox managed runtime", () => {
|
|||
const remoteWorkspaceDir = path.join(rootDir, "remote-workspace");
|
||||
const homeDir = path.join(rootDir, "codex-home");
|
||||
|
||||
// Git-backed workspace → git history + workspace overlay are two staging steps.
|
||||
// Git-backed workspace → git history + workspace overlay share one merged op.
|
||||
await mkdir(sourceRepoDir, { recursive: true });
|
||||
await git(sourceRepoDir, ["init"]);
|
||||
await git(sourceRepoDir, ["checkout", "-b", "main"]);
|
||||
|
|
@ -1458,30 +1612,31 @@ describe("sandbox managed runtime", () => {
|
|||
expect(directWrites).toEqual([]);
|
||||
expect(directRuns).toEqual([]);
|
||||
|
||||
// The collapsed count: exactly three inbound round-trips — git, workspace, home.
|
||||
expect(captured).toHaveLength(3);
|
||||
const gitOp = captured.find((op) =>
|
||||
op.files.some((mapping) => mapping.targetPath.endsWith("git-workspace-upload.tar")),
|
||||
);
|
||||
const workspaceOp = captured.find((op) =>
|
||||
op.files.some((mapping) => mapping.targetPath.endsWith("workspace-upload.tar")),
|
||||
);
|
||||
const homeOp = captured.find((op) =>
|
||||
op.files.some((mapping) => mapping.targetPath.endsWith("home-upload.tar")),
|
||||
);
|
||||
expect(gitOp).toBeDefined();
|
||||
// The collapsed count: exactly two inbound round-trips — the merged workspace
|
||||
// op (git history + overlay) and the home asset op.
|
||||
expect(captured).toHaveLength(2);
|
||||
const hasBase = (op: SandboxSyncOperation, base: string) =>
|
||||
op.files.some((mapping) => path.posix.basename(mapping.targetPath) === base);
|
||||
const workspaceOp = captured.find((op) => hasBase(op, "workspace-upload.tar"));
|
||||
const homeOp = captured.find((op) => hasBase(op, "home-upload.tar"));
|
||||
expect(workspaceOp).toBeDefined();
|
||||
expect(homeOp).toBeDefined();
|
||||
// The merged workspace op carries BOTH the git-history and overlay tars, with
|
||||
// both extract commands as ordered post-upload commands (git first, overlay
|
||||
// second). The two tars ride one native uploadFiles round-trip.
|
||||
expect(hasBase(workspaceOp!, "git-workspace-upload.tar")).toBe(true);
|
||||
expect(workspaceOp!.files).toHaveLength(2);
|
||||
expect((workspaceOp!.postUploadCommands ?? []).length).toBeGreaterThanOrEqual(2);
|
||||
|
||||
// Every operation is a single native uploadFiles (all `file` mappings) whose
|
||||
// Every operation is a native uploadFiles (all `file` mappings) whose
|
||||
// extract/merge rides as an ordered provider-executed post-upload command.
|
||||
for (const op of captured) {
|
||||
expect(op.files.length).toBeGreaterThanOrEqual(1);
|
||||
expect(op.files.every((mapping) => mapping.kind === "file")).toBe(true);
|
||||
expect(op.postUploadCommands ?? []).not.toHaveLength(0);
|
||||
}
|
||||
// Operation ids are distinct, so "3 operations" is 3 real round-trips.
|
||||
expect(new Set(captured.map((op) => op.operationId)).size).toBe(3);
|
||||
// Operation ids are distinct, so "2 operations" is 2 real round-trips.
|
||||
expect(new Set(captured.map((op) => op.operationId)).size).toBe(2);
|
||||
|
||||
// The credential asset actually materialized through the native seam.
|
||||
await expect(readFile(path.join(prepared.assetDirs.home, "auth.json"), "utf8"))
|
||||
|
|
|
|||
|
|
@ -785,64 +785,54 @@ export async function prepareSandboxManagedRuntime(input: {
|
|||
await upload.finish(params.progressBytes, params.progressBytes);
|
||||
};
|
||||
|
||||
// Upload a host tarball as a single `file` mapping with the extract/wipe/merge
|
||||
// steps as ordered post-upload commands. A thin wrapper over
|
||||
// `stageConfinedSyncIn` for the workspace/git anchor and asset paths.
|
||||
const stageTarball = async (input2: {
|
||||
tarPath: string;
|
||||
remoteTar: string;
|
||||
postUploadCommands: SandboxPostUploadCommand[];
|
||||
progressLabel: string;
|
||||
statusPhase: RuntimeStatusPhase;
|
||||
}): Promise<void> => {
|
||||
const tarSize = (await fs.stat(input2.tarPath)).size;
|
||||
await stageConfinedSyncIn({
|
||||
files: [{ sourcePath: input2.tarPath, targetPath: input2.remoteTar, kind: "file" }],
|
||||
postUploadCommands: input2.postUploadCommands,
|
||||
sourceRoots: [tempDir],
|
||||
targetRoots: [runtimeRootDir],
|
||||
progressLabel: input2.progressLabel,
|
||||
statusPhase: input2.statusPhase,
|
||||
progressBytes: tarSize,
|
||||
});
|
||||
};
|
||||
|
||||
if (syncWorkspace && gitSnapshot) {
|
||||
await emitRuntimeStatus(input.onRuntimeProgress, "git_sync", "Syncing git history to sandbox");
|
||||
await withShallowGitWorkspaceClone({
|
||||
localDir: input.workspaceLocalDir,
|
||||
snapshot: gitSnapshot,
|
||||
}, async (cloneDir) => {
|
||||
// git-workspace preserves `.paperclip-runtime` on the target and the
|
||||
// workspace overlay merges on top rather than replacing — expressed as
|
||||
// the operation's ordered post-upload commands, not a plain replace.
|
||||
const gitTarPath = path.join(tempDir, "git-workspace.tar");
|
||||
await createTarballFromDirectory({
|
||||
localDir: cloneDir,
|
||||
archivePath: gitTarPath,
|
||||
exclude: [".paperclip-runtime"],
|
||||
});
|
||||
const remoteGitTar = path.posix.join(runtimeRootDir, "git-workspace-upload.tar");
|
||||
await stageTarball({
|
||||
tarPath: gitTarPath,
|
||||
remoteTar: remoteGitTar,
|
||||
postUploadCommands: [{
|
||||
command: buildWorkspaceTarExtractCommand({
|
||||
workspaceRemoteDir,
|
||||
remoteTar: remoteGitTar,
|
||||
wipeExceptNames: [".paperclip-runtime"],
|
||||
}),
|
||||
}],
|
||||
progressLabel: "git history",
|
||||
statusPhase: "git_sync",
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
if (syncWorkspace) {
|
||||
// A git-backed workspace and a plain workspace both stage through ONE
|
||||
// confined `syncIn` operation. A git-backed workspace carries TWO host tars —
|
||||
// the git-history clone and the working-tree overlay — as two `file` mappings
|
||||
// on the SAME operation, with their extract commands as ordered
|
||||
// `postUploadCommands`. One operation shares one mkdir, one confine guard, one
|
||||
// `uploadFiles`, and one rename exec, so the second `syncIn` round trip is
|
||||
// removed. Build the whole merged file set and command list BEFORE the confine
|
||||
// guard runs (inside `stageConfinedSyncIn`); never append a mapping after it.
|
||||
const workspaceFiles: SandboxSyncFileMapping[] = [];
|
||||
const workspacePostUploadCommands: SandboxPostUploadCommand[] = [];
|
||||
let workspaceUploadBytes = 0;
|
||||
|
||||
// 1. git-history tar (git-backed workspace only). Both tar targets live under
|
||||
// `runtimeRootDir` (`.paperclip-runtime/<adapterKey>`). The git extract
|
||||
// wipes the target tree EXCEPT `.paperclip-runtime`, so the overlay tar,
|
||||
// which sits under `.paperclip-runtime`, survives to run its own extract.
|
||||
if (gitSnapshot) {
|
||||
await emitRuntimeStatus(input.onRuntimeProgress, "git_sync", "Syncing git history to sandbox");
|
||||
const gitTarPath = path.join(tempDir, "git-workspace.tar");
|
||||
const remoteGitTar = path.posix.join(runtimeRootDir, "git-workspace-upload.tar");
|
||||
await withShallowGitWorkspaceClone({
|
||||
localDir: input.workspaceLocalDir,
|
||||
snapshot: gitSnapshot,
|
||||
}, async (cloneDir) => {
|
||||
await createTarballFromDirectory({
|
||||
localDir: cloneDir,
|
||||
archivePath: gitTarPath,
|
||||
exclude: [".paperclip-runtime"],
|
||||
});
|
||||
});
|
||||
workspaceFiles.push({ sourcePath: gitTarPath, targetPath: remoteGitTar, kind: "file" });
|
||||
workspacePostUploadCommands.push({
|
||||
command: buildWorkspaceTarExtractCommand({
|
||||
workspaceRemoteDir,
|
||||
remoteTar: remoteGitTar,
|
||||
wipeExceptNames: [".paperclip-runtime"],
|
||||
}),
|
||||
});
|
||||
workspaceUploadBytes += (await fs.stat(gitTarPath)).size;
|
||||
}
|
||||
|
||||
// 2. workspace-overlay tar. A git-backed overlay merges on top of the just
|
||||
// extracted git tree (no wipe); a plain workspace wipes every child except
|
||||
// the preserved names first. The extract runs AFTER the git extract.
|
||||
await emitRuntimeStatus(input.onRuntimeProgress, "config_sync", "Syncing workspace to sandbox");
|
||||
const workspaceTarPath = path.join(tempDir, "workspace.tar");
|
||||
const workspaceArchiveDir = gitSnapshot ? path.join(tempDir, "workspace-overlay") : input.workspaceLocalDir;
|
||||
await emitRuntimeStatus(input.onRuntimeProgress, "config_sync", "Syncing workspace to sandbox");
|
||||
if (gitSnapshot) {
|
||||
await copySelectedWorkspaceEntries({
|
||||
sourceDir: input.workspaceLocalDir,
|
||||
|
|
@ -857,15 +847,15 @@ export async function prepareSandboxManagedRuntime(input: {
|
|||
exclude: gitSnapshot ? undefined : workspaceArchiveExclude,
|
||||
});
|
||||
const remoteWorkspaceTar = path.posix.join(runtimeRootDir, "workspace-upload.tar");
|
||||
// git overlay merges on top of the just-extracted git tree (no wipe);
|
||||
// non-git workspace wipes every child except the preserved names first.
|
||||
const workspacePostUploadCommands: SandboxPostUploadCommand[] = [{
|
||||
workspaceFiles.push({ sourcePath: workspaceTarPath, targetPath: remoteWorkspaceTar, kind: "file" });
|
||||
workspacePostUploadCommands.push({
|
||||
command: buildWorkspaceTarExtractCommand({
|
||||
workspaceRemoteDir,
|
||||
remoteTar: remoteWorkspaceTar,
|
||||
wipeExceptNames: gitSnapshot ? null : [...preservedNames],
|
||||
}),
|
||||
}];
|
||||
});
|
||||
// 3. Optional remove-deleted-paths command runs LAST, after both extracts.
|
||||
if (gitSnapshot && gitSnapshot.deletedPaths.length > 0) {
|
||||
workspacePostUploadCommands.push({
|
||||
command: buildRemoveDeletedPathsCommand({
|
||||
|
|
@ -874,12 +864,19 @@ export async function prepareSandboxManagedRuntime(input: {
|
|||
}),
|
||||
});
|
||||
}
|
||||
await stageTarball({
|
||||
tarPath: workspaceTarPath,
|
||||
remoteTar: remoteWorkspaceTar,
|
||||
workspaceUploadBytes += (await fs.stat(workspaceTarPath)).size;
|
||||
|
||||
// One confined `syncIn` for the whole merged workspace file set. The confine
|
||||
// guard covers every mapping BEFORE any bytes upload (fail-closed): a source
|
||||
// or target escape in EITHER tar mapping stops the upload of both.
|
||||
await stageConfinedSyncIn({
|
||||
files: workspaceFiles,
|
||||
postUploadCommands: workspacePostUploadCommands,
|
||||
sourceRoots: [tempDir],
|
||||
targetRoots: [runtimeRootDir],
|
||||
progressLabel: "workspace",
|
||||
statusPhase: "config_sync",
|
||||
progressBytes: workspaceUploadBytes,
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2956,6 +2956,159 @@ describe("daytona native file-sync hooks", () => {
|
|||
).toBe(false);
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Merged git-workspace operation. A git-backed workspace stage-sync rides ONE
|
||||
// operation whose `files` carry the git-history tar and the workspace-overlay
|
||||
// tar, with the two extract commands as ordered `postUploadCommands`. The
|
||||
// operation shares one mkdir, one confine guard, one `uploadFiles`, and one
|
||||
// rename exec.
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
it("stages a merged git-workspace operation as one uploadFiles batch and one rename exec, both extracts in order", async () => {
|
||||
const hostDir = await makeHostDir();
|
||||
const gitTar = path.join(hostDir, "git-workspace.tar");
|
||||
const overlayTar = path.join(hostDir, "workspace.tar");
|
||||
await fs.writeFile(gitTar, "git-bytes");
|
||||
await fs.writeFile(overlayTar, "overlay-bytes");
|
||||
const runtimeDir = `${REMOTE_DIR}/.paperclip-runtime/adapter`;
|
||||
|
||||
const sandbox = createMockSandbox();
|
||||
mockGet.mockResolvedValue(sandbox);
|
||||
|
||||
const gitExtract = "git-history-extract";
|
||||
const overlayExtract = "workspace-overlay-extract";
|
||||
const result = await plugin.definition.onEnvironmentSyncIn?.({
|
||||
driverKey: "daytona",
|
||||
companyId: "company-1",
|
||||
environmentId: "env-1",
|
||||
config: { timeoutMs: 300000, reuseLease: false },
|
||||
lease: syncLease(),
|
||||
operations: [
|
||||
{
|
||||
operationId: "merged-workspace",
|
||||
files: [
|
||||
{ sourcePath: gitTar, targetPath: `${runtimeDir}/git-workspace-upload.tar`, kind: "file" },
|
||||
{ sourcePath: overlayTar, targetPath: `${runtimeDir}/workspace-upload.tar`, kind: "file" },
|
||||
],
|
||||
postUploadCommands: [{ command: gitExtract }, { command: overlayExtract }],
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
// One bulk upload carries BOTH tars; one rename exec promotes both temps.
|
||||
expect(sandbox.fs.uploadFiles).toHaveBeenCalledTimes(1);
|
||||
const [uploads] = sandbox.fs.uploadFiles.mock.calls[0] as [Array<{ source: string; destination: string }>];
|
||||
expect(uploads).toHaveLength(2);
|
||||
const mvCalls = sandbox.process.executeCommand.mock.calls.filter(([cmd]: [string]) =>
|
||||
String(cmd).includes("mv -f"),
|
||||
);
|
||||
expect(mvCalls).toHaveLength(1);
|
||||
expect(String(mvCalls[0][0]).match(/mv -f /g)).toHaveLength(2);
|
||||
|
||||
// Both extract commands ran, in array order, AFTER the upload (git first).
|
||||
const orderOf = (cmd: string) => {
|
||||
const idx = sandbox.process.executeCommand.mock.calls.findIndex(([c]: [string]) => c === cmd);
|
||||
return sandbox.process.executeCommand.mock.invocationCallOrder[idx];
|
||||
};
|
||||
expect(orderOf(gitExtract)).toBeLessThan(orderOf(overlayExtract));
|
||||
expect(sandbox.fs.uploadFiles.mock.invocationCallOrder[0]).toBeLessThan(orderOf(gitExtract));
|
||||
|
||||
expect(result).toEqual({
|
||||
operations: [{
|
||||
operationId: "merged-workspace",
|
||||
filesTransferred: 2,
|
||||
bytesTransferred: "git-bytes".length + "overlay-bytes".length,
|
||||
}],
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects a merged operation when either tar mapping target escapes the remote dir, before uploading", async () => {
|
||||
const hostDir = await makeHostDir();
|
||||
const gitTar = path.join(hostDir, "git-workspace.tar");
|
||||
const overlayTar = path.join(hostDir, "workspace.tar");
|
||||
await fs.writeFile(gitTar, "git-bytes");
|
||||
await fs.writeFile(overlayTar, "overlay-bytes");
|
||||
const runtimeDir = `${REMOTE_DIR}/.paperclip-runtime/adapter`;
|
||||
|
||||
const sandbox = createMockSandbox();
|
||||
mockGet.mockResolvedValue(sandbox);
|
||||
|
||||
await expect(
|
||||
plugin.definition.onEnvironmentSyncIn?.({
|
||||
driverKey: "daytona",
|
||||
companyId: "company-1",
|
||||
environmentId: "env-1",
|
||||
config: { timeoutMs: 300000, reuseLease: false },
|
||||
lease: syncLease(),
|
||||
operations: [
|
||||
{
|
||||
operationId: "merged-escape",
|
||||
files: [
|
||||
{ sourcePath: gitTar, targetPath: `${runtimeDir}/git-workspace-upload.tar`, kind: "file" },
|
||||
// The overlay mapping target escapes the workspace remote dir.
|
||||
{ sourcePath: overlayTar, targetPath: `${REMOTE_DIR}/../../etc/workspace-upload.tar`, kind: "file" },
|
||||
],
|
||||
postUploadCommands: [{ command: "git-history-extract" }, { command: "workspace-overlay-extract" }],
|
||||
},
|
||||
],
|
||||
}),
|
||||
).rejects.toThrow(/escapes the workspace remote dir|not a confined absolute path/);
|
||||
|
||||
// Neither tar uploaded: the confine check on the escaping mapping trips first.
|
||||
expect(sandbox.fs.uploadFiles).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("stops the overlay and remove-deleted commands when the git extract fails (merged operation fail-fast)", async () => {
|
||||
const hostDir = await makeHostDir();
|
||||
const gitTar = path.join(hostDir, "git-workspace.tar");
|
||||
const overlayTar = path.join(hostDir, "workspace.tar");
|
||||
await fs.writeFile(gitTar, "git-bytes");
|
||||
await fs.writeFile(overlayTar, "overlay-bytes");
|
||||
const runtimeDir = `${REMOTE_DIR}/.paperclip-runtime/adapter`;
|
||||
|
||||
const sandbox = createMockSandbox();
|
||||
// The first (git-history) extract exits non-zero; every transfer/guard script
|
||||
// stays green so the fail-fast loop is the only thing that can trip this test.
|
||||
sandbox.process.executeCommand.mockImplementation(async (command: string) => {
|
||||
if (command === "git-history-extract") {
|
||||
return { exitCode: 5, result: "boom", artifacts: { stdout: "boom" } };
|
||||
}
|
||||
return { exitCode: 0, result: "", artifacts: { stdout: "" } };
|
||||
});
|
||||
mockGet.mockResolvedValue(sandbox);
|
||||
|
||||
await expect(
|
||||
plugin.definition.onEnvironmentSyncIn?.({
|
||||
driverKey: "daytona",
|
||||
companyId: "company-1",
|
||||
environmentId: "env-1",
|
||||
config: { timeoutMs: 300000, reuseLease: false },
|
||||
lease: syncLease(),
|
||||
operations: [
|
||||
{
|
||||
operationId: "merged-failfast",
|
||||
files: [
|
||||
{ sourcePath: gitTar, targetPath: `${runtimeDir}/git-workspace-upload.tar`, kind: "file" },
|
||||
{ sourcePath: overlayTar, targetPath: `${runtimeDir}/workspace-upload.tar`, kind: "file" },
|
||||
],
|
||||
postUploadCommands: [
|
||||
{ command: "git-history-extract" },
|
||||
{ command: "workspace-overlay-extract" },
|
||||
{ command: "remove-deleted-paths" },
|
||||
],
|
||||
},
|
||||
],
|
||||
}),
|
||||
).rejects.toThrow(/post-upload command failed \(exit 5\)/);
|
||||
|
||||
// Fail-fast: the overlay extract and the remove-deleted command never ran.
|
||||
const ran = (cmd: string) =>
|
||||
sandbox.process.executeCommand.mock.calls.some(([c]: [string]) => c === cmd);
|
||||
expect(ran("git-history-extract")).toBe(true);
|
||||
expect(ran("workspace-overlay-extract")).toBe(false);
|
||||
expect(ran("remove-deleted-paths")).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects a post-upload command cwd that escapes the remote dir lexically, before any exec (C2)", async () => {
|
||||
const hostDir = await makeHostDir();
|
||||
const source = path.join(hostDir, "config.txt");
|
||||
|
|
|
|||
Loading…
Reference in New Issue