diff --git a/packages/adapter-utils/src/acpx-engine/execute.test.ts b/packages/adapter-utils/src/acpx-engine/execute.test.ts index c1fe7c2c2d..29edf4a340 100644 --- a/packages/adapter-utils/src/acpx-engine/execute.test.ts +++ b/packages/adapter-utils/src/acpx-engine/execute.test.ts @@ -798,6 +798,155 @@ describe("shared ACPX engine runtime behavior", () => { expect(fp(rotatedKey)).not.toBe(fp(withKey)); }); + it("busts the session fingerprint when the referenced-project set or pinned checkout changes", async () => { + const root = await makeTempRoot(); + const stateDir = path.join(root, "state"); + const cwd = path.join(root, "workspace"); + const baseConfig = { agentCommand: "node ./fake-acp.js", stateDir }; + + // A compatible resume reuses the already-staged referenced-project trees, so + // the fingerprint must fold in the referenced-project set. Otherwise a resume + // reuses a stale tree after the set or a project's pinned checkout changes. + const withReferenced = (additional: unknown[]) => ({ + context: { + taskId: "issue-1", + wakeReason: "issue_assigned", + paperclipWorkspace: { cwd, realization: { additional } }, + }, + }); + const projectA = { + path: "/host/project-a", + projectId: "a", + projectWorkspaceId: "ws-a", + repoUrl: "https://example.test/a.git", + repoRef: "ref-a-1", + }; + const projectB = { + path: "/host/project-b", + projectId: "b", + projectWorkspaceId: "ws-b", + repoUrl: "https://example.test/b.git", + repoRef: "ref-b-1", + }; + + const one = await runExecutor(baseConfig, withReferenced([projectA])); + const two = await runExecutor(baseConfig, withReferenced([projectA, projectB])); + const reordered = await runExecutor(baseConfig, withReferenced([projectB, projectA])); + const repointed = await runExecutor( + baseConfig, + withReferenced([projectA, { ...projectB, repoRef: "ref-b-2" }]), + ); + + const fp = (r: { result: { sessionParams?: unknown } }) => + (r.result.sessionParams as { configFingerprint?: string } | undefined)?.configFingerprint; + + // Adding a referenced project changes the set identity, so the fingerprint + // busts and the next launch stages the current set. + expect(fp(one)).toBeDefined(); + expect(fp(two)).not.toBe(fp(one)); + // The identity depends on the set, not the order the records arrive in. + expect(fp(reordered)).toBe(fp(two)); + // Re-pinning one project's checkout ref busts the fingerprint, so the resume + // re-stages instead of reusing a stale tree. + expect(fp(repointed)).not.toBe(fp(two)); + }); + + it("busts the session fingerprint when referenced-project files change at the same host path", async () => { + const root = await makeTempRoot(); + const stateDir = path.join(root, "state"); + const cwd = path.join(root, "workspace"); + const projectDir = path.join(root, "project-a"); + await fs.mkdir(path.join(projectDir, "src"), { recursive: true }); + await fs.writeFile(path.join(projectDir, "src", "index.ts"), "export const value = 1;\n"); + const baseConfig = { agentCommand: "node ./fake-acp.js", stateDir }; + + // Same host path and same metadata across runs. Only the file bytes change, + // which mirrors a branch that moved or a re-checkout in place. The metadata + // identity stays constant, so without the content signature a compatible + // resume would reuse a stale staged tree. + const referencedProject = { + path: projectDir, + projectId: "a", + projectWorkspaceId: "ws-a", + repoUrl: "https://example.test/a.git", + repoRef: "main", + }; + const withProject = () => ({ + context: { + taskId: "issue-1", + wakeReason: "issue_assigned", + paperclipWorkspace: { cwd, realization: { additional: [referencedProject] } }, + }, + }); + const fp = (r: { result: { sessionParams?: unknown } }) => + (r.result.sessionParams as { configFingerprint?: string } | undefined)?.configFingerprint; + + const before = await runExecutor(baseConfig, withProject()); + // Change the tree content and its size while the host path and metadata stay + // identical. + await fs.writeFile(path.join(projectDir, "src", "index.ts"), "export const value = 2; // changed\n"); + const afterEdit = await runExecutor(baseConfig, withProject()); + // Add a new file to the tree. + await fs.writeFile(path.join(projectDir, "src", "extra.ts"), "export const extra = true;\n"); + const afterAdd = await runExecutor(baseConfig, withProject()); + + expect(fp(before)).toBeDefined(); + // A content-only edit busts the fingerprint, so the resume re-stages. + expect(fp(afterEdit)).not.toBe(fp(before)); + // A new file in the tree busts the fingerprint too. + expect(fp(afterAdd)).not.toBe(fp(afterEdit)); + }); + + it("busts the session fingerprint when referenced-project bytes change but size and mtime stay equal", async () => { + const root = await makeTempRoot(); + const stateDir = path.join(root, "state"); + const cwd = path.join(root, "workspace"); + const projectDir = path.join(root, "project-a"); + const filePath = path.join(projectDir, "src", "index.ts"); + await fs.mkdir(path.join(projectDir, "src"), { recursive: true }); + // The two contents have the same byte length, so a stat-only signature (size + // and modification time) would collide once the modification time also matches. + const firstContent = "export const value = 1;\n"; + const secondContent = "export const value = 2;\n"; + expect(secondContent.length).toBe(firstContent.length); + await fs.writeFile(filePath, firstContent); + + // A re-checkout can restore the same modification time, so pin it explicitly + // and reapply it after the edit. Only the file bytes change between runs. + const pinnedTime = new Date("2026-01-01T00:00:00.000Z"); + await fs.utimes(filePath, pinnedTime, pinnedTime); + + const referencedProject = { + path: projectDir, + projectId: "a", + projectWorkspaceId: "ws-a", + repoUrl: "https://example.test/a.git", + repoRef: "main", + }; + const withProject = () => ({ + context: { + taskId: "issue-1", + wakeReason: "issue_assigned", + paperclipWorkspace: { cwd, realization: { additional: [referencedProject] } }, + }, + }); + const fp = (r: { result: { sessionParams?: unknown } }) => + (r.result.sessionParams as { configFingerprint?: string } | undefined)?.configFingerprint; + + const baseConfig = { agentCommand: "node ./fake-acp.js", stateDir }; + const before = await runExecutor(baseConfig, withProject()); + // Replace the bytes with an equal-length string and restore the same + // modification time, so path, size, and mtime all stay identical across the two runs. + await fs.writeFile(filePath, secondContent); + await fs.utimes(filePath, pinnedTime, pinnedTime); + const after = await runExecutor(baseConfig, withProject()); + + expect(fp(before)).toBeDefined(); + // The content signature reads bytes, so an equal-size, equal-mtime edit busts + // the fingerprint and the resume re-stages instead of reusing a stale tree. + expect(fp(after)).not.toBe(fp(before)); + }); + it("shapes ACPX session env for remote execution identities", async () => { const root = await makeTempRoot(); const localCwd = path.join(root, "local"); diff --git a/packages/adapter-utils/src/acpx-engine/execute.ts b/packages/adapter-utils/src/acpx-engine/execute.ts index 67c4aae5a2..bc1b1947b8 100644 --- a/packages/adapter-utils/src/acpx-engine/execute.ts +++ b/packages/adapter-utils/src/acpx-engine/execute.ts @@ -28,6 +28,7 @@ import { type AdapterExecutionTargetTimeoutResolution, type AdapterManagedRuntimeAsset, type PreparedAdapterExecutionTargetRuntime, + type SandboxAdditionalSource, } from "@paperclipai/adapter-utils/execution-target"; import { DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE, @@ -418,6 +419,83 @@ function shortHash(value: unknown): string { return createHash("sha256").update(stableJson(value)).digest("hex").slice(0, 16); } +// Directory names the staging path never ships for a referenced project (heavy +// build/cache output and git history). The content signature skips them so it +// reflects only the staged tree and never reads their bytes. Keep this set equal +// to the staging excludes in the sandbox and remote runtimes. +const REFERENCED_SOURCE_SIGNATURE_SKIP_DIRS = new Set([ + "node_modules", + "vendor", + "dist", + "build", + "out", + "coverage", + ".next", + ".turbo", + ".cache", + ".git", +]); + +/** + * Content signature of a referenced-project host tree for the session fingerprint. + * + * The staged-runtime cache reuses an already-staged referenced-project tree on a + * compatible resume and does not re-sync it. Referenced-project metadata (id, host + * path, workspace id, repo url, pinned ref) can stay identical while the files at + * that host path change: a branch moved to a new commit, a re-checkout in place, or + * a dirty worktree. So the metadata identity alone lets a resume serve a stale tree. + * This signature folds the tree's own content state into the identity. + * + * The walk reads each file's relative path and bytes and folds them into the hash. + * It reads bytes, not only file stats. A stat-only signature (size and modification + * time) collides when an edit keeps the byte length and the modification time — a + * re-checkout that restores the same size and timestamp. The byte hash busts on any + * content change, so the fingerprint busts and the next launch stages the current + * tree. The walk skips the heavy build, cache, and git directories the staging path + * never ships, and records a symlink by its target text without following it. On a + * read error the function returns a stable marker, so the fingerprint does not churn + * while staging surfaces the real error. The walk runs only when the run carries + * referenced projects (the multi-project sync path). + */ +async function referencedSourceContentSignature(localPath: string): Promise { + const hash = createHash("sha256"); + const walk = async (relative: string): Promise => { + const current = relative ? path.join(localPath, relative) : localPath; + const dirents = await fs.readdir(current, { withFileTypes: true }); + dirents.sort((left, right) => (left.name < right.name ? -1 : left.name > right.name ? 1 : 0)); + for (const dirent of dirents) { + const next = relative ? path.posix.join(relative, dirent.name) : dirent.name; + if (dirent.isDirectory()) { + if (REFERENCED_SOURCE_SIGNATURE_SKIP_DIRS.has(dirent.name)) { + continue; + } + await walk(next); + continue; + } + const absolute = path.join(localPath, next); + const stats = await fs.lstat(absolute); + if (stats.isSymbolicLink()) { + const target = await fs.readlink(absolute); + hash.update(`symlink:${next}:${target}\n`); + continue; + } + if (!stats.isFile()) { + hash.update(`other:${next}:${stats.mode}\n`); + continue; + } + hash.update(`file:${next}:${stats.size}\n`); + hash.update(await fs.readFile(absolute)); + hash.update("\n"); + } + }; + try { + await walk(""); + } catch (error) { + return `unreadable:${String(error)}`; + } + return hash.digest("hex").slice(0, 16); +} + function defaultPaperclipInstanceDir(): string { const home = process.env.PAPERCLIP_HOME?.trim() || path.join(os.homedir(), ".paperclip"); const instanceId = process.env.PAPERCLIP_INSTANCE_ID?.trim() || "default"; @@ -1209,6 +1287,11 @@ async function stageAcpRemoteRuntime(input: { workspaceRemoteDir?: string; timeoutSec: number; assets?: AdapterManagedRuntimeAsset[]; + // Referenced (additional) projects to stage into the sandbox as plain, + // read-only trees alongside the anchor workspace. Empty unless run prep + // resolved referenced projects (gated upstream), so the anchor-only path is + // unchanged. + additionalSources?: SandboxAdditionalSource[]; onLog: AdapterExecutionContext["onLog"]; onRuntimeProgress: AdapterExecutionContext["onRuntimeProgress"]; }): Promise { @@ -1224,6 +1307,9 @@ async function stageAcpRemoteRuntime(input: { workspaceLocalDir: input.workspaceLocalDir, ...(input.workspaceRemoteDir ? { workspaceRemoteDir: input.workspaceRemoteDir } : {}), ...(input.assets && input.assets.length > 0 ? { assets: input.assets } : {}), + ...(input.additionalSources && input.additionalSources.length > 0 + ? { additionalSources: input.additionalSources } + : {}), onProgress: (line) => input.onLog("stdout", line), onRuntimeProgress: input.onRuntimeProgress, }); @@ -1271,6 +1357,43 @@ async function buildRuntime(input: { const useConfiguredInsteadOfAgentHome = workspaceSource === "agent_home" && configuredCwd.length > 0; const effectiveWorkspaceCwd = useConfiguredInsteadOfAgentHome ? "" : workspaceCwd; const cwd = effectiveWorkspaceCwd || configuredCwd || process.cwd(); + // Referenced (additional) projects to stage into the sandbox alongside the + // anchor workspace, read from the workspace realization record. The list is + // empty unless run prep resolved referenced projects — gated upstream by the + // multi-project workspace-sync kill-switch — so the anchor-only path is + // unchanged. + const realizationContext = parseObject(workspaceContext.realization); + const additionalSourceRecords = ( + Array.isArray(realizationContext.additional) ? realizationContext.additional : [] + ).map((entry) => parseObject(entry)); + const additionalSources: SandboxAdditionalSource[] = additionalSourceRecords + .map((entry) => ({ localPath: asString(entry.path, ""), projectId: asString(entry.projectId, "") })) + .filter((entry) => entry.localPath.length > 0 && entry.projectId.length > 0); + // Stable identity of the referenced-project set for the session fingerprint. + // The staged-runtime cache reuses already-staged referenced-project trees on a + // compatible resume, so the fingerprint must change when the set OR a project's + // pinned checkout changes. Without this, a resume reuses a stale staged tree. + // Fold in each project's id, host path, workspace id, and pinned ref; sort by + // projectId so the identity depends on the set, not the record order. + const additionalSourcesIdentityBase = additionalSourceRecords + .map((entry) => ({ + projectId: asString(entry.projectId, ""), + localPath: asString(entry.path, ""), + projectWorkspaceId: asString(entry.projectWorkspaceId, ""), + repoUrl: asString(entry.repoUrl, ""), + repoRef: asString(entry.repoRef, ""), + })) + .filter((entry) => entry.localPath.length > 0 && entry.projectId.length > 0) + .sort((a, b) => (a.projectId < b.projectId ? -1 : a.projectId > b.projectId ? 1 : 0)); + // Metadata alone does not change on a content-only checkout change (same host + // path and pinned ref, new file bytes). Fold in each tree's content signature so + // a file add, remove, or edit busts the fingerprint and the resume re-stages. + const additionalSourcesIdentity = await Promise.all( + additionalSourcesIdentityBase.map(async (entry) => ({ + ...entry, + contentSignature: await referencedSourceContentSignature(entry.localPath), + })), + ); const executionTarget = readAdapterExecutionTarget({ executionTarget: input.ctx.executionTarget, legacyRemoteExecution: input.ctx.executionTransport?.remoteExecution, @@ -1553,6 +1676,11 @@ async function buildRuntime(input: { requestedThinkingEffort, fastMode, remoteExecutionIdentity, + // Referenced-project set + pinned-checkout identity. A change here (a project + // added, removed, or re-pinned) invalidates a warm/resumable session so the + // next launch stages the current referenced-project trees instead of reusing + // a stale staged tree. + additionalSourcesIdentity, skillsIdentity, skillPromptInstructions, paperclipClaudeSettings: paperclipClaudeSettings @@ -1681,6 +1809,7 @@ async function buildRuntime(input: { workspaceRemoteDir: sessionCwd, timeoutSec, assets, + additionalSources, onLog: input.ctx.onLog, onRuntimeProgress: input.ctx.onRuntimeProgress, }); diff --git a/packages/adapter-utils/src/command-managed-runtime.test.ts b/packages/adapter-utils/src/command-managed-runtime.test.ts index a4b36fb49e..859ea12758 100644 --- a/packages/adapter-utils/src/command-managed-runtime.test.ts +++ b/packages/adapter-utils/src/command-managed-runtime.test.ts @@ -303,6 +303,59 @@ describe("command managed runtime", () => { ); }); + it("stages each additional project into an isolated dir on the base64/tar transport, one failure skipped", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-command-runtime-additional-")); + cleanupDirs.push(rootDir); + + const localWorkspaceDir = path.join(rootDir, "local-workspace"); + const remoteWorkspaceDir = path.join(rootDir, "remote-workspace"); + await mkdir(localWorkspaceDir, { recursive: true }); + await mkdir(remoteWorkspaceDir, { recursive: true }); + await writeFile(path.join(localWorkspaceDir, "README.md"), "anchor\n", "utf8"); + + const goodOne = path.join(rootDir, "src-one"); + const goodTwo = path.join(rootDir, "src-two"); + await mkdir(goodOne, { recursive: true }); + await mkdir(path.join(goodTwo, "nested"), { recursive: true }); + await writeFile(path.join(goodOne, "one.txt"), "one body\n", "utf8"); + await writeFile(path.join(goodTwo, "nested", "two.txt"), "two body\n", "utf8"); + + // The `makeSpawnRunner` runner exposes no native syncIn, so staging rides the + // base64/tar fallback. The middle source points at a missing directory, so + // its tar build fails; failure isolation skips only it. + const { runner } = makeSpawnRunner(); + const prepared = await prepareCommandManagedRuntime({ + runner, + spec: { + remoteCwd: remoteWorkspaceDir, + timeoutMs: 30_000, + }, + adapterKey: "claude", + workspaceLocalDir: localWorkspaceDir, + additionalSources: [ + { localPath: goodOne, projectId: "one" }, + { localPath: path.join(rootDir, "missing"), projectId: "broken" }, + { localPath: goodTwo, projectId: "two" }, + ], + }); + + const runtimeRootDir = path.posix.join(remoteWorkspaceDir, ".paperclip-runtime", "claude"); + expect(Object.keys(prepared.additionalSourceDirs).sort()).toEqual(["one", "two"]); + expect(prepared.additionalSourceDirs.one).toBe(path.posix.join(runtimeRootDir, "project-one")); + expect(prepared.additionalSourceDirs.two).toBe(path.posix.join(runtimeRootDir, "project-two")); + expect(prepared.additionalSourceDirs.broken).toBeUndefined(); + + // Each healthy project's tree materialized in its OWN dir (nested files kept). + await expect(readFile(path.join(prepared.additionalSourceDirs.one, "one.txt"), "utf8")).resolves.toBe("one body\n"); + await expect(readFile(path.join(prepared.additionalSourceDirs.two, "nested", "two.txt"), "utf8")).resolves.toBe( + "two body\n", + ); + // The broken project's dir was never created. + await expect(readFile(path.join(runtimeRootDir, "project-broken"), "utf8")).rejects.toMatchObject({ + code: "ENOENT", + }); + }); + it("keeps adapter detection on the profile-backed shell path", async () => { const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-command-runtime-detect-")); cleanupDirs.push(rootDir); diff --git a/packages/adapter-utils/src/command-managed-runtime.ts b/packages/adapter-utils/src/command-managed-runtime.ts index 58a5d08510..fc9b0f0cc8 100644 --- a/packages/adapter-utils/src/command-managed-runtime.ts +++ b/packages/adapter-utils/src/command-managed-runtime.ts @@ -6,6 +6,7 @@ import { createTarballFromDirectory, prepareSandboxManagedRuntime, type PreparedSandboxManagedRuntime, + type SandboxAdditionalSource, type SandboxManagedRuntimeAsset, type SandboxManagedRuntimeClient, type SandboxRemoteExecutionSpec, @@ -467,6 +468,8 @@ export async function prepareCommandManagedRuntime(input: { workspaceExclude?: string[]; preserveAbsentOnRestore?: string[]; assets?: CommandManagedRuntimeAsset[]; + /** Referenced (additional) projects to stage into the sandbox as plain, read-only trees. */ + additionalSources?: SandboxAdditionalSource[]; installCommand?: string | null; /** When provided alongside `installCommand`, skip the install if `command -v ` succeeds. */ detectCommand?: string | null; @@ -522,6 +525,7 @@ export async function prepareCommandManagedRuntime(input: { workspaceExclude: mergeRuntimeExcludes(input.workspaceExclude), preserveAbsentOnRestore: input.preserveAbsentOnRestore, assets: input.assets, + additionalSources: input.additionalSources, onProgress: input.onProgress, onRuntimeProgress: input.onRuntimeProgress, }); @@ -559,6 +563,7 @@ export async function prepareCommandManagedRuntime(input: { workspaceExclude: mergeRuntimeExcludes(input.workspaceExclude), preserveAbsentOnRestore: input.preserveAbsentOnRestore, assets: input.assets, + additionalSources: input.additionalSources, onProgress: input.onProgress, onRuntimeProgress: input.onRuntimeProgress, }); diff --git a/packages/adapter-utils/src/execution-target.ts b/packages/adapter-utils/src/execution-target.ts index 25777e5732..2ac0705bc3 100644 --- a/packages/adapter-utils/src/execution-target.ts +++ b/packages/adapter-utils/src/execution-target.ts @@ -14,6 +14,8 @@ import { prepareRemoteManagedRuntime, remoteExecutionSessionMatches, } from "./remote-managed-runtime.js"; +import type { SandboxAdditionalSource } from "./sandbox-managed-runtime.js"; +export type { SandboxAdditionalSource } from "./sandbox-managed-runtime.js"; import { createCommandManagedSandboxCallbackBridgeQueueClient, createSandboxCallbackBridgeAsset, @@ -114,6 +116,12 @@ export interface PreparedAdapterExecutionTargetRuntime { workspaceRemoteDir: string | null; runtimeRootDir: string | null; assetDirs: Record; + /** + * Remote directory of each additional (referenced) project that staged + * successfully, keyed by `projectId`. Empty for a local target or when no + * additional sources were requested. + */ + additionalSourceDirs: Record; restoreWorkspace(onProgress?: RuntimeProgressSink): Promise; } @@ -1107,6 +1115,8 @@ export async function prepareAdapterExecutionTargetRuntime(input: { workspaceExclude?: string[]; preserveAbsentOnRestore?: string[]; assets?: AdapterManagedRuntimeAsset[]; + /** Referenced (additional) projects to stage into the sandbox as plain, read-only trees. */ + additionalSources?: SandboxAdditionalSource[]; installCommand?: string | null; /** When provided alongside `installCommand`, skip the install if the binary is already on PATH. */ detectCommand?: string | null; @@ -1124,6 +1134,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: { workspaceRemoteDir: null, runtimeRootDir: null, assetDirs: {}, + additionalSourceDirs: {}, restoreWorkspace: async () => {}, }; } @@ -1137,6 +1148,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: { workspaceRemoteDir: input.workspaceRemoteDir, syncWorkspace: input.syncWorkspace, assets: input.assets, + additionalSources: input.additionalSources, onProgress: input.onProgress, }); return { @@ -1144,6 +1156,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: { workspaceRemoteDir: prepared.workspaceRemoteDir, runtimeRootDir: prepared.runtimeRootDir, assetDirs: prepared.assetDirs, + additionalSourceDirs: prepared.additionalSourceDirs, restoreWorkspace: prepared.restoreWorkspace, }; } @@ -1167,6 +1180,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: { workspaceExclude: input.workspaceExclude, preserveAbsentOnRestore: input.preserveAbsentOnRestore, assets: input.assets, + additionalSources: input.additionalSources, installCommand: input.installCommand, detectCommand: input.detectCommand, onProgress: input.onProgress, @@ -1177,6 +1191,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: { workspaceRemoteDir: prepared.workspaceRemoteDir, runtimeRootDir: prepared.runtimeRootDir, assetDirs: prepared.assetDirs, + additionalSourceDirs: prepared.additionalSourceDirs, restoreWorkspace: prepared.restoreWorkspace, }; } diff --git a/packages/adapter-utils/src/remote-managed-runtime.test.ts b/packages/adapter-utils/src/remote-managed-runtime.test.ts index 0d0c98fb0e..330b1cfcd6 100644 --- a/packages/adapter-utils/src/remote-managed-runtime.test.ts +++ b/packages/adapter-utils/src/remote-managed-runtime.test.ts @@ -15,7 +15,7 @@ const { stdout: Buffer.from('{"token":"remote"}\n').toString("base64"), stderr: "", })), - syncDirectoryToSsh: vi.fn(async () => undefined), + syncDirectoryToSsh: vi.fn(async (_input: { localDir: string }) => undefined), })); vi.mock("./ssh.js", () => ({ @@ -92,4 +92,95 @@ describe("remote managed runtime", () => { ); expect(restoredAuth).toBe('{"token":"remote"}\n'); }); + + it("stages each additional project into its own isolated SSH dir, isolating one failure", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-remote-runtime-additional-")); + cleanupDirs.push(rootDir); + const workspaceDir = path.join(rootDir, "workspace"); + const firstDir = path.join(rootDir, "referenced-first"); + const secondDir = path.join(rootDir, "referenced-second"); + const brokenDir = path.join(rootDir, "referenced-broken"); + await mkdir(workspaceDir, { recursive: true }); + + // The transfer rejects only for the broken project's directory. + syncDirectoryToSsh.mockImplementation(async (input: { localDir: string }) => { + if (input.localDir === brokenDir) throw new Error("ssh transfer failed"); + return undefined; + }); + + const prepared = await prepareRemoteManagedRuntime({ + spec: { + host: "127.0.0.1", + port: 2222, + username: "fixture", + remoteWorkspacePath: "/app", + remoteCwd: "/app", + privateKey: "PRIVATE KEY", + knownHosts: "KNOWN HOSTS", + strictHostKeyChecking: true, + }, + runId: "run-additional", + adapterKey: "codex", + workspaceLocalDir: workspaceDir, + workspaceRemoteDir: "/app", + syncWorkspace: false, + additionalSources: [ + { localPath: firstDir, projectId: "first" }, + { localPath: brokenDir, projectId: "broken" }, + { localPath: secondDir, projectId: "second" }, + ], + }); + + // Each healthy project staged into its OWN isolated dir under the runtime + // root; the broken one is skipped, not fatal. + expect(Object.keys(prepared.additionalSourceDirs).sort()).toEqual(["first", "second"]); + expect(prepared.additionalSourceDirs.first).toBe("/app/.paperclip-runtime/codex/project-first"); + expect(prepared.additionalSourceDirs.second).toBe("/app/.paperclip-runtime/codex/project-second"); + expect(prepared.additionalSourceDirs.broken).toBeUndefined(); + expect(syncDirectoryToSsh).toHaveBeenCalledWith(expect.objectContaining({ + localDir: firstDir, + remoteDir: "/app/.paperclip-runtime/codex/project-first", + })); + expect(syncDirectoryToSsh).toHaveBeenCalledWith(expect.objectContaining({ + localDir: secondDir, + remoteDir: "/app/.paperclip-runtime/codex/project-second", + })); + }); + + it("skips an additional project whose localPath is not absolute", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-remote-runtime-relative-")); + cleanupDirs.push(rootDir); + const workspaceDir = path.join(rootDir, "workspace"); + const healthyDir = path.join(rootDir, "referenced-healthy"); + await mkdir(workspaceDir, { recursive: true }); + + const prepared = await prepareRemoteManagedRuntime({ + spec: { + host: "127.0.0.1", + port: 2222, + username: "fixture", + remoteWorkspacePath: "/app", + remoteCwd: "/app", + privateKey: "PRIVATE KEY", + knownHosts: "KNOWN HOSTS", + strictHostKeyChecking: true, + }, + runId: "run-relative", + adapterKey: "codex", + workspaceLocalDir: workspaceDir, + workspaceRemoteDir: "/app", + syncWorkspace: false, + additionalSources: [ + { localPath: "relative/referenced", projectId: "relative" }, + { localPath: healthyDir, projectId: "healthy" }, + ], + }); + + // The relative-path project never reaches the transfer and is skipped; the + // absolute-path project still stages. + expect(Object.keys(prepared.additionalSourceDirs)).toEqual(["healthy"]); + expect(syncDirectoryToSsh).not.toHaveBeenCalledWith(expect.objectContaining({ + localDir: "relative/referenced", + })); + }); }); diff --git a/packages/adapter-utils/src/remote-managed-runtime.ts b/packages/adapter-utils/src/remote-managed-runtime.ts index 6d8cd3aa4b..0902e17a9c 100644 --- a/packages/adapter-utils/src/remote-managed-runtime.ts +++ b/packages/adapter-utils/src/remote-managed-runtime.ts @@ -7,10 +7,26 @@ import { restoreWorkspaceFromSshExecution, syncDirectoryToSsh, } from "./ssh.js"; -import type { SandboxManagedRuntimeAssetRestoreContext } from "./sandbox-managed-runtime.js"; +import type { + SandboxAdditionalSource, + SandboxManagedRuntimeAssetRestoreContext, +} from "./sandbox-managed-runtime.js"; import { captureDirectorySnapshot } from "./workspace-restore-merge.js"; import type { RuntimeProgressSink } from "./runtime-progress.js"; +const REMOTE_ADDITIONAL_SOURCE_HEAVY_DIR_EXCLUDES = [ + "node_modules", + "vendor", + "dist", + "build", + "out", + "coverage", + ".next", + ".turbo", + ".cache", + ".git", +].flatMap((entry) => [entry, `${entry}/*`, `*/${entry}`, `*/${entry}/*`]); + export interface RemoteManagedRuntimeAsset { key: string; localDir: string; @@ -25,6 +41,12 @@ export interface PreparedRemoteManagedRuntime { workspaceRemoteDir: string; runtimeRootDir: string; assetDirs: Record; + /** + * Remote directory of each additional (referenced) project that staged + * successfully, keyed by `projectId`. A project whose staging failed is + * absent (per-project failure isolation). + */ + additionalSourceDirs: Record; restoreWorkspace(onProgress?: RuntimeProgressSink): Promise; } @@ -86,6 +108,8 @@ export async function prepareRemoteManagedRuntime(input: { workspaceRemoteDir?: string; syncWorkspace?: boolean; assets?: RemoteManagedRuntimeAsset[]; + /** Referenced (additional) projects to stage as plain, read-only trees. */ + additionalSources?: SandboxAdditionalSource[]; // Upload progress sink. Threaded for the byte-counting transport rewrite; the // child task wires it into the workspace/asset transfers. onProgress?: RuntimeProgressSink; @@ -148,12 +172,50 @@ export async function prepareRemoteManagedRuntime(input: { throw error; } + // Stage each referenced (additional) project as a plain, read-only tree in its + // OWN isolated remote directory (`project-`). Additional sources + // never get the anchor's git-history/overlay semantics. Per-project failure + // isolation: one project's failure logs a warning and is skipped; the run and + // the other projects continue (no workspace restore, unlike an asset failure). + const additionalSourceDirs: Record = {}; + for (const source of input.additionalSources ?? []) { + const { localPath, projectId } = source; + try { + if (!path.posix.isAbsolute(localPath)) { + throw new Error(`additional source localPath is not an absolute path: ${localPath}`); + } + if ( + projectId.length === 0 || + projectId.includes("/") || + projectId.includes("\\") || + projectId.includes("..") + ) { + throw new Error(`additional source projectId is not a simple path segment: ${projectId}`); + } + const remoteDir = path.posix.join(runtimeRootDir, `project-${projectId}`); + await syncDirectoryToSsh({ + spec: input.spec, + localDir: localPath, + remoteDir, + exclude: REMOTE_ADDITIONAL_SOURCE_HEAVY_DIR_EXCLUDES, + onProgress: input.onProgress, + progressLabel: `project-${projectId}`, + }); + additionalSourceDirs[projectId] = remoteDir; + } catch (error) { + console.warn( + `[paperclip] Failed to stage referenced project ${projectId}; skipping it. ${String(error)}`, + ); + } + } + return { spec: input.spec, workspaceLocalDir: input.workspaceLocalDir, workspaceRemoteDir, runtimeRootDir, assetDirs, + additionalSourceDirs, restoreWorkspace: async (onProgress?: RuntimeProgressSink) => { if (preparedWorkspace && baselineSnapshot) { await restoreWorkspaceFromSshExecution({ diff --git a/packages/adapter-utils/src/sandbox-file-sync.test.ts b/packages/adapter-utils/src/sandbox-file-sync.test.ts index de3071c1c9..aeb642a645 100644 --- a/packages/adapter-utils/src/sandbox-file-sync.test.ts +++ b/packages/adapter-utils/src/sandbox-file-sync.test.ts @@ -231,6 +231,108 @@ describe("sandbox native file sync", () => { expect(await readFile(path.join(prepared.assetDirs.creds, "cred.txt"), "utf8")).toBe("secret\n"); }); + it("stages each additional project into its own isolated dir via a native directory syncIn", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-native-additional-")); + cleanupDirs.push(rootDir); + const localWorkspaceDir = path.join(rootDir, "local-workspace"); + const remoteWorkspaceDir = path.join(rootDir, "remote-workspace"); + await mkdir(localWorkspaceDir, { recursive: true }); + await writeFile(path.join(localWorkspaceDir, "README.md"), "anchor\n", "utf8"); + + // Three referenced projects, each with a distinctive file, staged as plain + // read-only trees. + const projects = [ + { projectId: "alpha", localDir: path.join(rootDir, "src-alpha"), file: "alpha.txt", body: "alpha body\n" }, + { projectId: "bravo", localDir: path.join(rootDir, "src-bravo"), file: "bravo.txt", body: "bravo body\n" }, + { projectId: "charlie", localDir: path.join(rootDir, "src-charlie"), file: "charlie.txt", body: "charlie body\n" }, + ]; + for (const project of projects) { + await mkdir(project.localDir, { recursive: true }); + await writeFile(path.join(project.localDir, project.file), project.body, "utf8"); + } + + const { client, syncInOps } = makeNativeClient(); + const prepared = await prepareSandboxManagedRuntime({ + spec: { transport: "sandbox", provider: "test", sandboxId: "s1", remoteCwd: remoteWorkspaceDir, timeoutMs: 30_000, apiKey: null }, + adapterKey: "test-adapter", + client, + workspaceLocalDir: localWorkspaceDir, + additionalSources: projects.map((project) => ({ localPath: project.localDir, projectId: project.projectId })), + }); + + // Each project lands in its OWN `project-` directory under the + // runtime root, and its file materializes there. + const projectDirs = projects.map((project) => { + const dir = prepared.additionalSourceDirs[project.projectId]; + expect(dir).toBe(path.posix.join(prepared.runtimeRootDir, `project-${project.projectId}`)); + return dir; + }); + for (const [index, project] of projects.entries()) { + expect(await readFile(path.join(projectDirs[index], project.file), "utf8")).toBe(project.body); + } + + // The target dirs are pairwise distinct and never nested inside one another. + for (const outer of projectDirs) { + for (const inner of projectDirs) { + if (outer === inner) continue; + expect(inner.startsWith(`${outer}/`)).toBe(false); + } + } + + // Each project rides its own `syncIn` operation as a single `directory` + // mapping, source = the host checkout dir, target = the isolated project dir. + const inboundOps = syncInOps.flat(); + for (const [index, project] of projects.entries()) { + const op = inboundOps.find((candidate) => + candidate.files.some((mapping) => mapping.targetPath === projectDirs[index]), + ); + expect(op).toBeDefined(); + expect(op!.operationId).toMatch(/^sync-op-\d+$/); + expect(op!.operationId).not.toContain(project.projectId); + expect(op!.files).toHaveLength(1); + expect(op!.files[0]).toMatchObject({ + sourcePath: project.localDir, + targetPath: projectDirs[index], + kind: "directory", + }); + // Plain read-only tree — no post-upload extract/wipe/merge command. + expect(op!.postUploadCommands ?? []).toHaveLength(0); + } + }); + + it("isolates one additional project's sync failure and stages the rest", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-native-additional-fail-")); + cleanupDirs.push(rootDir); + const localWorkspaceDir = path.join(rootDir, "local-workspace"); + const remoteWorkspaceDir = path.join(rootDir, "remote-workspace"); + const goodDir = path.join(rootDir, "src-good"); + await mkdir(localWorkspaceDir, { recursive: true }); + await mkdir(goodDir, { recursive: true }); + await writeFile(path.join(localWorkspaceDir, "README.md"), "anchor\n", "utf8"); + await writeFile(path.join(goodDir, "good.txt"), "good body\n", "utf8"); + + // The middle source points at a directory that does not exist, so its native + // transfer fails. Failure isolation must skip only it and stage the rest. + const { client } = makeNativeClient(); + const prepared = await prepareSandboxManagedRuntime({ + spec: { transport: "sandbox", provider: "test", sandboxId: "s1", remoteCwd: remoteWorkspaceDir, timeoutMs: 30_000, apiKey: null }, + adapterKey: "test-adapter", + client, + workspaceLocalDir: localWorkspaceDir, + additionalSources: [ + { localPath: goodDir, projectId: "good-a" }, + { localPath: path.join(rootDir, "does-not-exist"), projectId: "broken" }, + { localPath: goodDir, projectId: "good-b" }, + ], + }); + + // Both healthy projects staged; the broken one is absent, not fatal. + expect(Object.keys(prepared.additionalSourceDirs).sort()).toEqual(["good-a", "good-b"]); + expect(prepared.additionalSourceDirs.broken).toBeUndefined(); + expect(await readFile(path.join(prepared.additionalSourceDirs["good-a"], "good.txt"), "utf8")).toBe("good body\n"); + expect(await readFile(path.join(prepared.additionalSourceDirs["good-b"], "good.txt"), "utf8")).toBe("good body\n"); + }); + it("dereferences symlinks only when followSymlinks is true (native honors the flag)", async () => { const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-native-symlink-")); cleanupDirs.push(rootDir); diff --git a/packages/adapter-utils/src/sandbox-managed-runtime.test.ts b/packages/adapter-utils/src/sandbox-managed-runtime.test.ts index 8444f05915..fea9df5888 100644 --- a/packages/adapter-utils/src/sandbox-managed-runtime.test.ts +++ b/packages/adapter-utils/src/sandbox-managed-runtime.test.ts @@ -2,7 +2,7 @@ import { promises as fsPromises } from "node:fs"; import { lstat, mkdir, mkdtemp, readFile, readdir, rm, symlink, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { execFile as execFileCallback } from "node:child_process"; +import { execFile as execFileCallback, spawn } from "node:child_process"; import { promisify } from "node:util"; import { afterEach, describe, expect, it, vi } from "vitest"; import { resetLocalGitIndexToHead } from "./git-workspace-sync.js"; @@ -14,6 +14,11 @@ import { type SandboxSyncOperation, type SandboxSyncResult, } from "./sandbox-managed-runtime.js"; +import { + prepareCommandManagedRuntime, + type CommandManagedRuntimeRunner, +} from "./command-managed-runtime.js"; +import type { RunProcessResult } from "./server-utils.js"; function toArrayBuffer(bytes: Buffer): ArrayBuffer { return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer; @@ -1490,4 +1495,97 @@ describe("sandbox managed runtime", () => { expect(coreSource).not.toMatch(/auth\.json/i); expect(coreSource).not.toMatch(/merge-extract|merge-decision/i); }); + + // End-to-end: drive the WHOLE prepare path — `prepareCommandManagedRuntime` + // building the client, syncing the anchor workspace, then staging the + // referenced projects — through a runner that runs real shell commands on the + // host filesystem (host FS stands in for the sandbox FS). The runner exposes no + // native syncIn, so staging rides the base64/tar fallback, the same transport a + // provider without native sync uses. In production the kill-switch + // `PAPERCLIP_MULTI_PROJECT_WORKSPACE_SYNC` gates whether run prep resolves any + // referenced projects (OFF ⇒ none reach this layer). Enable it in-test only to + // model the ON scenario, and prove multi-project isolation plus one-failure + // isolation end-to-end. + it("stages multiple referenced projects into isolated sandbox dirs end-to-end, skipping a failing source", async () => { + const flagKey = "PAPERCLIP_MULTI_PROJECT_WORKSPACE_SYNC"; + const priorFlag = process.env[flagKey]; + process.env[flagKey] = "1"; + try { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-e2e-additional-")); + cleanupDirs.push(rootDir); + + const localWorkspaceDir = path.join(rootDir, "local-workspace"); + const remoteWorkspaceDir = path.join(rootDir, "remote-workspace"); + await mkdir(localWorkspaceDir, { recursive: true }); + await mkdir(remoteWorkspaceDir, { recursive: true }); + await writeFile(path.join(localWorkspaceDir, "README.md"), "anchor content\n", "utf8"); + + // Two real referenced-project checkouts (one with a nested file) plus a + // deliberately-missing source between them. + const first = path.join(rootDir, "referenced-first"); + const second = path.join(rootDir, "referenced-second"); + await mkdir(path.join(first, "docs"), { recursive: true }); + await mkdir(second, { recursive: true }); + await writeFile(path.join(first, "docs", "guide.md"), "first guide\n", "utf8"); + await writeFile(path.join(second, "notes.md"), "second notes\n", "utf8"); + + const runner: CommandManagedRuntimeRunner = { + execute: (input) => + new Promise((resolve) => { + const startedAt = new Date().toISOString(); + const command = + input.command === "sh" ? "/bin/sh" : input.command === "bash" ? "/bin/bash" : input.command; + const child = spawn(command, input.args ?? [], { cwd: input.cwd, env: { ...process.env, ...input.env } }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { stdout += chunk.toString("utf8"); }); + child.stderr.on("data", (chunk) => { stderr += chunk.toString("utf8"); }); + child.on("error", () => resolve({ exitCode: 127, signal: null, timedOut: false, stdout, stderr, pid: null, startedAt })); + child.on("close", (code) => resolve({ exitCode: code ?? 0, signal: null, timedOut: false, stdout, stderr, pid: child.pid ?? null, startedAt })); + if (input.stdin != null) child.stdin.write(input.stdin); + child.stdin.end(); + }), + }; + + const prepared = await prepareCommandManagedRuntime({ + runner, + spec: { remoteCwd: remoteWorkspaceDir, timeoutMs: 30_000 }, + adapterKey: "test-adapter", + workspaceLocalDir: localWorkspaceDir, + additionalSources: [ + { localPath: first, projectId: "proj-first" }, + { localPath: path.join(rootDir, "referenced-missing"), projectId: "proj-missing" }, + { localPath: second, projectId: "proj-second" }, + ], + }); + + const runtimeRootDir = path.posix.join(remoteWorkspaceDir, ".paperclip-runtime", "test-adapter"); + + // The anchor workspace synced normally and stays byte-identical. + await expect(readFile(path.join(remoteWorkspaceDir, "README.md"), "utf8")).resolves.toBe("anchor content\n"); + + // Each healthy referenced project landed in its OWN isolated dir; the + // missing one is skipped, not fatal. + expect(Object.keys(prepared.additionalSourceDirs).sort()).toEqual(["proj-first", "proj-second"]); + expect(prepared.additionalSourceDirs["proj-first"]).toBe(path.posix.join(runtimeRootDir, "project-proj-first")); + expect(prepared.additionalSourceDirs["proj-second"]).toBe(path.posix.join(runtimeRootDir, "project-proj-second")); + expect(prepared.additionalSourceDirs["proj-missing"]).toBeUndefined(); + + await expect(readFile(path.join(prepared.additionalSourceDirs["proj-first"], "docs", "guide.md"), "utf8")) + .resolves.toBe("first guide\n"); + await expect(readFile(path.join(prepared.additionalSourceDirs["proj-second"], "notes.md"), "utf8")) + .resolves.toBe("second notes\n"); + + // Neither project's tree leaked into the anchor workspace or into the other + // project's dir. + await expect(readFile(path.join(remoteWorkspaceDir, "notes.md"), "utf8")).rejects.toMatchObject({ code: "ENOENT" }); + await expect(readFile(path.join(prepared.additionalSourceDirs["proj-first"], "notes.md"), "utf8")).rejects + .toMatchObject({ code: "ENOENT" }); + await expect(readFile(path.join(runtimeRootDir, "project-proj-missing"), "utf8")).rejects + .toMatchObject({ code: "ENOENT" }); + } finally { + if (priorFlag === undefined) delete process.env[flagKey]; + else process.env[flagKey] = priorFlag; + } + }); }); diff --git a/packages/adapter-utils/src/sandbox-managed-runtime.ts b/packages/adapter-utils/src/sandbox-managed-runtime.ts index 50e7d92d3c..f41cd2d531 100644 --- a/packages/adapter-utils/src/sandbox-managed-runtime.ts +++ b/packages/adapter-utils/src/sandbox-managed-runtime.ts @@ -121,6 +121,22 @@ export interface SandboxManagedRuntimeAsset { restore?: (ctx: SandboxManagedRuntimeAssetRestoreContext) => Promise; } +/** + * A referenced (additional) project to stage into the run sandbox as a plain, + * read-only tree. `localPath` is the host checkout directory. Upstream code + * already authorized and realized this directory (`project:read`); this layer + * adds no authorization logic. `projectId` names the isolated remote + * subdirectory (`project-` under the runtime root) the tree lands in. + * + * Additional sources are plain trees only. They never carry the anchor + * workspace's git-history, overlay, or `.paperclip-runtime` preservation + * semantics — those stay anchor-only. + */ +export interface SandboxAdditionalSource { + localPath: string; + projectId: string; +} + /** * Per-call byte-level progress hook. `transferredBytes`/`totalBytes` are decoded * file bytes (not the base64 wire size). `totalBytes` is null when the size is @@ -259,6 +275,13 @@ export interface PreparedSandboxManagedRuntime { workspaceRemoteDir: string; runtimeRootDir: string; assetDirs: Record; + /** + * Remote directory of each additional (referenced) project that staged + * successfully, keyed by `projectId`. A project whose staging failed is + * absent (per-project failure isolation). Empty when no additional sources + * were requested. + */ + additionalSourceDirs: Record; restoreWorkspace(onProgress?: RuntimeProgressSink): Promise; } @@ -637,6 +660,12 @@ export async function prepareSandboxManagedRuntime(input: { workspaceExclude?: string[]; preserveAbsentOnRestore?: string[]; assets?: SandboxManagedRuntimeAsset[]; + /** + * Referenced (additional) projects to stage into the sandbox as plain, + * read-only trees, each in its own isolated `project-` directory. + * Defaults to none, so a legacy/anchor-only call is behavior-identical. + */ + additionalSources?: SandboxAdditionalSource[]; // Upload progress sink. Threaded for the byte-counting transport rewrite; the // child task wires it into writeFile/readFile. onProgress?: RuntimeProgressSink; @@ -686,6 +715,14 @@ export async function prepareSandboxManagedRuntime(input: { // Opaque, ordered, non-sensitive operation tokens — never a caller/asset id. const nextSyncOperationId = () => `sync-op-${++syncOperationSeq}`; + // Remote directory of each additional (referenced) project that stages + // successfully, keyed by projectId. A project that fails to stage is absent. + const additionalSourceDirs: Record = {}; + // Additional projects stage as plain trees. Drop the heavy build/cache dirs a + // reference tree does not need, and `.git` — additional sources never carry + // git-history semantics (anchor-only). + const additionalSourceExclude = mergeExcludes(SANDBOX_WORKSPACE_HEAVY_DIR_EXCLUDES, [".git"]); + // Every delegated post-upload command (extract/wipe/remove-deleted/asset merge) // must run under the run-specific timeout (`spec.timeoutMs`), not the provider // sync client's default timeout — the two can differ, and before staging was @@ -709,10 +746,48 @@ export async function prepareSandboxManagedRuntime(input: { ...(input.preserveAbsentOnRestore ?? []), ]); - // Build one `SandboxSyncOperation` uploading a host tarball as a single file - // mapping (rides native `uploadFiles`, or the base64-tar fallback) with the - // extract/wipe/merge steps carried as ordered `postUploadCommands`, confine - // it, and delegate to `syncIn`. `finish` emits the terminal progress line. + // Stage one source directory into an isolated remote subdirectory through the + // unified `syncIn` seam. Both the workspace/asset tar path and the additional + // (referenced) project path use it: build one `SandboxSyncOperation` from the + // caller's `files` mappings (a host tarball as a single `file` mapping, or a + // whole `directory` mapping), carry any extract/wipe/merge steps as ordered + // `postUploadCommands`, confine the operation's source and target to their + // own roots (fail-closed), and delegate to `syncIn` (native transfer, or the + // base64-tar fallback). `finish` emits the terminal progress line. + const stageConfinedSyncIn = async (params: { + files: SandboxSyncFileMapping[]; + postUploadCommands?: SandboxPostUploadCommand[]; + sourceRoots: string[]; + targetRoots: string[]; + progressLabel: string; + statusPhase: RuntimeStatusPhase; + progressBytes: number; + }): Promise => { + const operations: SandboxSyncOperation[] = [{ + operationId: nextSyncOperationId(), + files: params.files, + ...(params.postUploadCommands + ? { postUploadCommands: withRunTimeout(params.postUploadCommands) } + : {}), + }]; + assertSyncOperationsConfined(operations, { + sourceRoots: params.sourceRoots, + targetRoots: params.targetRoots, + }); + const upload = makeTransferProgress( + input.onProgress, + "Syncing", + "to", + params.progressLabel, + { sink: input.onRuntimeProgress, phase: params.statusPhase }, + ); + await syncIn(operations); + await upload.finish(params.progressBytes, params.progressBytes); + }; + + // Upload a host tarball as a single `file` mapping with the extract/wipe/merge + // steps as ordered post-upload commands. A thin wrapper over + // `stageConfinedSyncIn` for the workspace/git anchor and asset paths. const stageTarball = async (input2: { tarPath: string; remoteTar: string; @@ -721,24 +796,15 @@ export async function prepareSandboxManagedRuntime(input: { statusPhase: RuntimeStatusPhase; }): Promise => { const tarSize = (await fs.stat(input2.tarPath)).size; - const operations: SandboxSyncOperation[] = [{ - operationId: nextSyncOperationId(), + await stageConfinedSyncIn({ files: [{ sourcePath: input2.tarPath, targetPath: input2.remoteTar, kind: "file" }], - postUploadCommands: withRunTimeout(input2.postUploadCommands), - }]; - assertSyncOperationsConfined(operations, { + postUploadCommands: input2.postUploadCommands, sourceRoots: [tempDir], targetRoots: [runtimeRootDir], + progressLabel: input2.progressLabel, + statusPhase: input2.statusPhase, + progressBytes: tarSize, }); - const upload = makeTransferProgress( - input.onProgress, - "Syncing", - "to", - input2.progressLabel, - { sink: input.onRuntimeProgress, phase: input2.statusPhase }, - ); - await syncIn(operations); - await upload.finish(tarSize, tarSize); }; if (syncWorkspace && gitSnapshot) { @@ -860,25 +926,63 @@ export async function prepareSandboxManagedRuntime(input: { assetDir: remoteAssetDir, runtimeRootDir, }) ?? buildDefaultExtractRuntimeAssetCommand({ remoteAssetDir, remoteAssetTar }); - const operations: SandboxSyncOperation[] = [{ - operationId: nextSyncOperationId(), + const assetTarSize = (await fs.stat(assetTarPath)).size; + await stageConfinedSyncIn({ files, - postUploadCommands: withRunTimeout([{ command: postUploadCommand }]), - }]; - assertSyncOperationsConfined(operations, { + postUploadCommands: [{ command: postUploadCommand }], sourceRoots: [tempDir], targetRoots: [runtimeRootDir], + progressLabel: asset.key, + statusPhase: "config_sync", + progressBytes: assetTarSize, }); - const assetTarSize = (await fs.stat(assetTarPath)).size; - const assetUpload = makeTransferProgress( - input.onProgress, - "Syncing", - "to", - asset.key, - { sink: input.onRuntimeProgress, phase: "config_sync" }, - ); - await syncIn(operations); - await assetUpload.finish(assetTarSize, assetTarSize); + } + + // Stage each referenced (additional) project as a plain, read-only tree in + // its OWN isolated remote directory (`project-`). An additional + // project rides one confined `syncIn` directory mapping — a native directory + // transfer, or the base64-tar fallback — with source and target confined to + // their own roots. No workspace, git-history, or `.paperclip-runtime` + // semantics apply; those stay anchor-only. Per-project failure isolation: one + // project's confinement or sync failure logs a warning and is skipped, and + // the run plus the other projects continue. Only a project that stages + // successfully appears in `additionalSourceDirs`. + for (const source of input.additionalSources ?? []) { + const { localPath, projectId } = source; + const label = `project-${projectId}`; + try { + if (!path.posix.isAbsolute(localPath)) { + throw new Error(`additional source localPath is not an absolute path: ${localPath}`); + } + if ( + projectId.length === 0 || + projectId.includes("/") || + projectId.includes("\\") || + projectId.includes("..") + ) { + throw new Error(`additional source projectId is not a simple path segment: ${projectId}`); + } + const remoteProjectDir = path.posix.join(runtimeRootDir, label); + await emitRuntimeStatus(input.onRuntimeProgress, "config_sync", "Syncing referenced project to sandbox"); + await stageConfinedSyncIn({ + files: [{ + sourcePath: localPath, + targetPath: remoteProjectDir, + kind: "directory", + exclude: additionalSourceExclude, + }], + sourceRoots: [localPath], + targetRoots: [remoteProjectDir], + progressLabel: label, + statusPhase: "config_sync", + progressBytes: 0, + }); + additionalSourceDirs[projectId] = remoteProjectDir; + } catch (error) { + console.warn( + `[paperclip] Failed to stage referenced project ${projectId}; skipping it. ${String(error)}`, + ); + } } }); @@ -892,6 +996,7 @@ export async function prepareSandboxManagedRuntime(input: { workspaceRemoteDir, runtimeRootDir, assetDirs, + additionalSourceDirs, restoreWorkspace: async (onProgress?: RuntimeProgressSink) => { const restoreSink = onProgress ?? input.onProgress; if (!syncWorkspace) {