feat(sandbox): stage referenced projects into the run sandbox (#10469)
## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Runs can reference more than one project > - Each referenced project must land in its own sandbox tree so one run does not mix files across projects > - The anchor workspace must keep its own git history and overlay rules > - Fail closed on sync and confinement errors, and keep the other projects alive > - This pull request stages referenced projects into isolated project directories under the run sandbox root > - The benefit is safer multi-project runs with clear failure isolation ## Linked Issues or Issue Description No public issue exists. Related PR: #10448. ## What Changed - Thread additional referenced-project sources through the run prepare path and the runtime layers. - Stage each referenced project into its own `project-<projectId>` directory under the runtime root. - Keep the anchor workspace history and overlay semantics unchanged. - Fail closed on confinement or sync errors for one project, and keep the other projects running. - Keep the path inert by default behind the multi-project workspace-sync kill-switch. - No documentation update was needed for this code-only runtime change. ## Verification - `pnpm --filter @paperclipai/adapter-utils exec vitest run src/sandbox-file-sync.test.ts src/command-managed-runtime.test.ts src/sandbox-managed-runtime.test.ts src/remote-managed-runtime.test.ts` - `pnpm --filter @paperclipai/adapter-utils exec tsc --noEmit` - The branch points at `871378e4064a95adc8e4647e442ec1359768de61` on `origin/feat/stage-referenced-projects-into-sandbox`. - GitHub CI is green on PR #10469. ## Risks - A referenced project can skip if confinement or sync fails. - A new runtime tree layout can affect tools that assume one project root. - The kill-switch keeps the path inert until operators enable it. ## Model Used OpenAI Codex, GPT-5, tool use enabled. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes or confirmed no docs update was needed - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Nicky Leach
GitHub
parent
5c5366d0c1
commit
6a267e0328
|
|
@ -798,6 +798,155 @@ describe("shared ACPX engine runtime behavior", () => {
|
|||
expect(fp(rotatedKey)).not.toBe(fp(withKey));
|
||||
});
|
||||
|
||||
it("busts the session fingerprint when the referenced-project set or pinned checkout changes", async () => {
|
||||
const root = await makeTempRoot();
|
||||
const stateDir = path.join(root, "state");
|
||||
const cwd = path.join(root, "workspace");
|
||||
const baseConfig = { agentCommand: "node ./fake-acp.js", stateDir };
|
||||
|
||||
// A compatible resume reuses the already-staged referenced-project trees, so
|
||||
// the fingerprint must fold in the referenced-project set. Otherwise a resume
|
||||
// reuses a stale tree after the set or a project's pinned checkout changes.
|
||||
const withReferenced = (additional: unknown[]) => ({
|
||||
context: {
|
||||
taskId: "issue-1",
|
||||
wakeReason: "issue_assigned",
|
||||
paperclipWorkspace: { cwd, realization: { additional } },
|
||||
},
|
||||
});
|
||||
const projectA = {
|
||||
path: "/host/project-a",
|
||||
projectId: "a",
|
||||
projectWorkspaceId: "ws-a",
|
||||
repoUrl: "https://example.test/a.git",
|
||||
repoRef: "ref-a-1",
|
||||
};
|
||||
const projectB = {
|
||||
path: "/host/project-b",
|
||||
projectId: "b",
|
||||
projectWorkspaceId: "ws-b",
|
||||
repoUrl: "https://example.test/b.git",
|
||||
repoRef: "ref-b-1",
|
||||
};
|
||||
|
||||
const one = await runExecutor(baseConfig, withReferenced([projectA]));
|
||||
const two = await runExecutor(baseConfig, withReferenced([projectA, projectB]));
|
||||
const reordered = await runExecutor(baseConfig, withReferenced([projectB, projectA]));
|
||||
const repointed = await runExecutor(
|
||||
baseConfig,
|
||||
withReferenced([projectA, { ...projectB, repoRef: "ref-b-2" }]),
|
||||
);
|
||||
|
||||
const fp = (r: { result: { sessionParams?: unknown } }) =>
|
||||
(r.result.sessionParams as { configFingerprint?: string } | undefined)?.configFingerprint;
|
||||
|
||||
// Adding a referenced project changes the set identity, so the fingerprint
|
||||
// busts and the next launch stages the current set.
|
||||
expect(fp(one)).toBeDefined();
|
||||
expect(fp(two)).not.toBe(fp(one));
|
||||
// The identity depends on the set, not the order the records arrive in.
|
||||
expect(fp(reordered)).toBe(fp(two));
|
||||
// Re-pinning one project's checkout ref busts the fingerprint, so the resume
|
||||
// re-stages instead of reusing a stale tree.
|
||||
expect(fp(repointed)).not.toBe(fp(two));
|
||||
});
|
||||
|
||||
it("busts the session fingerprint when referenced-project files change at the same host path", async () => {
|
||||
const root = await makeTempRoot();
|
||||
const stateDir = path.join(root, "state");
|
||||
const cwd = path.join(root, "workspace");
|
||||
const projectDir = path.join(root, "project-a");
|
||||
await fs.mkdir(path.join(projectDir, "src"), { recursive: true });
|
||||
await fs.writeFile(path.join(projectDir, "src", "index.ts"), "export const value = 1;\n");
|
||||
const baseConfig = { agentCommand: "node ./fake-acp.js", stateDir };
|
||||
|
||||
// Same host path and same metadata across runs. Only the file bytes change,
|
||||
// which mirrors a branch that moved or a re-checkout in place. The metadata
|
||||
// identity stays constant, so without the content signature a compatible
|
||||
// resume would reuse a stale staged tree.
|
||||
const referencedProject = {
|
||||
path: projectDir,
|
||||
projectId: "a",
|
||||
projectWorkspaceId: "ws-a",
|
||||
repoUrl: "https://example.test/a.git",
|
||||
repoRef: "main",
|
||||
};
|
||||
const withProject = () => ({
|
||||
context: {
|
||||
taskId: "issue-1",
|
||||
wakeReason: "issue_assigned",
|
||||
paperclipWorkspace: { cwd, realization: { additional: [referencedProject] } },
|
||||
},
|
||||
});
|
||||
const fp = (r: { result: { sessionParams?: unknown } }) =>
|
||||
(r.result.sessionParams as { configFingerprint?: string } | undefined)?.configFingerprint;
|
||||
|
||||
const before = await runExecutor(baseConfig, withProject());
|
||||
// Change the tree content and its size while the host path and metadata stay
|
||||
// identical.
|
||||
await fs.writeFile(path.join(projectDir, "src", "index.ts"), "export const value = 2; // changed\n");
|
||||
const afterEdit = await runExecutor(baseConfig, withProject());
|
||||
// Add a new file to the tree.
|
||||
await fs.writeFile(path.join(projectDir, "src", "extra.ts"), "export const extra = true;\n");
|
||||
const afterAdd = await runExecutor(baseConfig, withProject());
|
||||
|
||||
expect(fp(before)).toBeDefined();
|
||||
// A content-only edit busts the fingerprint, so the resume re-stages.
|
||||
expect(fp(afterEdit)).not.toBe(fp(before));
|
||||
// A new file in the tree busts the fingerprint too.
|
||||
expect(fp(afterAdd)).not.toBe(fp(afterEdit));
|
||||
});
|
||||
|
||||
it("busts the session fingerprint when referenced-project bytes change but size and mtime stay equal", async () => {
|
||||
const root = await makeTempRoot();
|
||||
const stateDir = path.join(root, "state");
|
||||
const cwd = path.join(root, "workspace");
|
||||
const projectDir = path.join(root, "project-a");
|
||||
const filePath = path.join(projectDir, "src", "index.ts");
|
||||
await fs.mkdir(path.join(projectDir, "src"), { recursive: true });
|
||||
// The two contents have the same byte length, so a stat-only signature (size
|
||||
// and modification time) would collide once the modification time also matches.
|
||||
const firstContent = "export const value = 1;\n";
|
||||
const secondContent = "export const value = 2;\n";
|
||||
expect(secondContent.length).toBe(firstContent.length);
|
||||
await fs.writeFile(filePath, firstContent);
|
||||
|
||||
// A re-checkout can restore the same modification time, so pin it explicitly
|
||||
// and reapply it after the edit. Only the file bytes change between runs.
|
||||
const pinnedTime = new Date("2026-01-01T00:00:00.000Z");
|
||||
await fs.utimes(filePath, pinnedTime, pinnedTime);
|
||||
|
||||
const referencedProject = {
|
||||
path: projectDir,
|
||||
projectId: "a",
|
||||
projectWorkspaceId: "ws-a",
|
||||
repoUrl: "https://example.test/a.git",
|
||||
repoRef: "main",
|
||||
};
|
||||
const withProject = () => ({
|
||||
context: {
|
||||
taskId: "issue-1",
|
||||
wakeReason: "issue_assigned",
|
||||
paperclipWorkspace: { cwd, realization: { additional: [referencedProject] } },
|
||||
},
|
||||
});
|
||||
const fp = (r: { result: { sessionParams?: unknown } }) =>
|
||||
(r.result.sessionParams as { configFingerprint?: string } | undefined)?.configFingerprint;
|
||||
|
||||
const baseConfig = { agentCommand: "node ./fake-acp.js", stateDir };
|
||||
const before = await runExecutor(baseConfig, withProject());
|
||||
// Replace the bytes with an equal-length string and restore the same
|
||||
// modification time, so path, size, and mtime all stay identical across the two runs.
|
||||
await fs.writeFile(filePath, secondContent);
|
||||
await fs.utimes(filePath, pinnedTime, pinnedTime);
|
||||
const after = await runExecutor(baseConfig, withProject());
|
||||
|
||||
expect(fp(before)).toBeDefined();
|
||||
// The content signature reads bytes, so an equal-size, equal-mtime edit busts
|
||||
// the fingerprint and the resume re-stages instead of reusing a stale tree.
|
||||
expect(fp(after)).not.toBe(fp(before));
|
||||
});
|
||||
|
||||
it("shapes ACPX session env for remote execution identities", async () => {
|
||||
const root = await makeTempRoot();
|
||||
const localCwd = path.join(root, "local");
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ import {
|
|||
type AdapterExecutionTargetTimeoutResolution,
|
||||
type AdapterManagedRuntimeAsset,
|
||||
type PreparedAdapterExecutionTargetRuntime,
|
||||
type SandboxAdditionalSource,
|
||||
} from "@paperclipai/adapter-utils/execution-target";
|
||||
import {
|
||||
DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE,
|
||||
|
|
@ -418,6 +419,83 @@ function shortHash(value: unknown): string {
|
|||
return createHash("sha256").update(stableJson(value)).digest("hex").slice(0, 16);
|
||||
}
|
||||
|
||||
// Directory names the staging path never ships for a referenced project (heavy
|
||||
// build/cache output and git history). The content signature skips them so it
|
||||
// reflects only the staged tree and never reads their bytes. Keep this set equal
|
||||
// to the staging excludes in the sandbox and remote runtimes.
|
||||
const REFERENCED_SOURCE_SIGNATURE_SKIP_DIRS = new Set([
|
||||
"node_modules",
|
||||
"vendor",
|
||||
"dist",
|
||||
"build",
|
||||
"out",
|
||||
"coverage",
|
||||
".next",
|
||||
".turbo",
|
||||
".cache",
|
||||
".git",
|
||||
]);
|
||||
|
||||
/**
|
||||
* Content signature of a referenced-project host tree for the session fingerprint.
|
||||
*
|
||||
* The staged-runtime cache reuses an already-staged referenced-project tree on a
|
||||
* compatible resume and does not re-sync it. Referenced-project metadata (id, host
|
||||
* path, workspace id, repo url, pinned ref) can stay identical while the files at
|
||||
* that host path change: a branch moved to a new commit, a re-checkout in place, or
|
||||
* a dirty worktree. So the metadata identity alone lets a resume serve a stale tree.
|
||||
* This signature folds the tree's own content state into the identity.
|
||||
*
|
||||
* The walk reads each file's relative path and bytes and folds them into the hash.
|
||||
* It reads bytes, not only file stats. A stat-only signature (size and modification
|
||||
* time) collides when an edit keeps the byte length and the modification time — a
|
||||
* re-checkout that restores the same size and timestamp. The byte hash busts on any
|
||||
* content change, so the fingerprint busts and the next launch stages the current
|
||||
* tree. The walk skips the heavy build, cache, and git directories the staging path
|
||||
* never ships, and records a symlink by its target text without following it. On a
|
||||
* read error the function returns a stable marker, so the fingerprint does not churn
|
||||
* while staging surfaces the real error. The walk runs only when the run carries
|
||||
* referenced projects (the multi-project sync path).
|
||||
*/
|
||||
async function referencedSourceContentSignature(localPath: string): Promise<string> {
|
||||
const hash = createHash("sha256");
|
||||
const walk = async (relative: string): Promise<void> => {
|
||||
const current = relative ? path.join(localPath, relative) : localPath;
|
||||
const dirents = await fs.readdir(current, { withFileTypes: true });
|
||||
dirents.sort((left, right) => (left.name < right.name ? -1 : left.name > right.name ? 1 : 0));
|
||||
for (const dirent of dirents) {
|
||||
const next = relative ? path.posix.join(relative, dirent.name) : dirent.name;
|
||||
if (dirent.isDirectory()) {
|
||||
if (REFERENCED_SOURCE_SIGNATURE_SKIP_DIRS.has(dirent.name)) {
|
||||
continue;
|
||||
}
|
||||
await walk(next);
|
||||
continue;
|
||||
}
|
||||
const absolute = path.join(localPath, next);
|
||||
const stats = await fs.lstat(absolute);
|
||||
if (stats.isSymbolicLink()) {
|
||||
const target = await fs.readlink(absolute);
|
||||
hash.update(`symlink:${next}:${target}\n`);
|
||||
continue;
|
||||
}
|
||||
if (!stats.isFile()) {
|
||||
hash.update(`other:${next}:${stats.mode}\n`);
|
||||
continue;
|
||||
}
|
||||
hash.update(`file:${next}:${stats.size}\n`);
|
||||
hash.update(await fs.readFile(absolute));
|
||||
hash.update("\n");
|
||||
}
|
||||
};
|
||||
try {
|
||||
await walk("");
|
||||
} catch (error) {
|
||||
return `unreadable:${String(error)}`;
|
||||
}
|
||||
return hash.digest("hex").slice(0, 16);
|
||||
}
|
||||
|
||||
function defaultPaperclipInstanceDir(): string {
|
||||
const home = process.env.PAPERCLIP_HOME?.trim() || path.join(os.homedir(), ".paperclip");
|
||||
const instanceId = process.env.PAPERCLIP_INSTANCE_ID?.trim() || "default";
|
||||
|
|
@ -1209,6 +1287,11 @@ async function stageAcpRemoteRuntime(input: {
|
|||
workspaceRemoteDir?: string;
|
||||
timeoutSec: number;
|
||||
assets?: AdapterManagedRuntimeAsset[];
|
||||
// Referenced (additional) projects to stage into the sandbox as plain,
|
||||
// read-only trees alongside the anchor workspace. Empty unless run prep
|
||||
// resolved referenced projects (gated upstream), so the anchor-only path is
|
||||
// unchanged.
|
||||
additionalSources?: SandboxAdditionalSource[];
|
||||
onLog: AdapterExecutionContext["onLog"];
|
||||
onRuntimeProgress: AdapterExecutionContext["onRuntimeProgress"];
|
||||
}): Promise<PreparedAdapterExecutionTargetRuntime> {
|
||||
|
|
@ -1224,6 +1307,9 @@ async function stageAcpRemoteRuntime(input: {
|
|||
workspaceLocalDir: input.workspaceLocalDir,
|
||||
...(input.workspaceRemoteDir ? { workspaceRemoteDir: input.workspaceRemoteDir } : {}),
|
||||
...(input.assets && input.assets.length > 0 ? { assets: input.assets } : {}),
|
||||
...(input.additionalSources && input.additionalSources.length > 0
|
||||
? { additionalSources: input.additionalSources }
|
||||
: {}),
|
||||
onProgress: (line) => input.onLog("stdout", line),
|
||||
onRuntimeProgress: input.onRuntimeProgress,
|
||||
});
|
||||
|
|
@ -1271,6 +1357,43 @@ async function buildRuntime(input: {
|
|||
const useConfiguredInsteadOfAgentHome = workspaceSource === "agent_home" && configuredCwd.length > 0;
|
||||
const effectiveWorkspaceCwd = useConfiguredInsteadOfAgentHome ? "" : workspaceCwd;
|
||||
const cwd = effectiveWorkspaceCwd || configuredCwd || process.cwd();
|
||||
// Referenced (additional) projects to stage into the sandbox alongside the
|
||||
// anchor workspace, read from the workspace realization record. The list is
|
||||
// empty unless run prep resolved referenced projects — gated upstream by the
|
||||
// multi-project workspace-sync kill-switch — so the anchor-only path is
|
||||
// unchanged.
|
||||
const realizationContext = parseObject(workspaceContext.realization);
|
||||
const additionalSourceRecords = (
|
||||
Array.isArray(realizationContext.additional) ? realizationContext.additional : []
|
||||
).map((entry) => parseObject(entry));
|
||||
const additionalSources: SandboxAdditionalSource[] = additionalSourceRecords
|
||||
.map((entry) => ({ localPath: asString(entry.path, ""), projectId: asString(entry.projectId, "") }))
|
||||
.filter((entry) => entry.localPath.length > 0 && entry.projectId.length > 0);
|
||||
// Stable identity of the referenced-project set for the session fingerprint.
|
||||
// The staged-runtime cache reuses already-staged referenced-project trees on a
|
||||
// compatible resume, so the fingerprint must change when the set OR a project's
|
||||
// pinned checkout changes. Without this, a resume reuses a stale staged tree.
|
||||
// Fold in each project's id, host path, workspace id, and pinned ref; sort by
|
||||
// projectId so the identity depends on the set, not the record order.
|
||||
const additionalSourcesIdentityBase = additionalSourceRecords
|
||||
.map((entry) => ({
|
||||
projectId: asString(entry.projectId, ""),
|
||||
localPath: asString(entry.path, ""),
|
||||
projectWorkspaceId: asString(entry.projectWorkspaceId, ""),
|
||||
repoUrl: asString(entry.repoUrl, ""),
|
||||
repoRef: asString(entry.repoRef, ""),
|
||||
}))
|
||||
.filter((entry) => entry.localPath.length > 0 && entry.projectId.length > 0)
|
||||
.sort((a, b) => (a.projectId < b.projectId ? -1 : a.projectId > b.projectId ? 1 : 0));
|
||||
// Metadata alone does not change on a content-only checkout change (same host
|
||||
// path and pinned ref, new file bytes). Fold in each tree's content signature so
|
||||
// a file add, remove, or edit busts the fingerprint and the resume re-stages.
|
||||
const additionalSourcesIdentity = await Promise.all(
|
||||
additionalSourcesIdentityBase.map(async (entry) => ({
|
||||
...entry,
|
||||
contentSignature: await referencedSourceContentSignature(entry.localPath),
|
||||
})),
|
||||
);
|
||||
const executionTarget = readAdapterExecutionTarget({
|
||||
executionTarget: input.ctx.executionTarget,
|
||||
legacyRemoteExecution: input.ctx.executionTransport?.remoteExecution,
|
||||
|
|
@ -1553,6 +1676,11 @@ async function buildRuntime(input: {
|
|||
requestedThinkingEffort,
|
||||
fastMode,
|
||||
remoteExecutionIdentity,
|
||||
// Referenced-project set + pinned-checkout identity. A change here (a project
|
||||
// added, removed, or re-pinned) invalidates a warm/resumable session so the
|
||||
// next launch stages the current referenced-project trees instead of reusing
|
||||
// a stale staged tree.
|
||||
additionalSourcesIdentity,
|
||||
skillsIdentity,
|
||||
skillPromptInstructions,
|
||||
paperclipClaudeSettings: paperclipClaudeSettings
|
||||
|
|
@ -1681,6 +1809,7 @@ async function buildRuntime(input: {
|
|||
workspaceRemoteDir: sessionCwd,
|
||||
timeoutSec,
|
||||
assets,
|
||||
additionalSources,
|
||||
onLog: input.ctx.onLog,
|
||||
onRuntimeProgress: input.ctx.onRuntimeProgress,
|
||||
});
|
||||
|
|
|
|||
|
|
@ -303,6 +303,59 @@ describe("command managed runtime", () => {
|
|||
);
|
||||
});
|
||||
|
||||
it("stages each additional project into an isolated dir on the base64/tar transport, one failure skipped", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-command-runtime-additional-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
|
||||
const localWorkspaceDir = path.join(rootDir, "local-workspace");
|
||||
const remoteWorkspaceDir = path.join(rootDir, "remote-workspace");
|
||||
await mkdir(localWorkspaceDir, { recursive: true });
|
||||
await mkdir(remoteWorkspaceDir, { recursive: true });
|
||||
await writeFile(path.join(localWorkspaceDir, "README.md"), "anchor\n", "utf8");
|
||||
|
||||
const goodOne = path.join(rootDir, "src-one");
|
||||
const goodTwo = path.join(rootDir, "src-two");
|
||||
await mkdir(goodOne, { recursive: true });
|
||||
await mkdir(path.join(goodTwo, "nested"), { recursive: true });
|
||||
await writeFile(path.join(goodOne, "one.txt"), "one body\n", "utf8");
|
||||
await writeFile(path.join(goodTwo, "nested", "two.txt"), "two body\n", "utf8");
|
||||
|
||||
// The `makeSpawnRunner` runner exposes no native syncIn, so staging rides the
|
||||
// base64/tar fallback. The middle source points at a missing directory, so
|
||||
// its tar build fails; failure isolation skips only it.
|
||||
const { runner } = makeSpawnRunner();
|
||||
const prepared = await prepareCommandManagedRuntime({
|
||||
runner,
|
||||
spec: {
|
||||
remoteCwd: remoteWorkspaceDir,
|
||||
timeoutMs: 30_000,
|
||||
},
|
||||
adapterKey: "claude",
|
||||
workspaceLocalDir: localWorkspaceDir,
|
||||
additionalSources: [
|
||||
{ localPath: goodOne, projectId: "one" },
|
||||
{ localPath: path.join(rootDir, "missing"), projectId: "broken" },
|
||||
{ localPath: goodTwo, projectId: "two" },
|
||||
],
|
||||
});
|
||||
|
||||
const runtimeRootDir = path.posix.join(remoteWorkspaceDir, ".paperclip-runtime", "claude");
|
||||
expect(Object.keys(prepared.additionalSourceDirs).sort()).toEqual(["one", "two"]);
|
||||
expect(prepared.additionalSourceDirs.one).toBe(path.posix.join(runtimeRootDir, "project-one"));
|
||||
expect(prepared.additionalSourceDirs.two).toBe(path.posix.join(runtimeRootDir, "project-two"));
|
||||
expect(prepared.additionalSourceDirs.broken).toBeUndefined();
|
||||
|
||||
// Each healthy project's tree materialized in its OWN dir (nested files kept).
|
||||
await expect(readFile(path.join(prepared.additionalSourceDirs.one, "one.txt"), "utf8")).resolves.toBe("one body\n");
|
||||
await expect(readFile(path.join(prepared.additionalSourceDirs.two, "nested", "two.txt"), "utf8")).resolves.toBe(
|
||||
"two body\n",
|
||||
);
|
||||
// The broken project's dir was never created.
|
||||
await expect(readFile(path.join(runtimeRootDir, "project-broken"), "utf8")).rejects.toMatchObject({
|
||||
code: "ENOENT",
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps adapter detection on the profile-backed shell path", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-command-runtime-detect-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import {
|
|||
createTarballFromDirectory,
|
||||
prepareSandboxManagedRuntime,
|
||||
type PreparedSandboxManagedRuntime,
|
||||
type SandboxAdditionalSource,
|
||||
type SandboxManagedRuntimeAsset,
|
||||
type SandboxManagedRuntimeClient,
|
||||
type SandboxRemoteExecutionSpec,
|
||||
|
|
@ -467,6 +468,8 @@ export async function prepareCommandManagedRuntime(input: {
|
|||
workspaceExclude?: string[];
|
||||
preserveAbsentOnRestore?: string[];
|
||||
assets?: CommandManagedRuntimeAsset[];
|
||||
/** Referenced (additional) projects to stage into the sandbox as plain, read-only trees. */
|
||||
additionalSources?: SandboxAdditionalSource[];
|
||||
installCommand?: string | null;
|
||||
/** When provided alongside `installCommand`, skip the install if `command -v <detectCommand>` succeeds. */
|
||||
detectCommand?: string | null;
|
||||
|
|
@ -522,6 +525,7 @@ export async function prepareCommandManagedRuntime(input: {
|
|||
workspaceExclude: mergeRuntimeExcludes(input.workspaceExclude),
|
||||
preserveAbsentOnRestore: input.preserveAbsentOnRestore,
|
||||
assets: input.assets,
|
||||
additionalSources: input.additionalSources,
|
||||
onProgress: input.onProgress,
|
||||
onRuntimeProgress: input.onRuntimeProgress,
|
||||
});
|
||||
|
|
@ -559,6 +563,7 @@ export async function prepareCommandManagedRuntime(input: {
|
|||
workspaceExclude: mergeRuntimeExcludes(input.workspaceExclude),
|
||||
preserveAbsentOnRestore: input.preserveAbsentOnRestore,
|
||||
assets: input.assets,
|
||||
additionalSources: input.additionalSources,
|
||||
onProgress: input.onProgress,
|
||||
onRuntimeProgress: input.onRuntimeProgress,
|
||||
});
|
||||
|
|
|
|||
|
|
@ -14,6 +14,8 @@ import {
|
|||
prepareRemoteManagedRuntime,
|
||||
remoteExecutionSessionMatches,
|
||||
} from "./remote-managed-runtime.js";
|
||||
import type { SandboxAdditionalSource } from "./sandbox-managed-runtime.js";
|
||||
export type { SandboxAdditionalSource } from "./sandbox-managed-runtime.js";
|
||||
import {
|
||||
createCommandManagedSandboxCallbackBridgeQueueClient,
|
||||
createSandboxCallbackBridgeAsset,
|
||||
|
|
@ -114,6 +116,12 @@ export interface PreparedAdapterExecutionTargetRuntime {
|
|||
workspaceRemoteDir: string | null;
|
||||
runtimeRootDir: string | null;
|
||||
assetDirs: Record<string, string>;
|
||||
/**
|
||||
* Remote directory of each additional (referenced) project that staged
|
||||
* successfully, keyed by `projectId`. Empty for a local target or when no
|
||||
* additional sources were requested.
|
||||
*/
|
||||
additionalSourceDirs: Record<string, string>;
|
||||
restoreWorkspace(onProgress?: RuntimeProgressSink): Promise<void>;
|
||||
}
|
||||
|
||||
|
|
@ -1107,6 +1115,8 @@ export async function prepareAdapterExecutionTargetRuntime(input: {
|
|||
workspaceExclude?: string[];
|
||||
preserveAbsentOnRestore?: string[];
|
||||
assets?: AdapterManagedRuntimeAsset[];
|
||||
/** Referenced (additional) projects to stage into the sandbox as plain, read-only trees. */
|
||||
additionalSources?: SandboxAdditionalSource[];
|
||||
installCommand?: string | null;
|
||||
/** When provided alongside `installCommand`, skip the install if the binary is already on PATH. */
|
||||
detectCommand?: string | null;
|
||||
|
|
@ -1124,6 +1134,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: {
|
|||
workspaceRemoteDir: null,
|
||||
runtimeRootDir: null,
|
||||
assetDirs: {},
|
||||
additionalSourceDirs: {},
|
||||
restoreWorkspace: async () => {},
|
||||
};
|
||||
}
|
||||
|
|
@ -1137,6 +1148,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: {
|
|||
workspaceRemoteDir: input.workspaceRemoteDir,
|
||||
syncWorkspace: input.syncWorkspace,
|
||||
assets: input.assets,
|
||||
additionalSources: input.additionalSources,
|
||||
onProgress: input.onProgress,
|
||||
});
|
||||
return {
|
||||
|
|
@ -1144,6 +1156,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: {
|
|||
workspaceRemoteDir: prepared.workspaceRemoteDir,
|
||||
runtimeRootDir: prepared.runtimeRootDir,
|
||||
assetDirs: prepared.assetDirs,
|
||||
additionalSourceDirs: prepared.additionalSourceDirs,
|
||||
restoreWorkspace: prepared.restoreWorkspace,
|
||||
};
|
||||
}
|
||||
|
|
@ -1167,6 +1180,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: {
|
|||
workspaceExclude: input.workspaceExclude,
|
||||
preserveAbsentOnRestore: input.preserveAbsentOnRestore,
|
||||
assets: input.assets,
|
||||
additionalSources: input.additionalSources,
|
||||
installCommand: input.installCommand,
|
||||
detectCommand: input.detectCommand,
|
||||
onProgress: input.onProgress,
|
||||
|
|
@ -1177,6 +1191,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: {
|
|||
workspaceRemoteDir: prepared.workspaceRemoteDir,
|
||||
runtimeRootDir: prepared.runtimeRootDir,
|
||||
assetDirs: prepared.assetDirs,
|
||||
additionalSourceDirs: prepared.additionalSourceDirs,
|
||||
restoreWorkspace: prepared.restoreWorkspace,
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ const {
|
|||
stdout: Buffer.from('{"token":"remote"}\n').toString("base64"),
|
||||
stderr: "",
|
||||
})),
|
||||
syncDirectoryToSsh: vi.fn(async () => undefined),
|
||||
syncDirectoryToSsh: vi.fn(async (_input: { localDir: string }) => undefined),
|
||||
}));
|
||||
|
||||
vi.mock("./ssh.js", () => ({
|
||||
|
|
@ -92,4 +92,95 @@ describe("remote managed runtime", () => {
|
|||
);
|
||||
expect(restoredAuth).toBe('{"token":"remote"}\n');
|
||||
});
|
||||
|
||||
it("stages each additional project into its own isolated SSH dir, isolating one failure", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-remote-runtime-additional-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const workspaceDir = path.join(rootDir, "workspace");
|
||||
const firstDir = path.join(rootDir, "referenced-first");
|
||||
const secondDir = path.join(rootDir, "referenced-second");
|
||||
const brokenDir = path.join(rootDir, "referenced-broken");
|
||||
await mkdir(workspaceDir, { recursive: true });
|
||||
|
||||
// The transfer rejects only for the broken project's directory.
|
||||
syncDirectoryToSsh.mockImplementation(async (input: { localDir: string }) => {
|
||||
if (input.localDir === brokenDir) throw new Error("ssh transfer failed");
|
||||
return undefined;
|
||||
});
|
||||
|
||||
const prepared = await prepareRemoteManagedRuntime({
|
||||
spec: {
|
||||
host: "127.0.0.1",
|
||||
port: 2222,
|
||||
username: "fixture",
|
||||
remoteWorkspacePath: "/app",
|
||||
remoteCwd: "/app",
|
||||
privateKey: "PRIVATE KEY",
|
||||
knownHosts: "KNOWN HOSTS",
|
||||
strictHostKeyChecking: true,
|
||||
},
|
||||
runId: "run-additional",
|
||||
adapterKey: "codex",
|
||||
workspaceLocalDir: workspaceDir,
|
||||
workspaceRemoteDir: "/app",
|
||||
syncWorkspace: false,
|
||||
additionalSources: [
|
||||
{ localPath: firstDir, projectId: "first" },
|
||||
{ localPath: brokenDir, projectId: "broken" },
|
||||
{ localPath: secondDir, projectId: "second" },
|
||||
],
|
||||
});
|
||||
|
||||
// Each healthy project staged into its OWN isolated dir under the runtime
|
||||
// root; the broken one is skipped, not fatal.
|
||||
expect(Object.keys(prepared.additionalSourceDirs).sort()).toEqual(["first", "second"]);
|
||||
expect(prepared.additionalSourceDirs.first).toBe("/app/.paperclip-runtime/codex/project-first");
|
||||
expect(prepared.additionalSourceDirs.second).toBe("/app/.paperclip-runtime/codex/project-second");
|
||||
expect(prepared.additionalSourceDirs.broken).toBeUndefined();
|
||||
expect(syncDirectoryToSsh).toHaveBeenCalledWith(expect.objectContaining({
|
||||
localDir: firstDir,
|
||||
remoteDir: "/app/.paperclip-runtime/codex/project-first",
|
||||
}));
|
||||
expect(syncDirectoryToSsh).toHaveBeenCalledWith(expect.objectContaining({
|
||||
localDir: secondDir,
|
||||
remoteDir: "/app/.paperclip-runtime/codex/project-second",
|
||||
}));
|
||||
});
|
||||
|
||||
it("skips an additional project whose localPath is not absolute", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-remote-runtime-relative-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const workspaceDir = path.join(rootDir, "workspace");
|
||||
const healthyDir = path.join(rootDir, "referenced-healthy");
|
||||
await mkdir(workspaceDir, { recursive: true });
|
||||
|
||||
const prepared = await prepareRemoteManagedRuntime({
|
||||
spec: {
|
||||
host: "127.0.0.1",
|
||||
port: 2222,
|
||||
username: "fixture",
|
||||
remoteWorkspacePath: "/app",
|
||||
remoteCwd: "/app",
|
||||
privateKey: "PRIVATE KEY",
|
||||
knownHosts: "KNOWN HOSTS",
|
||||
strictHostKeyChecking: true,
|
||||
},
|
||||
runId: "run-relative",
|
||||
adapterKey: "codex",
|
||||
workspaceLocalDir: workspaceDir,
|
||||
workspaceRemoteDir: "/app",
|
||||
syncWorkspace: false,
|
||||
additionalSources: [
|
||||
{ localPath: "relative/referenced", projectId: "relative" },
|
||||
{ localPath: healthyDir, projectId: "healthy" },
|
||||
],
|
||||
});
|
||||
|
||||
// The relative-path project never reaches the transfer and is skipped; the
|
||||
// absolute-path project still stages.
|
||||
expect(Object.keys(prepared.additionalSourceDirs)).toEqual(["healthy"]);
|
||||
expect(syncDirectoryToSsh).not.toHaveBeenCalledWith(expect.objectContaining({
|
||||
localDir: "relative/referenced",
|
||||
}));
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -7,10 +7,26 @@ import {
|
|||
restoreWorkspaceFromSshExecution,
|
||||
syncDirectoryToSsh,
|
||||
} from "./ssh.js";
|
||||
import type { SandboxManagedRuntimeAssetRestoreContext } from "./sandbox-managed-runtime.js";
|
||||
import type {
|
||||
SandboxAdditionalSource,
|
||||
SandboxManagedRuntimeAssetRestoreContext,
|
||||
} from "./sandbox-managed-runtime.js";
|
||||
import { captureDirectorySnapshot } from "./workspace-restore-merge.js";
|
||||
import type { RuntimeProgressSink } from "./runtime-progress.js";
|
||||
|
||||
const REMOTE_ADDITIONAL_SOURCE_HEAVY_DIR_EXCLUDES = [
|
||||
"node_modules",
|
||||
"vendor",
|
||||
"dist",
|
||||
"build",
|
||||
"out",
|
||||
"coverage",
|
||||
".next",
|
||||
".turbo",
|
||||
".cache",
|
||||
".git",
|
||||
].flatMap((entry) => [entry, `${entry}/*`, `*/${entry}`, `*/${entry}/*`]);
|
||||
|
||||
export interface RemoteManagedRuntimeAsset {
|
||||
key: string;
|
||||
localDir: string;
|
||||
|
|
@ -25,6 +41,12 @@ export interface PreparedRemoteManagedRuntime {
|
|||
workspaceRemoteDir: string;
|
||||
runtimeRootDir: string;
|
||||
assetDirs: Record<string, string>;
|
||||
/**
|
||||
* Remote directory of each additional (referenced) project that staged
|
||||
* successfully, keyed by `projectId`. A project whose staging failed is
|
||||
* absent (per-project failure isolation).
|
||||
*/
|
||||
additionalSourceDirs: Record<string, string>;
|
||||
restoreWorkspace(onProgress?: RuntimeProgressSink): Promise<void>;
|
||||
}
|
||||
|
||||
|
|
@ -86,6 +108,8 @@ export async function prepareRemoteManagedRuntime(input: {
|
|||
workspaceRemoteDir?: string;
|
||||
syncWorkspace?: boolean;
|
||||
assets?: RemoteManagedRuntimeAsset[];
|
||||
/** Referenced (additional) projects to stage as plain, read-only trees. */
|
||||
additionalSources?: SandboxAdditionalSource[];
|
||||
// Upload progress sink. Threaded for the byte-counting transport rewrite; the
|
||||
// child task wires it into the workspace/asset transfers.
|
||||
onProgress?: RuntimeProgressSink;
|
||||
|
|
@ -148,12 +172,50 @@ export async function prepareRemoteManagedRuntime(input: {
|
|||
throw error;
|
||||
}
|
||||
|
||||
// Stage each referenced (additional) project as a plain, read-only tree in its
|
||||
// OWN isolated remote directory (`project-<projectId>`). Additional sources
|
||||
// never get the anchor's git-history/overlay semantics. Per-project failure
|
||||
// isolation: one project's failure logs a warning and is skipped; the run and
|
||||
// the other projects continue (no workspace restore, unlike an asset failure).
|
||||
const additionalSourceDirs: Record<string, string> = {};
|
||||
for (const source of input.additionalSources ?? []) {
|
||||
const { localPath, projectId } = source;
|
||||
try {
|
||||
if (!path.posix.isAbsolute(localPath)) {
|
||||
throw new Error(`additional source localPath is not an absolute path: ${localPath}`);
|
||||
}
|
||||
if (
|
||||
projectId.length === 0 ||
|
||||
projectId.includes("/") ||
|
||||
projectId.includes("\\") ||
|
||||
projectId.includes("..")
|
||||
) {
|
||||
throw new Error(`additional source projectId is not a simple path segment: ${projectId}`);
|
||||
}
|
||||
const remoteDir = path.posix.join(runtimeRootDir, `project-${projectId}`);
|
||||
await syncDirectoryToSsh({
|
||||
spec: input.spec,
|
||||
localDir: localPath,
|
||||
remoteDir,
|
||||
exclude: REMOTE_ADDITIONAL_SOURCE_HEAVY_DIR_EXCLUDES,
|
||||
onProgress: input.onProgress,
|
||||
progressLabel: `project-${projectId}`,
|
||||
});
|
||||
additionalSourceDirs[projectId] = remoteDir;
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
`[paperclip] Failed to stage referenced project ${projectId}; skipping it. ${String(error)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
spec: input.spec,
|
||||
workspaceLocalDir: input.workspaceLocalDir,
|
||||
workspaceRemoteDir,
|
||||
runtimeRootDir,
|
||||
assetDirs,
|
||||
additionalSourceDirs,
|
||||
restoreWorkspace: async (onProgress?: RuntimeProgressSink) => {
|
||||
if (preparedWorkspace && baselineSnapshot) {
|
||||
await restoreWorkspaceFromSshExecution({
|
||||
|
|
|
|||
|
|
@ -231,6 +231,108 @@ describe("sandbox native file sync", () => {
|
|||
expect(await readFile(path.join(prepared.assetDirs.creds, "cred.txt"), "utf8")).toBe("secret\n");
|
||||
});
|
||||
|
||||
it("stages each additional project into its own isolated dir via a native directory syncIn", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-native-additional-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const localWorkspaceDir = path.join(rootDir, "local-workspace");
|
||||
const remoteWorkspaceDir = path.join(rootDir, "remote-workspace");
|
||||
await mkdir(localWorkspaceDir, { recursive: true });
|
||||
await writeFile(path.join(localWorkspaceDir, "README.md"), "anchor\n", "utf8");
|
||||
|
||||
// Three referenced projects, each with a distinctive file, staged as plain
|
||||
// read-only trees.
|
||||
const projects = [
|
||||
{ projectId: "alpha", localDir: path.join(rootDir, "src-alpha"), file: "alpha.txt", body: "alpha body\n" },
|
||||
{ projectId: "bravo", localDir: path.join(rootDir, "src-bravo"), file: "bravo.txt", body: "bravo body\n" },
|
||||
{ projectId: "charlie", localDir: path.join(rootDir, "src-charlie"), file: "charlie.txt", body: "charlie body\n" },
|
||||
];
|
||||
for (const project of projects) {
|
||||
await mkdir(project.localDir, { recursive: true });
|
||||
await writeFile(path.join(project.localDir, project.file), project.body, "utf8");
|
||||
}
|
||||
|
||||
const { client, syncInOps } = makeNativeClient();
|
||||
const prepared = await prepareSandboxManagedRuntime({
|
||||
spec: { transport: "sandbox", provider: "test", sandboxId: "s1", remoteCwd: remoteWorkspaceDir, timeoutMs: 30_000, apiKey: null },
|
||||
adapterKey: "test-adapter",
|
||||
client,
|
||||
workspaceLocalDir: localWorkspaceDir,
|
||||
additionalSources: projects.map((project) => ({ localPath: project.localDir, projectId: project.projectId })),
|
||||
});
|
||||
|
||||
// Each project lands in its OWN `project-<projectId>` directory under the
|
||||
// runtime root, and its file materializes there.
|
||||
const projectDirs = projects.map((project) => {
|
||||
const dir = prepared.additionalSourceDirs[project.projectId];
|
||||
expect(dir).toBe(path.posix.join(prepared.runtimeRootDir, `project-${project.projectId}`));
|
||||
return dir;
|
||||
});
|
||||
for (const [index, project] of projects.entries()) {
|
||||
expect(await readFile(path.join(projectDirs[index], project.file), "utf8")).toBe(project.body);
|
||||
}
|
||||
|
||||
// The target dirs are pairwise distinct and never nested inside one another.
|
||||
for (const outer of projectDirs) {
|
||||
for (const inner of projectDirs) {
|
||||
if (outer === inner) continue;
|
||||
expect(inner.startsWith(`${outer}/`)).toBe(false);
|
||||
}
|
||||
}
|
||||
|
||||
// Each project rides its own `syncIn` operation as a single `directory`
|
||||
// mapping, source = the host checkout dir, target = the isolated project dir.
|
||||
const inboundOps = syncInOps.flat();
|
||||
for (const [index, project] of projects.entries()) {
|
||||
const op = inboundOps.find((candidate) =>
|
||||
candidate.files.some((mapping) => mapping.targetPath === projectDirs[index]),
|
||||
);
|
||||
expect(op).toBeDefined();
|
||||
expect(op!.operationId).toMatch(/^sync-op-\d+$/);
|
||||
expect(op!.operationId).not.toContain(project.projectId);
|
||||
expect(op!.files).toHaveLength(1);
|
||||
expect(op!.files[0]).toMatchObject({
|
||||
sourcePath: project.localDir,
|
||||
targetPath: projectDirs[index],
|
||||
kind: "directory",
|
||||
});
|
||||
// Plain read-only tree — no post-upload extract/wipe/merge command.
|
||||
expect(op!.postUploadCommands ?? []).toHaveLength(0);
|
||||
}
|
||||
});
|
||||
|
||||
it("isolates one additional project's sync failure and stages the rest", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-native-additional-fail-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const localWorkspaceDir = path.join(rootDir, "local-workspace");
|
||||
const remoteWorkspaceDir = path.join(rootDir, "remote-workspace");
|
||||
const goodDir = path.join(rootDir, "src-good");
|
||||
await mkdir(localWorkspaceDir, { recursive: true });
|
||||
await mkdir(goodDir, { recursive: true });
|
||||
await writeFile(path.join(localWorkspaceDir, "README.md"), "anchor\n", "utf8");
|
||||
await writeFile(path.join(goodDir, "good.txt"), "good body\n", "utf8");
|
||||
|
||||
// The middle source points at a directory that does not exist, so its native
|
||||
// transfer fails. Failure isolation must skip only it and stage the rest.
|
||||
const { client } = makeNativeClient();
|
||||
const prepared = await prepareSandboxManagedRuntime({
|
||||
spec: { transport: "sandbox", provider: "test", sandboxId: "s1", remoteCwd: remoteWorkspaceDir, timeoutMs: 30_000, apiKey: null },
|
||||
adapterKey: "test-adapter",
|
||||
client,
|
||||
workspaceLocalDir: localWorkspaceDir,
|
||||
additionalSources: [
|
||||
{ localPath: goodDir, projectId: "good-a" },
|
||||
{ localPath: path.join(rootDir, "does-not-exist"), projectId: "broken" },
|
||||
{ localPath: goodDir, projectId: "good-b" },
|
||||
],
|
||||
});
|
||||
|
||||
// Both healthy projects staged; the broken one is absent, not fatal.
|
||||
expect(Object.keys(prepared.additionalSourceDirs).sort()).toEqual(["good-a", "good-b"]);
|
||||
expect(prepared.additionalSourceDirs.broken).toBeUndefined();
|
||||
expect(await readFile(path.join(prepared.additionalSourceDirs["good-a"], "good.txt"), "utf8")).toBe("good body\n");
|
||||
expect(await readFile(path.join(prepared.additionalSourceDirs["good-b"], "good.txt"), "utf8")).toBe("good body\n");
|
||||
});
|
||||
|
||||
it("dereferences symlinks only when followSymlinks is true (native honors the flag)", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-native-symlink-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { promises as fsPromises } from "node:fs";
|
|||
import { lstat, mkdir, mkdtemp, readFile, readdir, rm, symlink, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { execFile as execFileCallback } from "node:child_process";
|
||||
import { execFile as execFileCallback, spawn } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { resetLocalGitIndexToHead } from "./git-workspace-sync.js";
|
||||
|
|
@ -14,6 +14,11 @@ import {
|
|||
type SandboxSyncOperation,
|
||||
type SandboxSyncResult,
|
||||
} from "./sandbox-managed-runtime.js";
|
||||
import {
|
||||
prepareCommandManagedRuntime,
|
||||
type CommandManagedRuntimeRunner,
|
||||
} from "./command-managed-runtime.js";
|
||||
import type { RunProcessResult } from "./server-utils.js";
|
||||
|
||||
function toArrayBuffer(bytes: Buffer): ArrayBuffer {
|
||||
return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer;
|
||||
|
|
@ -1490,4 +1495,97 @@ describe("sandbox managed runtime", () => {
|
|||
expect(coreSource).not.toMatch(/auth\.json/i);
|
||||
expect(coreSource).not.toMatch(/merge-extract|merge-decision/i);
|
||||
});
|
||||
|
||||
// End-to-end: drive the WHOLE prepare path — `prepareCommandManagedRuntime`
|
||||
// building the client, syncing the anchor workspace, then staging the
|
||||
// referenced projects — through a runner that runs real shell commands on the
|
||||
// host filesystem (host FS stands in for the sandbox FS). The runner exposes no
|
||||
// native syncIn, so staging rides the base64/tar fallback, the same transport a
|
||||
// provider without native sync uses. In production the kill-switch
|
||||
// `PAPERCLIP_MULTI_PROJECT_WORKSPACE_SYNC` gates whether run prep resolves any
|
||||
// referenced projects (OFF ⇒ none reach this layer). Enable it in-test only to
|
||||
// model the ON scenario, and prove multi-project isolation plus one-failure
|
||||
// isolation end-to-end.
|
||||
it("stages multiple referenced projects into isolated sandbox dirs end-to-end, skipping a failing source", async () => {
|
||||
const flagKey = "PAPERCLIP_MULTI_PROJECT_WORKSPACE_SYNC";
|
||||
const priorFlag = process.env[flagKey];
|
||||
process.env[flagKey] = "1";
|
||||
try {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-e2e-additional-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
|
||||
const localWorkspaceDir = path.join(rootDir, "local-workspace");
|
||||
const remoteWorkspaceDir = path.join(rootDir, "remote-workspace");
|
||||
await mkdir(localWorkspaceDir, { recursive: true });
|
||||
await mkdir(remoteWorkspaceDir, { recursive: true });
|
||||
await writeFile(path.join(localWorkspaceDir, "README.md"), "anchor content\n", "utf8");
|
||||
|
||||
// Two real referenced-project checkouts (one with a nested file) plus a
|
||||
// deliberately-missing source between them.
|
||||
const first = path.join(rootDir, "referenced-first");
|
||||
const second = path.join(rootDir, "referenced-second");
|
||||
await mkdir(path.join(first, "docs"), { recursive: true });
|
||||
await mkdir(second, { recursive: true });
|
||||
await writeFile(path.join(first, "docs", "guide.md"), "first guide\n", "utf8");
|
||||
await writeFile(path.join(second, "notes.md"), "second notes\n", "utf8");
|
||||
|
||||
const runner: CommandManagedRuntimeRunner = {
|
||||
execute: (input) =>
|
||||
new Promise<RunProcessResult>((resolve) => {
|
||||
const startedAt = new Date().toISOString();
|
||||
const command =
|
||||
input.command === "sh" ? "/bin/sh" : input.command === "bash" ? "/bin/bash" : input.command;
|
||||
const child = spawn(command, input.args ?? [], { cwd: input.cwd, env: { ...process.env, ...input.env } });
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
child.stdout.on("data", (chunk) => { stdout += chunk.toString("utf8"); });
|
||||
child.stderr.on("data", (chunk) => { stderr += chunk.toString("utf8"); });
|
||||
child.on("error", () => resolve({ exitCode: 127, signal: null, timedOut: false, stdout, stderr, pid: null, startedAt }));
|
||||
child.on("close", (code) => resolve({ exitCode: code ?? 0, signal: null, timedOut: false, stdout, stderr, pid: child.pid ?? null, startedAt }));
|
||||
if (input.stdin != null) child.stdin.write(input.stdin);
|
||||
child.stdin.end();
|
||||
}),
|
||||
};
|
||||
|
||||
const prepared = await prepareCommandManagedRuntime({
|
||||
runner,
|
||||
spec: { remoteCwd: remoteWorkspaceDir, timeoutMs: 30_000 },
|
||||
adapterKey: "test-adapter",
|
||||
workspaceLocalDir: localWorkspaceDir,
|
||||
additionalSources: [
|
||||
{ localPath: first, projectId: "proj-first" },
|
||||
{ localPath: path.join(rootDir, "referenced-missing"), projectId: "proj-missing" },
|
||||
{ localPath: second, projectId: "proj-second" },
|
||||
],
|
||||
});
|
||||
|
||||
const runtimeRootDir = path.posix.join(remoteWorkspaceDir, ".paperclip-runtime", "test-adapter");
|
||||
|
||||
// The anchor workspace synced normally and stays byte-identical.
|
||||
await expect(readFile(path.join(remoteWorkspaceDir, "README.md"), "utf8")).resolves.toBe("anchor content\n");
|
||||
|
||||
// Each healthy referenced project landed in its OWN isolated dir; the
|
||||
// missing one is skipped, not fatal.
|
||||
expect(Object.keys(prepared.additionalSourceDirs).sort()).toEqual(["proj-first", "proj-second"]);
|
||||
expect(prepared.additionalSourceDirs["proj-first"]).toBe(path.posix.join(runtimeRootDir, "project-proj-first"));
|
||||
expect(prepared.additionalSourceDirs["proj-second"]).toBe(path.posix.join(runtimeRootDir, "project-proj-second"));
|
||||
expect(prepared.additionalSourceDirs["proj-missing"]).toBeUndefined();
|
||||
|
||||
await expect(readFile(path.join(prepared.additionalSourceDirs["proj-first"], "docs", "guide.md"), "utf8"))
|
||||
.resolves.toBe("first guide\n");
|
||||
await expect(readFile(path.join(prepared.additionalSourceDirs["proj-second"], "notes.md"), "utf8"))
|
||||
.resolves.toBe("second notes\n");
|
||||
|
||||
// Neither project's tree leaked into the anchor workspace or into the other
|
||||
// project's dir.
|
||||
await expect(readFile(path.join(remoteWorkspaceDir, "notes.md"), "utf8")).rejects.toMatchObject({ code: "ENOENT" });
|
||||
await expect(readFile(path.join(prepared.additionalSourceDirs["proj-first"], "notes.md"), "utf8")).rejects
|
||||
.toMatchObject({ code: "ENOENT" });
|
||||
await expect(readFile(path.join(runtimeRootDir, "project-proj-missing"), "utf8")).rejects
|
||||
.toMatchObject({ code: "ENOENT" });
|
||||
} finally {
|
||||
if (priorFlag === undefined) delete process.env[flagKey];
|
||||
else process.env[flagKey] = priorFlag;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -121,6 +121,22 @@ export interface SandboxManagedRuntimeAsset {
|
|||
restore?: (ctx: SandboxManagedRuntimeAssetRestoreContext) => Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* A referenced (additional) project to stage into the run sandbox as a plain,
|
||||
* read-only tree. `localPath` is the host checkout directory. Upstream code
|
||||
* already authorized and realized this directory (`project:read`); this layer
|
||||
* adds no authorization logic. `projectId` names the isolated remote
|
||||
* subdirectory (`project-<projectId>` under the runtime root) the tree lands in.
|
||||
*
|
||||
* Additional sources are plain trees only. They never carry the anchor
|
||||
* workspace's git-history, overlay, or `.paperclip-runtime` preservation
|
||||
* semantics — those stay anchor-only.
|
||||
*/
|
||||
export interface SandboxAdditionalSource {
|
||||
localPath: string;
|
||||
projectId: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-call byte-level progress hook. `transferredBytes`/`totalBytes` are decoded
|
||||
* file bytes (not the base64 wire size). `totalBytes` is null when the size is
|
||||
|
|
@ -259,6 +275,13 @@ export interface PreparedSandboxManagedRuntime {
|
|||
workspaceRemoteDir: string;
|
||||
runtimeRootDir: string;
|
||||
assetDirs: Record<string, string>;
|
||||
/**
|
||||
* Remote directory of each additional (referenced) project that staged
|
||||
* successfully, keyed by `projectId`. A project whose staging failed is
|
||||
* absent (per-project failure isolation). Empty when no additional sources
|
||||
* were requested.
|
||||
*/
|
||||
additionalSourceDirs: Record<string, string>;
|
||||
restoreWorkspace(onProgress?: RuntimeProgressSink): Promise<void>;
|
||||
}
|
||||
|
||||
|
|
@ -637,6 +660,12 @@ export async function prepareSandboxManagedRuntime(input: {
|
|||
workspaceExclude?: string[];
|
||||
preserveAbsentOnRestore?: string[];
|
||||
assets?: SandboxManagedRuntimeAsset[];
|
||||
/**
|
||||
* Referenced (additional) projects to stage into the sandbox as plain,
|
||||
* read-only trees, each in its own isolated `project-<projectId>` directory.
|
||||
* Defaults to none, so a legacy/anchor-only call is behavior-identical.
|
||||
*/
|
||||
additionalSources?: SandboxAdditionalSource[];
|
||||
// Upload progress sink. Threaded for the byte-counting transport rewrite; the
|
||||
// child task wires it into writeFile/readFile.
|
||||
onProgress?: RuntimeProgressSink;
|
||||
|
|
@ -686,6 +715,14 @@ export async function prepareSandboxManagedRuntime(input: {
|
|||
// Opaque, ordered, non-sensitive operation tokens — never a caller/asset id.
|
||||
const nextSyncOperationId = () => `sync-op-${++syncOperationSeq}`;
|
||||
|
||||
// Remote directory of each additional (referenced) project that stages
|
||||
// successfully, keyed by projectId. A project that fails to stage is absent.
|
||||
const additionalSourceDirs: Record<string, string> = {};
|
||||
// Additional projects stage as plain trees. Drop the heavy build/cache dirs a
|
||||
// reference tree does not need, and `.git` — additional sources never carry
|
||||
// git-history semantics (anchor-only).
|
||||
const additionalSourceExclude = mergeExcludes(SANDBOX_WORKSPACE_HEAVY_DIR_EXCLUDES, [".git"]);
|
||||
|
||||
// Every delegated post-upload command (extract/wipe/remove-deleted/asset merge)
|
||||
// must run under the run-specific timeout (`spec.timeoutMs`), not the provider
|
||||
// sync client's default timeout — the two can differ, and before staging was
|
||||
|
|
@ -709,10 +746,48 @@ export async function prepareSandboxManagedRuntime(input: {
|
|||
...(input.preserveAbsentOnRestore ?? []),
|
||||
]);
|
||||
|
||||
// Build one `SandboxSyncOperation` uploading a host tarball as a single file
|
||||
// mapping (rides native `uploadFiles`, or the base64-tar fallback) with the
|
||||
// extract/wipe/merge steps carried as ordered `postUploadCommands`, confine
|
||||
// it, and delegate to `syncIn`. `finish` emits the terminal progress line.
|
||||
// Stage one source directory into an isolated remote subdirectory through the
|
||||
// unified `syncIn` seam. Both the workspace/asset tar path and the additional
|
||||
// (referenced) project path use it: build one `SandboxSyncOperation` from the
|
||||
// caller's `files` mappings (a host tarball as a single `file` mapping, or a
|
||||
// whole `directory` mapping), carry any extract/wipe/merge steps as ordered
|
||||
// `postUploadCommands`, confine the operation's source and target to their
|
||||
// own roots (fail-closed), and delegate to `syncIn` (native transfer, or the
|
||||
// base64-tar fallback). `finish` emits the terminal progress line.
|
||||
const stageConfinedSyncIn = async (params: {
|
||||
files: SandboxSyncFileMapping[];
|
||||
postUploadCommands?: SandboxPostUploadCommand[];
|
||||
sourceRoots: string[];
|
||||
targetRoots: string[];
|
||||
progressLabel: string;
|
||||
statusPhase: RuntimeStatusPhase;
|
||||
progressBytes: number;
|
||||
}): Promise<void> => {
|
||||
const operations: SandboxSyncOperation[] = [{
|
||||
operationId: nextSyncOperationId(),
|
||||
files: params.files,
|
||||
...(params.postUploadCommands
|
||||
? { postUploadCommands: withRunTimeout(params.postUploadCommands) }
|
||||
: {}),
|
||||
}];
|
||||
assertSyncOperationsConfined(operations, {
|
||||
sourceRoots: params.sourceRoots,
|
||||
targetRoots: params.targetRoots,
|
||||
});
|
||||
const upload = makeTransferProgress(
|
||||
input.onProgress,
|
||||
"Syncing",
|
||||
"to",
|
||||
params.progressLabel,
|
||||
{ sink: input.onRuntimeProgress, phase: params.statusPhase },
|
||||
);
|
||||
await syncIn(operations);
|
||||
await upload.finish(params.progressBytes, params.progressBytes);
|
||||
};
|
||||
|
||||
// Upload a host tarball as a single `file` mapping with the extract/wipe/merge
|
||||
// steps as ordered post-upload commands. A thin wrapper over
|
||||
// `stageConfinedSyncIn` for the workspace/git anchor and asset paths.
|
||||
const stageTarball = async (input2: {
|
||||
tarPath: string;
|
||||
remoteTar: string;
|
||||
|
|
@ -721,24 +796,15 @@ export async function prepareSandboxManagedRuntime(input: {
|
|||
statusPhase: RuntimeStatusPhase;
|
||||
}): Promise<void> => {
|
||||
const tarSize = (await fs.stat(input2.tarPath)).size;
|
||||
const operations: SandboxSyncOperation[] = [{
|
||||
operationId: nextSyncOperationId(),
|
||||
await stageConfinedSyncIn({
|
||||
files: [{ sourcePath: input2.tarPath, targetPath: input2.remoteTar, kind: "file" }],
|
||||
postUploadCommands: withRunTimeout(input2.postUploadCommands),
|
||||
}];
|
||||
assertSyncOperationsConfined(operations, {
|
||||
postUploadCommands: input2.postUploadCommands,
|
||||
sourceRoots: [tempDir],
|
||||
targetRoots: [runtimeRootDir],
|
||||
progressLabel: input2.progressLabel,
|
||||
statusPhase: input2.statusPhase,
|
||||
progressBytes: tarSize,
|
||||
});
|
||||
const upload = makeTransferProgress(
|
||||
input.onProgress,
|
||||
"Syncing",
|
||||
"to",
|
||||
input2.progressLabel,
|
||||
{ sink: input.onRuntimeProgress, phase: input2.statusPhase },
|
||||
);
|
||||
await syncIn(operations);
|
||||
await upload.finish(tarSize, tarSize);
|
||||
};
|
||||
|
||||
if (syncWorkspace && gitSnapshot) {
|
||||
|
|
@ -860,25 +926,63 @@ export async function prepareSandboxManagedRuntime(input: {
|
|||
assetDir: remoteAssetDir,
|
||||
runtimeRootDir,
|
||||
}) ?? buildDefaultExtractRuntimeAssetCommand({ remoteAssetDir, remoteAssetTar });
|
||||
const operations: SandboxSyncOperation[] = [{
|
||||
operationId: nextSyncOperationId(),
|
||||
const assetTarSize = (await fs.stat(assetTarPath)).size;
|
||||
await stageConfinedSyncIn({
|
||||
files,
|
||||
postUploadCommands: withRunTimeout([{ command: postUploadCommand }]),
|
||||
}];
|
||||
assertSyncOperationsConfined(operations, {
|
||||
postUploadCommands: [{ command: postUploadCommand }],
|
||||
sourceRoots: [tempDir],
|
||||
targetRoots: [runtimeRootDir],
|
||||
progressLabel: asset.key,
|
||||
statusPhase: "config_sync",
|
||||
progressBytes: assetTarSize,
|
||||
});
|
||||
const assetTarSize = (await fs.stat(assetTarPath)).size;
|
||||
const assetUpload = makeTransferProgress(
|
||||
input.onProgress,
|
||||
"Syncing",
|
||||
"to",
|
||||
asset.key,
|
||||
{ sink: input.onRuntimeProgress, phase: "config_sync" },
|
||||
);
|
||||
await syncIn(operations);
|
||||
await assetUpload.finish(assetTarSize, assetTarSize);
|
||||
}
|
||||
|
||||
// Stage each referenced (additional) project as a plain, read-only tree in
|
||||
// its OWN isolated remote directory (`project-<projectId>`). An additional
|
||||
// project rides one confined `syncIn` directory mapping — a native directory
|
||||
// transfer, or the base64-tar fallback — with source and target confined to
|
||||
// their own roots. No workspace, git-history, or `.paperclip-runtime`
|
||||
// semantics apply; those stay anchor-only. Per-project failure isolation: one
|
||||
// project's confinement or sync failure logs a warning and is skipped, and
|
||||
// the run plus the other projects continue. Only a project that stages
|
||||
// successfully appears in `additionalSourceDirs`.
|
||||
for (const source of input.additionalSources ?? []) {
|
||||
const { localPath, projectId } = source;
|
||||
const label = `project-${projectId}`;
|
||||
try {
|
||||
if (!path.posix.isAbsolute(localPath)) {
|
||||
throw new Error(`additional source localPath is not an absolute path: ${localPath}`);
|
||||
}
|
||||
if (
|
||||
projectId.length === 0 ||
|
||||
projectId.includes("/") ||
|
||||
projectId.includes("\\") ||
|
||||
projectId.includes("..")
|
||||
) {
|
||||
throw new Error(`additional source projectId is not a simple path segment: ${projectId}`);
|
||||
}
|
||||
const remoteProjectDir = path.posix.join(runtimeRootDir, label);
|
||||
await emitRuntimeStatus(input.onRuntimeProgress, "config_sync", "Syncing referenced project to sandbox");
|
||||
await stageConfinedSyncIn({
|
||||
files: [{
|
||||
sourcePath: localPath,
|
||||
targetPath: remoteProjectDir,
|
||||
kind: "directory",
|
||||
exclude: additionalSourceExclude,
|
||||
}],
|
||||
sourceRoots: [localPath],
|
||||
targetRoots: [remoteProjectDir],
|
||||
progressLabel: label,
|
||||
statusPhase: "config_sync",
|
||||
progressBytes: 0,
|
||||
});
|
||||
additionalSourceDirs[projectId] = remoteProjectDir;
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
`[paperclip] Failed to stage referenced project ${projectId}; skipping it. ${String(error)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
|
|
@ -892,6 +996,7 @@ export async function prepareSandboxManagedRuntime(input: {
|
|||
workspaceRemoteDir,
|
||||
runtimeRootDir,
|
||||
assetDirs,
|
||||
additionalSourceDirs,
|
||||
restoreWorkspace: async (onProgress?: RuntimeProgressSink) => {
|
||||
const restoreSink = onProgress ?? input.onProgress;
|
||||
if (!syncWorkspace) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue