From 23407236b460121c73201eb8a11a356e48375c23 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 7 Sep 2026 13:22:36 -0500 Subject: [PATCH] Preserve sandbox work through provider launches and failed saves Resume retained ephemeral leases, pin managed-agent imports against link replacement, carry scoped home settings through native provider boundaries, and keep file transfer arguments below Linux limits. Restrict acceptance exclusions to the engines explicitly deferred by the user. Co-Authored-By: Paperclip --- .../runner-core/src/acpx_sidecar_transport.rs | 2 + .../crates/runner-core/src/codex_provider.rs | 3 +- .../runner-core/src/process_supervisor.rs | 12 ++ .../src/drivers/acpx/environment.ts | 5 +- .../src/live/runnerd-codex-transport.test.ts | 13 +++ .../src/live/runnerd-codex-transport.ts | 12 +- .../src/work-folder-environment.test.ts | 8 ++ .../src/work-folder-environment.ts | 8 +- .../src/__tests__/environment-runtime.test.ts | 109 ++++++++++++++++++ .../__tests__/sandbox-work-folders.test.ts | 27 ++++- .../work-folder-agent-import.test.ts | 34 ++++++ .../__tests__/work-folder-transport.test.ts | 8 +- server/src/services/environment-runtime.ts | 16 ++- server/src/services/environments.ts | 8 +- server/src/services/sandbox-work-folders.ts | 36 +++--- .../src/services/work-folder-agent-import.ts | 48 ++++++++ server/src/services/work-folder-transport.ts | 7 +- tests/runner-e2e/deployed-stack.test.ts | 9 +- tests/runner-e2e/deployed-stack.ts | 17 ++- 19 files changed, 334 insertions(+), 48 deletions(-) create mode 100644 server/src/__tests__/work-folder-agent-import.test.ts create mode 100644 server/src/services/work-folder-agent-import.ts diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs index 55f7b25335..908519d176 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs @@ -13,6 +13,7 @@ use crate::generated_acpx_sidecar_contract::{ use crate::local_runner::LocalRunnerError; use crate::process_supervisor::{ BoundedLogBuffer, ProcessOutput, SupervisedProcess, VerifiedProcessLaunch, + WORK_FOLDER_ENVIRONMENT_KEYS, }; use crate::stable_identity::{is_stable_id, DURABLE_STABLE_ID_CHARS, SHORT_STABLE_ID_CHARS}; @@ -123,6 +124,7 @@ impl AcpxSidecarTransport { "PAPERCLIP_ACPX_PROVIDER_PACKAGE_MANIFEST", ]; keys.extend_from_slice(credential_keys); + keys.extend_from_slice(WORK_FOLDER_ENVIRONMENT_KEYS); Self::start_with_environment_keys(config, &keys) } diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs index ee04c64c96..f045c34453 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs @@ -16,7 +16,7 @@ use crate::durable::{redact_text, OpenCodeLaunchProfile}; use crate::local_runner::LocalRunnerError; use crate::process_supervisor::{ is_node_interpreter, BoundedLogBuffer, ProcessOutput, SupervisedProcess, - VerifiedProcessArgument, VerifiedProcessLaunch, + VerifiedProcessArgument, VerifiedProcessLaunch, WORK_FOLDER_ENVIRONMENT_KEYS, }; use crate::provider_bridge::{AuthorizedTool, DurableReplayFilter, ToolResult}; use crate::provider_events::normalized_codex_terminal_event_type; @@ -717,6 +717,7 @@ impl CodexProvider { .iter() .copied() .chain(provider_environment_keys.iter().copied()) + .chain(WORK_FOLDER_ENVIRONMENT_KEYS.iter().copied()) .chain(GITHUB_CREDENTIAL_ENVIRONMENT_KEYS.iter().copied()) .collect::>(); let process = if config.provider == "opencode" { diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/process_supervisor.rs b/packages/paperclip-runner/runner/crates/runner-core/src/process_supervisor.rs index 21eca672f7..d191f94805 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/process_supervisor.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/process_supervisor.rs @@ -26,6 +26,18 @@ use sha2::{Digest, Sha256}; use crate::local_runner::LocalRunnerError; const PROCESS_OUTPUT_QUEUE_CAPACITY: usize = 256; +// Host-validated scoped paths, carried only through sandbox provider launches. +pub(crate) const WORK_FOLDER_ENVIRONMENT_KEYS: &[&str] = &[ + "PAPERCLIP_RUNNER_EXTERNAL_SANDBOX", + "PAPERCLIP_TASK_DIR", + "PAPERCLIP_AGENT_DIR", + "PAPERCLIP_USER_DIR", + "PAPERCLIP_PROJECT_DIR", + "PAPERCLIP_REPOS_DIR", + "PAPERCLIP_PRIMARY_REPO", + "PAPERCLIP_WORKSPACE_CWD", + "AGENT_HOME", +]; const VERIFIED_RUNTIME_EXECUTABLE_ENV: &str = "PAPERCLIP_VERIFIED_RUNTIME_EXECUTABLE"; const VERIFIED_COMMONJS_ARTIFACT_LOADER: &str = r#"const fs=require("node:fs");const Module=require("node:module");const filename=process.argv[1];const source=fs.readFileSync(filename,"utf8").replace(/^#![^\r\n]*(?:\r?\n|$)/,"");const artifact=new Module(filename);artifact.filename=filename;artifact.paths=[];artifact._compile(source,filename);"#; diff --git a/packages/paperclip-runner/src/drivers/acpx/environment.ts b/packages/paperclip-runner/src/drivers/acpx/environment.ts index 57fef1b4e3..4c2f7e285a 100644 --- a/packages/paperclip-runner/src/drivers/acpx/environment.ts +++ b/packages/paperclip-runner/src/drivers/acpx/environment.ts @@ -1,4 +1,5 @@ import type { QualifiedAcpxAgent } from "./qualified-profiles.js"; +import { externalWorkFolderEnvironment } from "../../work-folder-environment.js"; declare const sanitizedAcpxSpawnInputBrand: unique symbol; @@ -23,6 +24,7 @@ export function createSanitizedAcpxSpawnInput( agent: QualifiedAcpxAgent, ): SanitizedAcpxSpawnInput { const source = environment ?? process.env; + const scoped = externalWorkFolderEnvironment(source); const result: NodeJS.ProcessEnv = {}; const credentialNames = agent === "pi" @@ -56,9 +58,10 @@ export function createSanitizedAcpxSpawnInput( "PAPERCLIP_NATIVE_MCP_NAME", "PAPERCLIP_NATIVE_MCP_URL", ...credentialNames, + ...Object.keys(scoped), ]); let retainedBytes = 0; - for (const [key, value] of Object.entries(source)) { + for (const [key, value] of Object.entries({ ...source, ...scoped })) { if (typeof value !== "string") continue; if (!allowed.has(key) && !/^LC_[A-Z0-9_]{1,32}$/.test(key)) continue; if (key.includes("\0") || value.includes("\0")) { diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index eb07c5bfa4..82e3a41219 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -1057,6 +1057,19 @@ it("allows trusted package-manager runtime roots without exposing HOME paths", ( ).toEqual(["/opt/homebrew", "/usr/local"]); }); +it.each(["codex", "opencode", "acpx"] as const)("keeps the natural sandbox home through the %s provider environment", (provider) => { + const home = "/home/daytona"; + const environment = { HOME: home, PAPERCLIP_RUNNER_EXTERNAL_SANDBOX: "1", PAPERCLIP_PRIMARY_REPO: `${home}/repos/main`, + PAPERCLIP_TASK_DIR: `${home}/task`, PAPERCLIP_AGENT_DIR: `${home}/agent`, PAPERCLIP_USER_DIR: `${home}/user`, + PAPERCLIP_PROJECT_DIR: `${home}/project`, PAPERCLIP_REPOS_DIR: `${home}/repos`, OPENAI_API_KEY: "fixture-provider-key" }; + const result = createCapabilityRunnerdProviderEnvironment({ provider, options: { provider, stateDirectory: "/controller/state", environment }, + identity: { runnerInstanceId: "runner", environmentLeaseId: "lease", runId: "run", normalizedSessionId: "session", turnId: "turn", itemId: "item" }, + codexHome: `${home}/.codex`, runtimeContextPath: "/runtime/context.json", hasRuntimeContext: false }); + expect(result).toMatchObject({ HOME: home, PAPERCLIP_RUNNER_EXTERNAL_SANDBOX: "1", PAPERCLIP_TASK_DIR: `${home}/task`, PAPERCLIP_USER_DIR: `${home}/user` }); + if (provider === "codex") expect(result.CODEX_HOME).toBe(`${home}/.codex`); + expect(() => createRunnerdCodexAppServerArgs({ environment, codexHome: `${home}/.codex` })).not.toThrow(); +}); + it("denies the isolated Codex home without denying a remote execution workspace", () => { const args = createRunnerdCodexAppServerArgs({ environment: { diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index 92ccd17a15..139fe1fb83 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -1,3 +1,4 @@ +import { externalWorkFolderEnvironment } from "../work-folder-environment.js"; import { execFileSync } from "node:child_process"; import { createHash, randomUUID } from "node:crypto"; import { @@ -1796,6 +1797,7 @@ export function createCapabilityRunnerdProviderEnvironment(input: { if (input.provider === "opencode") { return { ...createSanitizedOpenCodeRunnerEnvironment(input.options.environment), + ...externalWorkFolderEnvironment(input.options.environment ?? {}), PAPERCLIP_OPENCODE_PERMISSION_MODE: input.options.opencodePermissionMode ?? "ask", PAPERCLIP_OPENCODE_RUNTIME_DIR: @@ -1848,7 +1850,7 @@ export function createCapabilityRunnerdProviderEnvironment(input: { } const environment = createSanitizedCodexEnvironment({ ...input.options.environment, - HOME: input.codexHome, + HOME: externalWorkFolderEnvironment(input.options.environment ?? {}).HOME ?? input.codexHome, CODEX_HOME: input.codexHome, }); for (const key of ["OPENAI_API_KEY", "CODEX_API_KEY"] as const) { @@ -1949,7 +1951,7 @@ export function createRunnerdCodexAppServerArgs(input: { return createIsolatedCodexAppServerArgs( { ...input.environment, - HOME: input.codexHome, + HOME: externalWorkFolderEnvironment(input.environment ?? {}).HOME ?? input.codexHome, CODEX_HOME: input.codexHome, }, input.readOnlyRoots, @@ -2929,7 +2931,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { ); } const localCodexHome = resolve(this.#root, "codex-home"); - const codexHome = this.options.runnerFilesystemRoot + const scopedHome = externalWorkFolderEnvironment(this.options.environment ?? {}).HOME; + const codexHome = scopedHome ? resolve(scopedHome, ".codex") : this.options.runnerFilesystemRoot ? resolve(this.options.runnerFilesystemRoot, "codex-home") : localCodexHome; if (provider === "aws_agentcore") { @@ -3461,7 +3464,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { ); } const localCodexHome = resolve(this.#root, "codex-home"); - const codexHome = this.options.runnerFilesystemRoot + const scopedHome = externalWorkFolderEnvironment(this.options.environment ?? {}).HOME; + const codexHome = scopedHome ? resolve(scopedHome, ".codex") : this.options.runnerFilesystemRoot ? resolve(this.options.runnerFilesystemRoot, "codex-home") : localCodexHome; if (provider === "aws_agentcore") { diff --git a/packages/paperclip-runner/src/work-folder-environment.test.ts b/packages/paperclip-runner/src/work-folder-environment.test.ts index a0380f2002..ec2bde0df5 100644 --- a/packages/paperclip-runner/src/work-folder-environment.test.ts +++ b/packages/paperclip-runner/src/work-folder-environment.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { externalWorkFolderEnvironment } from "./work-folder-environment.js"; import { createSanitizedCodexEnvironment } from "./drivers/codex/app-server-transport.js"; import { codexCommandEnvironment } from "./drivers/codex/codex-security-config.js"; +import { createSanitizedAcpxSpawnInput } from "./drivers/acpx/environment.js"; describe("external sandbox work-folder environment", () => { const home = "/home/daytona"; @@ -17,4 +18,11 @@ describe("external sandbox work-folder environment", () => { expect(() => externalWorkFolderEnvironment({ ...environment, PAPERCLIP_USER_DIR: "/other/user" })).toThrow("does not match"); expect(() => externalWorkFolderEnvironment({ ...environment, PAPERCLIP_PRIMARY_REPO: `${home}/repos/../../.codex` })).toThrow("Invalid sandbox primary"); }); + it.each(["codex", "claude", "pi"] as const)("retains scoped identity across repeated ACPX %s launch boundaries", (agent) => { + const first = createSanitizedAcpxSpawnInput({ ...environment, UNRELATED_SECRET: "private" }, agent).env; + const second = createSanitizedAcpxSpawnInput(first, agent).env; + expect(second).toMatchObject(externalWorkFolderEnvironment(environment)); + expect(second).not.toHaveProperty("UNRELATED_SECRET"); + expect(externalWorkFolderEnvironment(second).HOME).toBe(home); + }); }); diff --git a/packages/paperclip-runner/src/work-folder-environment.ts b/packages/paperclip-runner/src/work-folder-environment.ts index e4451669e3..6d21b93b0f 100644 --- a/packages/paperclip-runner/src/work-folder-environment.ts +++ b/packages/paperclip-runner/src/work-folder-environment.ts @@ -4,8 +4,8 @@ import path from "node:path"; export function externalWorkFolderEnvironment(source: NodeJS.ProcessEnv): NodeJS.ProcessEnv { const home = source.HOME; if (source.PAPERCLIP_RUNNER_EXTERNAL_SANDBOX !== "1" || !home || !source.PAPERCLIP_TASK_DIR) return {}; - if (!path.isAbsolute(home) || path.resolve(home) !== home || home === "/") throw new Error("Invalid sandbox work-folder home"); - const result: NodeJS.ProcessEnv = { HOME: home }; + if (home.includes("\0") || home.length > 4096 || !path.isAbsolute(home) || path.resolve(home) !== home || home === "/") throw new Error("Invalid sandbox work-folder home"); + const result: NodeJS.ProcessEnv = { HOME: home, PAPERCLIP_RUNNER_EXTERNAL_SANDBOX: "1" }; for (const scope of ["task", "agent", "user", "project", "repos"] as const) { const key = `PAPERCLIP_${scope.toUpperCase()}_DIR`; const expected = path.join(home, scope); @@ -19,6 +19,10 @@ export function externalWorkFolderEnvironment(source: NodeJS.ProcessEnv): NodeJS throw new Error("Invalid sandbox primary repository path"); } result.PAPERCLIP_PRIMARY_REPO = primary; + if (source.PAPERCLIP_WORKSPACE_CWD !== undefined) { + if (source.PAPERCLIP_WORKSPACE_CWD !== primary) throw new Error("Sandbox primary repository does not match its workspace"); + result.PAPERCLIP_WORKSPACE_CWD = primary; + } } return result; } diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index 8ab3aaa32a..46d4efe06c 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -21,6 +21,7 @@ import { environments, executionWorkspaces, heartbeatRuns, + workFolderRuns, plugins, projects, } from "@paperclipai/db"; @@ -28,6 +29,7 @@ import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; +import { workFolderSandboxKey } from "../services/work-folder-retention.js"; import { resolveEnvironmentDriverConfigForRuntime } from "../services/environment-config.ts"; import { SANDBOX_CAPABILITY_KEYS, @@ -654,6 +656,113 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { ).toHaveLength(1); }); + it("recovers unsaved work from an originally ephemeral plugin sandbox", async () => { + const seeded = await seedReusablePluginSandboxLease(); + seeded.environment.config = { ...seeded.environment.config, reuseLease: false }; + await environmentService(db).update(seeded.environment.id, { config: seeded.environment.config }); + const workerManager = { + isRunning: vi.fn((id: string) => id === seeded.pluginId), + call: vi.fn(async (_pluginId: string, method: string) => { + if (method === "environmentAcquireLease") { + return { + providerLeaseId: "sandbox-exact-resume", + metadata: { + provider: "fake-plugin", + image: "fake:test", + timeoutMs: 1234, + reuseLease: false, + remoteCwd: "/workspace", + }, + }; + } + if (method === "environmentReleaseLease") return undefined; + if (method === "environmentResumeLease") { + return { + providerLeaseId: "sandbox-exact-resume", + metadata: { + provider: "fake-plugin", + image: "fake:test", + timeoutMs: 1234, + reuseLease: false, + remoteCwd: "/workspace", + }, + }; + } + throw new Error(`Unexpected plugin method during exact resume: ${method}`); + }), + getWorker: vi.fn(() => ({ + supportedMethods: [ + "environmentResumeLease", + "environmentReleaseLease", + "environmentDestroyLease", + ], + })), + } as unknown as PluginWorkerManager; + const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); + await environmentService(db).releaseLease(seeded.reusableLease.id, "expired"); + const first = await runtimeWithPlugin.acquireRunLease({ + companyId: seeded.companyId, + environment: seeded.environment, + issueId: null, + agentId: seeded.agentId, + heartbeatRunId: seeded.runId, + persistedExecutionWorkspace: { + id: seeded.executionWorkspaceId, + mode: "shared_workspace", + }, + }); + await db.insert(workFolderRuns).values({ runId: seeded.runId, companyId: seeded.companyId, state: "failed", + manifest: { version: 1, companyId: seeded.companyId, runId: seeded.runId, agentId: seeded.agentId, + taskId: null, projectId: null, responsibleUserId: null, leaseId: first.lease.id, + sandboxKey: workFolderSandboxKey(first.lease), home: "/home/sandbox", + folders: { task: null, agent: randomUUID(), user: null, project: null }, repositories: [] } }); + await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, seeded.runId)); + await runtimeWithPlugin.releaseRunLeases(seeded.runId, "failed", "save failed", "stop_and_retain"); + const retained = await environmentService(db).getLeaseById(first.lease.id); + expect(retained).toMatchObject({ status: "retained", expiresAt: null, metadata: { workFolderRecoveryRequired: true } }); + expect(workerManager.call).not.toHaveBeenCalledWith(seeded.pluginId, "environmentDestroyLease", expect.anything(), expect.anything()); + + const nextRunId = randomUUID(); + await db.insert(heartbeatRuns).values({ + id: nextRunId, + companyId: seeded.companyId, + agentId: seeded.agentId, + invocationSource: "manual", + status: "running", + createdAt: new Date(), + updatedAt: new Date(), + }); + const acquired = await runtimeWithPlugin.acquireRunLease({ + companyId: seeded.companyId, + environment: seeded.environment, + issueId: null, + agentId: seeded.agentId, + heartbeatRunId: nextRunId, + persistedExecutionWorkspace: { + id: seeded.executionWorkspaceId, + mode: "shared_workspace", + }, + }); + + expect(first.lease.metadata?.sandboxLeaseAcquisition).toEqual({ outcome: "created" }); + expect(acquired.lease.providerLeaseId).toBe("sandbox-exact-resume"); + expect(acquired.lease.metadata?.sandboxLeaseAcquisition).toEqual({ + outcome: "resumed", + }); + + await expect( + environmentService(db).getLeaseById(first.lease.id), + ).resolves.toMatchObject({ + status: "expired", + cleanupStatus: "success", + }); + expect( + workerManager.call.mock.calls.filter( + (call) => call[1] === "environmentAcquireLease", + ), + ).toHaveLength(1); + }); + it("reacquires one provider lease row idempotently during same-run recovery", async () => { const seeded = await seedReusablePluginSandboxLease(); const workerManager = { diff --git a/server/src/__tests__/sandbox-work-folders.test.ts b/server/src/__tests__/sandbox-work-folders.test.ts index 5301e80281..aa2cd53327 100644 --- a/server/src/__tests__/sandbox-work-folders.test.ts +++ b/server/src/__tests__/sandbox-work-folders.test.ts @@ -47,14 +47,15 @@ describe("shared sandbox work-folder lifecycle", () => { await database?.cleanup(); if (root) await fs.rm(root, { recursive: true, force: true }); }); async function prepare(home: string, leaseId: string, physicalId = leaseId, responsibleUserId: string | null = null, - options: { taskId?: string; branchName?: string } = {}) { + options: { taskId?: string; branchName?: string; agentId?: string } = {}) { await fs.mkdir(home, { recursive: true }); const runId = randomUUID(); - await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, responsibleUserId, status: "running" }); + const boundAgentId = options.agentId ?? agentId; + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId: boundAgentId, responsibleUserId, status: "running" }); const lease = { id: leaseId, companyId, environmentId, provider: "test", providerLeaseId: physicalId }; await db.insert(environmentLeases).values({ ...lease, heartbeatRunId: runId }).onConflictDoUpdate({ target: environmentLeases.id, set: { heartbeatRunId: runId } }); const [primary] = await db.select().from(projectWorkspaces).where(eq(projectWorkspaces.projectId, projectId)); - const run = await prepareSandboxWorkFolders({ db, companyId, agentId, projectId, taskId: options.taskId ?? taskId, runId, + const run = await prepareSandboxWorkFolders({ db, companyId, agentId: boundAgentId, projectId, taskId: options.taskId ?? taskId, runId, primaryWorkspaceId: primary!.id, primaryBranchName: options.branchName, responsibleUserId, storage, sandboxKey: workFolderSandboxKey(lease), target: { kind: "remote", transport: "sandbox", leaseId, remoteCwd: home, runner: { execute: (input) => localTestWorkFolderRunner.execute({ ...input, env: { ...input.env, HOME: home } }) } } }); @@ -220,6 +221,26 @@ describe("shared sandbox work-folder lifecycle", () => { await warm.stop(); active.splice(active.indexOf(warm), 1); }, 120_000); + it("restores task work across identities without carrying over private homes or sessions", async () => { + const otherAgentId = randomUUID(), firstUser = randomUUID(), secondUser = randomUUID(); + await db.insert(agents).values({ id: otherAgentId, companyId, name: "Replacement agent" }); + for (const userId of [firstUser, secondUser]) await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: userId, membershipRole: "member" }); + const lease = randomUUID(); const first = await prepare(path.join(root, "identity-one"), lease, lease, firstUser); + await fs.writeFile(path.join(first.home, "task/identity.txt"), "shared task work"); + await fs.writeFile(path.join(first.home, "agent/identity-private"), "old agent only"); + await fs.writeFile(path.join(first.home, "user/identity-private"), "old user only"); + await fs.writeFile(path.join(first.home, ".codex/session-private"), "old provider session"); + await first.stop(); active.splice(active.indexOf(first), 1); + await expect(prepare(first.home, randomUUID(), lease, secondUser, { agentId: otherAgentId })).rejects.toThrow("fresh sandbox"); + const replacementLease = randomUUID(); + const replacement = await prepare(path.join(root, "identity-two"), replacementLease, replacementLease, secondUser, { agentId: otherAgentId }); + expect(replacement.identityChanged).toBe(true); + expect(await fs.readFile(path.join(replacement.home, "task/identity.txt"), "utf8")).toBe("shared task work"); + for (const file of ["agent/identity-private", "user/identity-private", ".codex/session-private"]) await expect(fs.access(path.join(replacement.home, file))).rejects.toThrow(); + expect(replacement.manifest.repositories).toHaveLength(2); + await replacement.stop(); active.splice(active.indexOf(replacement), 1); + }, 120_000); + it("collects superseded repository objects while retaining the complete current checkpoint", async () => { const run = await prepare(path.join(root, "checkpoint-garbage"), randomUUID()); const filename = path.join(run.primaryRepo, "garbage-fixture"); diff --git a/server/src/__tests__/work-folder-agent-import.test.ts b/server/src/__tests__/work-folder-agent-import.test.ts new file mode 100644 index 0000000000..102f08e3df --- /dev/null +++ b/server/src/__tests__/work-folder-agent-import.test.ts @@ -0,0 +1,34 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, expect, it } from "vitest"; +import { managedAgentFiles } from "../services/work-folder-agent-import.js"; +const roots: string[] = []; +afterEach(async () => { for (const root of roots.splice(0)) await fs.rm(root, { recursive: true, force: true }); }); +async function fixture() { const root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "agent-import-"))); roots.push(root); return root; } +async function collect(root: string) { const results: Array<{ path: string; executable: boolean; text: string }> = []; + for await (const entry of managedAgentFiles(root)) { const chunks: Buffer[] = []; + if (entry.body) for await (const chunk of entry.body) chunks.push(Buffer.from(chunk)); + results.push({ path: entry.path, executable: entry.executable, text: Buffer.concat(chunks).toString() }); + } return results; } +it("imports nested empty and executable files while excluding credential homes", async () => { + const root = await fixture(); await fs.mkdir(path.join(root, "notes")); await fs.writeFile(path.join(root, "notes/run.sh"), "", { mode: 0o700 }); + await fs.mkdir(path.join(root, ".codex")); await fs.writeFile(path.join(root, ".codex/auth.json"), "private"); + expect(await collect(root)).toEqual([{ path: "notes", executable: false, text: "" }, { path: "notes/run.sh", executable: true, text: "" }]); + expect(await collect(path.join(root, "absent"))).toEqual([]); +}); +it("rejects symlink and hard-link imports", async () => { + const root = await fixture(); const other = await fixture(); await fs.writeFile(path.join(other, "private"), "private"); + await fs.symlink(path.join(other, "private"), path.join(root, "linked")); await expect(collect(root)).rejects.toThrow(); + await fs.unlink(path.join(root, "linked")); await fs.link(path.join(other, "private"), path.join(root, "linked")); + await expect(collect(root)).rejects.toThrow("hard links"); +}); +it.skipIf(process.platform !== "linux")("keeps an admitted parent pinned when replaced by a symlink", async () => { + const root = await fixture(); const other = await fixture(); await fs.mkdir(path.join(root, "notes")); + await fs.writeFile(path.join(root, "notes/file"), "original"); await fs.writeFile(path.join(other, "file"), "private"); + const iterator = managedAgentFiles(root); expect((await iterator.next()).value?.path).toBe("notes"); + await fs.rename(path.join(root, "notes"), path.join(root, "moved")); await fs.symlink(other, path.join(root, "notes")); + const file = (await iterator.next()).value!; const chunks: Buffer[] = []; + for await (const chunk of file.body!) chunks.push(Buffer.from(chunk)); + expect(Buffer.concat(chunks).toString()).toBe("original"); await iterator.return(undefined); +}); diff --git a/server/src/__tests__/work-folder-transport.test.ts b/server/src/__tests__/work-folder-transport.test.ts index d2fa0fc0aa..1b7198f2f6 100644 --- a/server/src/__tests__/work-folder-transport.test.ts +++ b/server/src/__tests__/work-folder-transport.test.ts @@ -24,7 +24,13 @@ describe("sandbox work folder transport with real Node and Git", () => { const body = Buffer.alloc(700_000, "x"); const entry = { path: "nested/file", kind: "file" as const, byteSize: body.length, sha256: createHash("sha256").update(body).digest("hex"), executable: true }; - await transport.write(dir, staging, entry, Readable.from([body])); + const boundedTransport = workFolderTransport({ async execute(input) { + // macOS permits larger argv entries than Linux; enforce the deployment + // bound here as well so this regression is caught on developer machines. + expect(Buffer.byteLength([input.command, ...(input.args ?? [])].join(" "))).toBeLessThan(120 * 1024); + return localTestWorkFolderRunner.execute(input); + } }); + await boundedTransport.write(dir, staging, entry, Readable.from([body])); expect(await readFile(path.join(dir, entry.path))).toEqual(body); const files = await transport.scan(dir); expect(files.find((file) => file.path === entry.path)).toEqual(entry); diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index e44fcdbe7a..76785ae56a 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -56,6 +56,7 @@ import { getSandboxProvider as getBuiltinSandboxProvider, isBuiltinSandboxProvider, releaseSandboxProviderLease, + resumeSandboxProviderLease, sandboxConfigFromLeaseMetadata, sandboxConfigFromLeaseMetadataLoose, } from "./sandbox-provider-runtime.js"; @@ -1893,7 +1894,8 @@ function createSandboxEnvironmentDriver( input.heartbeatRunId !== null && input.executionWorkspaceId !== null && input.agentId !== null - ? findReusableSandboxLeaseId({ config: storedConfig, leases: reusableExistingLeases }) + ? (reusableExistingLeases.find((lease) => lease.metadata?.workFolderRecoveryRequired === true)?.providerLeaseId + ?? findReusableSandboxLeaseId({ config: storedConfig, leases: reusableExistingLeases })) : null; const reusableLease = reusableProviderLeaseId ? reusableExistingLeases.find((lease) => lease.providerLeaseId === reusableProviderLeaseId) @@ -2254,7 +2256,8 @@ function createSandboxEnvironmentDriver( input.heartbeatRunId !== null && input.executionWorkspaceId !== null && input.agentId !== null - ? findReusableSandboxLeaseId({ config: parsed.config, leases: reusableExistingLeases }) + ? (reusableExistingLeases.find((lease) => lease.metadata?.workFolderRecoveryRequired === true)?.providerLeaseId + ?? findReusableSandboxLeaseId({ config: parsed.config, leases: reusableExistingLeases })) : null; const reusableLease = reusableProviderLeaseId ? reusableExistingLeases.find((lease) => lease.providerLeaseId === reusableProviderLeaseId) @@ -2262,7 +2265,14 @@ function createSandboxEnvironmentDriver( let providerLease; try { - providerLease = await acquireSandboxProviderLease({ + if (reusableLease?.metadata?.workFolderRecoveryRequired === true) { + // Recovery overrides the original ephemeral disposal policy, after + // the full host-owned identity/configuration fingerprint matched. + providerLease = await resumeSandboxProviderLease({ config: parsed.config, providerLeaseId: reusableLease.providerLeaseId! }); + if (!providerLease || providerLease.providerLeaseId !== reusableLease.providerLeaseId) { + throw new Error("Unsaved sandbox could not be resumed; original work was retained"); + } + } else providerLease = await acquireSandboxProviderLease({ config: parsed.config, environmentId: input.environment.id, heartbeatRunId: input.heartbeatRunId ?? randomUUID(), diff --git a/server/src/services/environments.ts b/server/src/services/environments.ts index 393b24bbf7..60312355f6 100644 --- a/server/src/services/environments.ts +++ b/server/src/services/environments.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import { and, desc, eq, inArray, ne, sql } from "drizzle-orm"; +import { and, desc, eq, inArray, ne, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { agents, @@ -1469,7 +1469,8 @@ export function environmentService(db: Db) { environmentLeases.executionWorkspaceId, input.executionWorkspaceId!, ), - eq(environmentLeases.leasePolicy, "reuse_by_environment"), + or(eq(environmentLeases.leasePolicy, "reuse_by_environment"), + sql`${environmentLeases.metadata}->'workFolderRecoveryRequired' = 'true'::jsonb`), eq( environmentLeases.providerLeaseId, input.providerLeaseId!, @@ -1513,7 +1514,8 @@ export function environmentService(db: Db) { environmentLeases.heartbeatRunId, input.heartbeatRunId!, ), - eq(environmentLeases.leasePolicy, "reuse_by_environment"), + or(eq(environmentLeases.leasePolicy, "reuse_by_environment"), + sql`${environmentLeases.metadata}->'workFolderRecoveryRequired' = 'true'::jsonb`), eq( environmentLeases.providerLeaseId, input.providerLeaseId!, diff --git a/server/src/services/sandbox-work-folders.ts b/server/src/services/sandbox-work-folders.ts index 019dc3a1d2..34af680892 100644 --- a/server/src/services/sandbox-work-folders.ts +++ b/server/src/services/sandbox-work-folders.ts @@ -1,7 +1,6 @@ import { createHash, randomUUID } from "node:crypto"; -import { createReadStream } from "node:fs"; -import fs from "node:fs/promises"; import path from "node:path"; +import { managedAgentFiles } from "./work-folder-agent-import.js"; import { and, asc, desc, eq, sql } from "drizzle-orm"; import { agents, assets, companyMemberships, heartbeatRuns, issues, projects, issueAttachments, projectWorkspaces, taskRepositoryBindings, workFileOperations, workFolderRuns, workFolders, type Db } from "@paperclipai/db"; import { WORK_FOLDER_SCOPES, type SandboxWorkFolderManifest, type WorkFolderScope } from "@paperclipai/shared"; @@ -15,6 +14,7 @@ import { workFolderService } from "./work-folders.js"; import { workFolderPaths, workFolderTransport, type WorkTreeEntry } from "./work-folder-transport.js"; import { workFolderRepositoryService } from "./work-folder-repositories.js"; import { startWorkFolderCheckpointer } from "./work-folder-checkpointer.js"; +import { logActivity } from "./activity-log.js"; import { assertWorkFolderAccess } from "./work-folder-access.js"; function signature(entry: WorkTreeEntry | undefined) { @@ -125,26 +125,12 @@ export async function prepareSandboxWorkFolders(input: { const folder = folders.agent!; if (folder.importedAt) return; const root = resolveDefaultAgentWorkspaceDir(input.agentId); - async function visit(relative: string) { - if (relative) { - const [receipt] = await db.select({ id: workFileOperations.id }).from(workFileOperations).where(and( - eq(workFileOperations.folderId, folder.id), eq(workFileOperations.operationId, `import:${relative}`))); - // Still descend into previously imported directories after an interrupted import. - if (receipt && !(await fs.lstat(path.join(root, relative))).isDirectory()) return; - } - const stat = await fs.lstat(path.join(root, relative)); - if (stat.isSymbolicLink()) throw new Error("Managed agent home contains an unsupported symbolic link"); - if (stat.isDirectory()) { - if (relative) await svc.write(folder, { path: relative, kind: "directory", operationId: `import:${relative}`, onlyIfMissing: true }); - for (const name of (await fs.readdir(path.join(root, relative))).sort()) { - if ([".codex", ".claude", ".cache", ".config", ".local", ".git", ".paperclip-runtime", ".ssh", ".aws", ".azure", ".netrc", ".git-credentials", ".npmrc", ".npm", "node_modules", ".venv"].includes(name)) continue; - await visit(relative ? `${relative}/${name}` : name); - } - } else if (stat.isFile()) { - await svc.write(folder, { path: relative, body: createReadStream(path.join(root, relative)), operationId: `import:${relative}`, onlyIfMissing: true, executable: Boolean(stat.mode & 0o111) }); - } else throw new Error("Managed agent home contains an unsupported file"); + for await (const file of managedAgentFiles(root)) { + const [receipt] = await db.select({ id: workFileOperations.id }).from(workFileOperations).where(and( + eq(workFileOperations.folderId, folder.id), eq(workFileOperations.operationId, `import:${file.path}`))); + if (receipt) continue; + await svc.write(folder, { ...file, operationId: `import:${file.path}`, onlyIfMissing: true }); } - try { await visit(""); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } await db.update(workFolders).set({ importedAt: new Date() }).where(eq(workFolders.id, folder.id)); } async function outgoing(scope: WorkFolderScope) { @@ -294,6 +280,12 @@ export async function prepareSandboxWorkFolders(input: { await db.update(workFolderRuns).set({ state, baselines, pendingOperations, manifest, error, updatedAt: new Date(), ...(state === "saved" ? { lastSavedAt: new Date() } : {}) }).where(eq(workFolderRuns.runId, input.runId)); } + async function recordCheckpoint(action: string) { + await logActivity(db, { companyId: input.companyId, actorType: "agent", actorId: input.agentId, + agentId: input.agentId, runId: input.runId, issueId: input.taskId, + responsibleUserIdOverride: input.responsibleUserId, action, entityType: "heartbeat_run", entityId: input.runId, + details: { scopes: WORK_FOLDER_SCOPES.filter((scope) => Boolean(folders[scope])), repositories: bindings.length } }); + } try { await seedAttachments(); await importAgentFiles(); @@ -301,6 +293,7 @@ export async function prepareSandboxWorkFolders(input: { if (previous) for (const scope of WORK_FOLDER_SCOPES) await outgoing(scope); for (const scope of WORK_FOLDER_SCOPES) await incoming(scope); await prepareRepositories(); + await recordCheckpoint("work_folder.prepared"); await saveState("starting"); if (previous?.refreshRequested) await db.update(workFolderRuns).set({ refreshRequested: false }) .where(eq(workFolderRuns.runId, previous.runId)); @@ -314,6 +307,7 @@ export async function prepareSandboxWorkFolders(input: { await saveState("saving"); for (const scope of WORK_FOLDER_SCOPES) await outgoing(scope); for (const { binding, root } of bindings) await repositories.checkpoint(binding, root); + await recordCheckpoint("work_folder.checkpoint"); await saveState("saved"); }, async onError() { await saveState("failed", "Files could not be saved; the sandbox must be retained for recovery"); }, diff --git a/server/src/services/work-folder-agent-import.ts b/server/src/services/work-folder-agent-import.ts new file mode 100644 index 0000000000..3805de6318 --- /dev/null +++ b/server/src/services/work-folder-agent-import.ts @@ -0,0 +1,48 @@ +import { constants } from "node:fs"; +import fs from "node:fs/promises"; +import path from "node:path"; +import type { Readable } from "node:stream"; + +const excluded = new Set([".codex", ".claude", ".cache", ".config", ".local", ".git", ".paperclip-runtime", ".ssh", ".aws", ".azure", ".netrc", ".git-credentials", ".npmrc", ".npm", "node_modules", ".venv"]); +export interface ManagedAgentFile { path: string; kind: "file" | "directory"; executable: boolean; body?: Readable } + +/** Pin directory descriptors on Linux so a concurrent rename cannot redirect imports. */ +export async function* managedAgentFiles(root: string): AsyncGenerator { + const absolute = path.resolve(root); + let current = path.parse(absolute).root; + let parent = await fs.open(current, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW); + const anchored = (fd: number, fallback: string) => process.platform === "linux" ? `/proc/self/fd/${fd}` : fallback; + try { + for (const segment of absolute.slice(current.length).split(path.sep).filter(Boolean)) { + const next = path.join(anchored(parent.fd, current), segment); + const child = await fs.open(next, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW).catch((error) => { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + }); + if (!child) return; + await parent.close(); parent = child; current = path.join(current, segment); + } + let entries = 0; + async function* visit(directory: typeof parent, fallback: string, relative: string): AsyncGenerator { + for (const name of (await fs.readdir(anchored(directory.fd, fallback))).sort()) { + if (excluded.has(name)) continue; + if (++entries > 100_000) throw new Error("Managed agent import exceeds its file limit"); + const filename = relative ? `${relative}/${name}` : name; + const source = path.join(anchored(directory.fd, fallback), name); + const handle = await fs.open(source, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const stat = await handle.stat(); + if (stat.isDirectory()) { + yield { path: filename, kind: "directory", executable: false }; + yield* visit(handle, path.join(fallback, name), filename); + } else if (stat.isFile() && stat.nlink === 1 && stat.size <= 1024 ** 3) { + const body = handle.createReadStream({ autoClose: false, highWaterMark: 256 * 1024 }); + try { yield { path: filename, kind: "file", executable: Boolean(stat.mode & 0o111), body }; } + finally { body.destroy(); } + } else throw new Error("Managed agent files cannot contain hard links or special files"); + } finally { await handle.close(); } + } + } + yield* visit(parent, absolute, ""); + } finally { await parent.close(); } +} diff --git a/server/src/services/work-folder-transport.ts b/server/src/services/work-folder-transport.ts index d1be428ebb..d8529775e8 100644 --- a/server/src/services/work-folder-transport.ts +++ b/server/src/services/work-folder-transport.ts @@ -11,6 +11,9 @@ const entrySchema = z.object({ path: z.string().refine((value) => { try { valida sha256: z.string().regex(/^[a-f0-9]{64}$/).nullable(), executable: z.boolean(), linkTarget: z.string().max(1024).optional() }); export type WorkTreeEntry = z.infer; let source: Promise | undefined; +// Requests are base64 encoded twice (file bytes, then JSON). Stay below +// Linux's 128 KiB single-argument limit, including a provider shell wrapper. +const WRITE_CHUNK_BYTES = 48 * 1024; export function workFolderTransport(runner: CommandManagedRuntimeRunner) { async function command(input: Record): Promise { @@ -44,8 +47,8 @@ export function workFolderTransport(runner: CommandManagedRuntimeRunner) { let offset = 0; for await (const value of body) { const chunk = Buffer.from(value); - for (let start = 0; start < chunk.length; start += 256 * 1024) { - const bytes = chunk.subarray(start, start + 256 * 1024); + for (let start = 0; start < chunk.length; start += WRITE_CHUNK_BYTES) { + const bytes = chunk.subarray(start, start + WRITE_CHUNK_BYTES); await command({ operation: "write", root: stagingRoot, path: stagingPath, offset, data: bytes.toString("base64") }); offset += bytes.length; } diff --git a/tests/runner-e2e/deployed-stack.test.ts b/tests/runner-e2e/deployed-stack.test.ts index 37b785863e..1382a16333 100644 --- a/tests/runner-e2e/deployed-stack.test.ts +++ b/tests/runner-e2e/deployed-stack.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { isStagingOrigin } from "./deployed-stack.js"; +import { isStagingOrigin, assertDeployedAdapterExclusions } from "./deployed-stack.js"; describe("deployed stack target", () => { it("requires an explicit HTTPS staging tenant and rejects credential-bearing URLs", () => { @@ -10,3 +10,10 @@ describe("deployed stack target", () => { } }); }); + +it("allows only the four explicitly deferred adapters to be excluded", () => { + expect(() => assertDeployedAdapterExclusions([{ adapterType: "cursor", reason: "Explicitly deferred by the user" }])).not.toThrow(); + for (const adapterType of ["codex_local", "claude_local", "opencode_local", "pi_local", "paperclip_runner"]) { + expect(() => assertDeployedAdapterExclusions([{ adapterType, reason: "skip" }])).toThrow("cannot be excluded"); + } +}); diff --git a/tests/runner-e2e/deployed-stack.ts b/tests/runner-e2e/deployed-stack.ts index 1a3ca295cf..a23f2d9dbf 100644 --- a/tests/runner-e2e/deployed-stack.ts +++ b/tests/runner-e2e/deployed-stack.ts @@ -29,12 +29,7 @@ export function loadDeployedStack(): DeployedStack { assert(/@sha256:[a-f0-9]{64}$/.test(manifest.sandboxImage), "Expected immutable sandbox image"); const uuid = /^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i; for (const key of ["companyId", "taskId", "agentId", "projectId"] as const) assert(uuid.test(manifest[key]), `Invalid ${key}`); - if (manifest.excludedAdapters !== undefined) { - assert(Array.isArray(manifest.excludedAdapters), "Invalid adapter exclusions"); - for (const exclusion of manifest.excludedAdapters) { - assert(typeof exclusion.adapterType === "string" && typeof exclusion.reason === "string" && exclusion.reason.trim().length > 0, "Exclusions require an explicit reason"); - } - } + assertDeployedAdapterExclusions(manifest.excludedAdapters); assert(Array.isArray(manifest.profiles) && manifest.profiles.length >= 7, "The seven baseline profiles are required"); for (const profile of manifest.profiles) { for (const key of ["id", "adapterType", "engine", "model", "qualification", "agentId"] as const) { @@ -78,3 +73,13 @@ export class DeployedStackApi { return response.json() as Promise; } } + +/** User-approved scope for this acceptance campaign; core engines cannot be excluded. */ +export function assertDeployedAdapterExclusions(exclusions: DeployedStack["excludedAdapters"]) { + if (exclusions === undefined) return; + assert(Array.isArray(exclusions), "Invalid adapter exclusions"); + for (const entry of exclusions) { + assert(["cursor", "gemini_local", "grok_local", "kimi_local"].includes(entry.adapterType), "Core acceptance adapters cannot be excluded"); + assert(typeof entry.reason === "string" && entry.reason.trim().length > 0, "Exclusions require an explicit reason"); + } +}