From fbf24941040f0c542abdf78ae5bb2b73113e8795 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 8 Sep 2026 19:27:56 -0500 Subject: [PATCH] Preserve existing sandbox workspaces across work-folder upgrades Keep established tasks on their original filesystem and session layout. Recover version-1 lease identity from host run records, preserve configuration checks, and retain old work when ownership or resume cannot be verified. Co-Authored-By: Paperclip --- doc/sandbox-work-folders.md | 25 ++++++ .../src/__tests__/environment-runtime.test.ts | 84 +++++++++++++++++++ server/src/services/environment-runtime.ts | 26 ++++-- server/src/services/heartbeat.ts | 6 +- .../src/services/legacy-sandbox-workspace.ts | 44 ++++++++++ 5 files changed, 179 insertions(+), 6 deletions(-) create mode 100644 server/src/services/legacy-sandbox-workspace.ts diff --git a/doc/sandbox-work-folders.md b/doc/sandbox-work-folders.md index c145be30a4..7c74b8b944 100644 --- a/doc/sandbox-work-folders.md +++ b/doc/sandbox-work-folders.md @@ -43,6 +43,31 @@ It preserves tool schemas, idempotent call IDs, cancellation, and the bridge's private-tool boundary. The bridge credential is injected at launch and excluded from persisted environment records; project extensions remain untrusted. +## Existing tasks and upgrade compatibility + +Tasks that have already completed a sandbox run without work-folder persistence +keep their original workspace, adapter file-sync/restore behavior, and provider +session directories. Upgrading does not move, clean, or replace those files. This +compatibility mode persists across turns and sandbox expiry; it does not claim +the new scoped-folder or repository-checkpoint durability guarantee for old tasks. +New tasks enter the scoped lifecycle below. Automatic migration of an old task's +working tree into scoped folders is not performed. + +Version-1 reusable leases obtain their missing task and responsible-user identity +from company-scoped host run records. Reuse still requires matching agent, task, +user, environment, workspace, provider, and configuration fingerprint. Missing or +conflicting identity records, configuration drift, or a failed resume retain the +old sandbox and report a recovery error instead of destroying its only copy. +The provider cannot opt a new task into this compatibility mode. + +Acceptance must resume representative pre-upgrade legacy and native tasks with +committed, staged, unstaged, and untracked work, verify their original paths and +usable continuation, and exercise their existing restore mechanism after a +sandbox restart. A newly created task passing the scoped-folder matrix does not +establish upgrade compatibility. + +## New sandbox tasks + Sandbox runs use the operating-system user's home directory. Both legacy adapters and the native runner enter the same host-owned lifecycle before dispatch. Local execution keeps its existing workspace and home behavior. diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index 46d4efe06c..e4e9e44e7f 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -21,6 +21,7 @@ import { environments, executionWorkspaces, heartbeatRuns, + issues, workFolderRuns, plugins, projects, @@ -29,6 +30,7 @@ import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; +import { bindLegacySandboxIdentity, taskUsesLegacySandboxWorkspace } from "../services/legacy-sandbox-workspace.js"; import { workFolderSandboxKey } from "../services/work-folder-retention.js"; import { resolveEnvironmentDriverConfigForRuntime } from "../services/environment-config.ts"; import { @@ -218,6 +220,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { } await db.delete(environmentLeases); await db.delete(heartbeatRuns); + await db.delete(issues); await db.delete(agents); await db.delete(environments); await db.delete(executionWorkspaces); @@ -467,6 +470,85 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { return { pluginId, companyId, agentId, environment, runId, executionWorkspaceId, reusableLease }; } + it("keeps an existing task's legacy sync contract after its sandbox expires, without affecting new tasks", async () => { + const seeded = await seedReusablePluginSandboxLease("codex_local"); + const taskId = randomUUID(); + await db.insert(issues).values({ id: taskId, companyId: seeded.companyId, title: "Pre-upgrade task" }); + await db.update(environmentLeases).set({ issueId: taskId, status: "expired" }).where(eq(environmentLeases.id, seeded.reusableLease.id)); + await db.update(heartbeatRuns).set({ status: "succeeded", contextSnapshot: { issueId: taskId } }).where(eq(heartbeatRuns.id, seeded.runId)); + expect(await taskUsesLegacySandboxWorkspace(db, seeded.companyId, taskId)).toBe(true); + expect(await taskUsesLegacySandboxWorkspace(db, randomUUID(), taskId)).toBe(false); + expect(await taskUsesLegacySandboxWorkspace(db, seeded.companyId, randomUUID())).toBe(false); + expect(await taskUsesLegacySandboxWorkspace(db, seeded.companyId, null)).toBe(false); + const scopedRunId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: scopedRunId, companyId: seeded.companyId, agentId: seeded.agentId, status: "succeeded" }); + await db.insert(workFolderRuns).values({ runId: scopedRunId, companyId: seeded.companyId, + manifest: { version: 1, companyId: seeded.companyId, runId: scopedRunId, taskId, agentId: seeded.agentId, + projectId: null, responsibleUserId: null, leaseId: seeded.reusableLease.id, sandboxKey: seeded.reusableLease.id, + home: "/home/sandbox", folders: { task: null, agent: null, user: null, project: null }, repositories: [] } }); + expect(await taskUsesLegacySandboxWorkspace(db, seeded.companyId, taskId)).toBe(false); + }); + + it("does not infer a legacy lease's private identity from conflicting or missing host records", async () => { + const seeded = await seedReusablePluginSandboxLease(); + const lease = { ...seeded.reusableLease, issueId: randomUUID(), metadata: { ...seeded.reusableLease.metadata, + reusableSandboxLease: { ...(seeded.reusableLease.metadata!.reusableSandboxLease as object), version: 1 } } }; + await db.update(heartbeatRuns).set({ contextSnapshot: { issueId: randomUUID() } }).where(eq(heartbeatRuns.id, seeded.runId)); + expect(await bindLegacySandboxIdentity(db, lease)).toBe(lease); + const missing = { ...lease, heartbeatRunId: randomUUID() }; + expect(await bindLegacySandboxIdentity(db, missing)).toBe(missing); + const foreign = { ...lease, companyId: randomUUID() }; + expect(await bindLegacySandboxIdentity(db, foreign)).toBe(foreign); + }); + + it.each(["codex_local", "paperclip_runner"])("preserves a pre-work-folders %s lease and rejects identity drift without destroying it", async (adapterType) => { + const seeded = await seedReusablePluginSandboxLease(adapterType); + const workerManager = { + isRunning: vi.fn((id: string) => id === seeded.pluginId), + call: vi.fn(async (_pluginId: string, method: string) => { + if (method === "environmentResumeLease") return { + providerLeaseId: seeded.reusableLease.providerLeaseId, + metadata: { remoteCwd: "/old/task/workspace", provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true }, + }; + if (method === "environmentReleaseLease") return undefined; + throw new Error(`Existing workspace must not be replaced: ${method}`); + }), + getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), + } as unknown as PluginWorkerManager; + const runtime = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); + const acquire = (runId: string) => runtime.acquireRunLease({ companyId: seeded.companyId, + environment: seeded.environment, issueId: null, agentId: seeded.agentId, heartbeatRunId: runId, adapterType, + persistedExecutionWorkspace: { id: seeded.executionWorkspaceId, mode: "shared_workspace" } }); + // Acquire once to obtain the same full config fingerprint the old release + // stored, then remove only the fields introduced by work folders. + const first = await acquire(seeded.runId); + const scope = first.lease.metadata!.reusableSandboxLease as Record; + const { responsibleUserId: _user, issueId: _issue, ...oldScope } = scope; + await db.update(environmentLeases).set({ status: "released", metadata: { ...first.lease.metadata, + reusableSandboxLease: { ...oldScope, version: 1 } } }).where(eq(environmentLeases.id, first.lease.id)); + const runId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: runId, companyId: seeded.companyId, agentId: seeded.agentId, status: "running" }); + const resumed = await acquire(runId); + expect(resumed.lease.providerLeaseId).toBe(first.lease.providerLeaseId); + expect(resumed.lease.metadata).toMatchObject({ workFolderLayout: "legacy", remoteCwd: "/old/task/workspace", + reusableSandboxLease: { version: 2, responsibleUserId: null, issueId: null } }); + // Compatibility persists on the next turn too; it is not a one-run bypass. + await db.update(environmentLeases).set({ status: "released" }).where(eq(environmentLeases.id, resumed.lease.id)); + const thirdId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: thirdId, companyId: seeded.companyId, agentId: seeded.agentId, status: "running" }); + const third = await acquire(thirdId); + expect(third.lease.metadata?.workFolderLayout).toBe("legacy"); + expect(third.lease.providerLeaseId).toBe(first.lease.providerLeaseId); + await db.update(environmentLeases).set({ status: "released" }).where(eq(environmentLeases.id, third.lease.id)); + const otherUserRun = randomUUID(); + await db.insert(heartbeatRuns).values({ id: otherUserRun, companyId: seeded.companyId, agentId: seeded.agentId, + responsibleUserId: "different-user", status: "running" }); + await expect(acquire(otherUserRun)).rejects.toThrow("original task, user, agent, and configuration"); + expect((await environmentService(db).getLeaseById(third.lease.id))?.status).toBe("retained"); + expect(workerManager.call).not.toHaveBeenCalledWith(seeded.pluginId, "environmentDestroyLease", expect.anything(), expect.anything()); + expect(workerManager.call).not.toHaveBeenCalledWith(seeded.pluginId, "environmentAcquireLease", expect.anything(), expect.anything()); + }); + it("retains a successful reusable sandbox lease without stopping the provider resource", async () => { const { pluginId, runId, reusableLease } = await seedReusablePluginSandboxLease(); const workerManager = { @@ -3436,6 +3518,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { timeoutMs: 1234, reuseLease: false, remoteCwd: "/workspace", + workFolderLayout: "legacy", // Provider metadata cannot bypass the host lifecycle. }, }; } @@ -3476,6 +3559,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { heartbeatRunId: runId, persistedExecutionWorkspace: null, }); + expect(acquired.lease.metadata?.workFolderLayout).toBe("scoped"); const executed = await runtimeWithPlugin.execute({ environment, lease: acquired.lease, diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index 76785ae56a..2570e940a7 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -33,6 +33,7 @@ import { runWithRuntimeParent, type StartupSpanContext, } from "@paperclipai/adapter-utils/acpx-engine/startup-timing"; +import { bindLegacySandboxIdentity, hasLegacySandboxWorkspace, taskUsesLegacySandboxWorkspace } from "./legacy-sandbox-workspace.js"; import { retainUnsavedWorkFolderLease } from "./work-folder-retention.js"; import { environmentService } from "./environments.js"; import { instanceSettingsService } from "./instance-settings.js"; @@ -1710,6 +1711,12 @@ function createSandboxEnvironmentDriver( for (const lease of input.leases) { if (reusableIds.has(lease.id)) continue; if (!reusableLeaseCanBeCleanedUp(lease)) continue; + if (hasLegacySandboxWorkspace(lease)) { + await db.update(environmentLeases).set({ status: "retained", expiresAt: null, + cleanupStatus: "failed", failureReason: "legacy_workspace_recovery_required", updatedAt: new Date() }) + .where(and(eq(environmentLeases.id, lease.id), eq(environmentLeases.companyId, lease.companyId))); + throw new Error("Existing sandbox workspace requires its original task, user, agent, and configuration; files were retained"); + } await destroyReusableSandboxLease({ environment: input.environment, lease, @@ -1758,6 +1765,7 @@ function createSandboxEnvironmentDriver( const [boundRun] = input.heartbeatRunId ? await db.select({ responsibleUserId: heartbeatRuns.responsibleUserId }) .from(heartbeatRuns).where(and(eq(heartbeatRuns.id, input.heartbeatRunId), eq(heartbeatRuns.companyId, input.companyId))) : []; const responsibleUserId = boundRun?.responsibleUserId ?? null; + const legacyWorkFolderLayout = await taskUsesLegacySandboxWorkspace(db, input.companyId, input.issueId); const storedParsed = parseEnvironmentDriverConfig(input.environment); const parsed = await resolveEnvironmentDriverConfigForRuntime(db, input.companyId, input.environment, { issueId: input.issueId, @@ -1860,7 +1868,8 @@ function createSandboxEnvironmentDriver( lease.metadata?.agentId === input.agentId, ) : []; - const reusableExistingLeases = reusableCandidateLeases.filter((lease) => + const identityBoundCandidates = await Promise.all(reusableCandidateLeases.map((lease) => bindLegacySandboxIdentity(db, lease))); + const reusableExistingLeases = identityBoundCandidates.filter((lease) => reusableSandboxLeaseScopeMatches({ lease, responsibleUserId, @@ -1879,7 +1888,7 @@ function createSandboxEnvironmentDriver( lease.heartbeatRunId === input.heartbeatRunId, }), ); - if (reusableCandidateLeases.some((lease) => lease.metadata?.workFolderRecoveryRequired === true && !reusableExistingLeases.includes(lease))) { + if (reusableCandidateLeases.some((lease) => lease.metadata?.workFolderRecoveryRequired === true && !reusableExistingLeases.some((candidate) => candidate.id === lease.id))) { throw new Error("Unsaved sandbox work requires recovery with its original run identity and configuration"); } if (reusableCandidateLeases.length > reusableExistingLeases.length) { @@ -1988,6 +1997,9 @@ function createSandboxEnvironmentDriver( }); } if (!providerLease) { + if (hasLegacySandboxWorkspace(reusableLease)) { + throw new Error("Existing sandbox could not be resumed; its original workspace was retained"); + } if (await retainUnsavedWorkFolderLease(db, reusableLease)) { throw new Error("Saved sandbox could not be resumed; unsaved work was retained for recovery"); } @@ -2080,6 +2092,7 @@ function createSandboxEnvironmentDriver( sandboxProviderPlugin: true, ...sandboxConfigForLeaseMetadata(storedConfig), ...sanitizedProviderMetadata, + workFolderLayout: legacyWorkFolderLayout || (reusableLease && hasLegacySandboxWorkspace(reusableLease)) ? "legacy" : "scoped", sandboxLeaseAcquisition: providerLease ? { outcome: "resumed", @@ -2222,7 +2235,8 @@ function createSandboxEnvironmentDriver( lease.metadata?.agentId === input.agentId, ) : []; - const reusableExistingLeases = reusableCandidateLeases.filter((lease) => + const identityBoundCandidates = await Promise.all(reusableCandidateLeases.map((lease) => bindLegacySandboxIdentity(db, lease))); + const reusableExistingLeases = identityBoundCandidates.filter((lease) => reusableSandboxLeaseScopeMatches({ lease, responsibleUserId, @@ -2241,7 +2255,7 @@ function createSandboxEnvironmentDriver( lease.heartbeatRunId === input.heartbeatRunId, }), ); - if (reusableCandidateLeases.some((lease) => lease.metadata?.workFolderRecoveryRequired === true && !reusableExistingLeases.includes(lease))) { + if (reusableCandidateLeases.some((lease) => lease.metadata?.workFolderRecoveryRequired === true && !reusableExistingLeases.some((candidate) => candidate.id === lease.id))) { throw new Error("Unsaved sandbox work requires recovery with its original run identity and configuration"); } if (reusableCandidateLeases.length > reusableExistingLeases.length) { @@ -2265,7 +2279,7 @@ function createSandboxEnvironmentDriver( let providerLease; try { - if (reusableLease?.metadata?.workFolderRecoveryRequired === true) { + if (reusableLease && (reusableLease.metadata?.workFolderRecoveryRequired === true || hasLegacySandboxWorkspace(reusableLease))) { // Recovery overrides the original ephemeral disposal policy, after // the full host-owned identity/configuration fingerprint matched. providerLease = await resumeSandboxProviderLease({ config: parsed.config, providerLeaseId: reusableLease.providerLeaseId! }); @@ -2328,6 +2342,7 @@ function createSandboxEnvironmentDriver( driver: input.environment.driver, executionWorkspaceMode: input.executionWorkspaceMode, ...providerLease.metadata, + workFolderLayout: legacyWorkFolderLayout || (reusableLease && hasLegacySandboxWorkspace(reusableLease)) ? "legacy" : "scoped", sandboxLeaseAcquisition: reusableLease && providerLease.providerLeaseId === reusableLease.providerLeaseId ? { @@ -3228,6 +3243,7 @@ function readString(value: unknown): string | null { // directly, so the worker never needs it as config. Drop every key here before // the runtime sends a config to a lifecycle RPC. const INTERNAL_PLUGIN_SANDBOX_CONFIG_KEYS = new Set([ + "workFolderLayout", "driver", "executionWorkspaceMode", "pluginId", diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 077d719f51..c3f44643e0 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -3,6 +3,7 @@ import { githubBrokerEnvironment } from "@paperclipai/adapter-utils/github-launc import { cleanupGitHubOperationLaunchers, prepareGitHubOperationLaunchers, startAdapterExecutionTargetPaperclipBridge } from "@paperclipai/adapter-utils/execution-target"; import fs from "node:fs/promises"; import { retainUnsavedWorkFolderLease, workFolderSandboxKey } from "./work-folder-retention.js"; +import { hasLegacySandboxWorkspace } from "./legacy-sandbox-workspace.js"; import { prepareSandboxWorkFolders } from "./sandbox-work-folders.js"; import { bindWarmSandboxWorkspace } from "./sandbox-workspace-binding.js"; import path from "node:path"; @@ -19760,7 +19761,10 @@ export function heartbeatService( await bindIssueToPersistedExecutionWorkspace(persistedExecutionWorkspace); const workspaceRealization = realizationResult.workspaceRealization; const executionTarget = realizationResult.executionTarget; - if (executionTarget?.kind === "remote" && executionTarget.transport === "sandbox") { + if (executionTarget?.kind === "remote" && executionTarget.transport === "sandbox" + && !hasLegacySandboxWorkspace(activeEnvironmentLease.lease)) { + // Existing task sandboxes keep their original adapter sync, cwd and CLI + // session homes. Do not migrate their only working copy during startup. // The coordinator owns folder identity, hydration and durability for // both legacy and native dispatch. Local execution never enters here. workFolderSaveFailed = true; diff --git a/server/src/services/legacy-sandbox-workspace.ts b/server/src/services/legacy-sandbox-workspace.ts new file mode 100644 index 0000000000..537c27d164 --- /dev/null +++ b/server/src/services/legacy-sandbox-workspace.ts @@ -0,0 +1,44 @@ +import { and, eq, isNull, notExists, sql } from "drizzle-orm"; +import { environmentLeases, heartbeatRuns, workFolderRuns, type Db } from "@paperclipai/db"; +import type { EnvironmentLease } from "@paperclipai/shared"; + +function record(value: unknown): Record | null { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? value as Record : null; +} + +export function hasLegacySandboxWorkspace(lease: Pick) { + return lease.metadata?.workFolderLayout === "legacy" + || record(lease.metadata?.reusableSandboxLease)?.version === 1; +} + +/** Keep the old sync/restore contract even after its provider sandbox expires. */ +export async function taskUsesLegacySandboxWorkspace(db: Db, companyId: string, issueId: string | null) { + if (!issueId) return false; + const [previous] = await db.select({ id: environmentLeases.id }).from(environmentLeases) + .innerJoin(heartbeatRuns, and(eq(heartbeatRuns.id, environmentLeases.heartbeatRunId), eq(heartbeatRuns.companyId, environmentLeases.companyId))) + .leftJoin(workFolderRuns, eq(workFolderRuns.runId, heartbeatRuns.id)) + .where(and(eq(environmentLeases.companyId, companyId), eq(environmentLeases.issueId, issueId), + sql`${environmentLeases.metadata}->>'driver' = 'sandbox'`, + eq(heartbeatRuns.status, "succeeded"), isNull(workFolderRuns.runId), + notExists(db.select({ id: workFolderRuns.runId }).from(workFolderRuns).where(and( + eq(workFolderRuns.companyId, companyId), sql`${workFolderRuns.manifest}->>'taskId' = ${issueId}`))))).limit(1); + return Boolean(previous); +} + +/** Recover the missing identity from host records, never from provider claims. */ +export async function bindLegacySandboxIdentity(db: Db, lease: EnvironmentLease): Promise { + const scope = record(lease.metadata?.reusableSandboxLease); + if (scope?.version !== 1 || !lease.heartbeatRunId) return lease; + const [run] = await db.select({ agentId: heartbeatRuns.agentId, + responsibleUserId: heartbeatRuns.responsibleUserId, context: heartbeatRuns.contextSnapshot }) + .from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, lease.companyId), eq(heartbeatRuns.id, lease.heartbeatRunId))); + if (!run || run.agentId !== scope.agentId || scope.companyId !== lease.companyId) return lease; + const context = run.context ?? {}; + const taskIds = [lease.issueId, context.issueId, context.taskId, context.taskKey] + .filter((value): value is string => typeof value === "string" && value.length > 0); + if (new Set(taskIds).size > 1) return lease; + const issueId = taskIds[0] ?? null; + return { ...lease, metadata: { ...lease.metadata, workFolderLayout: "legacy", + reusableSandboxLease: { ...scope, version: 2, responsibleUserId: run.responsibleUserId, issueId } } }; +}