diff --git a/doc/sandbox-work-folders.md b/doc/sandbox-work-folders.md index 4889de996f..d3d333b6f2 100644 --- a/doc/sandbox-work-folders.md +++ b/doc/sandbox-work-folders.md @@ -13,6 +13,24 @@ such as Daytona's region when the environment leaves them unspecified. Existing configuration and identity checks still reject incompatible reuse; stopping and resuming a compatible lease must reopen the same account-scoped provider handle. +Task workspace binding follows the sandbox's `reuseLease` setting independently +of the native runner's process lifecycle. Both `warm` and `per_turn` retain the +task's execution workspace and compatible provider session; restarting the +provider process is not a request for a fresh sandbox. Local execution and +sandboxes with reuse disabled retain their existing behavior. + +Session compatibility compares effective workspace settings. A project title or +description edit, and startup pinning an inherited workspace mode to the same +already-effective mode, do not invalidate the session. Actual project policy, +network settings, model, and identity changes remain compatibility boundaries. + +Periodic saves are best effort for files that continue changing during a scan; +the required final flush must still save the settled working copy or visibly +retain it for recovery. A continuously rewritten file missing one periodic save +and the existing file-preview polling lag are accepted limitations. Startup +latency should be compared with the legacy folder path on an equivalent workload, +rather than treating isolated cold-start timings as an independent acceptance gate. + The deployed acceptance entry point is `pnpm test:e2e:work-folders:deployed`. Set `PAPERCLIP_DEPLOYED_STACK_MANIFEST` to a JSON manifest matching `tests/runner-e2e/deployed-stack.ts`, `PAPERCLIP_DEPLOYED_STACK_AUTH` to a private @@ -63,6 +81,47 @@ from persisted environment records; project extensions remain untrusted. ## Existing tasks and upgrade compatibility +Session fingerprint normalization accepts exact fingerprints produced by the +previous algorithm for the same effective configuration, including an inherited +workspace mode that startup subsequently pinned. It does not waive model, secret +version, or workspace-policy changes. Subsequent session publication writes the +normalized fingerprint through the existing persistence path. + +Legacy Codex and Claude session codecs retain their remote execution identity. +Sandbox conversations bind to the physical provider sandbox and environment, +so creating a new host lease record for another turn does not discard the +conversation. Older records missing this metadata can be repaired from their +last successful host run only when the company, agent, responsible user, task, +workspace, environment, provider sandbox, working directory, and conversation +all match. An explicit conflicting identity is never overwritten. Local and SSH +session matching remain separate; a replacement sandbox cannot inherit a +conversation merely because its working-directory path is the same. +Codex configuration refresh replaces only the managed auth/config/skills entries; +it preserves the sandbox's rollout files and SQLite state, including WAL files. +Those provider-session files stay outside shared work-folder collections. +Claude also retains the MCP server identity used by its conversation. A +host-verified old record missing that identity may migrate with only the built-in +Paperclip server; unknown external MCP server sets do not bypass the existing +session compatibility check. +Failed sandbox acquisition retains its provisional resume claim immediately. +Startup also recovers provisional claims left active by older terminal runs; +it does not reclaim an executing run or a pending provider release. A temporary +provider startup failure therefore cannot permanently block the task's next run. +If folder preparation fails before publishing a new manifest, heartbeat explicitly +retains the lease even when the previous run completed a successful final save. + +An intentional native session reset (for example, changing the model) can start a +new conversation inside the task's retained sandbox. The host authorizes this +only when it mints a new logical session ID; the runner atomically claims an +absent session directory. Existing task files and previous conversation state +remain in place. A normal continuation, restart recovery, existing partial +session directory, or saved checkpoint cannot take this fresh-session path. + +Older reusable per-turn sandboxes may have a task-owned execution workspace with +no explicit reuse preference. Startup can recover that default binding only for +the same company, project, and source task. Explicit workspace preferences remain +authoritative, and workspace freshness and lease identity checks still apply. + Tasks that have already completed a sandbox run without work-folder persistence keep their original workspace, adapter file-sync/restore behavior, and provider session directories. Upgrading does not move, clean, or replace those files. This diff --git a/packages/adapter-utils/src/execution-target.ts b/packages/adapter-utils/src/execution-target.ts index 7058f63841..5cd3f73e6f 100644 --- a/packages/adapter-utils/src/execution-target.ts +++ b/packages/adapter-utils/src/execution-target.ts @@ -1287,6 +1287,9 @@ export function adapterExecutionTargetSessionIdentity( providerKey: target.providerKey ?? null, environmentId: target.environmentId ?? null, leaseId: target.leaseId ?? null, + ...(target.sandboxLeaseAcquisition?.providerLeaseId + ? { providerLeaseId: target.sandboxLeaseAcquisition.providerLeaseId } + : {}), remoteCwd: target.remoteCwd, }; } @@ -1305,7 +1308,9 @@ export function adapterExecutionTargetSessionMatches( readStringMeta(parsedSaved, "transport") === current?.transport && readStringMeta(parsedSaved, "providerKey") === current?.providerKey && readStringMeta(parsedSaved, "environmentId") === current?.environmentId && - readStringMeta(parsedSaved, "leaseId") === current?.leaseId && + (readStringMeta(parsedSaved, "providerLeaseId") + ? readStringMeta(parsedSaved, "providerLeaseId") === current?.providerLeaseId + : readStringMeta(parsedSaved, "leaseId") === current?.leaseId) && readStringMeta(parsedSaved, "remoteCwd") === current?.remoteCwd ); } diff --git a/packages/adapter-utils/src/session-execution-identity.ts b/packages/adapter-utils/src/session-execution-identity.ts new file mode 100644 index 0000000000..6a6cf40b6d --- /dev/null +++ b/packages/adapter-utils/src/session-execution-identity.ts @@ -0,0 +1,13 @@ +/** Keep only execution identity fields; never persist credentials or runner objects. */ +export function serializeSessionExecutionIdentity(value: unknown): Record | null { + if (value === undefined || value === null) return null; + if (typeof value !== "object" || Array.isArray(value)) return { transport: "invalid" }; + const record = value as Record; + const result: Record = {}; + for (const key of ["transport", "host", "username", "remoteCwd", "providerKey", "environmentId", "leaseId", "providerLeaseId"]) { + if (typeof record[key] === "string") result[key] = record[key]; + } + if (typeof record.port === "number" && Number.isFinite(record.port)) result.port = record.port; + // Malformed remote state must not become an empty, local-compatible identity. + return Object.keys(result).length > 0 ? result : { transport: "invalid" }; +} diff --git a/packages/adapters/claude-local/src/server/execute.ts b/packages/adapters/claude-local/src/server/execute.ts index 1d25cc9313..8fd70d5767 100644 --- a/packages/adapters/claude-local/src/server/execute.ts +++ b/packages/adapters/claude-local/src/server/execute.ts @@ -127,6 +127,19 @@ interface ClaudeRuntimeConfig { extraArgs: string[]; } +export function claudeSessionMcpServersMatch(input: { + savedIdentity: string; + currentIdentity: string; + currentConnectionIds: readonly (string | null | undefined)[]; + legacyPlatformSession: boolean; +}): boolean { + if (input.savedIdentity.length > 0) return input.savedIdentity === input.currentIdentity; + return input.currentConnectionIds.length === 0 || ( + input.legacyPlatformSession + && input.currentConnectionIds.every((id) => id === "paperclip-runtime-tools") + ); +} + export function claudeSessionCwdMatchesExecutionTarget(input: { runtimeSessionCwd: string; effectiveExecutionCwd: string; @@ -769,10 +782,14 @@ export async function execute(ctx: AdapterExecutionContext): Promise server.connectionId), + legacyPlatformSession: runtimeSessionParams.legacyPlatformMcpSession === true, + }); const isValidUuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(runtimeSessionId); const canResumeSession = runtimeSessionId.length > 0 && diff --git a/packages/adapters/claude-local/src/server/index.ts b/packages/adapters/claude-local/src/server/index.ts index bb677eb6f1..ea532cfaf5 100644 --- a/packages/adapters/claude-local/src/server/index.ts +++ b/packages/adapters/claude-local/src/server/index.ts @@ -1,4 +1,4 @@ -export { claudeSessionCwdMatchesExecutionTarget, execute, runClaudeLogin } from "./execute.js"; +export { claudeSessionCwdMatchesExecutionTarget, claudeSessionMcpServersMatch, execute, runClaudeLogin } from "./execute.js"; export * from "./acp.js"; export { getConfigSchema } from "./config-schema.js"; export { listClaudeSkills, syncClaudeSkills } from "./skills.js"; @@ -62,6 +62,7 @@ export type { SetupTokenLoginResult, RunSetupTokenLoginOptions, } from "./setup-token-runner.js"; +import { serializeSessionExecutionIdentity } from "@paperclipai/adapter-utils/session-execution-identity"; import type { AdapterSessionCodec } from "@paperclipai/adapter-utils"; import { sessionCodec as acpxSessionCodec } from "@paperclipai/adapter-utils/acpx-engine/session-codec"; @@ -85,9 +86,13 @@ export const sessionCodec: AdapterSessionCodec = { const workspaceId = readNonEmptyString(record.workspaceId) ?? readNonEmptyString(record.workspace_id); const repoUrl = readNonEmptyString(record.repoUrl) ?? readNonEmptyString(record.repo_url); const repoRef = readNonEmptyString(record.repoRef) ?? readNonEmptyString(record.repo_ref); + const remoteExecution = serializeSessionExecutionIdentity(record.remoteExecution); + const mcpServerIdentity = readNonEmptyString(record.mcpServerIdentity); return { sessionId, ...(cwd ? { cwd } : {}), + ...(remoteExecution ? { remoteExecution } : {}), + ...(mcpServerIdentity ? { mcpServerIdentity } : {}), ...(promptBundleKey ? { promptBundleKey } : {}), ...(workspaceId ? { workspaceId } : {}), ...(repoUrl ? { repoUrl } : {}), @@ -108,9 +113,13 @@ export const sessionCodec: AdapterSessionCodec = { const workspaceId = readNonEmptyString(params.workspaceId) ?? readNonEmptyString(params.workspace_id); const repoUrl = readNonEmptyString(params.repoUrl) ?? readNonEmptyString(params.repo_url); const repoRef = readNonEmptyString(params.repoRef) ?? readNonEmptyString(params.repo_ref); + const remoteExecution = serializeSessionExecutionIdentity(params.remoteExecution); + const mcpServerIdentity = readNonEmptyString(params.mcpServerIdentity); return { sessionId, ...(cwd ? { cwd } : {}), + ...(remoteExecution ? { remoteExecution } : {}), + ...(mcpServerIdentity ? { mcpServerIdentity } : {}), ...(promptBundleKey ? { promptBundleKey } : {}), ...(workspaceId ? { workspaceId } : {}), ...(repoUrl ? { repoUrl } : {}), diff --git a/packages/adapters/codex-local/src/server/codex-auth-merge-extract.sh b/packages/adapters/codex-local/src/server/codex-auth-merge-extract.sh index fdb1be03f4..4a772b6d29 100755 --- a/packages/adapters/codex-local/src/server/codex-auth-merge-extract.sh +++ b/packages/adapters/codex-local/src/server/codex-auth-merge-extract.sh @@ -47,8 +47,15 @@ if [ "$keep_sandbox" -eq 1 ]; then fi fi -rm -rf "$asset_dir" || exit 1 +# Refresh the managed config contribution, not the sandbox's provider state. +# Removing CODEX_HOME also removed sessions/ and the SQLite rollout index before +# every resume. Keep sandbox-created state in place, including WAL sidecars. +# These names match CODEX_SYNC_ALLOWLIST; omitted managed entries are revoked. +[ ! -L "$asset_dir" ] || exit 1 mkdir -p "$asset_dir" || exit 1 +for managed_entry in auth.json config.json config.toml instructions.md skills; do + rm -rf "$asset_dir/$managed_entry" || exit 1 +done find "$stage_dir" -mindepth 1 -maxdepth 1 ! -name "$auth_name" -exec mv -f -- {} "$asset_dir/" \; || exit 1 source_auth="$host_auth" diff --git a/packages/adapters/codex-local/src/server/codex-auth-merge.test.ts b/packages/adapters/codex-local/src/server/codex-auth-merge.test.ts index 502c069e94..f6b167f599 100644 --- a/packages/adapters/codex-local/src/server/codex-auth-merge.test.ts +++ b/packages/adapters/codex-local/src/server/codex-auth-merge.test.ts @@ -12,6 +12,7 @@ import { type SandboxSyncOperation, } from "@paperclipai/adapter-utils/sandbox-managed-runtime"; import { buildCodexAuthInboundProvision } from "./codex-auth-merge-scripts.js"; +import { CODEX_SYNC_ALLOWLIST } from "./codex-home.js"; const execFile = promisify(execFileCallback); @@ -50,12 +51,14 @@ describe("codex home auth merge on sandbox asset extract", () => { sandboxAuth?: string; hostAuth?: string; imageAuth?: string; + sandboxFiles?: Record; }): Promise<{ commandText: string; writtenPaths: string[]; finalAuth: string; finalMode: number; combinedOutput: string; + remoteHomeDir: string; }> { const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-codex-auth-merge-")); cleanupDirs.push(rootDir); @@ -67,6 +70,10 @@ describe("codex home auth merge on sandbox asset extract", () => { await mkdir(localWorkspaceDir, { recursive: true }); await mkdir(localHomeDir, { recursive: true }); await mkdir(remoteHomeDir, { recursive: true }); + for (const [name, contents] of Object.entries(input.sandboxFiles ?? {})) { + await mkdir(path.dirname(path.join(remoteHomeDir, name)), { recursive: true }); + await writeFile(path.join(remoteHomeDir, name), contents); + } await writeFile(path.join(localWorkspaceDir, "README.md"), "workspace\n", "utf8"); if (input.hostAuth !== undefined) { await writeFile(path.join(localHomeDir, "auth.json"), input.hostAuth, { mode: 0o600 }); @@ -162,9 +169,36 @@ describe("codex home auth merge on sandbox asset extract", () => { finalAuth: await readFile(finalAuthPath, "utf8"), finalMode: (await lstat(finalAuthPath)).mode & 0o777, combinedOutput: outputs.join("\n"), + remoteHomeDir, }; } + it("refreshes exactly the managed Codex home allowlist", async () => { + const script = await readFile(new URL("./codex-auth-merge-extract.sh", import.meta.url), "utf8"); + const names = script.match(/for managed_entry in ([^;]+); do/)?.[1].split(" ").sort(); + expect(names).toEqual([...CODEX_SYNC_ALLOWLIST].sort()); + }); + + it("preserves sandbox rollout files and SQLite state while revoking omitted managed config", async () => { + const state = { + "sessions/2026/09/11/rollout.jsonl": "existing conversation\n", + "state_5.sqlite": "database", "state_5.sqlite-wal": "pending changes", + "state_5.sqlite-shm": "shared memory", "session_index.jsonl": "index\n", + }; + const result = await runCodexHomeAssetExtract({ + hostAuth: apiKeyAuth("host"), sandboxFiles: { + ...state, "config.toml": "old config", "skills/old/SKILL.md": "revoked skill", + "instructions.md": "revoked instructions", + }, + }); + for (const [name, contents] of Object.entries(state)) { + expect(await readFile(path.join(result.remoteHomeDir, name), "utf8")).toBe(contents); + } + expect(await readFile(path.join(result.remoteHomeDir, "config.toml"), "utf8")).toBe('model = "gpt"\n'); + await expect(lstat(path.join(result.remoteHomeDir, "skills"))).rejects.toMatchObject({ code: "ENOENT" }); + await expect(lstat(path.join(result.remoteHomeDir, "instructions.md"))).rejects.toMatchObject({ code: "ENOENT" }); + }); + it("keeps a newer same-account sandbox auth.json and installs it atomically with mode 0600", async () => { const sandboxAuth = subscriptionAuth({ accountId: "acct-same", diff --git a/packages/adapters/codex-local/src/server/index.ts b/packages/adapters/codex-local/src/server/index.ts index cdbfb01822..f5fe218f66 100644 --- a/packages/adapters/codex-local/src/server/index.ts +++ b/packages/adapters/codex-local/src/server/index.ts @@ -60,6 +60,7 @@ export { fetchWithTimeout, codexHomeDir, } from "./quota.js"; +import { serializeSessionExecutionIdentity } from "@paperclipai/adapter-utils/session-execution-identity"; import type { AdapterSessionCodec } from "@paperclipai/adapter-utils"; import { sessionCodec as acpxSessionCodec } from "@paperclipai/adapter-utils/acpx-engine/session-codec"; @@ -80,9 +81,11 @@ export const sessionCodec: AdapterSessionCodec = { const workspaceId = readNonEmptyString(record.workspaceId) ?? readNonEmptyString(record.workspace_id); const repoUrl = readNonEmptyString(record.repoUrl) ?? readNonEmptyString(record.repo_url); const repoRef = readNonEmptyString(record.repoRef) ?? readNonEmptyString(record.repo_ref); + const remoteExecution = serializeSessionExecutionIdentity(record.remoteExecution); return { sessionId, ...(cwd ? { cwd } : {}), + ...(remoteExecution ? { remoteExecution } : {}), ...(workspaceId ? { workspaceId } : {}), ...(repoUrl ? { repoUrl } : {}), ...(repoRef ? { repoRef } : {}), @@ -99,9 +102,11 @@ export const sessionCodec: AdapterSessionCodec = { const workspaceId = readNonEmptyString(params.workspaceId) ?? readNonEmptyString(params.workspace_id); const repoUrl = readNonEmptyString(params.repoUrl) ?? readNonEmptyString(params.repo_url); const repoRef = readNonEmptyString(params.repoRef) ?? readNonEmptyString(params.repo_ref); + const remoteExecution = serializeSessionExecutionIdentity(params.remoteExecution); return { sessionId, ...(cwd ? { cwd } : {}), + ...(remoteExecution ? { remoteExecution } : {}), ...(workspaceId ? { workspaceId } : {}), ...(repoUrl ? { repoUrl } : {}), ...(repoRef ? { repoRef } : {}), diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index 0047733639..de6f5565ae 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -626,8 +626,14 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { expect(call).toHaveBeenCalledTimes(1); }); - it.each(["codex_local", "paperclip_runner"])("retains a failed %s resume through run cleanup and retries the original sandbox", async (adapterType) => { + it.each([["codex_local", true], ["paperclip_runner", true], ["codex_local", false], ["paperclip_runner", false]] as const)("retains a failed %s resume and retries the original sandbox (cleanup: %s)", async (adapterType, cleanupRan) => { const seeded = await seedReusablePluginSandboxLease(adapterType); + const taskId = randomUUID(); + await db.insert(issues).values({ id: taskId, companyId: seeded.companyId, title: "Resume recovery" }); + await db.update(environmentLeases).set({ issueId: taskId, metadata: { + ...seeded.reusableLease.metadata, + reusableSandboxLease: { ...(seeded.reusableLease.metadata!.reusableSandboxLease as object), issueId: taskId }, + } }).where(eq(environmentLeases.id, seeded.reusableLease.id)); let failResume = false; const workerManager = { isRunning: vi.fn(() => true), @@ -643,7 +649,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { const runtime = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const acquire = (runId: string) => runtime.acquireRunLease({ companyId: seeded.companyId, environment: seeded.environment, agentId: seeded.agentId, - heartbeatRunId: runId, issueId: null, adapterType, + heartbeatRunId: runId, issueId: taskId, adapterType, persistedExecutionWorkspace: { id: seeded.executionWorkspaceId, mode: "shared_workspace" }, }); const first = await acquire(seeded.runId); @@ -654,10 +660,19 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { failResume = true; await expect(acquire(failedRun)).rejects.toThrow("the lease was preserved"); await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, failedRun)); - const released = await runtime.releaseRunLeases(failedRun, "failed"); - expect(released).toHaveLength(1); - expect(released[0]!.lease).toMatchObject({ status: "retained", expiresAt: null, + const claimed = (await environmentService(db).listLeases(seeded.environment.id)) + .find(lease => lease.heartbeatRunId === failedRun)!; + expect(claimed).toMatchObject({ status: "retained", expiresAt: null, providerLeaseId: first.lease.providerLeaseId, metadata: { sandboxResumePending: true } }); + if (cleanupRan) { + const released = await runtime.releaseRunLeases(failedRun, "failed"); + expect(released).toHaveLength(0); // Already retained at the failed acquisition boundary. + expect((await environmentService(db).getLeaseById(claimed.id))?.status).toBe("retained"); + } else { + // Reproduce an older server's orphaned active claim, whose acquisition + // threw before heartbeat obtained a lease to release. + await db.update(environmentLeases).set({ status: "active" }).where(eq(environmentLeases.id, claimed.id)); + } expect(vi.mocked(workerManager.call).mock.calls.every((call) => call[1] === "environmentResumeLease")).toBe(true); const retryRun = randomUUID(); await db.insert(heartbeatRuns).values({ id: retryRun, companyId: seeded.companyId, agentId: seeded.agentId, status: "running" }); @@ -5168,7 +5183,8 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { expect.anything(), ); await expect(environmentService(db).getLeaseById(staleLease.id)).resolves.toMatchObject({ - status: "active", + status: "retained", failureReason: "sandbox_resume_incomplete", + providerLeaseId: "stale-plugin-lease", }); }); diff --git a/server/src/__tests__/fixtures/pre-normalization-session-fingerprints.json b/server/src/__tests__/fixtures/pre-normalization-session-fingerprints.json new file mode 100644 index 0000000000..5143cb5e62 --- /dev/null +++ b/server/src/__tests__/fixtures/pre-normalization-session-fingerprints.json @@ -0,0 +1,342 @@ +{ + "baseInput": { + "adapterType": "paperclip_runner", + "effectiveAdapterConfig": { + "command": "codex", + "model": "gpt-5.4-mini", + "env": { + "OPENAI_API_KEY": "resolved-secret-value", + "PLAIN_FLAG": "plain-value" + } + }, + "agentRuntimeConfig": { + "heartbeat": { + "maxConcurrentRuns": 1 + } + }, + "issueOverrides": null, + "workspaceConfig": { + "requestedMode": "shared_workspace", + "effectiveMode": "shared_workspace", + "projectConfigRevisionAt": "2026-06-01T00:00:00.000Z", + "projectPolicy": { + "enabled": true, + "defaultMode": "shared_workspace" + }, + "issueSettings": { + "mode": "shared_workspace" + } + }, + "environment": { + "selectionSource": "default", + "selectedEnvironmentId": "environment-1", + "selectedEnvironment": { + "id": "environment-1", + "driver": "local", + "configRevisionAt": "2026-06-01T00:00:00.000Z" + } + }, + "environmentEnv": { + "ENVIRONMENT_FLAG": "enabled" + }, + "projectEnv": { + "PROJECT_FLAG": "enabled" + }, + "routineEnv": null, + "secretManifest": [ + { + "configPath": "env.OPENAI_API_KEY", + "envKey": "OPENAI_API_KEY", + "secretId": "secret-1", + "bindingId": "binding-1", + "secretKey": "openai-api-key", + "version": 7, + "provider": "local_encrypted", + "outcome": "success" + } + ], + "runtimeSkills": [ + { + "key": "paperclip", + "runtimeName": "paperclip", + "source": "/tmp/paperclip/runtime-skills/paperclip", + "versionId": null, + "currentVersionId": "skill-version-1", + "sourceStatus": "available", + "missingDetail": null + } + ], + "agentConfigRevision": { + "id": "agent-config-revision-1", + "changedKeys": [ + "adapterConfig" + ], + "configRevisionAt": "2026-06-01T00:00:00.000Z" + } + }, + "cases": [ + { + "overrides": { + "adapterType": "codex_local", + "workspaceConfig": { + "requestedMode": "shared_workspace", + "effectiveMode": "shared_workspace", + "projectConfigRevisionAt": "2026-06-01T00:00:00.000Z", + "projectPolicy": { + "enabled": true, + "defaultMode": "shared_workspace" + }, + "issueSettings": null + } + }, + "metadata": { + "version": 1, + "fingerprint": "v1:sha256:65ff7f85f57a0c224b10d24d160fcbeff2a0cd3765d6c3953611a60289830ca0", + "categories": [ + "adapter", + "adapterConfig", + "agentRuntimeConfig", + "instructions", + "issueOverrides", + "workspaceConfig", + "environment", + "envBindings", + "secrets", + "runtimeSkills" + ], + "categoryFingerprints": { + "adapter": "v1:sha256:29804f650528878073c2438dfce2a079a9dd31cc2518c0fba591828dfbc67322", + "adapterConfig": "v1:sha256:127c71e718fe73bf91c94dcc8879f17ed484ea736b0f2e90d81bc873468ea634", + "agentRuntimeConfig": "v1:sha256:3c37c19f6928661cb205435737aa3283039119567a08df6d1a06c7e785326594", + "instructions": "v1:sha256:8ab53c8385958d7cc11bcd9833f77cb82f02f6b4061954c5ac2cf9d27630d4f6", + "issueOverrides": "v1:sha256:54a9d829899ea51ff0fa3b2b7ce8fa0e3f5b46b26862931646c05cc2bb623b48", + "workspaceConfig": "v1:sha256:6eb9a524164e629c0bd3690c30bb0580ff744a169333126c399415975b09ac27", + "environment": "v1:sha256:b2c8a01983988b1d799b543a0f98dc57c4d557f4ea1d9c018be1dec039a82815", + "envBindings": "v1:sha256:a93a8befb4e04e7cd65dbb9a4eb32b585b9f9a03171513576bcc6872aa30dea8", + "secrets": "v1:sha256:65d3dc5cc91f9076003b489701585236e7f54e68c5cdfa08b625a4536b4b7f96", + "runtimeSkills": "v1:sha256:9c70a22f33f7b6672f688675bb9bc5b7b9c9a2e3b4b3b3a9d313b6748cdc592d" + } + } + }, + { + "overrides": { + "adapterType": "codex_local", + "workspaceConfig": { + "requestedMode": "shared_workspace", + "effectiveMode": "shared_workspace", + "projectConfigRevisionAt": "2026-06-01T00:00:00.000Z", + "projectPolicy": { + "enabled": true, + "defaultMode": "shared_workspace" + }, + "issueSettings": {} + } + }, + "metadata": { + "version": 1, + "fingerprint": "v1:sha256:cfb12b7b170c2b1b16e6f179e201add4287d04f1d468044be32d42e01d4a903a", + "categories": [ + "adapter", + "adapterConfig", + "agentRuntimeConfig", + "instructions", + "issueOverrides", + "workspaceConfig", + "environment", + "envBindings", + "secrets", + "runtimeSkills" + ], + "categoryFingerprints": { + "adapter": "v1:sha256:29804f650528878073c2438dfce2a079a9dd31cc2518c0fba591828dfbc67322", + "adapterConfig": "v1:sha256:127c71e718fe73bf91c94dcc8879f17ed484ea736b0f2e90d81bc873468ea634", + "agentRuntimeConfig": "v1:sha256:3c37c19f6928661cb205435737aa3283039119567a08df6d1a06c7e785326594", + "instructions": "v1:sha256:8ab53c8385958d7cc11bcd9833f77cb82f02f6b4061954c5ac2cf9d27630d4f6", + "issueOverrides": "v1:sha256:54a9d829899ea51ff0fa3b2b7ce8fa0e3f5b46b26862931646c05cc2bb623b48", + "workspaceConfig": "v1:sha256:3c2a1ecc7d58ac2006e3be9d7af9ce37567fb439aab52c9cecfda254ec63e045", + "environment": "v1:sha256:b2c8a01983988b1d799b543a0f98dc57c4d557f4ea1d9c018be1dec039a82815", + "envBindings": "v1:sha256:a93a8befb4e04e7cd65dbb9a4eb32b585b9f9a03171513576bcc6872aa30dea8", + "secrets": "v1:sha256:65d3dc5cc91f9076003b489701585236e7f54e68c5cdfa08b625a4536b4b7f96", + "runtimeSkills": "v1:sha256:9c70a22f33f7b6672f688675bb9bc5b7b9c9a2e3b4b3b3a9d313b6748cdc592d" + } + } + }, + { + "overrides": { + "adapterType": "codex_local", + "workspaceConfig": { + "requestedMode": "shared_workspace", + "effectiveMode": "shared_workspace", + "projectConfigRevisionAt": "2026-06-01T00:00:00.000Z", + "projectPolicy": { + "enabled": true, + "defaultMode": "shared_workspace" + }, + "issueSettings": { + "mode": "shared_workspace" + } + } + }, + "metadata": { + "version": 1, + "fingerprint": "v1:sha256:c26eb71a3919d667731d1af324628cfbcdd8252fb83dc208502b2c85c10d6e80", + "categories": [ + "adapter", + "adapterConfig", + "agentRuntimeConfig", + "instructions", + "issueOverrides", + "workspaceConfig", + "environment", + "envBindings", + "secrets", + "runtimeSkills" + ], + "categoryFingerprints": { + "adapter": "v1:sha256:29804f650528878073c2438dfce2a079a9dd31cc2518c0fba591828dfbc67322", + "adapterConfig": "v1:sha256:127c71e718fe73bf91c94dcc8879f17ed484ea736b0f2e90d81bc873468ea634", + "agentRuntimeConfig": "v1:sha256:3c37c19f6928661cb205435737aa3283039119567a08df6d1a06c7e785326594", + "instructions": "v1:sha256:8ab53c8385958d7cc11bcd9833f77cb82f02f6b4061954c5ac2cf9d27630d4f6", + "issueOverrides": "v1:sha256:54a9d829899ea51ff0fa3b2b7ce8fa0e3f5b46b26862931646c05cc2bb623b48", + "workspaceConfig": "v1:sha256:601f668086c690a142041137dfc69543eabb0c2f73ae28f57eed51d31b7911c2", + "environment": "v1:sha256:b2c8a01983988b1d799b543a0f98dc57c4d557f4ea1d9c018be1dec039a82815", + "envBindings": "v1:sha256:a93a8befb4e04e7cd65dbb9a4eb32b585b9f9a03171513576bcc6872aa30dea8", + "secrets": "v1:sha256:65d3dc5cc91f9076003b489701585236e7f54e68c5cdfa08b625a4536b4b7f96", + "runtimeSkills": "v1:sha256:9c70a22f33f7b6672f688675bb9bc5b7b9c9a2e3b4b3b3a9d313b6748cdc592d" + } + } + }, + { + "overrides": { + "adapterType": "paperclip_runner", + "workspaceConfig": { + "requestedMode": "shared_workspace", + "effectiveMode": "shared_workspace", + "projectConfigRevisionAt": "2026-06-01T00:00:00.000Z", + "projectPolicy": { + "enabled": true, + "defaultMode": "shared_workspace" + }, + "issueSettings": null + } + }, + "metadata": { + "version": 1, + "fingerprint": "v1:sha256:aefbeafb1378268238414c7510376bb93c4f647f1a052fc766f93fb9997b6b13", + "categories": [ + "adapter", + "adapterConfig", + "agentRuntimeConfig", + "instructions", + "issueOverrides", + "workspaceConfig", + "environment", + "envBindings", + "secrets", + "runtimeSkills" + ], + "categoryFingerprints": { + "adapter": "v1:sha256:f6d7d8a401559cb95feafadaba9f60d56d2c80cca8295d6a3f650e2ddc9c5c1e", + "adapterConfig": "v1:sha256:127c71e718fe73bf91c94dcc8879f17ed484ea736b0f2e90d81bc873468ea634", + "agentRuntimeConfig": "v1:sha256:3c37c19f6928661cb205435737aa3283039119567a08df6d1a06c7e785326594", + "instructions": "v1:sha256:8ab53c8385958d7cc11bcd9833f77cb82f02f6b4061954c5ac2cf9d27630d4f6", + "issueOverrides": "v1:sha256:54a9d829899ea51ff0fa3b2b7ce8fa0e3f5b46b26862931646c05cc2bb623b48", + "workspaceConfig": "v1:sha256:6eb9a524164e629c0bd3690c30bb0580ff744a169333126c399415975b09ac27", + "environment": "v1:sha256:b2c8a01983988b1d799b543a0f98dc57c4d557f4ea1d9c018be1dec039a82815", + "envBindings": "v1:sha256:a93a8befb4e04e7cd65dbb9a4eb32b585b9f9a03171513576bcc6872aa30dea8", + "secrets": "v1:sha256:65d3dc5cc91f9076003b489701585236e7f54e68c5cdfa08b625a4536b4b7f96", + "runtimeSkills": "v1:sha256:9c70a22f33f7b6672f688675bb9bc5b7b9c9a2e3b4b3b3a9d313b6748cdc592d" + } + } + }, + { + "overrides": { + "adapterType": "paperclip_runner", + "workspaceConfig": { + "requestedMode": "shared_workspace", + "effectiveMode": "shared_workspace", + "projectConfigRevisionAt": "2026-06-01T00:00:00.000Z", + "projectPolicy": { + "enabled": true, + "defaultMode": "shared_workspace" + }, + "issueSettings": {} + } + }, + "metadata": { + "version": 1, + "fingerprint": "v1:sha256:d923a618fb1bd04e30a63f5d98bbc46508c330ef36a1a1187c2a2752e0b7e5cf", + "categories": [ + "adapter", + "adapterConfig", + "agentRuntimeConfig", + "instructions", + "issueOverrides", + "workspaceConfig", + "environment", + "envBindings", + "secrets", + "runtimeSkills" + ], + "categoryFingerprints": { + "adapter": "v1:sha256:f6d7d8a401559cb95feafadaba9f60d56d2c80cca8295d6a3f650e2ddc9c5c1e", + "adapterConfig": "v1:sha256:127c71e718fe73bf91c94dcc8879f17ed484ea736b0f2e90d81bc873468ea634", + "agentRuntimeConfig": "v1:sha256:3c37c19f6928661cb205435737aa3283039119567a08df6d1a06c7e785326594", + "instructions": "v1:sha256:8ab53c8385958d7cc11bcd9833f77cb82f02f6b4061954c5ac2cf9d27630d4f6", + "issueOverrides": "v1:sha256:54a9d829899ea51ff0fa3b2b7ce8fa0e3f5b46b26862931646c05cc2bb623b48", + "workspaceConfig": "v1:sha256:3c2a1ecc7d58ac2006e3be9d7af9ce37567fb439aab52c9cecfda254ec63e045", + "environment": "v1:sha256:b2c8a01983988b1d799b543a0f98dc57c4d557f4ea1d9c018be1dec039a82815", + "envBindings": "v1:sha256:a93a8befb4e04e7cd65dbb9a4eb32b585b9f9a03171513576bcc6872aa30dea8", + "secrets": "v1:sha256:65d3dc5cc91f9076003b489701585236e7f54e68c5cdfa08b625a4536b4b7f96", + "runtimeSkills": "v1:sha256:9c70a22f33f7b6672f688675bb9bc5b7b9c9a2e3b4b3b3a9d313b6748cdc592d" + } + } + }, + { + "overrides": { + "adapterType": "paperclip_runner", + "workspaceConfig": { + "requestedMode": "shared_workspace", + "effectiveMode": "shared_workspace", + "projectConfigRevisionAt": "2026-06-01T00:00:00.000Z", + "projectPolicy": { + "enabled": true, + "defaultMode": "shared_workspace" + }, + "issueSettings": { + "mode": "shared_workspace" + } + } + }, + "metadata": { + "version": 1, + "fingerprint": "v1:sha256:649054ef19224addf2d28d1d145cdfab310fcd6aa5c7b81892b3717f6071a01e", + "categories": [ + "adapter", + "adapterConfig", + "agentRuntimeConfig", + "instructions", + "issueOverrides", + "workspaceConfig", + "environment", + "envBindings", + "secrets", + "runtimeSkills" + ], + "categoryFingerprints": { + "adapter": "v1:sha256:f6d7d8a401559cb95feafadaba9f60d56d2c80cca8295d6a3f650e2ddc9c5c1e", + "adapterConfig": "v1:sha256:127c71e718fe73bf91c94dcc8879f17ed484ea736b0f2e90d81bc873468ea634", + "agentRuntimeConfig": "v1:sha256:3c37c19f6928661cb205435737aa3283039119567a08df6d1a06c7e785326594", + "instructions": "v1:sha256:8ab53c8385958d7cc11bcd9833f77cb82f02f6b4061954c5ac2cf9d27630d4f6", + "issueOverrides": "v1:sha256:54a9d829899ea51ff0fa3b2b7ce8fa0e3f5b46b26862931646c05cc2bb623b48", + "workspaceConfig": "v1:sha256:601f668086c690a142041137dfc69543eabb0c2f73ae28f57eed51d31b7911c2", + "environment": "v1:sha256:b2c8a01983988b1d799b543a0f98dc57c4d557f4ea1d9c018be1dec039a82815", + "envBindings": "v1:sha256:a93a8befb4e04e7cd65dbb9a4eb32b585b9f9a03171513576bcc6872aa30dea8", + "secrets": "v1:sha256:65d3dc5cc91f9076003b489701585236e7f54e68c5cdfa08b625a4536b4b7f96", + "runtimeSkills": "v1:sha256:9c70a22f33f7b6672f688675bb9bc5b7b9c9a2e3b4b3b3a9d313b6748cdc592d" + } + } + } + ], + "sourceCommit": "b58c10611ae5fe5b2f41e4c67e24835195fb604d" +} diff --git a/server/src/__tests__/heartbeat-workspace-session.test.ts b/server/src/__tests__/heartbeat-workspace-session.test.ts index 1da2071e2c..e8c4497285 100644 --- a/server/src/__tests__/heartbeat-workspace-session.test.ts +++ b/server/src/__tests__/heartbeat-workspace-session.test.ts @@ -1806,6 +1806,23 @@ describe("effective run execution workspace config freshness", () => { }, ); + it("recovers an older unpinned sandbox binding without overriding explicit workspace choices", () => { + const input = { + issueExecutionWorkspaceId: "workspace-old", + existingExecutionWorkspaceStatus: "active", + reuseDefaultSandboxWorkspace: true, + }; + expect(resolveExecutionWorkspaceReuseRequestForIssue(input).requestedShouldReuseExisting).toBe(true); + for (const issueExecutionWorkspacePreference of ["isolated_workspace", "operator_branch", "agent_default", "inherit"]) { + expect(resolveExecutionWorkspaceReuseRequestForIssue({ ...input, issueExecutionWorkspacePreference }) + .requestedShouldReuseExisting).toBe(false); + } + expect(resolveExecutionWorkspaceReuseRequestForIssue({ ...input, reuseDefaultSandboxWorkspace: false }) + .requestedShouldReuseExisting).toBe(false); + expect(resolveExecutionWorkspaceReuseRequestForIssue({ ...input, requestedExistingBranch: "different-branch" }) + .requestedShouldReuseExisting).toBe(false); + }); + it("keeps reusing an inherited workspace whose branch matches the pinned existing branch", () => { expect(resolveExecutionWorkspaceReuseRequestForIssue({ issueExecutionWorkspaceId: "workspace-old", @@ -2211,6 +2228,49 @@ function sessionParamsWithConfigMetadata( } describe("effective run session config freshness", () => { + it("upgrades actual pre-normalization native and legacy session fingerprints without resetting", async () => { + const fixture = JSON.parse(await fs.readFile( + new URL("./fixtures/pre-normalization-session-fingerprints.json", import.meta.url), "utf8", + )); + for (const prior of fixture.cases) { + const input = { ...fixture.baseInput, ...prior.overrides }; + const next = await buildEffectiveRunSessionConfigMetadata({ + ...input, + workspaceConfig: { + ...input.workspaceConfig, + issueSettings: { mode: "shared_workspace" }, + }, + }); + expect(next.fingerprint).not.toBe(prior.metadata.fingerprint); + expect(resolveTaskSessionConfigFreshness({ + hasTaskSession: true, configuredModel: "gpt-5.4-mini", + taskSessionParams: sessionParamsWithConfigMetadata(prior.metadata), configMetadata: next, + })).toMatchObject({ reset: false, reasons: [], nextFingerprint: next.fingerprint }); + // Once published, the normalized fingerprint is reusable without the old + // project timestamp: a later description edit remains compatible. + const later = await buildEffectiveRunSessionConfigMetadata({ + ...input, + workspaceConfig: { ...input.workspaceConfig, projectConfigRevisionAt: "2026-06-02T00:00:00Z" }, + }); + expect(resolveTaskSessionConfigFreshness({ + hasTaskSession: true, configuredModel: "gpt-5.4-mini", + taskSessionParams: sessionParamsWithConfigMetadata(next), configMetadata: later, + }).reset).toBe(false); + for (const change of [ + { effectiveAdapterConfig: { ...input.effectiveAdapterConfig, model: "other-model" } }, + { secretManifest: input.secretManifest.map((entry: Record) => ({ ...entry, version: 8 })) }, + { workspaceConfig: { ...input.workspaceConfig, issueSettings: { mode: "shared_workspace", networkEgress: "restricted" } } }, + { workspaceConfig: { ...input.workspaceConfig, projectPolicy: { enabled: true, defaultMode: "isolated_workspace" } } }, + ]) { + const incompatible = await buildEffectiveRunSessionConfigMetadata({ ...input, ...change }); + expect(resolveTaskSessionConfigFreshness({ + hasTaskSession: true, configuredModel: "gpt-5.4-mini", + taskSessionParams: sessionParamsWithConfigMetadata(prior.metadata), configMetadata: incompatible, + }).reset).toBe(true); + } + } + }); + it("resets when effective adapter config changes after model/profile/env resolution", async () => { const base = await buildSessionConfigMetadata(); const next = await buildSessionConfigMetadata({ @@ -2286,6 +2346,56 @@ describe("effective run session config freshness", () => { }); }); + it.each(["codex_local", "paperclip_runner"])( + "preserves %s sessions across project metadata updates but detects policy changes", + async (adapterType) => { + const workspaceConfig = { + requestedMode: "shared_workspace", + effectiveMode: "shared_workspace", + projectPolicy: { enabled: true, defaultMode: "shared_workspace" }, + issueSettings: null, + }; + const base = await buildSessionConfigMetadata({ + adapterType, + workspaceConfig: { + ...workspaceConfig, + projectConfigRevisionAt: "2026-06-01T00:00:00.000Z", + }, + }); + const metadataUpdate = await buildSessionConfigMetadata({ + adapterType, + workspaceConfig: { + ...workspaceConfig, + projectConfigRevisionAt: "2026-06-01T00:05:00.000Z", + }, + }); + const policyUpdate = await buildSessionConfigMetadata({ + adapterType, + workspaceConfig: { + ...workspaceConfig, + projectConfigRevisionAt: "2026-06-01T00:05:00.000Z", + projectPolicy: { enabled: true, defaultMode: "isolated_workspace" }, + }, + }); + const decide = (configMetadata: SessionConfigMetadata) => + resolveTaskSessionConfigFreshness({ + hasTaskSession: true, + configuredModel: "gpt-5.4-mini", + taskSessionParams: sessionParamsWithConfigMetadata(base), + configMetadata, + }); + expect(decide(metadataUpdate)).toMatchObject({ + reset: false, + changedCategories: [], + reasons: [], + }); + expect(decide(policyUpdate)).toMatchObject({ + reset: true, + changedCategories: ["workspaceConfig"], + }); + }, + ); + it("does not reset when a reusable execution workspace becomes realized", async () => { const base = await buildSessionConfigMetadata({ workspaceConfig: { @@ -2325,6 +2435,48 @@ describe("effective run session config freshness", () => { }); }); + it("keeps native session identity when startup pins the already-effective workspace mode", async () => { + const workspaceConfig = { + requestedMode: "shared_workspace", + effectiveMode: "shared_workspace", + projectPolicy: { enabled: true, defaultMode: "shared_workspace" }, + }; + const before = await buildSessionConfigMetadata({ + adapterType: "paperclip_runner", + workspaceConfig: { ...workspaceConfig, issueSettings: null }, + }); + const pinned = await buildSessionConfigMetadata({ + adapterType: "paperclip_runner", + workspaceConfig: { + ...workspaceConfig, + issueSettings: { mode: "shared_workspace" }, + }, + }); + const policyChanged = await buildSessionConfigMetadata({ + adapterType: "paperclip_runner", + workspaceConfig: { + ...workspaceConfig, + issueSettings: { mode: "shared_workspace", networkEgress: "restricted" }, + }, + }); + const decide = (configMetadata: SessionConfigMetadata) => + resolveTaskSessionConfigFreshness({ + hasTaskSession: true, + configuredModel: "gpt-5.4-mini", + taskSessionParams: sessionParamsWithConfigMetadata(before), + configMetadata, + }); + expect(decide(pinned)).toMatchObject({ + reset: false, + reasons: [], + changedCategories: [], + }); + expect(decide(policyChanged)).toMatchObject({ + reset: true, + changedCategories: ["workspaceConfig"], + }); + }); + it("keeps model-only compatibility as an additional reset reason", async () => { const base = await buildSessionConfigMetadata(); diff --git a/server/src/__tests__/legacy-sandbox-session.test.ts b/server/src/__tests__/legacy-sandbox-session.test.ts new file mode 100644 index 0000000000..0296e2ede1 --- /dev/null +++ b/server/src/__tests__/legacy-sandbox-session.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, it } from "vitest"; +import { sessionCodec as codex } from "@paperclipai/adapter-codex-local/server"; +import { sessionCodec as claude, claudeSessionMcpServersMatch } from "@paperclipai/adapter-claude-local/server"; +import { adapterExecutionTargetSessionIdentity, adapterExecutionTargetSessionMatches, type AdapterSandboxExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; +import { recoverLegacySandboxSession } from "../services/legacy-sandbox-session.js"; + +const target: AdapterSandboxExecutionTarget = { + kind: "remote", transport: "sandbox", providerKey: "daytona", environmentId: "env", + leaseId: "new-run-lease", remoteCwd: "/home/daytona/repos/main", + sandboxLeaseAcquisition: { outcome: "resumed", providerLeaseId: "physical-sandbox" }, +}; +function fixture(): Parameters[0] { + return { + adapterType: "codex_local", params: { sessionId: "conversation", cwd: target.remoteCwd }, target, + companyId: "company", agentId: "agent", taskId: "task", responsibleUserId: "user", + executionWorkspaceId: "workspace", + previousRun: { + companyId: "company", agentId: "agent", responsibleUserId: "user", status: "succeeded", + sessionIdAfter: "conversation", contextSnapshot: { + taskId: "task", executionWorkspaceId: "workspace", + paperclipEnvironment: { + driver: "sandbox", id: "env", leaseId: "old-run-lease", remoteCwd: target.remoteCwd, + workspaceRealization: { provider: "daytona", providerLeaseId: "physical-sandbox" }, + }, + }, + }, + }; +} + +describe("legacy sandbox conversation persistence", () => { + it("migrates only the built-in MCP server for a host-verified old Claude session", () => { + const input = fixture(); input.adapterType = "claude_local"; + const recovered = recoverLegacySandboxSession(input)!; + expect(recovered.legacyPlatformMcpSession).toBe(true); + const match = { savedIdentity: "", currentIdentity: "current", currentConnectionIds: ["paperclip-runtime-tools"], legacyPlatformSession: true }; + expect(claudeSessionMcpServersMatch(match)).toBe(true); + expect(claudeSessionMcpServersMatch({ ...match, legacyPlatformSession: false })).toBe(false); + expect(claudeSessionMcpServersMatch({ ...match, currentConnectionIds: ["external"] })).toBe(false); + expect(claudeSessionMcpServersMatch({ ...match, savedIdentity: "old" })).toBe(false); + const persisted = claude.serialize({ ...recovered, mcpServerIdentity: "current" }); + expect(persisted?.legacyPlatformMcpSession).toBeUndefined(); + expect(claude.deserialize(persisted)?.mcpServerIdentity).toBe("current"); + expect(claudeSessionMcpServersMatch({ ...match, savedIdentity: "current", legacyPlatformSession: false })).toBe(true); + }); + for (const [name, codec] of [["codex_local", codex], ["claude_local", claude]] as const) { + it(`${name} recovers old metadata, persists it, and resumes across host leases`, () => { + const input = fixture(); input.adapterType = name; + const recovered = recoverLegacySandboxSession(input); + const next = codec.deserialize(codec.serialize(recovered)); + expect(next?.sessionId).toBe("conversation"); + expect(next?.remoteExecution).toEqual(adapterExecutionTargetSessionIdentity(target)); + expect(adapterExecutionTargetSessionMatches(next?.remoteExecution, { ...target, leaseId: "third-run-lease" })).toBe(true); + expect(adapterExecutionTargetSessionMatches(next?.remoteExecution, { ...target, + sandboxLeaseAcquisition: { outcome: "replacement", providerLeaseId: "another-sandbox" }, + })).toBe(false); + expect(adapterExecutionTargetSessionMatches(next?.remoteExecution, { ...target, environmentId: "other-env" })).toBe(false); + expect(adapterExecutionTargetSessionMatches(next?.remoteExecution, { ...target, remoteCwd: "/other" })).toBe(false); + expect(adapterExecutionTargetSessionMatches(next?.remoteExecution, { kind: "local" })).toBe(false); + }); + + it(`${name} retains SSH identity and leaves local sessions unchanged`, () => { + const local = { sessionId: "local-session", cwd: "/tmp/work" }; + expect(codec.deserialize(codec.serialize(local))).toEqual(local); + const remoteExecution = { transport: "ssh", host: "host", username: "user", port: 22, remoteCwd: "/work" }; + expect(codec.deserialize(codec.serialize({ ...local, remoteExecution: { ...remoteExecution, secret: "discard-me" } }))?.remoteExecution).toEqual(remoteExecution); + expect(adapterExecutionTargetSessionMatches(codec.serialize({ ...local, remoteExecution: {} })?.remoteExecution, { kind: "local" })).toBe(false); + }); + } + + for (const field of ["companyId", "agentId", "taskId", "responsibleUserId", "executionWorkspaceId"] as const) { + it(`does not recover across a different ${field}`, () => { + const input = fixture(); input[field] = "another"; + expect(recoverLegacySandboxSession(input)).toBe(input.params); + }); + } + it("does not invent a resume after replacement, a changed cwd, an unknown run, or an explicit identity", () => { + for (const mutate of [ + (input: ReturnType) => { input.target = { ...target, sandboxLeaseAcquisition: { outcome: "replacement", providerLeaseId: "physical-sandbox" } }; }, + (input: ReturnType) => { input.target = { ...target, sandboxLeaseAcquisition: { outcome: "resumed", providerLeaseId: "other" } }; }, + (input: ReturnType) => { input.params!.cwd = "/other"; }, + (input: ReturnType) => { input.previousRun = null; }, + (input: ReturnType) => { input.previousRun!.sessionIdAfter = "other"; }, + (input: ReturnType) => { input.previousRun!.status = "failed"; }, + (input: ReturnType) => { input.params!.remoteExecution = { transport: "ssh" }; }, + (input: ReturnType) => { input.params!.remoteExecution = { transport: "sandbox", leaseId: "foreign" }; }, + (input: ReturnType) => { input.target = { kind: "local" }; }, + ]) { + const input = fixture(); mutate(input); + expect(recoverLegacySandboxSession(input)).toBe(input.params); + } + }); + it("keeps legacy host-lease matching when no physical identity is available", () => { + const oldTarget = { ...target, sandboxLeaseAcquisition: undefined }; + const saved = adapterExecutionTargetSessionIdentity(oldTarget); + expect(adapterExecutionTargetSessionMatches(saved, oldTarget)).toBe(true); + expect(adapterExecutionTargetSessionMatches(saved, { ...oldTarget, leaseId: "other" })).toBe(false); + }); +}); diff --git a/server/src/__tests__/sandbox-work-folders.test.ts b/server/src/__tests__/sandbox-work-folders.test.ts index 36304bbe2c..a45cc38c2f 100644 --- a/server/src/__tests__/sandbox-work-folders.test.ts +++ b/server/src/__tests__/sandbox-work-folders.test.ts @@ -11,7 +11,7 @@ import os from "node:os"; import { agents, assets, companyMemberships, issueAttachments, companies, createDb, heartbeatRuns, issues, environments, environmentLeases, executionWorkspaces, projects, projectWorkspaces, taskRepositoryBindings, workFolderObjects, workFolderRuns, startEmbeddedPostgresTestDatabase, type Db } from "@paperclipai/db"; import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js"; import { prepareSandboxWorkFolders } from "../services/sandbox-work-folders.js"; -import { bindWarmSandboxWorkspace } from "../services/sandbox-workspace-binding.js"; +import { bindReusableSandboxWorkspace, shouldBindReusableSandboxWorkspace } from "../services/sandbox-workspace-binding.js"; import { findUnboundLegacyTaskWorkspace } from "../services/legacy-sandbox-workspace.js"; import { retainUnsavedWorkFolderLease, workFolderSandboxKey } from "../services/work-folder-retention.js"; import * as activityLog from "../services/activity-log.js"; @@ -22,6 +22,27 @@ import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/com import { localTestWorkFolderRunner } from "./helpers/work-folder-runner.js"; const exec = promisify(execFile); +describe("sandbox workspace reuse policy", () => { + it.each(["warm", "per_turn", undefined])( + "pins reusable task workspaces independently of provider lifecycle %s", + (runnerLifecycleMode) => { + expect(shouldBindReusableSandboxWorkspace({ + driver: "sandbox", config: { reuseLease: true, runnerLifecycleMode }, + })).toBe(true); + }, + ); + + it.each([ + null, + { driver: "local", config: { reuseLease: true } }, + { driver: "ssh", config: { reuseLease: true } }, + { driver: "sandbox", config: { reuseLease: false, runnerLifecycleMode: "warm" } }, + { driver: "sandbox", config: {} }, + ])("leaves local and non-reusable execution unchanged: %j", (environment) => { + expect(shouldBindReusableSandboxWorkspace(environment)).toBe(false); + }); +}); + describe("shared sandbox work-folder lifecycle", () => { let database: Awaited>; let db: Db; @@ -113,14 +134,14 @@ describe("shared sandbox work-folder lifecycle", () => { await db.insert(executionWorkspaces).values({ id: workspaceId, companyId, projectId, sourceIssueId: task, mode: "shared_workspace", strategyType: "project_primary", name: "Warm binding" }); const input = { companyId, issueId: task, runId, agentId, workspaceId }; - await bindWarmSandboxWorkspace(db, input); + await bindReusableSandboxWorkspace(db, input); const [bound] = await db.select().from(issues).where(eq(issues.id, task)); expect(bound).toMatchObject({ executionWorkspaceId: workspaceId, executionWorkspacePreference: "reuse_existing", executionWorkspaceSettings: { mode: "shared_workspace" } }); for (const bad of [{ companyId: randomUUID() }, { agentId: randomUUID() }, { issueId: taskId }, { runId: randomUUID() }]) { - await expect(bindWarmSandboxWorkspace(db, { ...input, ...bad })).rejects.toThrow("active task run"); + await expect(bindReusableSandboxWorkspace(db, { ...input, ...bad })).rejects.toThrow("active task run"); } await db.update(executionWorkspaces).set({ sourceIssueId: taskId }).where(eq(executionWorkspaces.id, workspaceId)); - await expect(bindWarmSandboxWorkspace(db, input)).rejects.toThrow("active task run"); + await expect(bindReusableSandboxWorkspace(db, input)).rejects.toThrow("active task run"); await db.update(executionWorkspaces).set({ sourceIssueId: task }).where(eq(executionWorkspaces.id, workspaceId)); const originalLogActivity = activityLog.logActivity; let release!: () => void; @@ -130,7 +151,7 @@ describe("shared sandbox work-folder lifecycle", () => { const audit = vi.spyOn(activityLog, "logActivity").mockImplementationOnce(async (...args) => { entered(); await gate; return originalLogActivity(...args); }); - const pendingBinding = bindWarmSandboxWorkspace(db, input); + const pendingBinding = bindReusableSandboxWorkspace(db, input); try { await reached; // These state changes must wait until the validated binding commits. @@ -145,7 +166,7 @@ describe("shared sandbox work-folder lifecycle", () => { release(); await pendingBinding; audit.mockRestore(); } await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, runId)); - await expect(bindWarmSandboxWorkspace(db, input)).rejects.toThrow("active task run"); + await expect(bindReusableSandboxWorkspace(db, input)).rejects.toThrow("active task run"); }); async function prepare(home: string, leaseId: string, physicalId = leaseId, responsibleUserId: string | null = null, options: { taskId?: string; branchName?: string; agentId?: string; bulkStdin?: boolean } = {}) { @@ -535,6 +556,13 @@ describe("shared sandbox work-folder lifecycle", () => { expect(await retainUnsavedWorkFolderLease(db, { id: leaseId, companyId })).toBe(true); await run.stop(); active.splice(active.indexOf(run), 1); expect(await retainUnsavedWorkFolderLease(db, { id: leaseId, companyId })).toBe(false); + // The next startup may fail before writing a manifest; the old final save + // must not leave its working copy active and permanently blocking retries. + await db.update(environmentLeases).set({ status: "active" }).where(eq(environmentLeases.id, leaseId)); + expect(await retainUnsavedWorkFolderLease(db, { id: leaseId, companyId }, { runSaveFailed: true })).toBe(true); + const [retained] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, leaseId)); + expect(retained).toMatchObject({ status: "retained", expiresAt: null, + metadata: { workFolderRecoveryRequired: true } }); }, 120_000); it("reconciles file-directory replacements and preserves deleted children in trash", async () => { const svc = workFolderService(db, storage); diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index ee49428295..b15613547f 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -698,7 +698,7 @@ export class SandboxOrphanCleanupWriteError extends Error { } } -async function retainIncompleteSandboxResume(db: Db, lease: EnvironmentLease): Promise { +async function retainIncompleteSandboxResume(db: Db, lease: EnvironmentLease, requireTerminalRun = false): Promise { if (lease.metadata?.sandboxResumePending !== true || !lease.heartbeatRunId) return null; // A resume failure is not proof that the existing workspace is disposable. // Keep the claimed reference eligible for retry, without a provider release @@ -715,6 +715,11 @@ async function retainIncompleteSandboxResume(db: Db, lease: EnvironmentLease): P eq(environmentLeases.heartbeatRunId, lease.heartbeatRunId), inArray(environmentLeases.status, ["active", "retained"]), sql`${environmentLeases.metadata}->'sandboxResumePending' = 'true'::jsonb`, + ...(requireTerminalRun ? [sql`exists ( + select 1 from ${heartbeatRuns} where ${heartbeatRuns.id} = ${lease.heartbeatRunId} + and ${heartbeatRuns.companyId} = ${lease.companyId} + and ${heartbeatRuns.status} in ('succeeded', 'interrupted', 'failed', 'cancelled', 'timed_out') + )`] : []), )).returning(); return retained ? toEnvironmentLeaseSnapshot(retained) : null; } @@ -1837,8 +1842,10 @@ function createSandboxEnvironmentDriver( const handoffDeadline = Date.now() + 30_000; while (true) { const [holder] = await db.select({ + id: environmentLeases.id, providerLeaseId: environmentLeases.providerLeaseId, runStatus: heartbeatRuns.status, + resumePending: sql`${environmentLeases.metadata}->'sandboxResumePending' = 'true'::jsonb`, releaseFailureReason: environmentLeases.failureReason, releasePending: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) ? 'sandboxReleasePending'`, }).from(environmentLeases).innerJoin(heartbeatRuns, and( @@ -1859,6 +1866,14 @@ function createSandboxEnvironmentDriver( sql`${heartbeatRuns.responsibleUserId} is not distinct from ${responsibleUserId}`, )).limit(1); if (!holder) break; + // Older acquisition failures could leave an active provisional claim + // before the caller received a lease to clean up. Recover only a + // terminal owner's incomplete resume, never an executing/releasing run. + if (holder.resumePending && !holder.releasePending + && ["succeeded", "interrupted", "failed", "cancelled", "timed_out"].includes(holder.runStatus)) { + const incomplete = await environmentsSvc.getLeaseById(holder.id); + if (incomplete && await retainIncompleteSandboxResume(db, incomplete, true)) continue; + } if (holder.releasePending && (holder.releaseFailureReason === "sandbox_release_recovery_required" || Date.now() >= handoffDeadline)) throw new Error("sandbox_release_recovery_required"); if (!["succeeded", "interrupted", "failed", "cancelled", "timed_out"].includes(holder.runStatus) || Date.now() >= handoffDeadline) { throw new ReusableSandboxResumeError({ @@ -2096,6 +2111,7 @@ function createSandboxEnvironmentDriver( : "expired"; } } catch (error) { + await retainIncompleteSandboxResume(db, reusableLease); throw new ReusableSandboxResumeError({ provider: parsed.config.provider, providerLeaseId: reusableLease.providerLeaseId, @@ -2408,6 +2424,7 @@ function createSandboxEnvironmentDriver( }); } catch (error) { if (reusableLease) { + await retainIncompleteSandboxResume(db, reusableLease); throw new ReusableSandboxResumeError({ provider: parsed.config.provider, providerLeaseId: reusableLease.providerLeaseId!, diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 6236b7f306..ee71706282 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -7,8 +7,9 @@ import { cleanupGitHubOperationLaunchers, prepareGitHubOperationLaunchers } from import fs from "node:fs/promises"; import { retainUnsavedWorkFolderLease, workFolderSandboxKey } from "./work-folder-retention.js"; import { findUnboundLegacyTaskWorkspace, hasLegacySandboxWorkspace } from "./legacy-sandbox-workspace.js"; +import { recoverLegacySandboxSession } from "./legacy-sandbox-session.js"; import { prepareSandboxWorkFolders } from "./sandbox-work-folders.js"; -import { bindWarmSandboxWorkspace } from "./sandbox-workspace-binding.js"; +import { bindReusableSandboxWorkspace, shouldBindReusableSandboxWorkspace } from "./sandbox-workspace-binding.js"; import path from "node:path"; import { execFile as execFileCallback } from "node:child_process"; import { promisify } from "node:util"; @@ -5473,6 +5474,7 @@ type EffectiveRunSessionConfigMetadata = { categories: EffectiveRunSessionConfigCategory[]; categoryFingerprints: Record; fingerprints: EffectiveRunConfigFingerprints; + compatibleFingerprints: string[]; }; type TaskSessionConfigFreshnessDecision = { @@ -5556,6 +5558,7 @@ export function resolveExecutionWorkspaceReuseRequestForIssue(input: { existingExecutionWorkspaceStatus?: string | null; requestedExistingBranch?: string | null; existingExecutionWorkspaceBranchName?: string | null; + reuseDefaultSandboxWorkspace?: boolean; }): ExecutionWorkspaceReuseRequestForIssue { const requestedExecutionWorkspaceId = readNonEmptyString( input.issueExecutionWorkspaceId, @@ -5571,7 +5574,8 @@ export function resolveExecutionWorkspaceReuseRequestForIssue(input: { readNonEmptyString(input.existingExecutionWorkspaceBranchName) === requestedExistingBranch; const requestedShouldReuseExisting = - input.issueExecutionWorkspacePreference === "reuse_existing" && + (input.issueExecutionWorkspacePreference === "reuse_existing" || + (input.issueExecutionWorkspacePreference == null && input.reuseDefaultSandboxWorkspace === true)) && requestedExecutionWorkspaceId !== null && existingWorkspaceMatchesRequestedBranch; @@ -6109,6 +6113,20 @@ function buildSessionConfigCategoryValues(input: { // the timestamp here makes every comment invalidate an otherwise reusable // task session. delete workspaceConfig.issueConfigRevisionAt; + // A project's updatedAt also changes for its title, description, and other + // metadata. Compare the effective project policy and workspace settings + // below, not that timestamp, or an unrelated edit drops the native session + // while its reusable sandbox still contains the original provider state. + delete workspaceConfig.projectConfigRevisionAt; + // First sandbox startup pins the effective mode onto the issue. An inherited + // mode and that identical explicit pin describe the same execution contract. + // Keep other issue settings (network, credentials, etc.) in the fingerprint. + if (typeof workspaceConfig.effectiveMode === "string") { + workspaceConfig.issueSettings = { + ...parseObject(workspaceConfig.issueSettings), + mode: workspaceConfig.effectiveMode, + }; + } // This row is runtime state, not requested configuration. It is absent // before the first reusable run is realized and present on the next turn; // fingerprinting that transition would rotate the native session exactly @@ -6181,12 +6199,38 @@ export async function buildEffectiveRunSessionConfigMetadata(input: { subcategories: EFFECTIVE_RUN_SESSION_CONFIG_CATEGORIES, secretManifest, }); + // Pre-normalization releases hashed the project timestamp and the issue's + // raw mode setting. Accept only an exact old-algorithm hash of the current + // effective configuration, never a blanket exemption for workspace changes. + // A successful run publishes the new fingerprint through the usual path. + const legacyWorkspace: Record = { + ...categoryValues.workspaceConfig, + projectConfigRevisionAt: parseObject(input.workspaceConfig).projectConfigRevisionAt, + issueSettings: parseObject(input.workspaceConfig).issueSettings, + }; + const legacyWorkspaceVariants = [legacyWorkspace]; + const legacyIssueSettings = parseObject(legacyWorkspace.issueSettings); + if (legacyIssueSettings.mode === legacyWorkspace.effectiveMode) { + const inheritedSettings = { ...legacyIssueSettings }; + delete inheritedSettings.mode; + legacyWorkspaceVariants.push({ ...legacyWorkspace, issueSettings: inheritedSettings }); + if (Object.keys(inheritedSettings).length === 0) { + legacyWorkspaceVariants.push({ ...legacyWorkspace, issueSettings: null }); + } + } + const compatibleFingerprints = [...new Set(legacyWorkspaceVariants.map((workspaceConfig) => + createEffectiveRunConfigFingerprints({ + session: { ...categoryValues, workspaceConfig }, + secretManifest, + }).sessionFingerprint.fingerprint, + ))]; return { version: EFFECTIVE_RUN_CONFIG_FINGERPRINT_VERSION, fingerprint: fingerprints.sessionFingerprint.fingerprint, categories: [...EFFECTIVE_RUN_SESSION_CONFIG_CATEGORIES], categoryFingerprints, fingerprints, + compatibleFingerprints, }; } @@ -6534,7 +6578,8 @@ export function resolveTaskSessionConfigFreshness(input: { ); } else if ( storedConfig && - storedConfig.fingerprint !== input.configMetadata.fingerprint + storedConfig.fingerprint !== input.configMetadata.fingerprint && + !input.configMetadata.compatibleFingerprints.includes(storedConfig.fingerprint) ) { changedCategories = changedEffectiveRunSessionConfigCategories({ previous: storedConfig.categoryFingerprints, @@ -18849,6 +18894,15 @@ export function heartbeatService( : (issueRef?.executionWorkspacePreference ?? null), existingExecutionWorkspaceStatus: existingExecutionWorkspace?.status ?? null, + // Older per-turn runs recorded a task-owned workspace but left its + // reuse preference unset. Recover that binding without interpreting + // an explicit workspace choice as an upgrade default. Lease identity + // and configuration freshness still gate the actual restoration. + reuseDefaultSandboxWorkspace: + shouldBindReusableSandboxWorkspace(selectedEnvironmentForConfig) && + existingExecutionWorkspace?.companyId === agent.companyId && + existingExecutionWorkspace?.sourceIssueId === issueId && + existingExecutionWorkspace?.projectId === issueRef?.projectId, }); const requestedShouldReuseExisting = workspaceReuseRequest.requestedShouldReuseExisting; @@ -19415,10 +19469,8 @@ export function heartbeatService( issueRef?.executionWorkspacePreference ?? null; let issueExecutionWorkspaceModeForRun = issueExecutionWorkspaceSettings?.mode ?? null; - const warmReusableExecutionWorkspace = - selectedEnvironmentForConfig?.driver === "sandbox" && - selectedEnvironmentConfigForFingerprint.reuseLease === true && - selectedEnvironmentConfigForFingerprint.runnerLifecycleMode === "warm"; + const reusableSandboxExecutionWorkspace = + shouldBindReusableSandboxWorkspace(selectedEnvironmentForConfig); const bindIssueToPersistedExecutionWorkspace = async ( workspace: ExecutionWorkspace | null, ) => { @@ -19432,7 +19484,7 @@ export function heartbeatService( issueRef?.executionWorkspacePreference === "reuse_existing" || requestedExecutionWorkspaceMode === "isolated_workspace" || requestedExecutionWorkspaceMode === "operator_branch" || - warmReusableExecutionWorkspace; + reusableSandboxExecutionWorkspace; const nextIssuePatch: Record = {}; if (issueExecutionWorkspaceIdForRun !== workspace.id) { nextIssuePatch.executionWorkspaceId = workspace.id; @@ -19455,8 +19507,8 @@ export function heartbeatService( }; } if (Object.keys(nextIssuePatch).length > 0) { - if (warmReusableExecutionWorkspace && !isolatedWorkspacesEnabled) { - await measureSandboxOperation("heartbeat.bind_warm_sandbox_workspace", { operationIndex: 58 }, async () => (bindWarmSandboxWorkspace(db, { + if (reusableSandboxExecutionWorkspace && !isolatedWorkspacesEnabled) { + await measureSandboxOperation("heartbeat.bind_reusable_sandbox_workspace", { operationIndex: 58 }, async () => (bindReusableSandboxWorkspace(db, { companyId: agent.companyId, issueId, runId: run.id, agentId: agent.id, workspaceId: workspace.id, }))); } else { @@ -20037,6 +20089,24 @@ export function heartbeatService( updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, run.id)))); + if (taskSessionForRun?.lastRunId && previousSessionParams && issueId + && !explicitResumeSessionParams && !explicitResumeSessionDisplayId + && ["codex_local", "claude_local"].includes(agent.adapterType) + && executionTarget?.kind === "remote" && executionTarget.transport === "sandbox" + && executionTarget.sandboxLeaseAcquisition?.outcome === "resumed" + && !parseObject(previousSessionParams.remoteExecution).providerLeaseId) { + const previousRun = await db.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, taskSessionForRun.lastRunId), + eq(heartbeatRuns.companyId, agent.companyId), + eq(heartbeatRuns.agentId, agent.id), + )).then((rows) => rows[0] ?? null); + previousSessionParams = recoverLegacySandboxSession({ + adapterType: agent.adapterType, params: previousSessionParams, target: executionTarget, + companyId: agent.companyId, agentId: agent.id, taskId: issueId, + responsibleUserId: run.responsibleUserId ?? null, + executionWorkspaceId: persistedExecutionWorkspace?.id ?? null, previousRun, + }); + } const runtimeSessionResolution = resolveRuntimeSessionParamsForWorkspace({ agentId: agent.id, previousSessionParams, @@ -20600,6 +20670,7 @@ export function heartbeatService( workspaceId: persistedExecutionWorkspace?.id ?? null, }); let nativeExecution: NativeExecutionInput | null = null; + let freshNativeSessionAuthority: { runId: string; normalizedSessionId: string } | undefined; let nativeRunnerInstanceId: string | null = null; if (nativeRuntimeResolution.kind === "native") { if (!issueRef) { @@ -20977,6 +21048,13 @@ export function heartbeatService( } } } + // A retained physical sandbox may host a newly minted conversation + // after an intentional reset. Resume and restart paths never receive + // this authority; the executor also requires an absent session root. + if (!persistedNativeExecutionInput && !run.nativeSessionId && + nativeSessionId !== resumableTaskSessionId) { + freshNativeSessionAuthority = { runId: run.id, normalizedSessionId: nativeSessionId }; + } const nativeSandboxLifecycle = resolveNativeSandboxLifecycle({ adapterType: agent.adapterType, lifecyclePolicy: nativeExecution.session.lifecyclePolicy, @@ -21548,6 +21626,7 @@ export function heartbeatService( runnerInstanceId: nativeRunnerInstanceId, leaseOwner: runOptions.nativeLeaseOwner, restartRecovery: runOptions.nativeRestartRecovery, + freshSessionAuthority: freshNativeSessionAuthority, backend: options.nativeSessionBackendFactory?.(nativeExecution), useRunnerd: agent.adapterType === "paperclip_runner", @@ -23081,7 +23160,7 @@ export function heartbeatService( } if (workFolderSaveFailed && workFolderLeaseId) { const unsavedLeaseId = workFolderLeaseId; - await measureSandboxOperation("heartbeat.retain_unsaved_work_folder_lease.catch", { operationIndex: 250 }, async () => (retainUnsavedWorkFolderLease(db, { id: unsavedLeaseId, companyId: run.companyId }).catch((error) => { + await measureSandboxOperation("heartbeat.retain_unsaved_work_folder_lease.catch", { operationIndex: 250 }, async () => (retainUnsavedWorkFolderLease(db, { id: unsavedLeaseId, companyId: run.companyId }, { runSaveFailed: true }).catch((error) => { logger.error({ err: error, runId: run.id }, "Could not record work folder retention; lease remains active"); }))); } diff --git a/server/src/services/legacy-sandbox-session.ts b/server/src/services/legacy-sandbox-session.ts new file mode 100644 index 0000000000..55ea742246 --- /dev/null +++ b/server/src/services/legacy-sandbox-session.ts @@ -0,0 +1,58 @@ +import { + adapterExecutionTargetSessionIdentity, + type AdapterExecutionTarget, +} from "@paperclipai/adapter-utils/execution-target"; + +function record(value: unknown): Record { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? value as Record : {}; +} + +/** Recover metadata discarded by old CLI codecs only from host-owned run evidence. */ +export function recoverLegacySandboxSession(input: { + adapterType: string; + params: Record | null; + target: AdapterExecutionTarget | null; + companyId: string; + agentId: string; + taskId: string; + responsibleUserId: string | null; + executionWorkspaceId: string | null; + previousRun: { + companyId: string; + agentId: string; + responsibleUserId: string | null; + status: string; + sessionIdAfter: string | null; + contextSnapshot: unknown; + } | null; +}): Record | null { + const { params, target, previousRun: previous } = input; + if (!params || !previous || !["codex_local", "claude_local"].includes(input.adapterType) + || target?.kind !== "remote" || target.transport !== "sandbox" + || target.sandboxLeaseAcquisition?.outcome !== "resumed") return params; + const saved = record(params.remoteExecution); + if (saved.providerLeaseId || (params.remoteExecution != null && saved.transport !== "sandbox")) return params; + const context = record(previous.contextSnapshot); + const environment = record(context.paperclipEnvironment); + const realization = record(environment.workspaceRealization); + if (previous.status !== "succeeded" + || previous.companyId !== input.companyId || previous.agentId !== input.agentId + || previous.responsibleUserId !== input.responsibleUserId + || !params.sessionId || previous.sessionIdAfter !== params.sessionId + || context.taskId !== input.taskId + || !input.executionWorkspaceId || context.executionWorkspaceId !== input.executionWorkspaceId + || environment.driver !== "sandbox" || environment.id !== target.environmentId + || realization.provider !== target.providerKey + || realization.providerLeaseId !== target.sandboxLeaseAcquisition.providerLeaseId + || params.cwd !== target.remoteCwd || environment.remoteCwd !== params.cwd + || (params.remoteExecution != null && ( + saved.leaseId !== environment.leaseId || saved.environmentId !== environment.id + || saved.providerKey !== target.providerKey || saved.remoteCwd !== params.cwd + ))) return params; + return { + ...params, remoteExecution: adapterExecutionTargetSessionIdentity(target), + ...(input.adapterType === "claude_local" && !params.mcpServerIdentity + ? { legacyPlatformMcpSession: true } : {}), + }; +} diff --git a/server/src/services/native-runtime/fresh-native-sandbox-session.test.ts b/server/src/services/native-runtime/fresh-native-sandbox-session.test.ts new file mode 100644 index 0000000000..ab0a1aa5cf --- /dev/null +++ b/server/src/services/native-runtime/fresh-native-sandbox-session.test.ts @@ -0,0 +1,67 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { mkdtemp, mkdir, readFile, rm, symlink, writeFile, stat } from "node:fs/promises"; +import { spawnSync } from "node:child_process"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; +import { claimFreshNativeSandboxSession } from "./fresh-native-sandbox-session.js"; + +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +async function fixture() { + const root = await mkdtemp(join(tmpdir(), "native fresh ' session-")); + roots.push(root); + const execute = vi.fn(async input => { + const result = spawnSync(input.command, input.args ?? [], { encoding: "utf8" }); + return { pid: result.pid, startedAt: new Date().toISOString(), exitCode: result.status, signal: null, timedOut: false, stdout: result.stdout, stderr: result.stderr }; + }); + return { + authority: { runId: "run-new", normalizedSessionId: "session-new" }, + runId: "run-new", normalizedSessionId: "session-new", hasPriorState: false, + runner: { execute }, sessionRoot: join(root, "sessions", "new"), + }; +} + +describe("fresh native session on a retained sandbox", () => { + it("claims a separate private directory without changing old conversation or task files", async () => { + const input = await fixture(); + const old = join(input.sessionRoot, "..", "old"); + await mkdir(old, { recursive: true }); + await writeFile(join(old, "conversation"), "keep conversation"); + await claimFreshNativeSandboxSession(input); + expect((await stat(input.sessionRoot)).mode & 0o777).toBe(0o700); + expect(await readFile(join(old, "conversation"), "utf8")).toBe("keep conversation"); + await expect(claimFreshNativeSandboxSession(input)).rejects.toThrow("runner_harness_state_mismatch"); + }); + + it.each(["missing authority", "other run", "other session", "prior state"])("refuses %s before touching the sandbox", async scenario => { + const input = await fixture(); + const authority = scenario === "missing authority" ? undefined : { + runId: scenario === "other run" ? "old-run" : input.runId, + normalizedSessionId: scenario === "other session" ? "old-session" : input.normalizedSessionId, + }; + await expect(claimFreshNativeSandboxSession({ ...input, authority, hasPriorState: scenario === "prior state" })) + .rejects.toThrow("runner_harness_state_mismatch"); + expect(input.runner.execute).not.toHaveBeenCalled(); + }); + + it.each(["partial directory", "file", "symlink"])("preserves an existing %s", async scenario => { + const input = await fixture(); + await mkdir(join(input.sessionRoot, ".."), { recursive: true }); + if (scenario === "partial directory") { + await mkdir(input.sessionRoot); + await writeFile(join(input.sessionRoot, "partial"), "keep"); + } else if (scenario === "file") await writeFile(input.sessionRoot, "keep"); + else await symlink("missing-target", input.sessionRoot); + await expect(claimFreshNativeSandboxSession(input)).rejects.toThrow("runner_harness_state_mismatch"); + if (scenario === "partial directory") expect(await readFile(join(input.sessionRoot, "partial"), "utf8")).toBe("keep"); + if (scenario === "file") expect(await readFile(input.sessionRoot, "utf8")).toBe("keep"); + }); + + it.each(["timeout", "transport failure"])("fails closed on %s", async scenario => { + const input = await fixture(); + if (scenario === "timeout") input.runner.execute.mockResolvedValue({ pid: null, startedAt: new Date().toISOString(), exitCode: 0, signal: null, timedOut: true, stdout: "", stderr: "" }); + else input.runner.execute.mockRejectedValue(new Error("transport unavailable")); + await expect(claimFreshNativeSandboxSession(input)).rejects.toThrow(); + }); +}); diff --git a/server/src/services/native-runtime/fresh-native-sandbox-session.ts b/server/src/services/native-runtime/fresh-native-sandbox-session.ts new file mode 100644 index 0000000000..9f32aa2f90 --- /dev/null +++ b/server/src/services/native-runtime/fresh-native-sandbox-session.ts @@ -0,0 +1,35 @@ +import { posix } from "node:path"; +import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; + +/** Internal dispatch authority, never taken from adapter configuration or model input. */ +export interface FreshNativeSessionAuthority { + runId: string; + normalizedSessionId: string; +} + +export async function claimFreshNativeSandboxSession(input: { + authority?: FreshNativeSessionAuthority; + runId: string; + normalizedSessionId: string; + hasPriorState: boolean; + runner: CommandManagedRuntimeRunner; + sessionRoot: string; +}): Promise { + if (input.hasPriorState || !input.authority || + input.authority.runId !== input.runId || + input.authority.normalizedSessionId !== input.normalizedSessionId) { + throw new Error("runner_harness_state_mismatch"); + } + const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; + // mkdir without -p atomically proves the new session has no existing state, + // including an incomplete bootstrap or a symlink. Never clear an old root. + const claimed = await input.runner.execute({ + command: "sh", + args: ["-c", `umask 077; mkdir -p ${quote(posix.dirname(input.sessionRoot))} && mkdir ${quote(input.sessionRoot)}`], + bypassSession: true, + timeoutMs: 10_000, + }); + if (claimed.exitCode !== 0 || claimed.timedOut) { + throw new Error("runner_harness_state_mismatch"); + } +} diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index e4c28c4819..2c1dce8771 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -1,5 +1,6 @@ import { createHash, randomUUID } from "node:crypto"; import { execFileSync } from "node:child_process"; +import { claimFreshNativeSandboxSession, type FreshNativeSessionAuthority } from "./fresh-native-sandbox-session.js"; import { hasSandboxPerformanceTrace } from "../sandbox-performance.js"; import { adoptVerifiedRemoteRunner, @@ -3727,6 +3728,8 @@ export async function executePaperclipNativeSession(input: { runnerInstanceId: string; leaseOwner?: string; restartRecovery?: NativeRestartRecoveryClaim; + /** Host-issued only when this dispatch minted a new logical session. */ + freshSessionAuthority?: FreshNativeSessionAuthority; onSpawn?: (meta: { pid: number; processGroupId: number | null; @@ -6414,6 +6417,8 @@ export async function createRunnerdBackend(input: { execution: NativeExecutionInput; runnerInstanceId: string; restartRecovery?: NativeRestartRecoveryClaim; + /** Host-issued only when this dispatch minted a new logical session. */ + freshSessionAuthority?: FreshNativeSessionAuthority; durableEnvironmentLeaseId?: string; onSpawn?: (meta: { pid: number; @@ -7494,12 +7499,20 @@ async function createRunnerdBackendWithinSessionClaim( !state.runnerState || !state.providerSessionIdentity ) { - throw new Error("runner_harness_state_mismatch"); + await claimFreshNativeSandboxSession({ + authority: input.freshSessionAuthority, + runId: input.execution.binding.runId, + normalizedSessionId: nativeSessionKey(input.execution), + hasPriorState: Boolean(durableBinding) || backupAvailable || Boolean(input.restartRecovery), + runner: remoteCommandRunner, + sessionRoot: remoteSessionRoot!, + }); + } else { + await recordInPlaceHarnessReuse( + state.providerSessionIdentity, + reuseStartedAtMs, + ); } - await recordInPlaceHarnessReuse( - state.providerSessionIdentity, - reuseStartedAtMs, - ); } else if ( sandboxLeaseAcquisition?.outcome === "replacement" ) { diff --git a/server/src/services/sandbox-workspace-binding.ts b/server/src/services/sandbox-workspace-binding.ts index 9034fe3416..e34e406fd2 100644 --- a/server/src/services/sandbox-workspace-binding.ts +++ b/server/src/services/sandbox-workspace-binding.ts @@ -3,8 +3,21 @@ import { executionWorkspaces, heartbeatRuns, issues, type Db } from "@paperclipa import { logActivity, publishActivity, type ActivityPublication } from "./activity-log.js"; import { issueExecutionWorkspaceModeForPersistedWorkspace } from "./execution-workspace-policy.js"; +/** Sandbox reuse is independent of whether the provider process stays warm. */ +export function shouldBindReusableSandboxWorkspace(environment: { + driver: string; + config: unknown; +} | null | undefined): boolean { + const config = environment?.config; + return environment?.driver === "sandbox" + && typeof config === "object" + && config !== null + && !Array.isArray(config) + && (config as Record).reuseLease === true; +} + /** Host runtime state must survive even when user-configurable worktrees are disabled. */ -export async function bindWarmSandboxWorkspace(db: Db, input: { +export async function bindReusableSandboxWorkspace(db: Db, input: { companyId: string; issueId: string; runId: string; agentId: string; workspaceId: string; }) { const publications: ActivityPublication[] = []; @@ -22,7 +35,7 @@ export async function bindWarmSandboxWorkspace(db: Db, input: { )).for("update"); if (!issue || !run || !workspace || workspace.projectId !== issue.projectId || workspace.status !== "active" || (workspace.sourceIssueId !== null && workspace.sourceIssueId !== issue.id)) { - throw new Error("Warm sandbox workspace no longer belongs to this active task run"); + throw new Error("Reusable sandbox workspace no longer belongs to this active task run"); } await tx.update(issues).set({ executionWorkspaceId: workspace.id, executionWorkspacePreference: "reuse_existing", @@ -37,7 +50,7 @@ export async function bindWarmSandboxWorkspace(db: Db, input: { companyId: input.companyId, actorType: "agent", actorId: input.agentId, agentId: input.agentId, runId: input.runId, issueId: issue.id, action: "execution_workspace.sandbox_bound", entityType: "execution_workspace", entityId: workspace.id, - details: { issueId: issue.id, reason: "warm_sandbox_reuse" }, + details: { issueId: issue.id, reason: "sandbox_reuse" }, }, publications); }); for (const publication of publications) publishActivity(publication); diff --git a/server/src/services/work-folder-retention.ts b/server/src/services/work-folder-retention.ts index a80330bd58..0c2988c4e5 100644 --- a/server/src/services/work-folder-retention.ts +++ b/server/src/services/work-folder-retention.ts @@ -11,7 +11,7 @@ export function workFolderSandboxKey(lease: { id: string; companyId: string; env * A transferred old lease is protected without mutating or resurrecting its row. * A periodic checkpoint is not permission to discard edits made after it. */ -export async function retainUnsavedWorkFolderLease(db: Db, lease: { id: string; companyId: string }) { +export async function retainUnsavedWorkFolderLease(db: Db, lease: { id: string; companyId: string }, options: { runSaveFailed?: boolean } = {}) { const [row] = await db.select().from(environmentLeases).where(and(eq(environmentLeases.id, lease.id), eq(environmentLeases.companyId, lease.companyId))); if (!row) return false; // This old row no longer owns the physical resource. Protect it from stale @@ -22,7 +22,9 @@ export async function retainUnsavedWorkFolderLease(db: Db, lease: { id: string; .from(workFolderRuns).where(and(eq(workFolderRuns.companyId, lease.companyId), sql`(${workFolderRuns.manifest}->>'sandboxKey' = ${sandboxKey} or ${workFolderRuns.manifest}->>'leaseId' = ${lease.id})`)) .orderBy(desc(workFolderRuns.updatedAt)).limit(1); - if (!run || (run.state === "saved" && run.manifest.finalCheckpointAt)) return false; + // A startup failure can precede this run's manifest. The previous run's final + // save cannot attest that partial hydration or the current working copy is safe. + if (!options.runSaveFailed && (!run || (run.state === "saved" && run.manifest.finalCheckpointAt))) return false; await db.update(environmentLeases).set({ status: "retained", expiresAt: null, failureReason: "work_folder_save_required", cleanupStatus: "failed", metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || '{"workFolderRecoveryRequired":true}'::jsonb`,