diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.yml index 0c4fa59387..3179f5c8e1 100644 --- a/.github/workflows/runner-full-stack-e2e.yml +++ b/.github/workflows/runner-full-stack-e2e.yml @@ -14,7 +14,7 @@ on: type: boolean default: true group: - description: "Comma-separated groups (AND semantics: legacy,native,local,daytona,core,breadth)" + description: "Comma-separated groups (AND semantics: legacy,native,local,daytona,warm,core,breadth)" type: string required: false suite: diff --git a/packages/adapter-utils/src/command-managed-runtime.ts b/packages/adapter-utils/src/command-managed-runtime.ts index 36500fdc70..cdbae889cd 100644 --- a/packages/adapter-utils/src/command-managed-runtime.ts +++ b/packages/adapter-utils/src/command-managed-runtime.ts @@ -12,11 +12,15 @@ import { type SandboxRemoteExecutionSpec, type SandboxSyncOperation, type SandboxSyncResult, + type WorkspaceDurableSeedPaths, + type WorkspaceInboundMode, } from "./sandbox-managed-runtime.js"; import { preferredShellForSandbox, shellCommandArgs } from "./sandbox-shell.js"; import type { RunProcessResult } from "./server-utils.js"; import type { RuntimeProgressSink, RuntimeStatusSink } from "./runtime-progress.js"; import type { RuntimeSpanRunner } from "./acpx-engine/startup-timing.js"; +import type { GitWorkspaceSnapshot } from "./git-workspace-sync.js"; +import type { DirectorySnapshot } from "./workspace-restore-merge.js"; /** * Input for a duplex channel open. The caller supplies only the command argument @@ -516,6 +520,10 @@ export async function prepareCommandManagedRuntime(input: { workspaceLocalDir: string; workspaceRemoteDir?: string; syncWorkspace?: boolean; + workspaceInboundMode?: WorkspaceInboundMode; + workspaceDurableSeed?: WorkspaceDurableSeedPaths; + workspaceBaseline?: DirectorySnapshot; + workspaceGitSnapshot?: GitWorkspaceSnapshot | null; workspaceExclude?: string[]; preserveAbsentOnRestore?: string[]; assets?: CommandManagedRuntimeAsset[]; @@ -577,6 +585,10 @@ export async function prepareCommandManagedRuntime(input: { workspaceLocalDir: input.workspaceLocalDir, workspaceRemoteDir, syncWorkspace: input.syncWorkspace, + workspaceInboundMode: input.workspaceInboundMode, + workspaceDurableSeed: input.workspaceDurableSeed, + workspaceBaseline: input.workspaceBaseline, + workspaceGitSnapshot: input.workspaceGitSnapshot, workspaceExclude: mergeRuntimeExcludes(input.workspaceExclude), preserveAbsentOnRestore: input.preserveAbsentOnRestore, assets: input.assets, @@ -616,6 +628,10 @@ export async function prepareCommandManagedRuntime(input: { workspaceLocalDir: input.workspaceLocalDir, workspaceRemoteDir, syncWorkspace: input.syncWorkspace, + workspaceInboundMode: input.workspaceInboundMode, + workspaceDurableSeed: input.workspaceDurableSeed, + workspaceBaseline: input.workspaceBaseline, + workspaceGitSnapshot: input.workspaceGitSnapshot, workspaceExclude: mergeRuntimeExcludes(input.workspaceExclude), preserveAbsentOnRestore: input.preserveAbsentOnRestore, assets: input.assets, diff --git a/packages/adapter-utils/src/execution-target.ts b/packages/adapter-utils/src/execution-target.ts index db0fed0d72..9ea8730cb2 100644 --- a/packages/adapter-utils/src/execution-target.ts +++ b/packages/adapter-utils/src/execution-target.ts @@ -18,10 +18,12 @@ import { import type { AdditionalSourceStagingFailure, SandboxAdditionalSource, + WorkspaceDurableSeedPaths, + WorkspaceInboundMode, } from "./sandbox-managed-runtime.js"; -export { - resolveReferencedSourceIgnore, -} from "./sandbox-managed-runtime.js"; +import type { GitWorkspaceSnapshot } from "./git-workspace-sync.js"; +import type { DirectorySnapshot } from "./workspace-restore-merge.js"; +export { resolveReferencedSourceIgnore } from "./sandbox-managed-runtime.js"; export type { AdditionalSourceStagingFailure, ReferencedSourceIgnoreResolution, @@ -249,6 +251,10 @@ export interface PreparedAdapterExecutionTargetRuntime { * stage referenced projects, or when every requested project staged. */ additionalSourceFailures: AdditionalSourceStagingFailure[]; + workspaceSyncSnapshot: { + baseline: DirectorySnapshot; + gitSnapshot: GitWorkspaceSnapshot | null; + } | null; restoreWorkspace(onProgress?: RuntimeProgressSink): Promise; } @@ -1374,6 +1380,10 @@ export async function prepareAdapterExecutionTargetRuntime(input: { timeoutSec?: number; workspaceRemoteDir?: string; syncWorkspace?: boolean; + workspaceInboundMode?: WorkspaceInboundMode; + workspaceDurableSeed?: WorkspaceDurableSeedPaths; + workspaceBaseline?: DirectorySnapshot; + workspaceGitSnapshot?: GitWorkspaceSnapshot | null; workspaceExclude?: string[]; preserveAbsentOnRestore?: string[]; assets?: AdapterManagedRuntimeAsset[]; @@ -1403,6 +1413,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: { assetDirs: {}, additionalSourceDirs: {}, additionalSourceFailures: [], + workspaceSyncSnapshot: null, restoreWorkspace: async () => {}, }; } @@ -1428,6 +1439,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: { // The SSH transport does not stage referenced projects (it is out of scope), so it never // reports a per-project staging failure. additionalSourceFailures: [], + workspaceSyncSnapshot: null, restoreWorkspace: prepared.restoreWorkspace, }; } @@ -1448,6 +1460,10 @@ export async function prepareAdapterExecutionTargetRuntime(input: { workspaceLocalDir: input.workspaceLocalDir, workspaceRemoteDir: input.workspaceRemoteDir, syncWorkspace: input.syncWorkspace, + workspaceInboundMode: input.workspaceInboundMode, + workspaceDurableSeed: input.workspaceDurableSeed, + workspaceBaseline: input.workspaceBaseline, + workspaceGitSnapshot: input.workspaceGitSnapshot, workspaceExclude: input.workspaceExclude, preserveAbsentOnRestore: input.preserveAbsentOnRestore, assets: input.assets, @@ -1465,6 +1481,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: { assetDirs: prepared.assetDirs, additionalSourceDirs: prepared.additionalSourceDirs, additionalSourceFailures: prepared.additionalSourceFailures, + workspaceSyncSnapshot: prepared.workspaceSyncSnapshot, restoreWorkspace: prepared.restoreWorkspace, }; } diff --git a/packages/adapter-utils/src/sandbox-managed-runtime.test.ts b/packages/adapter-utils/src/sandbox-managed-runtime.test.ts index be36585a1e..387ac215df 100644 --- a/packages/adapter-utils/src/sandbox-managed-runtime.test.ts +++ b/packages/adapter-utils/src/sandbox-managed-runtime.test.ts @@ -348,6 +348,129 @@ describe("sandbox managed runtime", () => { } }); + it("adopts a warm remote workspace without inbound overwrite and still merges outbound changes", async () => { + const rootDir = await mkdtemp( + path.join(os.tmpdir(), "paperclip-sandbox-adopt-"), + ); + cleanupDirs.push(rootDir); + const localWorkspaceDir = path.join(rootDir, "local-workspace"); + const remoteWorkspaceDir = path.join(rootDir, "remote-workspace"); + await mkdir(localWorkspaceDir, { recursive: true }); + await mkdir(remoteWorkspaceDir, { recursive: true }); + await writeFile( + path.join(localWorkspaceDir, "continuity.txt"), + "host baseline\n", + "utf8", + ); + await writeFile( + path.join(remoteWorkspaceDir, "continuity.txt"), + "remote retained\n", + "utf8", + ); + const client = makeFilesystemClient(); + const syncIn = vi.spyOn(client, "syncIn"); + + const prepared = await prepareSandboxManagedRuntime({ + spec: { + transport: "sandbox", + provider: "test", + sandboxId: "sandbox-warm", + remoteCwd: remoteWorkspaceDir, + timeoutMs: 30_000, + apiKey: null, + }, + adapterKey: "test-adapter", + client, + workspaceLocalDir: localWorkspaceDir, + workspaceInboundMode: "adopt_remote", + }); + + expect(syncIn).not.toHaveBeenCalled(); + await expect( + readFile(path.join(remoteWorkspaceDir, "continuity.txt"), "utf8"), + ).resolves.toBe("remote retained\n"); + expect(prepared.workspaceSyncSnapshot).not.toBeNull(); + + await writeFile( + path.join(remoteWorkspaceDir, "continuity.txt"), + "remote finalized\n", + "utf8", + ); + await prepared.restoreWorkspace(); + await expect( + readFile(path.join(localWorkspaceDir, "continuity.txt"), "utf8"), + ).resolves.toBe("remote finalized\n"); + }); + + it("reconstructs a replacement workspace from the exact durable pre-turn seed", async () => { + const rootDir = await mkdtemp( + path.join(os.tmpdir(), "paperclip-sandbox-durable-seed-"), + ); + cleanupDirs.push(rootDir); + const localWorkspaceDir = path.join(rootDir, "local-workspace"); + const firstRemoteDir = path.join(rootDir, "first-remote"); + const replacementRemoteDir = path.join(rootDir, "replacement-remote"); + const durableSeed = { + workspaceArchivePath: path.join(rootDir, "state", "workspace.tar"), + gitArchivePath: path.join(rootDir, "state", "git.tar"), + }; + await mkdir(localWorkspaceDir, { recursive: true }); + await writeFile( + path.join(localWorkspaceDir, "continuity.txt"), + "durable pre-turn bytes\n", + "utf8", + ); + + const first = await prepareSandboxManagedRuntime({ + spec: { + transport: "sandbox", + provider: "test", + sandboxId: "sandbox-first", + remoteCwd: firstRemoteDir, + timeoutMs: 30_000, + apiKey: null, + }, + adapterKey: "test-adapter", + client: makeFilesystemClient(), + workspaceLocalDir: localWorkspaceDir, + workspaceInboundMode: "host_current", + workspaceDurableSeed: durableSeed, + }); + expect(first.workspaceSyncSnapshot).not.toBeNull(); + await expect(stat(durableSeed.workspaceArchivePath)).resolves.toMatchObject( + { + mode: expect.any(Number), + }, + ); + + await writeFile( + path.join(localWorkspaceDir, "continuity.txt"), + "concurrent host edit must not enter replacement\n", + "utf8", + ); + await prepareSandboxManagedRuntime({ + spec: { + transport: "sandbox", + provider: "test", + sandboxId: "sandbox-replacement", + remoteCwd: replacementRemoteDir, + timeoutMs: 30_000, + apiKey: null, + }, + adapterKey: "test-adapter", + client: makeFilesystemClient(), + workspaceLocalDir: localWorkspaceDir, + workspaceInboundMode: "durable_seed", + workspaceDurableSeed: durableSeed, + workspaceBaseline: first.workspaceSyncSnapshot!.baseline, + workspaceGitSnapshot: first.workspaceSyncSnapshot!.gitSnapshot, + }); + + await expect( + readFile(path.join(replacementRemoteDir, "continuity.txt"), "utf8"), + ).resolves.toBe("durable pre-turn bytes\n"); + }); + it("preserves excluded local workspace artifacts during restore mirroring", async () => { const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-sandbox-restore-")); cleanupDirs.push(rootDir); diff --git a/packages/adapter-utils/src/sandbox-managed-runtime.ts b/packages/adapter-utils/src/sandbox-managed-runtime.ts index d3de812500..937c812ebf 100644 --- a/packages/adapter-utils/src/sandbox-managed-runtime.ts +++ b/packages/adapter-utils/src/sandbox-managed-runtime.ts @@ -1,6 +1,10 @@ import { execFile as execFileCallback } from "node:child_process"; -import { randomUUID } from "node:crypto"; -import { constants as fsConstants, promises as fs } from "node:fs"; +import { createHash, randomUUID } from "node:crypto"; +import { + constants as fsConstants, + createReadStream, + promises as fs, +} from "node:fs"; import os from "node:os"; import path from "node:path"; import { promisify } from "node:util"; @@ -12,6 +16,7 @@ import { deleteLocalGitRef, fetchGitBundleIntoLocalRef, GIT_ARCHIVE_EXCLUDES, + type GitWorkspaceSnapshot, integrateImportedGitHead, readGitWorkspaceSnapshot, ReferencedSourceIgnoreScanLimitExceededError, @@ -20,7 +25,11 @@ import { withShallowGitWorkspaceClone, WORKSPACE_GIT_SCAN_SATURATED_CODE, } from "./git-workspace-sync.js"; -import { captureDirectorySnapshot, mergeDirectoryWithBaseline } from "./workspace-restore-merge.js"; +import { + captureDirectorySnapshot, + mergeDirectoryWithBaseline, + type DirectorySnapshot, +} from "./workspace-restore-merge.js"; import { createRuntimeProgressReporter, type RuntimeProgressDirection, @@ -553,9 +562,29 @@ export interface PreparedSandboxManagedRuntime { * staged, or when no additional sources were requested. */ additionalSourceFailures: AdditionalSourceStagingFailure[]; + /** Durable merge inputs used to resume an outbound restore after host restart. */ + workspaceSyncSnapshot: { + baseline: DirectorySnapshot; + gitSnapshot: GitWorkspaceSnapshot | null; + } | null; restoreWorkspace(onProgress?: RuntimeProgressSink): Promise; } +export type WorkspaceInboundMode = + "host_current" | "durable_seed" | "adopt_remote"; + +/** + * Controller-owned archives for replaying the exact pre-turn workspace into a + * replacement sandbox. Paths are never sent to the provider as credentials or + * persisted in database metadata. + */ +export interface WorkspaceDurableSeedPaths { + workspaceArchivePath: string; + workspaceArchiveSha256?: string; + gitArchivePath?: string | null; + gitArchiveSha256?: string | null; +} + /** One additional (referenced) project that failed to stage into the sandbox. */ export interface AdditionalSourceStagingFailure { projectId: string; @@ -719,6 +748,57 @@ async function withTempDir(prefix: string, fn: (dir: string) => Promise): } } +async function sha256File(filePath: string): Promise { + return await new Promise((resolveDigest, rejectDigest) => { + const digest = createHash("sha256"); + const stream = createReadStream(filePath); + stream.on("data", (chunk) => digest.update(chunk)); + stream.on("error", rejectDigest); + stream.on("end", () => resolveDigest(digest.digest("hex"))); + }); +} + +async function copyDurableSeedArchive(input: { + sourcePath: string; + targetPath: string; + expectedSha256?: string | null; +}): Promise { + const source = await fs.lstat(input.sourcePath); + if (source.isSymbolicLink() || !source.isFile()) { + throw new Error("workspace_durable_seed_invalid"); + } + if ( + input.expectedSha256 && + (await sha256File(input.sourcePath)) !== input.expectedSha256 + ) { + throw new Error("workspace_durable_seed_digest_mismatch"); + } + await fs.copyFile(input.sourcePath, input.targetPath); +} + +async function persistDurableSeedArchive(input: { + sourcePath: string; + targetPath: string; +}): Promise { + const parent = path.dirname(input.targetPath); + await fs.mkdir(parent, { recursive: true, mode: 0o700 }); + const parentStat = await fs.lstat(parent); + if (parentStat.isSymbolicLink() || !parentStat.isDirectory()) { + throw new Error("workspace_durable_seed_root_invalid"); + } + const temporary = path.join( + parent, + `.${path.basename(input.targetPath)}.${randomUUID()}.tmp`, + ); + try { + await fs.copyFile(input.sourcePath, temporary); + await fs.chmod(temporary, 0o600); + await fs.rename(temporary, input.targetPath); + } finally { + await fs.rm(temporary, { force: true }).catch(() => undefined); + } +} + async function execTar(args: string[]): Promise { await execFile("tar", args, { env: { @@ -976,6 +1056,12 @@ export async function prepareSandboxManagedRuntime(input: { workspaceLocalDir: string; workspaceRemoteDir?: string; syncWorkspace?: boolean; + /** Selects authoritative host staging, exact durable-seed replay, or no-overwrite adoption. */ + workspaceInboundMode?: WorkspaceInboundMode; + workspaceDurableSeed?: WorkspaceDurableSeedPaths; + /** Durable snapshots supplied when reconstructing an interrupted restore. */ + workspaceBaseline?: DirectorySnapshot; + workspaceGitSnapshot?: GitWorkspaceSnapshot | null; workspaceExclude?: string[]; preserveAbsentOnRestore?: string[]; assets?: SandboxManagedRuntimeAsset[]; @@ -999,6 +1085,16 @@ export async function prepareSandboxManagedRuntime(input: { const workspaceRemoteDir = input.workspaceRemoteDir ?? input.spec.remoteCwd; const runtimeRootDir = path.posix.join(workspaceRemoteDir, ".paperclip-runtime", input.adapterKey); const syncWorkspace = input.syncWorkspace !== false; + const workspaceInboundMode = input.workspaceInboundMode ?? "host_current"; + const stageWorkspace = + syncWorkspace && workspaceInboundMode !== "adopt_remote"; + const prepareWorkspaceSeed = + syncWorkspace && + workspaceInboundMode === "adopt_remote" && + input.workspaceDurableSeed !== undefined; + if (workspaceInboundMode === "durable_seed" && !input.workspaceDurableSeed) { + throw new Error("workspace_durable_seed_missing"); + } // Reject any unsafe asset key before an archive path or an asset directory is // built from it. This runs before the git snapshot work so a bad key fails fast. @@ -1025,7 +1121,11 @@ export async function prepareSandboxManagedRuntime(input: { // It reads git's own bookkeeping to decide what to include/exclude, so it is // usually fast, but on a large working tree the `--ignored` walk is not free. const gitSnapshot = syncWorkspace - ? await runStepSpan("snapshot.git", () => readGitWorkspaceSnapshot(input.workspaceLocalDir)) + ? input.workspaceGitSnapshot !== undefined + ? input.workspaceGitSnapshot + : await runStepSpan("snapshot.git", () => + readGitWorkspaceSnapshot(input.workspaceLocalDir), + ) : null; const gitIgnoredExcludes = gitSnapshot?.ignoredPaths; const workspaceArchiveExclude = mergeExcludes( @@ -1047,9 +1147,12 @@ export async function prepareSandboxManagedRuntime(input: { // dominant cost in the pre-`pack` window — it reads the content of every // non-excluded file, serially — so it earns its own span. const baselineSnapshot = syncWorkspace - ? await runStepSpan("snapshot.baseline", () => - captureDirectorySnapshot(input.workspaceLocalDir, { exclude: restoreExclude }), - ) + ? (input.workspaceBaseline ?? + (await runStepSpan("snapshot.baseline", () => + captureDirectorySnapshot(input.workspaceLocalDir, { + exclude: restoreExclude, + }), + ))) : null; // Every inbound staging step delegates to the provider through `client.syncIn`: @@ -1172,7 +1275,7 @@ export async function prepareSandboxManagedRuntime(input: { // records its own failure and never rejects. const inboundTaskIsRequired: boolean[] = []; - if (syncWorkspace) { + if (stageWorkspace || prepareWorkspaceSeed) { inboundTaskIsRequired.push(true); inboundTasks.push(() => runStepSpan("stage.workspace", async () => { @@ -1203,18 +1306,49 @@ export async function prepareSandboxManagedRuntime(input: { if (gitSnapshot) { await emitRuntimeStatus(input.onRuntimeProgress, "git_sync", "Syncing git history to environment"); const gitTarPath = path.join(tempDir, "git-workspace.tar"); - const remoteGitTar = path.posix.join(runtimeRootDir, "git-workspace-upload.tar"); - await withShallowGitWorkspaceClone({ - localDir: input.workspaceLocalDir, - snapshot: gitSnapshot, - }, async (cloneDir) => { - await createTarballFromDirectory({ - localDir: cloneDir, - archivePath: gitTarPath, - exclude: [".paperclip-runtime"], + const remoteGitTar = path.posix.join( + runtimeRootDir, + "git-workspace-upload.tar", + ); + if (workspaceInboundMode === "durable_seed") { + const durableGitArchive = + input.workspaceDurableSeed?.gitArchivePath; + if (!durableGitArchive) { + throw new Error("workspace_durable_seed_git_missing"); + } + await copyDurableSeedArchive({ + sourcePath: durableGitArchive, + targetPath: gitTarPath, + expectedSha256: input.workspaceDurableSeed?.gitArchiveSha256, }); + } else { + await withShallowGitWorkspaceClone( + { + localDir: input.workspaceLocalDir, + snapshot: gitSnapshot, + }, + async (cloneDir) => { + await createTarballFromDirectory({ + localDir: cloneDir, + archivePath: gitTarPath, + exclude: [".paperclip-runtime"], + }); + }, + ); + if (input.workspaceDurableSeed?.gitArchivePath) { + await persistDurableSeedArchive({ + sourcePath: gitTarPath, + targetPath: input.workspaceDurableSeed.gitArchivePath, + }); + } + } + workspaceFiles.push({ + sourcePath: gitTarPath, + targetPath: remoteGitTar, + kind: "file", + access: "rw", + writablePath: workspaceRemoteDir, }); - workspaceFiles.push({ sourcePath: gitTarPath, targetPath: remoteGitTar, kind: "file", access: "rw", writablePath: workspaceRemoteDir }); workspacePostUploadCommands.push({ command: buildWorkspaceTarExtractCommand({ workspaceRemoteDir, @@ -1230,22 +1364,48 @@ export async function prepareSandboxManagedRuntime(input: { // the preserved names first. The extract runs AFTER the git extract. await emitRuntimeStatus(input.onRuntimeProgress, "config_sync", "Syncing workspace to environment"); const workspaceTarPath = path.join(tempDir, "workspace.tar"); - const workspaceArchiveDir = gitSnapshot ? path.join(tempDir, "workspace-overlay") : input.workspaceLocalDir; - if (gitSnapshot) { - await copySelectedWorkspaceEntries({ - sourceDir: input.workspaceLocalDir, - targetDir: workspaceArchiveDir, - relativePaths: gitSnapshot.overlayPaths, - exclude: workspaceArchiveExclude, + if (workspaceInboundMode === "durable_seed") { + await copyDurableSeedArchive({ + sourcePath: input.workspaceDurableSeed!.workspaceArchivePath, + targetPath: workspaceTarPath, + expectedSha256: + input.workspaceDurableSeed!.workspaceArchiveSha256, }); + } else { + const workspaceArchiveDir = gitSnapshot + ? path.join(tempDir, "workspace-overlay") + : input.workspaceLocalDir; + if (gitSnapshot) { + await copySelectedWorkspaceEntries({ + sourceDir: input.workspaceLocalDir, + targetDir: workspaceArchiveDir, + relativePaths: gitSnapshot.overlayPaths, + exclude: workspaceArchiveExclude, + }); + } + await createTarballFromDirectory({ + localDir: workspaceArchiveDir, + archivePath: workspaceTarPath, + exclude: gitSnapshot ? undefined : workspaceArchiveExclude, + }); + if (input.workspaceDurableSeed) { + await persistDurableSeedArchive({ + sourcePath: workspaceTarPath, + targetPath: input.workspaceDurableSeed.workspaceArchivePath, + }); + } } - await createTarballFromDirectory({ - localDir: workspaceArchiveDir, - archivePath: workspaceTarPath, - exclude: gitSnapshot ? undefined : workspaceArchiveExclude, + const remoteWorkspaceTar = path.posix.join( + runtimeRootDir, + "workspace-upload.tar", + ); + workspaceFiles.push({ + sourcePath: workspaceTarPath, + targetPath: remoteWorkspaceTar, + kind: "file", + access: "rw", + writablePath: workspaceRemoteDir, }); - const remoteWorkspaceTar = path.posix.join(runtimeRootDir, "workspace-upload.tar"); - workspaceFiles.push({ sourcePath: workspaceTarPath, targetPath: remoteWorkspaceTar, kind: "file", access: "rw", writablePath: workspaceRemoteDir }); workspacePostUploadCommands.push({ command: buildWorkspaceTarExtractCommand({ workspaceRemoteDir, @@ -1265,6 +1425,8 @@ export async function prepareSandboxManagedRuntime(input: { workspaceUploadBytes += (await fs.stat(workspaceTarPath)).size; }); + if (!stageWorkspace) return; + // One confined `syncIn` for the whole merged workspace file set. The confine // guard covers every mapping BEFORE any bytes upload (fail-closed): a source // or target escape in EITHER tar mapping stops the upload of both. @@ -1457,6 +1619,10 @@ export async function prepareSandboxManagedRuntime(input: { assetDirs, additionalSourceDirs, additionalSourceFailures, + workspaceSyncSnapshot: + syncWorkspace && baselineSnapshot + ? { baseline: baselineSnapshot, gitSnapshot } + : null, restoreWorkspace: async (onProgress?: RuntimeProgressSink) => { const restoreSink = onProgress ?? input.onProgress; diff --git a/packages/adapter-utils/src/workspace-restore-merge.test.ts b/packages/adapter-utils/src/workspace-restore-merge.test.ts index d3a06c08bc..516c6dcc81 100644 --- a/packages/adapter-utils/src/workspace-restore-merge.test.ts +++ b/packages/adapter-utils/src/workspace-restore-merge.test.ts @@ -9,9 +9,12 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { resolvePaperclipInstanceRootForAdapter } from "./server-utils.js"; import { captureDirectorySnapshot, + directorySnapshotSha256, classifyWorkspaceRestoreFailure, describeWorkspaceRestoreFailure, mergeDirectoryWithBaseline, + parseDirectorySnapshot, + serializeDirectorySnapshot, withDirectoryMergeLock, WORKSPACE_RESTORE_LOCK_TIMEOUT_CODE, } from "./workspace-restore-merge.js"; @@ -27,6 +30,36 @@ describe("workspace restore merge", () => { } }); + it("round-trips a deterministic durable snapshot and rejects traversal", async () => { + const rootDir = await mkdtemp( + path.join(os.tmpdir(), "paperclip-snapshot-"), + ); + cleanupDirs.push(rootDir); + await mkdir(path.join(rootDir, "nested"), { recursive: true }); + await writeFile(path.join(rootDir, "b.txt"), "bravo\n", "utf8"); + await writeFile(path.join(rootDir, "nested", "a.txt"), "alpha\n", "utf8"); + + const snapshot = await captureDirectorySnapshot(rootDir, { exclude: [] }); + const serialized = serializeDirectorySnapshot(snapshot); + const restored = parseDirectorySnapshot(serialized); + + expect(serialized.entries.map(([relativePath]) => relativePath)).toEqual([ + "b.txt", + "nested", + "nested/a.txt", + ]); + expect(restored).not.toBeNull(); + expect(directorySnapshotSha256(restored!)).toBe( + directorySnapshotSha256(snapshot), + ); + expect( + parseDirectorySnapshot({ + ...serialized, + entries: [["../escape", serialized.entries[0]![1]]], + }), + ).toBeNull(); + }); + it("preserves sibling files when sequential stale-baseline restores create the same nested directory tree", async () => { const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-restore-merge-")); cleanupDirs.push(rootDir); diff --git a/packages/adapter-utils/src/workspace-restore-merge.ts b/packages/adapter-utils/src/workspace-restore-merge.ts index 8d59cd06db..92992e684c 100644 --- a/packages/adapter-utils/src/workspace-restore-merge.ts +++ b/packages/adapter-utils/src/workspace-restore-merge.ts @@ -5,7 +5,7 @@ import path from "node:path"; import { shouldExcludePath } from "./exclude-patterns.js"; import { resolvePaperclipInstanceRootForAdapter } from "./server-utils.js"; -type SnapshotEntry = +export type SnapshotEntry = | { kind: "dir" } | { kind: "file"; mode: number; hash: string } | { kind: "symlink"; target: string }; @@ -15,6 +15,87 @@ export interface DirectorySnapshot { entries: Map; } +export interface SerializedDirectorySnapshot { + version: 1; + exclude: string[]; + entries: Array<[string, SnapshotEntry]>; +} + +function isSafeSnapshotRelativePath(value: string): boolean { + if (!value || path.posix.isAbsolute(value) || path.win32.isAbsolute(value)) { + return false; + } + return !value.split(/[\\/]/).some((segment) => segment === ".."); +} + +function parseSnapshotEntry(value: unknown): SnapshotEntry | null { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const candidate = value as Record; + if (candidate.kind === "dir") return { kind: "dir" }; + if (candidate.kind === "symlink" && typeof candidate.target === "string") { + return { kind: "symlink", target: candidate.target }; + } + if ( + candidate.kind === "file" && + typeof candidate.mode === "number" && + Number.isInteger(candidate.mode) && + candidate.mode >= 0 && + typeof candidate.hash === "string" && + /^[0-9a-f]{64}$/.test(candidate.hash) + ) { + return { kind: "file", mode: candidate.mode, hash: candidate.hash }; + } + return null; +} + +export function serializeDirectorySnapshot( + snapshot: DirectorySnapshot, +): SerializedDirectorySnapshot { + return { + version: 1, + exclude: [...snapshot.exclude], + entries: [...snapshot.entries.entries()].sort(([left], [right]) => + left.localeCompare(right), + ), + }; +} + +export function parseDirectorySnapshot( + value: unknown, +): DirectorySnapshot | null { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const candidate = value as Record; + if ( + candidate.version !== 1 || + !Array.isArray(candidate.exclude) || + !candidate.exclude.every((entry) => typeof entry === "string") || + !Array.isArray(candidate.entries) + ) { + return null; + } + const entries = new Map(); + for (const rawEntry of candidate.entries) { + if (!Array.isArray(rawEntry) || rawEntry.length !== 2) return null; + const [relative, rawSnapshotEntry] = rawEntry; + if (typeof relative !== "string" || !isSafeSnapshotRelativePath(relative)) { + return null; + } + const entry = parseSnapshotEntry(rawSnapshotEntry); + if (!entry || entries.has(relative)) return null; + entries.set(relative, entry); + } + return { + exclude: [...new Set(candidate.exclude as string[])], + entries, + }; +} + +export function directorySnapshotSha256(snapshot: DirectorySnapshot): string { + return createHash("sha256") + .update(JSON.stringify(serializeDirectorySnapshot(snapshot))) + .digest("hex"); +} + async function hashFile(filePath: string): Promise { return await new Promise((resolve, reject) => { const hash = createHash("sha256"); diff --git a/packages/adapters/codex-local/src/server/execute.test.ts b/packages/adapters/codex-local/src/server/execute.test.ts index a49073d8b0..375687c89f 100644 --- a/packages/adapters/codex-local/src/server/execute.test.ts +++ b/packages/adapters/codex-local/src/server/execute.test.ts @@ -117,11 +117,10 @@ describe("codex execute — outbound auth copy-back restore contribution", () => ); } - async function runTeardown(input: { - sandboxAuth: string; - hostAuth: string; - }): Promise<{ finalHostAuth: string; finalHostMode: number }> { - const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-codex-copyback-e2e-")); + async function runTeardown(input: { sandboxAuth: string; hostAuth: string }) { + const rootDir = await mkdtemp( + path.join(os.tmpdir(), "paperclip-codex-copyback-e2e-"), + ); cleanupDirs.push(rootDir); const workspaceDir = path.join(rootDir, "workspace"); // The shared host home is what `resolveSharedCodexHomeDir` returns @@ -137,7 +136,7 @@ describe("codex execute — outbound auth copy-back restore contribution", () => process.env.CODEX_HOME = sharedHostHome; sandboxAuthFixture.bytes = Buffer.from(input.sandboxAuth, "utf8"); - await execute({ + const executionResult = await execute({ runId: "run-copyback-e2e", agent: { id: "agent-1", @@ -178,6 +177,7 @@ describe("codex execute — outbound auth copy-back restore contribution", () => return { finalHostAuth: await readFile(hostAuthPath, "utf8"), finalHostMode: (await lstat(hostAuthPath)).mode & 0o777, + executionResult, }; } @@ -231,4 +231,50 @@ describe("codex execute — outbound auth copy-back restore contribution", () => expect(result.finalHostMode, entry.name).toBe(0o600); } }); + + it("surfaces workspace restore failure after successful provider execution", async () => { + prepareAdapterExecutionTargetRuntime.mockResolvedValueOnce({ + target: { kind: "remote", transport: "ssh" }, + workspaceRemoteDir: "/remote/workspace", + runtimeRootDir: REMOTE_RUNTIME_ROOT, + assetDirs: { home: `${REMOTE_RUNTIME_ROOT}/home` }, + restoreWorkspace: async () => { + throw new Error("workspace copy-back failed"); + }, + }); + + await expect( + runTeardown({ + sandboxAuth: subscriptionAuth({ accountId: "acct", marker: "sandbox" }), + hostAuth: subscriptionAuth({ accountId: "acct", marker: "host" }), + }), + ).rejects.toThrow("workspace copy-back failed"); + }); + + it("preserves a provider failure when workspace restore also fails", async () => { + runChildProcess.mockResolvedValueOnce({ + exitCode: 1, + signal: null, + timedOut: false, + stdout: "", + stderr: "provider failed first", + pid: 321, + startedAt: new Date().toISOString(), + }); + prepareAdapterExecutionTargetRuntime.mockResolvedValueOnce({ + target: { kind: "remote", transport: "ssh" }, + workspaceRemoteDir: "/remote/workspace", + runtimeRootDir: REMOTE_RUNTIME_ROOT, + assetDirs: { home: `${REMOTE_RUNTIME_ROOT}/home` }, + restoreWorkspace: async () => { + throw new Error("workspace copy-back failed second"); + }, + }); + + const result = await runTeardown({ + sandboxAuth: subscriptionAuth({ accountId: "acct", marker: "sandbox" }), + hostAuth: subscriptionAuth({ accountId: "acct", marker: "host" }), + }); + expect(result.executionResult.errorMessage).toBe("provider failed first"); + }); }); diff --git a/packages/adapters/codex-local/src/server/execute.ts b/packages/adapters/codex-local/src/server/execute.ts index 430a77bcc2..436cd585b4 100644 --- a/packages/adapters/codex-local/src/server/execute.ts +++ b/packages/adapters/codex-local/src/server/execute.ts @@ -1526,6 +1526,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise undefined); + // A provider failure remains the primary outcome. When provider work + // succeeded, however, silently accepting a failed copy-back can lose + // the only workspace edits before a replacement sandbox starts. + if (executionError === null) throw error; } } } diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs index 402b71a094..92e7323b49 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs @@ -1345,6 +1345,13 @@ impl CommandExecutor for AcpxCommandExecutor { } } + fn rotate_authority(&mut self, config: &DurableRunnerConfig) { + self.context.run_id = config.run_id.clone(); + self.context.normalized_session_id = config.normalized_session_id.clone(); + self.context.turn_id = config.turn_id.clone(); + self.context.item_id = config.item_id.clone(); + } + fn poll_events(&mut self) -> Result, DurableRunnerError> { self.restore()?; if self diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs index 41f1685cb0..06b9d0d0fd 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs @@ -54,7 +54,7 @@ pub(crate) const MAX_SETTLED_PROVIDER_TURN_IDS: usize = 4_096; type QuestionOptionLabels = BTreeMap>; type QuestionSetMapping = (String, Value, QuestionOptionLabels); -#[derive(Clone)] +#[derive(Clone, PartialEq)] struct ProviderCompletionContract { revision: String, criterion_ids: Vec, @@ -807,6 +807,46 @@ impl CodexProvider { self.durable_tool_call_replays = true; } + pub(crate) fn attach_run_in_place( + &mut self, + authorized_tools: impl IntoIterator, + completion_contract: Option<(&str, &[String])>, + ) -> Result { + let authorized_tools = authorized_tools.into_iter().collect::>(); + let completion_contract = + completion_contract.map(|(revision, criterion_ids)| ProviderCompletionContract { + revision: revision.to_owned(), + criterion_ids: criterion_ids.to_vec(), + }); + if authorized_tools != self.authorized_tools + || completion_contract != self.completion_contract + { + return Ok(false); + } + if self.process.try_wait()?.is_some() + || self.quarantined + || self.active_provider_turn_id.is_some() + || self.ambiguous_turn_start_pending + || !self.pending_messages.is_empty() + || !self.deferred_ambiguous_messages.is_empty() + || !self.pending_tool_requests.is_empty() + || !self.pending_runtime_requests.is_empty() + { + return Err(LocalRunnerError::invalid( + "Codex warm run attachment requires an idle live provider with no pending work", + )); + } + // The provider process and its thread remain authoritative. Exact + // settled-turn identities stay in memory so delayed output from an + // earlier run cannot be accepted as the next turn. A changed semantic + // tool or completion contract returns false so the caller can preserve + // the existing cold-resume behavior for that incompatible boundary. + self.completed_turn_authority = None; + self.completion_reconciliation_pending = false; + self.expected_shutdown = false; + Ok(true) + } + pub(crate) fn restore_completed_turn_authority( &mut self, authoritative: bool, diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs index 6d0ae23dfc..f88dc0bbc2 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs @@ -116,9 +116,121 @@ impl CommandLifecycle { } } +fn next_authority_config( + command: &Command, + current: &DurableRunnerConfig, +) -> Result, DurableRunnerError> { + if command.command_type != "run.attach" { + return Ok(None); + } + let Some(boundary) = command.payload.get("paperclipNextAuthority") else { + return Ok(None); + }; + let identity = boundary + .get("identity") + .and_then(Value::as_object) + .ok_or_else(|| DurableRunnerError::invalid("run.attach authority identity is required"))?; + let read_identity = |key: &str| { + identity + .get(key) + .and_then(Value::as_str) + .filter(|value| !value.is_empty()) + .map(str::to_owned) + .ok_or_else(|| { + DurableRunnerError::invalid(format!( + "run.attach authority identity field {key} is required" + )) + }) + }; + let connection = boundary + .get("connection") + .and_then(Value::as_object) + .ok_or_else(|| { + DurableRunnerError::invalid("run.attach authority connection is required") + })?; + let connect_url = match connection.get("mode").and_then(Value::as_str) { + Some("connect") => connection + .get("connectUrl") + .and_then(Value::as_str) + .filter(|value| !value.is_empty()) + .map(str::to_owned) + .ok_or_else(|| DurableRunnerError::invalid("run.attach connect URL is required"))?, + Some("listen") => { + let address = connection + .get("listenAddress") + .and_then(Value::as_str) + .ok_or_else(|| { + DurableRunnerError::invalid("run.attach listen address is required") + })?; + let port = connection + .get("listenPort") + .and_then(Value::as_u64) + .ok_or_else(|| DurableRunnerError::invalid("run.attach listen port is required"))?; + let path = connection + .get("listenPath") + .and_then(Value::as_str) + .ok_or_else(|| DurableRunnerError::invalid("run.attach listen path is required"))?; + format!("listen://{address}:{port}{path}") + } + _ => { + return Err(DurableRunnerError::invalid( + "run.attach authority connection mode is invalid", + )); + } + }; + let mut next = current.clone(); + next.connect_url = connect_url; + next.ca_bundle_path = connection + .get("caBundlePath") + .and_then(Value::as_str) + .filter(|value| !value.is_empty()) + .map(Into::into); + next.runner_instance_id = read_identity("runnerInstanceId")?; + next.environment_lease_id = read_identity("environmentLeaseId")?; + next.run_id = read_identity("runId")?; + next.normalized_session_id = read_identity("normalizedSessionId")?; + next.turn_id = read_identity("turnId")?; + next.item_id = read_identity("itemId")?; + next.validate()?; + if next.runner_instance_id != current.runner_instance_id + || next.environment_lease_id != current.environment_lease_id + || next.normalized_session_id != current.normalized_session_id + || next.run_id == current.run_id + { + return Err(DurableRunnerError::invalid( + "run.attach authority changed an immutable session binding", + )); + } + Ok(Some(next)) +} + +fn apply_authority_rotation( + state: &mut DurableState, + store: &DurableStateStore, + config: &mut DurableRunnerConfig, + endpoint: &mut RunnerTransportEndpoint, + next: DurableRunnerConfig, +) -> Result<(), DurableRunnerError> { + let reconnect_count = state.reconnect_count.saturating_add(1); + let mut diagnostics = std::mem::take(&mut state.diagnostics); + *endpoint = RunnerTransportEndpoint::new(&next.connect_url, &next.run_id)?; + *config = next; + let mut rotated = DurableState::new(config); + rotated.reconnect_count = reconnect_count; + rotated.diagnostics.append(&mut diagnostics); + rotated.record_diagnostic("runner advanced to a new warm run authority"); + *state = rotated; + store.save(state) +} + pub trait CommandExecutor { fn execute(&mut self, command: &Command) -> Result; + /// Advances provider-side event correlation after a durable `run.attach` + /// has moved runnerd to the next run-bound authority. The runner validates + /// and persists the new authority before invoking this infallible hook. + fn rotate_authority(&mut self, _config: &DurableRunnerConfig) {} + fn poll_events(&mut self) -> Result, DurableRunnerError> { Ok(Vec::new()) } @@ -136,7 +248,7 @@ pub trait CommandExecutor { } pub fn run_durable_runner( - config: DurableRunnerConfig, + mut config: DurableRunnerConfig, bootstrap_ticket: BootstrapTicket, mut executor: E, ) -> Result<(), DurableRunnerError> { @@ -162,7 +274,7 @@ pub fn run_durable_runner( // Bind listener mode or resolve dial mode before processing commands. Dial // reconnects retain the same validated addresses so DNS cannot redirect a // retry after the trust decision. - let endpoint = RunnerTransportEndpoint::new(&config.connect_url, &config.run_id)?; + let mut endpoint = RunnerTransportEndpoint::new(&config.connect_url, &config.run_id)?; let started = Instant::now(); let mut bootstrap_ticket = Some(bootstrap_ticket); let mut lease: Option = None; @@ -287,8 +399,10 @@ pub fn run_durable_runner( let mut sent_source_seq = state.acked_source_seq; let mut lifecycle_after_reply = CommandLifecycle::Continue; + let mut authority_rotation = None; let mut disconnected = false; for command in welcome.pending_commands { + let next_authority = next_authority_config(&command, &config)?; let (result, lifecycle) = process_command(&mut state, &store, &config, &mut executor, &command)?; if let Some(durable_lifecycle) = lifecycle.durable_state() { @@ -333,6 +447,16 @@ pub fn run_durable_runner( // then release the executor without observing later commands. break; } + if next_authority.is_some() { + authority_rotation = next_authority; + break; + } + } + if let Some(next) = authority_rotation { + apply_authority_rotation(&mut state, &store, &mut config, &mut endpoint, next)?; + executor.rotate_authority(&config); + disconnected_since = Some(Instant::now()); + continue; } if !disconnected { if let Err(error) = send_outbox(&mut transport, &state, &mut sent_source_seq) { @@ -425,6 +549,7 @@ pub fn run_durable_runner( .map_err(|error| { DurableRunnerError::invalid(format!("command is malformed: {error}")) })?; + let next_authority = next_authority_config(&command, &config)?; let (result, lifecycle) = process_command(&mut state, &store, &config, &mut executor, &command)?; if let Some(durable_lifecycle) = lifecycle.durable_state() { @@ -468,6 +593,18 @@ pub fn run_durable_runner( ); } } + if let Some(next) = next_authority { + apply_authority_rotation( + &mut state, + &store, + &mut config, + &mut endpoint, + next, + )?; + executor.rotate_authority(&config); + disconnected_since = Some(Instant::now()); + break; + } if let Err(error) = send_outbox(&mut transport, &state, &mut sent_source_seq) { state.record_diagnostic( "outbox delivery failed; unacknowledged suffix remains durable", @@ -1018,6 +1155,59 @@ mod tests { } } + #[test] + fn warm_run_attachment_rotates_only_the_run_authority() { + let directory = std::env::temp_dir().join(format!( + "paperclip-runner-warm-authority-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&directory); + let mut current = config(directory.clone()); + current.runner_digest = format!("sha256:{}", "a".repeat(64)); + let mut attach = command("run.attach"); + attach.payload = json!({ + "paperclipNextAuthority": { + "identity": { + "runnerInstanceId": current.runner_instance_id, + "environmentLeaseId": current.environment_lease_id, + "runId": "run_2", + "normalizedSessionId": current.normalized_session_id, + "turnId": "turn_2", + "itemId": "item_2" + }, + "connection": { + "mode": "connect", + "connectUrl": "ws://127.0.0.1:3001/path" + } + } + }); + + let next = next_authority_config(&attach, ¤t) + .unwrap() + .expect("attachment should carry a new authority"); + assert_eq!(next.run_id, "run_2"); + assert_eq!(next.connect_url, "ws://127.0.0.1:3001/path"); + + let store = DurableStateStore::new(&directory).unwrap(); + let (mut state, _) = store.load_or_create(¤t).unwrap(); + state.outbox.push(crate::durable::state::StoredOutboxEvent { + source_seq: 1, + priority: 0, + event_type: "run.attached".to_owned(), + byte_size: 1, + envelope: json!({}), + }); + let mut endpoint = + RunnerTransportEndpoint::new(¤t.connect_url, ¤t.run_id).unwrap(); + apply_authority_rotation(&mut state, &store, &mut current, &mut endpoint, next).unwrap(); + + assert_eq!(state.run_id, "run_2"); + assert_eq!(state.next_source_seq, 1); + assert!(state.outbox.is_empty()); + assert_eq!(current.run_id, "run_2"); + fs::remove_dir_all(directory).unwrap(); + } + #[test] fn terminal_lifecycle_is_durable_before_fallible_cleanup() { let directory = std::env::temp_dir().join(format!( diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs index 5456fc9824..dcdfb5ed0e 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs @@ -1670,6 +1670,10 @@ impl CommandExecutor for ManagedProviderCommandExecutor { } } + fn rotate_authority(&mut self, config: &DurableRunnerConfig) { + self.config = config.clone(); + } + fn poll_events(&mut self) -> Result, DurableRunnerError> { self.poll_provider()?; Ok(self diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/native_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/native_provider_backend.rs index 959b06c8d1..4d54b48a20 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/native_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/native_provider_backend.rs @@ -35,6 +35,14 @@ impl CommandExecutor for SelectedExecutor { } } + fn rotate_authority(&mut self, config: &DurableRunnerConfig) { + match self { + Self::LocalFacade(executor) => executor.rotate_authority(config), + Self::Acpx(executor) => executor.rotate_authority(config), + Self::Managed(executor) => executor.rotate_authority(config), + } + } + fn acknowledge_events(&mut self, count: usize) -> Result<(), DurableRunnerError> { match self { Self::LocalFacade(executor) => executor.acknowledge_events(count), @@ -163,6 +171,13 @@ impl CommandExecutor for NativeProviderCommandExecutor { .map_or_else(|| Ok(Vec::new()), CommandExecutor::poll_events) } + fn rotate_authority(&mut self, config: &DurableRunnerConfig) { + self.config = config.clone(); + if let Some(executor) = self.selected.as_mut() { + executor.rotate_authority(config); + } + } + fn acknowledge_events(&mut self, count: usize) -> Result<(), DurableRunnerError> { self.select_recovery()?; if let Some(executor) = self.selected.as_mut() { diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs index f11fe402df..c335667184 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs @@ -1675,19 +1675,40 @@ impl CodexCommandExecutor { next_state.active_provider_result_fingerprint = None; next_state.active_provider_result_disposition = None; next_state.last_agent_message = None; - if let Some(provider) = self.provider.as_mut() { + let provider = self.provider.as_mut().ok_or_else(|| { + DurableRunnerError::invalid("run.attach requires the restored Codex provider process") + })?; + let retained_provider = provider + .attach_run_in_place( + next_state.tool_bridge.authorized_tools().cloned(), + next_state.completion_contract.as_ref().map(|contract| { + ( + contract.revision.as_str(), + contract.criterion_ids.as_slice(), + ) + }), + ) + .map_err(|error| { + DurableRunnerError::invalid(format!( + "failed to retain Codex for warm run attachment: {error}" + )) + })?; + if !retained_provider { provider.shutdown().map_err(|error| { DurableRunnerError::invalid(format!( "failed to checkpoint Codex before attaching a new run: {error}" )) })?; + self.provider = None; } - self.provider = None; next_state.pending_events.clear(); - // Persist the checkpoint as not-open before open_session resumes it for - // the new authority. Otherwise recovery emits a second session.resumed - // notice into the provider queue in addition to the command event. - next_state.lifecycle = "prepared".to_owned(); + next_state.lifecycle = if retained_provider { + "session_open".to_owned() + } else { + // The next provider command restores the same checkpointed session + // with the rotated tool/completion authority. + "prepared".to_owned() + }; self.persist_state(&next_state)?; self.state = Some(next_state); Ok(()) @@ -3009,6 +3030,10 @@ impl CommandExecutor for CodexCommandExecutor { } } + fn rotate_authority(&mut self, config: &DurableRunnerConfig) { + self.event_identity = Some(ProviderEventIdentity::from_config(config)); + } + fn poll_events(&mut self) -> Result, DurableRunnerError> { self.poll_provider()?; Ok(self diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts index fb76579f69..85ee0ab0e1 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts @@ -939,7 +939,7 @@ class AuthorityConnection { /** Authenticated, replay-safe PRP transport authority. Business operations are caller supplied. */ export class DurablePrpControlPlane { - readonly #identity: DurableRecoveryIdentity; + #identity: DurableRecoveryIdentity; readonly #store: DurableCoreStore; #expectedRunnerVersion: string; #expectedRunnerDigest: string; @@ -1041,6 +1041,36 @@ export class DurablePrpControlPlane { ).length; } + /** + * Atomically advances a settled reusable runner to a new run authority while + * retaining its existing connection lease secret. The runner performs the + * matching state transition only after acknowledging `run.attach`. + */ + rotateRunIdentity(identity: DurableRecoveryIdentity): void { + if ( + !Object.values(identity).every( + (value) => typeof value === "string" && stableIdPattern.test(value), + ) || + identity.runnerInstanceId !== this.#identity.runnerInstanceId || + identity.environmentLeaseId !== this.#identity.environmentLeaseId || + identity.normalizedSessionId !== this.#identity.normalizedSessionId || + identity.runId === this.#identity.runId || + this.#store.state.commands.some((command) => command.status === "pending") + ) { + throw new Error("Durable PRP run identity rotation is invalid."); + } + this.disconnectActiveRunner(); + const leases = Object.fromEntries( + Object.entries(this.#store.state.leases).map(([key, lease]) => [ + key, + { ...lease, identity: structuredClone(identity) }, + ]), + ); + Object.assign(this.#store.state, initialCoreState(identity), { leases }); + this.#identity = structuredClone(identity); + this.#store.save(); + } + issueBootstrapTicket(ttlMs = 5_000): string { if (!Number.isInteger(ttlMs) || ttlMs < 1_000 || ttlMs > 60_000) { throw new Error("Durable PRP bootstrap TTL is invalid."); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index dcbfc49159..b8ffc2d367 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -1965,7 +1965,7 @@ it("steers the active provider turn through the durable PRP command path", async } }, 30_000); -it("does not expose cross-run attachment before PRP authority can rotate atomically", async () => { +it("rotates PRP authority in place for a warm cross-run attachment", async () => { const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-warm-attach-")); const bundle = createCapabilityRunnerdCodexTransport({ runnerBinary: defaultCapabilityRunnerdBinary(), @@ -1978,22 +1978,36 @@ it("does not expose cross-run attachment before PRP authority can rotate atomica success: true, contentItems: [], })); + const within = async (label: string, promise: Promise) => + await Promise.race([ + promise, + new Promise((_, reject) => + setTimeout(() => reject(new Error(`${label} timeout`)), 5_000), + ), + ]); try { - await bundle.transport.request("initialize", {}); - await bundle.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: [ - { - name: "get_task_context", - description: "Read the active task.", - inputSchema: { - type: "object", - properties: {}, - additionalProperties: false, + await within("initialize", bundle.transport.request("initialize", {})); + await within( + "thread start", + bundle.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [ + { + name: "get_task_context", + description: "Read the active task.", + inputSchema: { + type: "object", + properties: {}, + additionalProperties: false, + }, }, + ], + completionContract: { + revision: "sha256:warm-three-turn-contract", + criterionIds: ["objective"], }, - ], - }); + }), + ); const runnerPid = bundle.evidence().runnerPid; const providerPid = bundle.evidence().codexPid; const notifications = bundle.transport @@ -2015,12 +2029,45 @@ it("does not expose cross-run attachment before PRP authority can rotate atomica } throw new Error(`${label} completion timeout`); }; - await bundle.transport.request("turn/start", { - input: [{ type: "text", text: "first run" }], - }); + await within( + "first turn start", + bundle.transport.request("turn/start", { + input: [{ type: "text", text: "first run" }], + }), + ); await waitForCompletion("first run"); - expect(bundle.transport.attachRun).toBeUndefined(); + await within( + "warm attach", + bundle.transport.attachRun!({ + runId: "run-warm-second", + turnId: "turn-warm-second", + itemId: "item-warm-second", + }), + ); + await within( + "second turn start", + bundle.transport.request("turn/start", { + input: [{ type: "text", text: "second run" }], + }), + ); + await waitForCompletion("second run"); + + await within( + "second warm attach", + bundle.transport.attachRun!({ + runId: "run-warm-third", + turnId: "turn-warm-third", + itemId: "item-warm-third", + }), + ); + await within( + "third turn start", + bundle.transport.request("turn/start", { + input: [{ type: "text", text: "third run" }], + }), + ); + await waitForCompletion("third run"); expect(bundle.evidence()).toMatchObject({ runnerPid, @@ -2033,6 +2080,99 @@ it("does not expose cross-run attachment before PRP authority can rotate atomica } }, 30_000); +it("releases both PRP authorities when warm rotation activation fails", async () => { + const stateDirectory = await mkdtemp( + join(tmpdir(), "runnerd-warm-attach-activation-failure-"), + ); + const server = createServer(); + const authorities = new Map(); + const released: string[] = []; + server.on("upgrade", (request, socket, head) => { + const route = request.url ?? ""; + const authority = authorities.get(route); + if (!authority) { + socket.destroy(); + return; + } + authority.handleUpgrade(request, socket, route, head); + }); + await new Promise((resolveListen) => + server.listen(0, "127.0.0.1", resolveListen), + ); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("Expected warm activation failure test listener"); + } + let registrationCount = 0; + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory), + stateDirectory, + lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, + controlPlaneRegistration: async (authority) => { + registrationCount += 1; + const route = `/runner-${registrationCount}`; + authorities.set(route, authority); + return { + connectUrl: `ws://127.0.0.1:${address.port}${route}`, + ...(registrationCount === 1 + ? {} + : { + activate: () => { + throw new Error("rotation activation failed"); + }, + }), + release: () => { + released.push(route); + if (authorities.get(route) === authority) authorities.delete(route); + }, + }; + }, + }); + bundle.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + let runnerPid: number | null = null; + try { + await bundle.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: codexSemanticToolSpecs(), + }); + runnerPid = bundle.evidence().runnerPid; + + await expect( + bundle.transport.attachRun!({ + runId: "run-warm-activation-failure", + turnId: "turn-warm-activation-failure", + itemId: "item-warm-activation-failure", + }), + ).rejects.toThrow("rotation activation failed"); + expect(new Set(released)).toEqual(new Set(["/runner-1", "/runner-2"])); + expect(authorities.size).toBe(0); + await expect(bundle.transport.request("thread/read", {})).rejects.toThrow( + "rotation activation failed", + ); + } finally { + await bundle.transport.close().catch(() => undefined); + if (runnerPid) { + try { + process.kill(-runnerPid, "SIGKILL"); + } catch { + // A successful durable close already stopped the runner process group. + } + } + server.closeAllConnections(); + if (server.listening) { + await new Promise((resolveClose) => + server.close(() => resolveClose()), + ); + } + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 30_000); + it.each([ { binding: "runner instance", diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index 5ab150c78f..fe384d7336 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -400,7 +400,7 @@ async function rotateExternalAuthorityEpoch( } function rotatedRunAttachPayload( - state: Record, + state: { commands?: unknown }, desired: DurableRecoveryIdentity, authorizedTools: Record | null, completionContract: @@ -418,7 +418,22 @@ function rotatedRunAttachPayload( ); if (!seed) throw new Error("native_runner_authority_rotation_seed_unavailable"); - const payload = structuredClone(record(seed.payload)); + return retargetRunAttachPayload( + record(seed.payload), + desired, + authorizedTools, + completionContract, + ); +} + +function retargetRunAttachPayload( + seedPayload: Record, + desired: DurableRecoveryIdentity, + authorizedTools: Record | null, + completionContract: + { revision: string; criterionIds: readonly string[] } | undefined, +): Record { + const payload = structuredClone(seedPayload); const provider = record(payload.provider); if (provider.kind === "acpx" || provider.provider === "acpx") { provider.runId = desired.runId; @@ -958,7 +973,10 @@ export interface CapabilityRunnerdCodexTransportOptions { itemId: string; }; /** Registers the run-bound PRP authority on Paperclip's shared HTTP server. */ - controlPlaneRegistration?: (authority: DurablePrpControlPlane) => Promise<{ + controlPlaneRegistration?: ( + authority: DurablePrpControlPlane, + identity?: DurableRecoveryIdentity, + ) => Promise<{ connectUrl?: string; connection?: RunnerProcessConnection; activate?: () => Promise | void; @@ -1942,6 +1960,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { #expectedProviderTurnId: string | null = null; #durableTurnId = ""; #authorizedTools: Record | null = null; + #runAttachTemplate: Record | null = null; #closed = false; #closePromise: Promise | null = null; #failure: Error | null = null; @@ -2216,6 +2235,94 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#handler = handler; } + async attachRun(input: { + runId: string; + turnId: string; + itemId: string; + }): Promise { + const core = this.#core; + if (!core || !this.#startupComplete) { + throw new Error("native_runner_prp_run_rotation_unavailable"); + } + const prior = core.store.state.identity; + const desired: DurableRecoveryIdentity = { + ...prior, + runId: input.runId, + turnId: input.turnId, + itemId: input.itemId, + }; + const registration = this.options.controlPlaneRegistration + ? await this.options.controlPlaneRegistration(core, desired) + : null; + const connection: RunnerProcessConnection = + registration?.connection ?? + (registration?.connectUrl + ? { mode: "connect", connectUrl: registration.connectUrl } + : { mode: "connect", connectUrl: core.connectUrl }); + const commandId = `command_attach_${createHash("sha256") + .update(`${prior.runId}:${desired.runId}:${desired.turnId}`) + .digest("hex") + .slice(0, 32)}`; + const runAttachTemplate = this.#runAttachTemplate + ? retargetRunAttachPayload( + this.#runAttachTemplate, + desired, + this.#authorizedTools, + this.options.resumeCompletionContract, + ) + : rotatedRunAttachPayload( + core.store.state, + desired, + this.#authorizedTools, + this.options.resumeCompletionContract, + ); + this.#runAttachTemplate = structuredClone(runAttachTemplate); + const payload = { + ...runAttachTemplate, + paperclipNextAuthority: { identity: desired, connection }, + }; + core.queueCommand("run.attach", payload, commandId, true); + await this.#waitCommand("run.attach", commandId); + const attached = core.store.state.commands.find( + (command) => command.commandId === commandId, + ); + if (attached?.status !== "completed") { + await Promise.resolve(registration?.release()).catch(() => undefined); + throw new Error("native_runner_prp_run_rotation_failed"); + } + + const previousRelease = this.#controlPlaneRelease; + core.rotateRunIdentity(desired); + this.#eventIndex = 0; + this.#durableTurnId = desired.turnId; + this.#controlPlaneRelease = registration?.release ?? null; + let previousReleased = false; + try { + await registration?.activate?.(); + if (registration?.failure) { + void registration.failure.catch((error: unknown) => { + this.#failTransport( + error instanceof Error ? error : new Error(String(error)), + ); + }); + } + await previousRelease?.(); + previousReleased = true; + await this.#awaitRegistrationReady(registration?.ready); + } catch (error) { + const failure = error instanceof Error ? error : new Error(String(error)); + this.#controlPlaneRelease = null; + await Promise.allSettled([ + Promise.resolve().then(() => registration?.release()), + ...(previousReleased + ? [] + : [Promise.resolve().then(() => previousRelease?.())]), + ]); + this.#failTransport(failure); + throw failure; + } + } + async resolveRuntimeRequest(input: { requestId: string; turnId: string; @@ -3298,15 +3405,14 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { }); this.#core = core; if (rotatedAuthority) { - core.queueCommand( - "run.attach", - rotatedRunAttachPayload( - controlPlaneState, - desiredIdentity, - this.#authorizedTools, - this.options.resumeCompletionContract, - ), + const runAttachTemplate = rotatedRunAttachPayload( + controlPlaneState, + desiredIdentity, + this.#authorizedTools, + this.options.resumeCompletionContract, ); + this.#runAttachTemplate = structuredClone(runAttachTemplate); + core.queueCommand("run.attach", runAttachTemplate); } const committedEvents = core.store.state.committedEvents; const runAttachment = recoveredRunAttachment(core.store.state); diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts index 41da01fb5a..84787dff01 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts @@ -1240,6 +1240,8 @@ describe("Daytona sandbox provider plugin", () => { providerLeaseId: "sandbox-reuse", metadata: { resumedLease: true, + resumedFromState: "stopped", + sandboxState: "started", workspaceSentinel: { result: "matched", token: "sentinel-token", @@ -4060,6 +4062,35 @@ describe("daytona native file-sync hooks", () => { }); }); + it("classifies a deleted sandbox during syncOut with a stable unrecoverable code", async () => { + const hostDir = await makeHostDir(); + mockGet.mockRejectedValue( + new MockDaytonaNotFoundError("provider detail must not escape"), + ); + + await expect( + plugin.definition.onEnvironmentSyncOut?.({ + driverKey: "daytona", + companyId: "company-1", + environmentId: "env-1", + config: { timeoutMs: 300000, reuseLease: true }, + lease: syncLease(), + operations: [ + { + operationId: "sync-op-missing-sandbox", + files: [ + { + sourcePath: `${REMOTE_DIR}/out/result.txt`, + targetPath: path.join(hostDir, "result.txt"), + kind: "file", + }, + ], + }, + ], + }), + ).rejects.toThrow("daytona_sandbox_not_found"); + }); + it("syncOut snapshot guard re-checks the resolved source is a non-symlink regular file immediately before copying (validation→copy TOCTOU)", async () => { const hostDir = await makeHostDir(); const sandbox = createMockSandbox(); diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.ts index 52819929b5..afb988d64e 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.ts @@ -619,6 +619,8 @@ function leaseMetadata(input: { shellCommand: "bash" | "sh"; remoteCwd: string; resumedLease: boolean; + resumedFromState?: string | null; + sandboxState?: string | null; workspaceSentinel?: WorkspaceSentinelResult; }) { return { @@ -626,7 +628,7 @@ function leaseMetadata(input: { shellCommand: input.shellCommand, sandboxId: input.sandbox.id, sandboxName: input.sandbox.name, - sandboxState: input.sandbox.state ?? null, + sandboxState: input.sandboxState ?? input.sandbox.state ?? null, image: input.config.image, snapshot: input.config.snapshot, target: input.sandbox.target, @@ -637,6 +639,9 @@ function leaseMetadata(input: { ...(input.config.archiveOnRelease ? { archiveOnRelease: true } : {}), remoteCwd: input.remoteCwd, resumedLease: input.resumedLease, + ...(input.resumedLease + ? { resumedFromState: input.resumedFromState ?? null } + : {}), // Record the resources Paperclip attempted to request so future diagnosis // can compare requested allocation against what Daytona provisioned. ...(input.config.cpu != null ? { cpu: input.config.cpu } : {}), @@ -2187,61 +2192,72 @@ const plugin = definePlugin({ return { providerLeaseId: null, metadata: { expired: true } }; } - // A stopped sandbox loses its session shell, so the stored session id is - // stale after a real restart. Clear the id only when the sandbox is not - // already running, and clear it before the restart. A stopped sandbox has - // no live session, so the clear drops a dead id and a later command opens - // a fresh session. A running sandbox keeps its live session, so the resume - // leaves the id in place; a concurrent command still finds it and teardown - // deletes one session. An unconditional clear would drop the id of a live - // session and leak its shell until sandbox reaping. - if (sandbox.state !== "started") { - sandboxHandleSessionStore.clear(scope); - // A stopped sandbox loses its pseudo-terminals, so a stored duplex channel - // is dead after a real restart. Close and drop every channel on this lease - // before the restart, so no stale channel id survives the resume. - await closeDaytonaDuplexChannelsForLease(params.providerLeaseId); - } - await ensureSandboxStarted(sandbox, toTimeoutSeconds(config.timeoutMs)); - try { - const remoteCwd = await resolveSandboxWorkingDirectory(sandbox); - // C3: a resumed lease must clear the workspace sentinel before it is - // trusted, even when the handle came from the cache. On any non-match we - // evict the cached handle and expire the lease so a stale/foreign sandbox - // is never reused on the subsequent (sentinel-skipping) exec path. - const workspaceSentinel = await verifyWorkspaceSentinel({ - sandbox, - remoteCwd, - leaseMetadata: params.leaseMetadata, - timeoutSeconds: toTimeoutSeconds(config.timeoutMs), - }); - if (workspaceSentinel.result !== "matched") { - evictSandboxHandle(scope); - return { providerLeaseId: null, metadata: { expired: true, workspaceSentinel } }; - } - const shellCommand = await detectSandboxShellCommand(sandbox, toTimeoutSeconds(config.timeoutMs)); - sandboxHandleCache.markFresh(scope); - sandboxHandleLeaseAdmissionStates.open(scope); - return { - providerLeaseId: sandbox.id, - metadata: leaseMetadata({ - config, - sandbox, - shellCommand, - remoteCwd, - resumedLease: true, - workspaceSentinel, - }), - }; - } catch (error) { - evictSandboxHandle(scope); - // A timeout, rate limit, or provider 5xx does not prove this sandbox is - // lost. Preserve the exact resource and let the host retry its recorded - // lease; replacement is permitted only after an explicit not-found or - // an immutable workspace identity mismatch. - throw error; - } - }, { allowClosed: true }); + // A stopped sandbox loses its session shell, so the stored session id is + // stale after a real restart. Clear the id only when the sandbox is not + // already running, and clear it before the restart. A stopped sandbox has + // no live session, so the clear drops a dead id and a later command opens + // a fresh session. A running sandbox keeps its live session, so the resume + // leaves the id in place; a concurrent command still finds it and teardown + // deletes one session. An unconditional clear would drop the id of a live + // session and leak its shell until sandbox reaping. + if (sandbox.state !== "started") { + sandboxHandleSessionStore.clear(scope); + // A stopped sandbox loses its pseudo-terminals, so a stored duplex channel + // is dead after a real restart. Close and drop every channel on this lease + // before the restart, so no stale channel id survives the resume. + await closeDaytonaDuplexChannelsForLease(params.providerLeaseId); + } + const resumedFromState = sandbox.state ?? null; + await ensureSandboxStarted(sandbox, toTimeoutSeconds(config.timeoutMs)); + try { + const remoteCwd = await resolveSandboxWorkingDirectory(sandbox); + // C3: a resumed lease must clear the workspace sentinel before it is + // trusted, even when the handle came from the cache. On any non-match we + // evict the cached handle and expire the lease so a stale/foreign sandbox + // is never reused on the subsequent (sentinel-skipping) exec path. + const workspaceSentinel = await verifyWorkspaceSentinel({ + sandbox, + remoteCwd, + leaseMetadata: params.leaseMetadata, + timeoutSeconds: toTimeoutSeconds(config.timeoutMs), + }); + if (workspaceSentinel.result !== "matched") { + evictSandboxHandle(scope); + return { + providerLeaseId: null, + metadata: { expired: true, workspaceSentinel }, + }; + } + const shellCommand = await detectSandboxShellCommand( + sandbox, + toTimeoutSeconds(config.timeoutMs), + ); + sandboxHandleCache.markFresh(scope); + sandboxHandleLeaseAdmissionStates.open(scope); + return { + providerLeaseId: sandbox.id, + metadata: leaseMetadata({ + config, + sandbox, + shellCommand, + remoteCwd, + resumedLease: true, + resumedFromState, + sandboxState: "started", + workspaceSentinel, + }), + }; + } catch (error) { + evictSandboxHandle(scope); + // A timeout, rate limit, or provider 5xx does not prove this sandbox is + // lost. Preserve the exact resource and let the host retry its recorded + // lease; replacement is permitted only after an explicit not-found or + // an immutable workspace identity mismatch. + throw error; + } + }, + { allowClosed: true }, + ); }, async onEnvironmentReleaseLease( @@ -2828,18 +2844,28 @@ const plugin = definePlugin({ providerLeaseId: params.lease.providerLeaseId, config, }; - return await withSandboxActivityGate(scope, async () => { - const sandbox = await getSandbox(scope, { bypassTeardownGate: true }); - await ensureSandboxStarted(sandbox, timeoutSeconds); - const result = await performSyncOut({ - sandbox, - operations: params.operations, - remoteDir, - timeoutSeconds, + try { + return await withSandboxActivityGate(scope, async () => { + const sandbox = await getSandbox(scope, { bypassTeardownGate: true }); + await ensureSandboxStarted(sandbox, timeoutSeconds); + const result = await performSyncOut({ + sandbox, + operations: params.operations, + remoteDir, + timeoutSeconds, + }); + sandboxHandleCache.markFresh(scope); + return result; }); - sandboxHandleCache.markFresh(scope); - return result; - }); + } catch (error) { + // A deleted Daytona sandbox is the one provider failure that proves its + // unexported workspace bytes no longer exist. Convert the SDK class to a + // stable cross-worker message; every other error remains retryable. + if (error instanceof DaytonaNotFoundError) { + throw new Error("daytona_sandbox_not_found"); + } + throw error; + } }, // Open one live login pseudo-terminal. Resolve the cached sandbox by the diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index e69e2fa9a9..d24350465f 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -561,6 +561,18 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { mode: "shared_workspace", }, }); + const workspaceSyncStamp = { + schema: "paperclip.native-workspace-stamp/v1", + workspaceId: seeded.executionWorkspaceId, + providerLeaseId: "sandbox-exact-resume", + remoteCwd: "/workspace", + hostSha256: "a".repeat(64), + finalizedRunId: seeded.runId, + }; + await environmentService(db).updateLeaseMetadata(first.lease.id, { + ...(first.lease.metadata ?? {}), + nativeWorkspaceSync: workspaceSyncStamp, + }); await runtimeWithPlugin.releaseRunLeases( seeded.runId, "released", @@ -592,9 +604,17 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { expect(first.lease.metadata?.sandboxLeaseAcquisition).toEqual({ outcome: "created" }); expect(acquired.lease.providerLeaseId).toBe("sandbox-exact-resume"); - expect(acquired.lease.metadata?.sandboxLeaseAcquisition).toEqual({ outcome: "resumed" }); - expect(workerManager.call.mock.calls.filter((call) => call[1] === "environmentAcquireLease")) - .toHaveLength(1); + expect(acquired.lease.metadata?.sandboxLeaseAcquisition).toEqual({ + outcome: "resumed", + }); + expect(acquired.lease.metadata?.nativeWorkspaceSync).toEqual( + workspaceSyncStamp, + ); + expect( + workerManager.call.mock.calls.filter( + (call) => call[1] === "environmentAcquireLease", + ), + ).toHaveLength(1); }); it("destroys a disposable paperclip_runner sandbox after the turn", async () => { diff --git a/server/src/__tests__/native-sandbox-lifecycle.test.ts b/server/src/__tests__/native-sandbox-lifecycle.test.ts index 4cc88c1a20..13b2553b9f 100644 --- a/server/src/__tests__/native-sandbox-lifecycle.test.ts +++ b/server/src/__tests__/native-sandbox-lifecycle.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from "vitest"; -import { resolveNativeSandboxLifecycle } from "../services/heartbeat.js"; +import { + providerResourceDispositionForTerminalRun, + resolveNativeSandboxLifecycle, + resolveReusableSandboxLifecycle, +} from "../services/heartbeat.js"; const reusableSandbox = { kind: "remote" as const, @@ -21,6 +25,19 @@ describe("paperclip_runner sandbox lifecycle", () => { }); }); + it("keeps the same warm reusable sandbox for a legacy adapter", () => { + expect( + resolveReusableSandboxLifecycle({ + lifecyclePolicy: { mode: "warm", idleTimeoutMs: 300_000 }, + target: reusableSandbox, + }), + ).toEqual({ + runnerProcess: "warm", + sandboxResource: "keep_running", + failoverBackup: "verified", + }); + }); + it("stops and reuses a per-turn reusable sandbox", () => { expect(resolveNativeSandboxLifecycle({ adapterType: "paperclip_runner", @@ -71,4 +88,22 @@ describe("paperclip_runner sandbox lifecycle", () => { target: { kind: "local" }, })).toBeNull(); }); + + it("keeps a warm sandbox only after a successful turn", () => { + expect( + providerResourceDispositionForTerminalRun("keep_running", "succeeded"), + ).toBe("keep_running"); + expect( + providerResourceDispositionForTerminalRun("keep_running", "failed"), + ).toBe("stop_and_retain"); + expect( + providerResourceDispositionForTerminalRun("keep_running", "cancelled"), + ).toBe("stop_and_retain"); + expect( + providerResourceDispositionForTerminalRun("keep_running", "timed_out"), + ).toBe("stop_and_retain"); + expect(providerResourceDispositionForTerminalRun("destroy", "failed")).toBe( + "destroy", + ); + }); }); diff --git a/server/src/__tests__/native-workspace-sync.test.ts b/server/src/__tests__/native-workspace-sync.test.ts new file mode 100644 index 0000000000..a55ee16512 --- /dev/null +++ b/server/src/__tests__/native-workspace-sync.test.ts @@ -0,0 +1,297 @@ +import { mkdtemp, readdir, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + directorySnapshotSha256, + serializeDirectorySnapshot, +} from "@paperclipai/adapter-utils/workspace-restore-merge"; + +import { + classifyNativeWorkspaceInbound, + nativeWorkspaceSyncInternals, + readNativeWorkspaceSyncReference, + resumeNativeWorkspaceSync, +} from "../services/native-runtime/native-workspace-sync.js"; + +const digest = "a".repeat(64); + +describe("native workspace sync durable metadata", () => { + const originalPaperclipHome = process.env.PAPERCLIP_HOME; + const originalPaperclipInstanceId = process.env.PAPERCLIP_INSTANCE_ID; + const cleanupDirs: string[] = []; + + afterEach(async () => { + if (originalPaperclipHome === undefined) delete process.env.PAPERCLIP_HOME; + else process.env.PAPERCLIP_HOME = originalPaperclipHome; + if (originalPaperclipInstanceId === undefined) + delete process.env.PAPERCLIP_INSTANCE_ID; + else process.env.PAPERCLIP_INSTANCE_ID = originalPaperclipInstanceId; + await Promise.all( + cleanupDirs + .splice(0) + .map((directory) => rm(directory, { recursive: true, force: true })), + ); + }); + + it("classifies fresh, warm, replacement, and same-run recovery inputs", () => { + expect( + classifyNativeWorkspaceInbound({ + kind: "new_run", + acquisition: "created", + hasPriorStamp: false, + }), + ).toBe("host_current"); + expect( + classifyNativeWorkspaceInbound({ + kind: "new_run", + acquisition: "resumed", + hasPriorStamp: true, + }), + ).toBe("adopt_remote"); + expect( + classifyNativeWorkspaceInbound({ + kind: "new_run", + acquisition: "resumed", + hasPriorStamp: false, + }), + ).toBe("host_current"); + expect( + classifyNativeWorkspaceInbound({ + kind: "new_run", + acquisition: "replacement", + hasPriorStamp: true, + }), + ).toBe("host_current"); + expect( + classifyNativeWorkspaceInbound({ + kind: "existing_run", + restartRecovery: true, + sameProviderLease: true, + }), + ).toBe("adopt_remote"); + expect( + classifyNativeWorkspaceInbound({ + kind: "existing_run", + restartRecovery: true, + sameProviderLease: false, + }), + ).toBe("durable_seed"); + expect(() => + classifyNativeWorkspaceInbound({ + kind: "existing_run", + restartRecovery: false, + sameProviderLease: true, + }), + ).toThrow("native_workspace_sync_unexpected_existing_descriptor"); + }); + + it("reads backward-compatible references and the resource disposition", () => { + const base = { + schema: "paperclip.native-workspace-sync/v1", + state: "prepared", + descriptorSha256: digest, + baselineSha256: digest, + finalHostSha256: null, + workspaceId: "workspace-1", + leaseId: "lease-1", + providerLeaseId: "sandbox-1", + remoteCwd: "/workspace", + }; + + expect(readNativeWorkspaceSyncReference(base)).toEqual({ + ...base, + resourceDisposition: null, + }); + expect( + readNativeWorkspaceSyncReference({ + ...base, + resourceDisposition: "keep_running", + }), + ).toEqual({ ...base, resourceDisposition: "keep_running" }); + expect( + readNativeWorkspaceSyncReference({ + ...base, + resourceDisposition: "delete_everything", + }), + ).toBeNull(); + }); + + it("rejects traversal before constructing a durable state path", () => { + expect(() => + nativeWorkspaceSyncInternals.descriptorPath("../run", digest), + ).toThrow("native_workspace_sync_invalid_run_id"); + expect(() => + nativeWorkspaceSyncInternals.descriptorPath("run-1", "../descriptor"), + ).toThrow("native_workspace_sync_descriptor_digest_invalid"); + }); + + it("writes one immutable descriptor when the same state is replayed", async () => { + const paperclipHome = await mkdtemp( + path.join(os.tmpdir(), "paperclip-native-workspace-sync-"), + ); + cleanupDirs.push(paperclipHome); + process.env.PAPERCLIP_HOME = paperclipHome; + process.env.PAPERCLIP_INSTANCE_ID = "descriptor-test"; + const baseline = { + exclude: [".paperclip-runtime"], + entries: new Map([ + [ + "continuity.txt", + { kind: "file" as const, mode: 0o644, hash: digest }, + ], + ]), + }; + const descriptor = { + schema: "paperclip.native-workspace-sync/v1" as const, + binding: { + runId: "run-idempotent", + companyId: "company-1", + workspaceId: "workspace-1", + leaseId: "lease-1", + providerLeaseId: "sandbox-1", + localCwd: path.join(paperclipHome, "workspace"), + remoteCwd: "/workspace", + }, + state: "prepared" as const, + baselineSha256: directorySnapshotSha256(baseline), + baseline: serializeDirectorySnapshot(baseline), + gitSnapshot: null, + seed: null, + createdAt: "2026-01-01T00:00:00.000Z", + finalizedAt: null, + finalHostSha256: null, + resourceDisposition: "keep_running" as const, + }; + + const first = + await nativeWorkspaceSyncInternals.writeDescriptor(descriptor); + const second = + await nativeWorkspaceSyncInternals.writeDescriptor(descriptor); + + expect(second).toEqual(first); + const files = await readdir( + path.dirname( + nativeWorkspaceSyncInternals.descriptorPath( + descriptor.binding.runId, + first.descriptorSha256, + ), + ), + ); + expect(files.filter((file) => file.endsWith(".json"))).toEqual([ + `descriptor.${first.descriptorSha256}.json`, + ]); + await expect( + nativeWorkspaceSyncInternals.readDescriptor({ + runId: descriptor.binding.runId, + reference: first, + }), + ).resolves.toMatchObject({ descriptor }); + }); + + it("repairs finalized remote and lease stamps after an interrupted commit", async () => { + const paperclipHome = await mkdtemp( + path.join(os.tmpdir(), "paperclip-native-workspace-sync-repair-"), + ); + cleanupDirs.push(paperclipHome); + process.env.PAPERCLIP_HOME = paperclipHome; + process.env.PAPERCLIP_INSTANCE_ID = "descriptor-repair-test"; + const baseline = { + exclude: [".paperclip-runtime"], + entries: new Map([ + [ + "continuity.txt", + { kind: "file" as const, mode: 0o644, hash: digest }, + ], + ]), + }; + const finalHostSha256 = "b".repeat(64); + const descriptor = { + schema: "paperclip.native-workspace-sync/v1" as const, + binding: { + runId: "run-finalized-repair", + companyId: "company-1", + workspaceId: "workspace-1", + leaseId: "lease-1", + providerLeaseId: "sandbox-1", + localCwd: path.join(paperclipHome, "workspace"), + remoteCwd: "/workspace", + }, + state: "finalized" as const, + baselineSha256: directorySnapshotSha256(baseline), + baseline: serializeDirectorySnapshot(baseline), + gitSnapshot: null, + seed: null, + createdAt: "2026-01-01T00:00:00.000Z", + finalizedAt: "2026-01-01T00:01:00.000Z", + finalHostSha256, + resourceDisposition: "keep_running" as const, + }; + const reference = + await nativeWorkspaceSyncInternals.writeDescriptor(descriptor); + const rows = (values: unknown[]) => { + const query = { + from: () => query, + where: () => query, + for: () => query, + limit: () => query, + then: ( + onfulfilled?: + ((value: unknown[]) => TResult1 | PromiseLike) | null, + onrejected?: + ((reason: unknown) => TResult2 | PromiseLike) | null, + ) => Promise.resolve(values).then(onfulfilled, onrejected), + }; + return query; + }; + let persistedLeaseMetadata: Record | null = null; + const db = { + select: () => + rows([{ runnerProfileJson: { nativeWorkspaceSync: reference } }]), + transaction: async (callback: (tx: unknown) => Promise) => + callback({ + select: () => rows([{ metadata: { retained: true } }]), + update: () => ({ + set: (value: { metadata: Record }) => ({ + where: async () => { + persistedLeaseMetadata = value.metadata; + }, + }), + }), + }), + }; + const execute = vi + .fn() + .mockResolvedValueOnce({ timedOut: false, exitCode: 1, stdout: "" }) + .mockResolvedValueOnce({ timedOut: false, exitCode: 0, stdout: "" }); + + await expect( + resumeNativeWorkspaceSync({ + db: db as never, + runId: descriptor.binding.runId, + target: { + kind: "remote", + transport: "sandbox", + remoteCwd: descriptor.binding.remoteCwd, + sandboxLeaseAcquisition: { + providerLeaseId: descriptor.binding.providerLeaseId, + }, + runner: { execute }, + } as never, + }), + ).resolves.toBe(true); + + expect(execute).toHaveBeenCalledTimes(2); + expect(persistedLeaseMetadata).toMatchObject({ + retained: true, + nativeWorkspaceSync: { + schema: "paperclip.native-workspace-stamp/v1", + workspaceId: descriptor.binding.workspaceId, + providerLeaseId: descriptor.binding.providerLeaseId, + remoteCwd: descriptor.binding.remoteCwd, + hostSha256: finalHostSha256, + finalizedRunId: descriptor.binding.runId, + }, + }); + }); +}); diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index 5451d53c0c..f35b16fb40 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -2060,6 +2060,11 @@ function createSandboxEnvironmentDriver( ...(reusableLease?.metadata?.nativeHarnessBackup ? { nativeHarnessBackup: reusableLease.metadata.nativeHarnessBackup } : {}), + ...(providerLease && reusableLease?.metadata?.nativeWorkspaceSync + ? { + nativeWorkspaceSync: reusableLease.metadata.nativeWorkspaceSync, + } + : {}), ...(reusableScope ? { reusableSandboxLease: reusableScope } : {}), }; try { @@ -2282,6 +2287,13 @@ function createSandboxEnvironmentDriver( ...(reusableLease?.metadata?.nativeHarnessBackup ? { nativeHarnessBackup: reusableLease.metadata.nativeHarnessBackup } : {}), + ...(reusableLease && + providerLease.providerLeaseId === reusableLease.providerLeaseId && + reusableLease.metadata?.nativeWorkspaceSync + ? { + nativeWorkspaceSync: reusableLease.metadata.nativeWorkspaceSync, + } + : {}), ...(reusableScope ? { reusableSandboxLease: reusableScope } : {}), }; try { @@ -3154,6 +3166,7 @@ const INTERNAL_PLUGIN_SANDBOX_CONFIG_KEYS = new Set([ "sandboxProviderPlugin", "sandboxLeaseAcquisition", "nativeHarnessBackup", + "nativeWorkspaceSync", ]); // Drop the host-internal and per-lease runtime keys from a sandbox config diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 195ad4810b..94a52139ee 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -142,6 +142,9 @@ import { materializeLegacyQuestionResponseWakeProjection, materializeNativeInteractionResponses, NativeCancellationPendingRecoveryError, + prepareNativeWorkspaceSync, + readNativeWorkspaceSyncReference, + recordNativeFinalizationFailure, type NativeRestartRecoveryClaim, rebindNativeSessionCheckpoint, reconcileNativeFinalizations, @@ -1746,12 +1749,53 @@ export function leaseReleaseStatusForRunStatus( return status === "failed" || status === "timed_out" ? "failed" : "released"; } +export function providerResourceDispositionForTerminalRun( + desired: ProviderResourceDisposition | undefined, + status: string | null | undefined, +): ProviderResourceDisposition | undefined { + return desired === "keep_running" && status !== "succeeded" + ? "stop_and_retain" + : desired; +} + export interface NativeSandboxLifecycle { runnerProcess: "per_turn" | "warm"; sandboxResource: "keep_running" | "stop_and_reuse" | "destroy_after_turn"; failoverBackup: "verified"; } +export function resolveReusableSandboxLifecycle(input: { + lifecyclePolicy: + | { mode: "per_turn"; idleTimeoutMs: null } + | { mode: "warm"; idleTimeoutMs: number }; + target: { + kind: "local" | "remote"; + transport?: string; + reusableLeaseConfigured?: boolean; + effectiveCapabilities?: { reusableLeases: boolean } | null; + } | null; +}): NativeSandboxLifecycle | null { + if (input.target?.kind !== "remote" || input.target.transport !== "sandbox") { + return null; + } + const reusableLease = + input.target.reusableLeaseConfigured === true && + input.target.effectiveCapabilities?.reusableLeases === true; + if (input.lifecyclePolicy.mode === "warm" && !reusableLease) { + throw new Error("runner_warm_lifecycle_requires_reusable_provider_lease"); + } + return { + runnerProcess: input.lifecyclePolicy.mode, + sandboxResource: + input.lifecyclePolicy.mode === "warm" + ? "keep_running" + : reusableLease + ? "stop_and_reuse" + : "destroy_after_turn", + failoverBackup: "verified", + }; +} + export function resolveNativeSandboxLifecycle(input: { adapterType: string; lifecyclePolicy: @@ -1770,22 +1814,7 @@ export function resolveNativeSandboxLifecycle(input: { input.target.transport !== "sandbox" ) return null; - const reusableLease = - input.target.reusableLeaseConfigured === true && - input.target.effectiveCapabilities?.reusableLeases === true; - if (input.lifecyclePolicy.mode === "warm" && !reusableLease) { - throw new Error("runner_warm_lifecycle_requires_reusable_provider_lease"); - } - return { - runnerProcess: input.lifecyclePolicy.mode, - sandboxResource: - input.lifecyclePolicy.mode === "warm" - ? "keep_running" - : reusableLease - ? "stop_and_reuse" - : "destroy_after_turn", - failoverBackup: "verified", - }; + return resolveReusableSandboxLifecycle(input); } export function applyPersistedExecutionWorkspaceConfig(input: { @@ -8170,6 +8199,18 @@ class NativeSessionResumeScheduledError extends Error { } } +class NativeWorkspaceFinalizeScheduledError extends Error { + constructor( + readonly original: unknown, + readonly terminalFailure: boolean, + readonly reasonCode: + "workspace_sync_out_failed" | "workspace_sync_out_unrecoverable", + ) { + super("Native workspace finalization recovery has been scheduled."); + this.name = "NativeWorkspaceFinalizeScheduledError"; + } +} + type WorkspaceReadyCommentWriter = { addComment: ( issueId: string, @@ -12945,7 +12986,10 @@ export function heartbeatService( // A result committed before the old controller stopped outranks process // recovery. Finish its durable workspace/status suffix before deciding // whether any provider authority needs to be reopened. - await reconcileNativeFinalizations(db); + await reconcileNativeFinalizations(db, undefined, { + environmentRuntime, + onWorkspaceSettled: settleRecoveredNativeWorkspace, + }); const intent = await readHotRestartIntent().catch((error) => { logger.warn( { err: error }, @@ -16854,6 +16898,41 @@ export function heartbeatService( return blocked; } + async function settleRecoveredNativeWorkspace(input: { + runId: string; + companyId: string; + agentId: string; + succeeded: boolean; + }) { + const settledRun = await getRun(input.runId); + const workspaceSyncReference = readNativeWorkspaceSyncReference( + parseObject(settledRun?.runnerProfileJson).nativeWorkspaceSync, + ); + await releaseEnvironmentLeasesForRun({ + runId: input.runId, + companyId: input.companyId, + agentId: input.agentId, + status: settledRun?.status, + failureReason: settledRun?.error ?? undefined, + providerResourceDisposition: input.succeeded + ? (workspaceSyncReference?.resourceDisposition ?? "stop_and_retain") + : "stop_and_retain", + }); + await releaseRuntimeServicesForRun(input.runId).catch(() => undefined); + await finalizeAgentStatus( + input.agentId, + input.succeeded ? "succeeded" : "failed", + input.succeeded + ? null + : (settledRun?.error ?? "native_workspace_sync_out_failed"), + { + wasFirstHeartbeat: settledRun + ? timerClaimWasFirstHeartbeat(settledRun) + : undefined, + }, + ).catch(() => undefined); + } + async function reapOrphanedRuns(opts?: { staleThresholdMs?: number }) { const staleThresholdMs = opts?.staleThresholdMs ?? 0; const now = new Date(); @@ -16861,7 +16940,10 @@ export function heartbeatService( // Complete persisted native results before generic orphan recovery. The // reconciler reads the durable workspace barrier and persisted runtime // mode, never the current feature flag. - await reconcileNativeFinalizations(db).catch((error) => { + await reconcileNativeFinalizations(db, undefined, { + environmentRuntime, + onWorkspaceSettled: settleRecoveredNativeWorkspace, + }).catch((error) => { logger.warn( { err: error }, "failed to reconcile persisted native finalizations before orphan reaping", @@ -17876,6 +17958,10 @@ export function heartbeatService( activeRunExecutions.add(run.id); let runScratch: HeartbeatRunScratch | null = null; let nativeSessionResumeScheduled = false; + let nativeWorkspaceFinalizeScheduled = false; + let nativeWorkspaceSync: Awaited< + ReturnType + > = null; let providerResourceDispositionForRun: ProviderResourceDisposition | undefined; let nativeLifecycleTelemetryForRun: @@ -19659,6 +19745,9 @@ export function heartbeatService( driver: selectedEnvironment.driver, leaseId: activeEnvironmentLease.lease.id, workspaceRealization, + sandboxLeaseAcquisition: + activeEnvironmentLease.lease.metadata?.sandboxLeaseAcquisition ?? + null, ...(typeof activeEnvironmentLease.lease.metadata?.remoteCwd === "string" ? { remoteCwd: activeEnvironmentLease.lease.metadata.remoteCwd, @@ -20363,6 +20452,9 @@ export function heartbeatService( agentId: heartbeatRuns.agentId, runnerInstanceId: heartbeatRuns.runnerInstanceId, nativeSessionId: heartbeatRuns.nativeSessionId, + processPid: heartbeatRuns.processPid, + processGroupId: heartbeatRuns.processGroupId, + processStartedAt: heartbeatRuns.processStartedAt, runnerProfileJson: heartbeatRuns.runnerProfileJson, }) .from(heartbeatRuns) @@ -20718,6 +20810,21 @@ export function heartbeatService( ? previousNativeRun.runnerInstanceId : (lockedRun.runnerInstanceId ?? nativeRunnerInstanceId), nativeSessionId: lockedRun.nativeSessionId ?? nativeSessionId, + processPid: + lockedRun.processPid ?? + (previousNativeRun?.nativeSessionId === nativeSessionId + ? previousNativeRun.processPid + : null), + processGroupId: + lockedRun.processGroupId ?? + (previousNativeRun?.nativeSessionId === nativeSessionId + ? previousNativeRun.processGroupId + : null), + processStartedAt: + lockedRun.processStartedAt ?? + (previousNativeRun?.nativeSessionId === nativeSessionId + ? previousNativeRun.processStartedAt + : null), nativeIssueId: lockedRun.nativeIssueId ?? issueRef.id, driverKind: lockedRun.driverKind ?? @@ -20745,7 +20852,30 @@ export function heartbeatService( }) .onConflictDoNothing(); }); + nativeWorkspaceSync = await prepareNativeWorkspaceSync({ + db, + runId: run.id, + companyId: agent.companyId, + workspaceId: nativeExecutionWorkspaceId, + workspaceLocalDir: executionWorkspace.cwd, + target: executionTarget, + lease: activeEnvironmentLease.lease, + restartRecovery: runOptions.nativeRestartRecovery, + resourceDisposition: providerResourceDispositionForRun, + }); } else { + const legacyWarmLifecycle = + executionTarget?.kind === "remote" && + executionTarget.transport === "sandbox" && + executionTarget.runnerLifecyclePolicy?.mode === "warm" + ? resolveReusableSandboxLifecycle({ + lifecyclePolicy: executionTarget.runnerLifecyclePolicy, + target: executionTarget, + }) + : null; + if (legacyWarmLifecycle?.sandboxResource === "keep_running") { + providerResourceDispositionForRun = "keep_running"; + } await db .update(heartbeatRuns) .set({ @@ -21315,19 +21445,26 @@ export function heartbeatService( // If recording the barrier itself fails, propagate as a run failure // rather than silently leaving dependents stranded behind a missing // finalize row. + if (nativeWorkspaceSync) { + await nativeWorkspaceSync.restoreWorkspace(); + } await recordWorkspaceFinalize("succeeded"); if (adapterResult.nativeFinalization) { adapterResult.nativeFinalization.workspaceFinalizeStatus = "succeeded"; try { - await finalizeNativeRun({ + const finalized = await finalizeNativeRun({ db, runId: run.id, workspaceFinalizeStatus: "succeeded", + preserveProviderAttempt: Boolean(nativeWorkspaceSync), }); await dispatchPendingNativeStatusWakeups({ companyId: run.companyId, }); + if (finalized.phase === "committed") { + await nativeWorkspaceSync?.cleanup(); + } } catch (finalizeErr) { logger.warn( { err: finalizeErr, runId: run.id }, @@ -21379,6 +21516,40 @@ export function heartbeatService( ); } if (nativeRuntimeResolution.kind === "native") { + const proposedResult = await db + .select({ resultId: nativeRunFinalizations.resultId }) + .from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, run.id)) + .limit(1) + .then((rows) => rows[0]?.resultId ?? null); + if (proposedResult && nativeWorkspaceSync) { + const workspaceFailureMessage = + adapterErr instanceof Error ? adapterErr.message : ""; + const unrecoverable = + workspaceFailureMessage === + "workspace_sync_out_unrecoverable" || + workspaceFailureMessage.includes("daytona_sandbox_not_found"); + const failure = await recordNativeFinalizationFailure({ + db, + runId: run.id, + error: new Error( + unrecoverable + ? "native_workspace_sync_out_unrecoverable" + : "native_workspace_sync_out_failed", + ), + projectRunStatus: true, + failureScope: "workspace", + permanent: unrecoverable, + }); + nativeWorkspaceFinalizeScheduled = true; + throw new NativeWorkspaceFinalizeScheduledError( + adapterErr, + failure.phase === "terminal_failure", + unrecoverable + ? "workspace_sync_out_unrecoverable" + : "workspace_sync_out_failed", + ); + } try { await finalizeNativeRun({ db, @@ -22064,6 +22235,45 @@ export function heartbeatService( } return; } + if (err instanceof NativeWorkspaceFinalizeScheduledError) { + const coordinator = await db + .select({ + nextAttemptAt: nativeRunFinalizations.nextAttemptAt, + attempt: nativeRunFinalizations.attempt, + }) + .from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, run.id)) + .limit(1) + .then((rows) => rows[0] ?? null); + await appendRunEvent(run, { + eventType: "lifecycle", + stream: "system", + level: err.terminalFailure ? "error" : "warn", + message: err.terminalFailure + ? "native result is durable, but the sandbox containing unexported workspace changes is unrecoverable" + : "native result is durable; workspace copy-back will retry without another provider turn", + payload: { + attempt: coordinator?.attempt ?? null, + nextAttemptAt: coordinator?.nextAttemptAt?.toISOString() ?? null, + fallbackSuppressed: true, + retryReasonCode: err.reasonCode, + }, + }).catch(() => undefined); + if (err.terminalFailure) { + // The durable coordinator already failed the run, blocked the + // issue, and cleared its execution lock. Let ordinary teardown + // release the now-useless lease and return the agent to service. + nativeWorkspaceFinalizeScheduled = false; + providerResourceDispositionForRun = "stop_and_retain"; + await finalizeAgentStatus( + run.agentId, + "failed", + "native_workspace_sync_out_unrecoverable", + { wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run) }, + ).catch(() => undefined); + } + return; + } const message = redactCurrentUserText( err instanceof Error ? err.message : "Unknown adapter failure", await getCurrentUserRedactionOptions(), @@ -22456,7 +22666,11 @@ export function heartbeatService( // terminal". When the teardown reaches this point with the run still // running or queued, force a terminal status before the lease is // released, so the UI never shows a finished task as "Live". - if (latestRun && !nativeSessionResumeScheduled) { + if ( + latestRun && + !nativeSessionResumeScheduled && + !nativeWorkspaceFinalizeScheduled + ) { latestRun = await terminalizeRunOnLeaseRelease(latestRun).catch( (terminalizeErr) => { logger.error( @@ -22467,7 +22681,15 @@ export function heartbeatService( }, ); } - if (!nativeSessionResumeScheduled) { + // Warm retention is earned only by a fully successful turn. A failed, + // cancelled, or timed-out run stops the reusable sandbox so the next + // acquisition must revalidate and explicitly resume it. + providerResourceDispositionForRun = + providerResourceDispositionForTerminalRun( + providerResourceDispositionForRun, + latestRun?.status, + ); + if (!nativeSessionResumeScheduled && !nativeWorkspaceFinalizeScheduled) { await releaseEnvironmentLeasesForRun({ runId: run.id, companyId: run.companyId, @@ -22539,7 +22761,11 @@ export function heartbeatService( } } activeRunExecutions.delete(run.id); - if (!nativeSessionResumeScheduled && !shutdownInProgress) { + if ( + !nativeSessionResumeScheduled && + !nativeWorkspaceFinalizeScheduled && + !shutdownInProgress + ) { await startNextQueuedRunForAgent(run.agentId); } } diff --git a/server/src/services/native-runtime/index.ts b/server/src/services/native-runtime/index.ts index f5edaaacb5..643e18b9a9 100644 --- a/server/src/services/native-runtime/index.ts +++ b/server/src/services/native-runtime/index.ts @@ -9,4 +9,5 @@ export * from "./paperclip-control-plane-port.js"; export * from "./native-run-finalizer.js"; export * from "./native-finalization-reconciler.js"; export * from "./native-restart-recovery.js"; +export * from "./native-workspace-sync.js"; export * from "./status-arbiter.js"; diff --git a/server/src/services/native-runtime/native-finalization-reconciler.ts b/server/src/services/native-runtime/native-finalization-reconciler.ts index 4e7073c42c..294a92b985 100644 --- a/server/src/services/native-runtime/native-finalization-reconciler.ts +++ b/server/src/services/native-runtime/native-finalization-reconciler.ts @@ -24,6 +24,11 @@ import { issueRecoveryActionService } from "../issue-recovery-actions.js"; import { issueService } from "../issues.js"; import { emitAgentTaskRun } from "../agent-task-run-telemetry.js"; import { resumeNativeWorkspaceFinalization } from "./native-workspace-finalizer.js"; +import { + cleanupNativeWorkspaceSync, + readNativeWorkspaceSyncReference, +} from "./native-workspace-sync.js"; +import type { EnvironmentRuntimeService } from "../environment-runtime.js"; import { classifyNativeEvidence } from "./evidence-classifier.js"; import { recordNativeWorkAssessment } from "./work-assessments.js"; import { @@ -373,19 +378,33 @@ export async function claimNativeSessionResumptions(input: { } /** Recovery is keyed only by persisted mode/coordinator state, never the live flag. */ -export async function reconcileNativeFinalizations(db: Db, runIds?: string[]) { - const rows = await db.select({ - runId: heartbeatRuns.id, - companyId: heartbeatRuns.companyId, - agentId: heartbeatRuns.agentId, - issueId: nativeRunFinalizations.issueId, - issueStatus: issues.status, - issueStatusVersion: issues.statusVersion, - issueDecisionId: issues.lastStatusDecisionId, - coordinatorPhase: nativeRunFinalizations.phase, - assessmentId: nativeRunFinalizations.assessmentId, - decisionId: nativeRunFinalizations.decisionId, - }) +export async function reconcileNativeFinalizations( + db: Db, + runIds?: string[], + options: { + environmentRuntime?: EnvironmentRuntimeService; + onWorkspaceSettled?: (input: { + runId: string; + companyId: string; + agentId: string; + succeeded: boolean; + }) => Promise; + } = {}, +) { + const rows = await db + .select({ + runId: heartbeatRuns.id, + companyId: heartbeatRuns.companyId, + agentId: heartbeatRuns.agentId, + issueId: nativeRunFinalizations.issueId, + issueStatus: issues.status, + issueStatusVersion: issues.statusVersion, + issueDecisionId: issues.lastStatusDecisionId, + coordinatorPhase: nativeRunFinalizations.phase, + assessmentId: nativeRunFinalizations.assessmentId, + decisionId: nativeRunFinalizations.decisionId, + runnerProfileJson: heartbeatRuns.runnerProfileJson, + }) .from(heartbeatRuns) .innerJoin(nativeRunFinalizations, eq(nativeRunFinalizations.runId, heartbeatRuns.id)) .innerJoin(issues, and( @@ -617,14 +636,59 @@ export async function reconcileNativeFinalizations(db: Db, runIds?: string[]) { if (!recoveryDecision.effects.some((effect) => effect.kind === "resume_workspace_operation")) { throw new Error("native_reconciliation_workspace_resume_policy_missing"); } - const operation = await resumeNativeWorkspaceFinalization({ db, runId: row.runId }); - const workspaceFinalizeStatus = operation.status === "succeeded" ? "succeeded" : "failed"; + const operation = await resumeNativeWorkspaceFinalization({ + db, + runId: row.runId, + environmentRuntime: options.environmentRuntime, + }); + const workspaceFinalizeStatus = + operation.status === "succeeded" ? "succeeded" : "failed"; + if (workspaceFinalizeStatus === "failed") { + const unrecoverable = operation.stderrExcerpt?.includes( + "workspace_sync_out_unrecoverable", + ); + const failure = await recordNativeFinalizationFailure({ + db, + runId: row.runId, + error: new Error( + unrecoverable + ? "native_workspace_sync_out_unrecoverable" + : "native_workspace_sync_out_failed", + ), + projectRunStatus: true, + failureScope: "workspace", + permanent: unrecoverable, + }); + results.push({ + ...failure, + reconciliationAction: "resume_workspace_operation" as const, + workspaceOperationId: operation.id, + workspaceFinalizeStatus, + reconciliationDecision: recoveryDecision, + }); + if (failure.phase === "terminal_failure") { + await options + .onWorkspaceSettled?.({ + runId: row.runId, + companyId: row.companyId, + agentId: row.agentId, + succeeded: false, + }) + .catch(() => undefined); + } + continue; + } try { const finalized = await finalizeNativeRun({ db, runId: row.runId, workspaceFinalizeStatus, projectRunStatus: true, + preserveProviderAttempt: Boolean( + readNativeWorkspaceSyncReference( + record(row.runnerProfileJson).nativeWorkspaceSync, + ), + ), }); results.push({ ...finalized, @@ -633,6 +697,20 @@ export async function reconcileNativeFinalizations(db: Db, runIds?: string[]) { workspaceFinalizeStatus, reconciliationDecision: recoveryDecision, }); + if ( + workspaceFinalizeStatus === "succeeded" && + finalized.phase === "committed" + ) { + await cleanupNativeWorkspaceSync(row.runId).catch(() => undefined); + await options + .onWorkspaceSettled?.({ + runId: row.runId, + companyId: row.companyId, + agentId: row.agentId, + succeeded: true, + }) + .catch(() => undefined); + } } catch (error) { const failure = await recordNativeFinalizationFailure({ db, @@ -659,12 +737,29 @@ export async function reconcileNativeFinalizations(db: Db, runIds?: string[]) { throw new Error("native_reconciliation_replay_policy_invalid"); } try { - results.push(await finalizeNativeRun({ + const finalized = await finalizeNativeRun({ db, runId: row.runId, workspaceFinalizeStatus: barrier.status as "succeeded" | "failed", projectRunStatus: true, - })); + preserveProviderAttempt: Boolean( + readNativeWorkspaceSyncReference( + record(row.runnerProfileJson).nativeWorkspaceSync, + ), + ), + }); + results.push(finalized); + if (finalized.phase === "committed") { + await cleanupNativeWorkspaceSync(row.runId).catch(() => undefined); + await options + .onWorkspaceSettled?.({ + runId: row.runId, + companyId: row.companyId, + agentId: row.agentId, + succeeded: true, + }) + .catch(() => undefined); + } } catch (error) { results.push(await recordNativeFinalizationFailure({ db, diff --git a/server/src/services/native-runtime/native-run-finalizer-telemetry.test.ts b/server/src/services/native-runtime/native-run-finalizer-telemetry.test.ts index ab0446d59a..c06e11c78b 100644 --- a/server/src/services/native-runtime/native-run-finalizer-telemetry.test.ts +++ b/server/src/services/native-runtime/native-run-finalizer-telemetry.test.ts @@ -279,6 +279,93 @@ describeEmbeddedPostgres("native run finalizer / status decision committer — a expect(run?.status).toBe("running"); }); + it("bounds workspace-only retries without consuming the provider attempt", async () => { + const fixture = await seedNativeRun(); + await db.insert(nativeRunFinalizations).values({ + runId: fixture.runId, + companyId, + issueId: fixture.issueId, + phase: "observed", + attempt: 1, + }); + + for ( + let expectedAttempt = 1; + expectedAttempt <= 3; + expectedAttempt += 1 + ) { + await recordNativeFinalizationFailure({ + db, + runId: fixture.runId, + error: new Error("native_workspace_sync_out_failed"), + projectRunStatus: true, + failureScope: "workspace", + }); + const coordinator = await db + .select() + .from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, fixture.runId)) + .then((rows) => rows[0]!); + expect(coordinator.attempt).toBe(1); + expect(coordinator.failureDetail).toMatchObject({ + workspaceFinalizeAttempt: expectedAttempt, + }); + expect(coordinator.phase).toBe( + expectedAttempt === 3 ? "terminal_failure" : "retryable_failure", + ); + } + + await expect( + db + .select({ status: heartbeatRuns.status }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, fixture.runId)) + .then((rows) => rows[0]?.status), + ).resolves.toBe("failed"); + }); + + it("blocks immediately when the sandbox with unexported changes is gone", async () => { + const fixture = await seedNativeRun(); + await db.insert(nativeRunFinalizations).values({ + runId: fixture.runId, + companyId, + issueId: fixture.issueId, + phase: "workspace_finalizing", + attempt: 1, + resultId: null, + }); + + const failure = await recordNativeFinalizationFailure({ + db, + runId: fixture.runId, + error: new Error("native_workspace_sync_out_unrecoverable"), + projectRunStatus: true, + failureScope: "workspace", + permanent: true, + }); + + expect(failure).toMatchObject({ + phase: "terminal_failure", + failureCode: "native_workspace_sync_out_unrecoverable", + nextAttemptAt: null, + attempt: 1, + }); + await expect( + db + .select({ status: heartbeatRuns.status }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, fixture.runId)) + .then((rows) => rows[0]?.status), + ).resolves.toBe("failed"); + await expect( + db + .select({ status: issues.status }) + .from(issues) + .where(eq(issues.id, fixture.issueId)) + .then((rows) => rows[0]?.status), + ).resolves.toBe("blocked"); + }); + it("emits exactly one event for a cancel_continuations write (trap 2: :685/:518 overlap)", async () => { const fixture = await seedNativeRun(); // Build the minimal real rows commitNativeStatusDecision's foreign keys @@ -329,7 +416,10 @@ describeEmbeddedPostgres("native run finalizer / status decision committer — a toStatus: "cancelled", reasonCode: "cancellation_issue_authorized", unblockDescriptor: null, - effects: [{ kind: "release_checkout" }, { kind: "cancel_continuations" }], + effects: [ + { kind: "release_checkout" }, + { kind: "cancel_continuations" }, + ], }; const callsBefore = mockTelemetryClient.track.mock.calls.length; diff --git a/server/src/services/native-runtime/native-run-finalizer.ts b/server/src/services/native-runtime/native-run-finalizer.ts index 979013c4d5..6042c748fb 100644 --- a/server/src/services/native-runtime/native-run-finalizer.ts +++ b/server/src/services/native-runtime/native-run-finalizer.ts @@ -126,7 +126,11 @@ async function acceptedInteractionFromRun(input: { .then((rows) => rows[0] ?? null); } -async function claimCoordinator(input: { db: Db; runId: string }) { +async function claimCoordinator(input: { + db: Db; + runId: string; + preserveProviderAttempt?: boolean; +}) { const leaseOwner = `native-finalizer:${randomUUID()}`; const now = new Date(); const claimed = await input.db.transaction(async (tx) => { @@ -152,23 +156,33 @@ async function claimCoordinator(input: { db: Db; runId: string }) { } if (coordinator.phase === "terminal_failure") throw new Error("native_finalization_terminal_failure"); if ( - coordinator.leaseOwner - && coordinator.leaseExpiresAt - && coordinator.leaseExpiresAt > now - && coordinator.leaseOwner !== leaseOwner - ) throw new Error("native_finalization_lease_busy"); - const [updated] = await tx.update(nativeRunFinalizations).set({ - leaseOwner, - leaseExpiresAt: new Date(now.getTime() + 5 * 60_000), - attempt: coordinator.attempt + 1, - phase: coordinator.phase === "retryable_failure" - ? coordinator.assessmentId ? "arbitrating" : "workspace_finalizing" - : coordinator.phase, - failureCode: null, - failureDetail: null, - nextAttemptAt: null, - updatedAt: now, - }).where(eq(nativeRunFinalizations.runId, input.runId)).returning(); + coordinator.leaseOwner && + coordinator.leaseExpiresAt && + coordinator.leaseExpiresAt > now && + coordinator.leaseOwner !== leaseOwner + ) + throw new Error("native_finalization_lease_busy"); + const [updated] = await tx + .update(nativeRunFinalizations) + .set({ + leaseOwner, + leaseExpiresAt: new Date(now.getTime() + 5 * 60_000), + attempt: input.preserveProviderAttempt + ? coordinator.attempt + : coordinator.attempt + 1, + phase: + coordinator.phase === "retryable_failure" + ? coordinator.assessmentId + ? "arbitrating" + : "workspace_finalizing" + : coordinator.phase, + failureCode: null, + failureDetail: null, + nextAttemptAt: null, + updatedAt: now, + }) + .where(eq(nativeRunFinalizations.runId, input.runId)) + .returning(); if (!updated) throw new Error("native_finalization_claim_failed"); return { coordinator: updated, leaseOwner }; }); @@ -183,6 +197,8 @@ async function recordRetryableFailure(input: { message: string; nextAction: string; projectRunStatus?: boolean; + failureScope?: "provider" | "workspace"; + permanent?: boolean; }) { const now = new Date(); const nextAttemptAt = new Date(now.getTime() + 30_000); @@ -228,50 +244,87 @@ async function recordRetryableFailure(input: { const supersededByNewerRun = Boolean( latestDecisionRun && latestDecisionRun.createdAt > input.run.createdAt, ); - const exhausted = input.coordinator.attempt >= 3; - const phase = supersededByNewerRun || exhausted - ? "terminal_failure" as const - : "retryable_failure" as const; + const priorFailureDetail = record(input.coordinator.failureDetail); + const workspaceFinalizeAttempt = + input.failureScope === "workspace" + ? (typeof priorFailureDetail.workspaceFinalizeAttempt === "number" && + Number.isInteger(priorFailureDetail.workspaceFinalizeAttempt) && + priorFailureDetail.workspaceFinalizeAttempt >= 0 + ? priorFailureDetail.workspaceFinalizeAttempt + : 0) + 1 + : null; + const exhausted = + input.permanent === true || + (workspaceFinalizeAttempt !== null + ? workspaceFinalizeAttempt >= 3 + : input.coordinator.attempt >= 3); + const phase = + supersededByNewerRun || exhausted + ? ("terminal_failure" as const) + : ("retryable_failure" as const); const failureCode = supersededByNewerRun ? "native_finalization_superseded" - : exhausted - ? "native_finalization_retry_exhausted" - : input.failureCode; - await tx.update(nativeRunFinalizations).set({ - phase, - leaseOwner: null, - leaseExpiresAt: null, - failureCode, - failureDetail: { - message: input.message.slice(0, 2_000), - originalFailureCode: input.failureCode, - recoveryOwner: supersededByNewerRun - ? { kind: "none", reason: "newer_native_decision" } - : exhausted - ? { kind: "board" } - : { kind: "agent", agentId: input.run.agentId }, - nextAction: input.nextAction, - }, - nextAttemptAt: supersededByNewerRun || exhausted ? null : nextAttemptAt, - updatedAt: now, - }).where(eq(nativeRunFinalizations.runId, input.run.id)); - const projectsTerminalStatus = exhausted && !supersededByNewerRun && input.projectRunStatus; - const [updatedRun] = await tx.update(heartbeatRuns).set({ - ...(projectsTerminalStatus ? { - status: exhaustedRunStatus, - finishedAt: input.run.finishedAt ?? now, - } : {}), - nativePhase: phase, - nativePhaseUpdatedAt: now, - resultJson: { - ...record(input.run.resultJson), - finalizationPhase: phase, + : input.permanent + ? input.failureCode + : exhausted + ? input.failureScope === "workspace" + ? "native_workspace_sync_out_retry_exhausted" + : "native_finalization_retry_exhausted" + : input.failureCode; + await tx + .update(nativeRunFinalizations) + .set({ + phase, + leaseOwner: null, + leaseExpiresAt: null, failureCode, - originalFailureCode: input.failureCode, - nextAttemptAt: supersededByNewerRun || exhausted ? null : nextAttemptAt.toISOString(), - }, - updatedAt: now, - }).where(eq(heartbeatRuns.id, input.run.id)).returning(); + failureDetail: { + message: input.message.slice(0, 2_000), + originalFailureCode: input.failureCode, + ...(workspaceFinalizeAttempt === null + ? {} + : { workspaceFinalizeAttempt }), + recoveryOwner: supersededByNewerRun + ? { kind: "none", reason: "newer_native_decision" } + : exhausted + ? { kind: "board" } + : { kind: "agent", agentId: input.run.agentId }, + nextAction: input.nextAction, + }, + nextAttemptAt: supersededByNewerRun || exhausted ? null : nextAttemptAt, + updatedAt: now, + }) + .where(eq(nativeRunFinalizations.runId, input.run.id)); + const projectsTerminalStatus = + exhausted && !supersededByNewerRun && input.projectRunStatus; + const [updatedRun] = await tx + .update(heartbeatRuns) + .set({ + ...(projectsTerminalStatus + ? { + status: + input.failureScope === "workspace" + ? "failed" + : exhaustedRunStatus, + finishedAt: input.run.finishedAt ?? now, + } + : {}), + nativePhase: phase, + nativePhaseUpdatedAt: now, + resultJson: { + ...record(input.run.resultJson), + finalizationPhase: phase, + failureCode, + originalFailureCode: input.failureCode, + nextAttemptAt: + supersededByNewerRun || exhausted + ? null + : nextAttemptAt.toISOString(), + }, + updatedAt: now, + }) + .where(eq(heartbeatRuns.id, input.run.id)) + .returning(); if (projectsTerminalStatus) terminalRunToEmit = updatedRun ?? null; if (supersededByNewerRun) { await issueRecoveryActionService(tx as unknown as Db).resolveActiveForIssue({ @@ -284,9 +337,16 @@ async function recordRetryableFailure(input: { resolutionNote: "A newer native run already committed the authoritative issue decision; the stale finalizer was retired without changing issue state.", }, tx); } else if (exhausted) { - await issueService(tx as unknown as Db).update(input.coordinator.issueId, { - status: "in_review", - }, tx); + await issueService(tx as unknown as Db).update( + input.coordinator.issueId, + { + status: + input.permanent && input.failureScope === "workspace" + ? "blocked" + : "in_review", + }, + tx, + ); } if (!supersededByNewerRun) { await issueRecoveryActionService(tx as unknown as Db).upsertSourceScoped({ @@ -301,10 +361,15 @@ async function recordRetryableFailure(input: { evidence: { runId: input.run.id, coordinatorAttempt: input.coordinator.attempt, + ...(workspaceFinalizeAttempt === null + ? {} + : { workspaceFinalizeAttempt }), originalFailureCode: input.failureCode, }, nextAction: exhausted - ? `Finalization retry budget exhausted. ${input.nextAction}` + ? input.permanent + ? input.nextAction + : `Finalization retry budget exhausted. ${input.nextAction}` : input.nextAction, wakePolicy: exhausted ? null @@ -328,6 +393,8 @@ export async function recordNativeFinalizationFailure(input: { runId: string; error: unknown; projectRunStatus?: boolean; + failureScope?: "provider" | "workspace"; + permanent?: boolean; }) { const [run, coordinator] = await Promise.all([ input.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId)) @@ -344,8 +411,15 @@ export async function recordNativeFinalizationFailure(input: { coordinator, failureCode, message, - nextAction: "Repair the persisted native result or contract discriminator, then resume finalization from the coordinator.", + nextAction: + input.failureScope === "workspace" + ? input.permanent + ? "Restore the exact sandbox containing the unexported workspace changes, or resolve the task manually from durable evidence." + : "Retry workspace export and merge from the retained sandbox; do not submit another provider turn." + : "Repair the persisted native result or contract discriminator, then resume finalization from the coordinator.", projectRunStatus: input.projectRunStatus, + failureScope: input.failureScope, + permanent: input.permanent, }); } @@ -395,12 +469,23 @@ export async function finalizeNativeRun(input: { workspaceFinalizeStatus: "succeeded" | "failed"; /** Reconciliation owns terminal run projection; the live heartbeat does it afterward. */ projectRunStatus?: boolean; + /** Workspace-only replay must not consume the provider recovery budget. */ + preserveProviderAttempt?: boolean; failpoint?: NativeStatusCommitFailpoint; }) { - const run = await input.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId)) - .limit(1).then((rows) => rows[0] ?? null); - if (!run || run.runtimeMode !== "native") throw new Error("native_finalization_run_missing"); - const claim = await claimCoordinator({ db: input.db, runId: input.runId }); + const run = await input.db + .select() + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, input.runId)) + .limit(1) + .then((rows) => rows[0] ?? null); + if (!run || run.runtimeMode !== "native") + throw new Error("native_finalization_run_missing"); + const claim = await claimCoordinator({ + db: input.db, + runId: input.runId, + preserveProviderAttempt: input.preserveProviderAttempt, + }); const coordinator = claim.coordinator; if (!claim.leaseOwner && coordinator.phase === "committed") { if (input.projectRunStatus) await projectCommittedRun({ db: input.db, run, coordinator }); diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 7a9c857026..e6e70a53d5 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -2880,7 +2880,7 @@ describe("native warm session supervision", () => { ); }); - it("rehydrates a runnerd warm session from its checkpoint under a fresh run authority", async () => { + it("reattaches a live runnerd warm session under a fresh run authority", async () => { const stateBase = await mkdtemp( join(tmpdir(), "paperclip-runnerd-warm-authority-"), ); @@ -2889,9 +2889,7 @@ describe("native warm session supervision", () => { process.env.PAPERCLIP_RUNNER_STATE_DIR = stateBase; process.env.PAPERCLIP_HOME = stateBase; const firstClose = vi.fn(async () => undefined); - const secondClose = vi.fn(async () => undefined); const firstSession = { close: firstClose }; - const secondSession = { close: secondClose }; const first = { ...execution, binding: { @@ -2947,18 +2945,8 @@ describe("native warm session supervision", () => { return result; }) .mockImplementationOnce(async (options) => { - expect(options.existingSession).toBeUndefined(); - expect(options.persistedSession).toEqual( - expect.objectContaining({ - identity: expect.objectContaining({ - runId: second.binding.runId, - sessionId: second.session.normalizedSessionId, - }), - providerSessionId: "provider-runnerd-warm", - activeTurnId: null, - }), - ); - options.onSession?.(secondSession); + expect(options.existingSession).toBe(firstSession); + expect(options.persistedSession).toBeUndefined(); return result; }); @@ -3014,10 +3002,10 @@ describe("native warm session supervision", () => { runnerInstanceId: "runner-runnerd-warm", useRunnerd: true, }); - expect(firstClose).toHaveBeenCalledWith({ - reason: "warm native session authority epoch rotated", - }); - await vi.waitFor(() => expect(secondClose).toHaveBeenCalled(), { + expect(firstClose).not.toHaveBeenCalled(); + await vi.waitFor(() => expect(firstClose).toHaveBeenCalledWith({ + reason: "warm native session idle timeout", + }), { timeout: 500, }); } finally { @@ -5392,9 +5380,13 @@ describe("runnerd provider runtime wiring", () => { const firstOptions = state.createBackend.mock.calls[0]![1]; const continuationOptions = state.createBackend.mock.calls[1]![1]; - await expect(firstOptions.dynamicToolHandler!({})).rejects.toThrow( - "native_tool_authority_epoch_revoked", - ); + // The retained runner backend owns one stable callback. After run.attach, + // that callback routes through the session-scope authority registry to + // the new run; stale provider calls are rejected earlier by runnerd's + // turn identity boundary. + await expect(firstOptions.dynamicToolHandler!({})).resolves.toEqual({ + runId: continuation.binding.runId, + }); await expect( continuationOptions.dynamicToolHandler!({}), ).resolves.toEqual({ runId: continuation.binding.runId }); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index f7fc0f2a3e..5af22bd551 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -2623,7 +2623,13 @@ export async function nativeProviderRecoveryEvidence(input: { const durableEvents = await input.db .select({ eventType: heartbeatRunEvents.eventType }) .from(heartbeatRunEvents) - .where(eq(heartbeatRunEvents.runId, input.runId)); + .where( + and( + eq(heartbeatRunEvents.runId, input.runId), + inArray(heartbeatRunEvents.eventType, [...PROVIDER_DURABLE_EVENT_TYPES]), + ), + ) + .limit(1); const providerEventsExist = durableEvents.some((event) => PROVIDER_DURABLE_EVENT_TYPES.has(event.eventType), ); @@ -4171,23 +4177,7 @@ async function executePaperclipNativeSessionWithinScope( entry.busy = true; if (entry.idleTimer !== null) clearTimeout(entry.idleTimer); entry.idleTimer = null; - if (input.useRunnerd) { - // A retained driver closes over the tool authority from the run that - // created it. Preserve provider continuity through its durable - // checkpoint, but rebuild the runnerd backend so the next run gets a - // fresh, immutable authority epoch. Reusing the live driver would - // either retain stale authority or require rebinding old callbacks. - warmNativeSessions.delete(warmSessionId); - await entry.session.close({ - reason: "warm native session authority epoch rotated", - }); - persistedWarmSession = loadWarmNativeCheckpoint( - input.execution, - warmConfigDigest, - ); - } else { - existingWarmSession = entry.session; - } + existingWarmSession = entry.session; } } else { persistedWarmSession = loadWarmNativeCheckpoint( @@ -7235,6 +7225,13 @@ async function createRunnerdBackendWithinSessionClaim( input.restartRecovery?.kind === "reattach_existing_runner" ? input.restartRecovery.process : null; + const executeCurrentToolAuthority = ( + call: Parameters[0], + ) => { + const current = sessionToolAuthorityEpochs.get(sessionScopeId); + if (!current) throw new Error("native_session_tool_authority_unavailable"); + return current.execute(call); + }; const backend = createNativeSessionBackend(runnerExecution, { runnerInstanceId: input.runnerInstanceId, environment: effectiveRunnerEnvironment, @@ -7243,8 +7240,8 @@ async function createRunnerdBackendWithinSessionClaim( : "local_filesystem", onSpawn: input.onSpawn, dynamicTools, - dynamicToolHandler: (call) => authorityEpoch.execute(call), - acpxDynamicToolHandler: (call) => authorityEpoch.execute(call), + dynamicToolHandler: executeCurrentToolAuthority, + acpxDynamicToolHandler: executeCurrentToolAuthority, opencodeRuntimeDirectory: resolve( resolvePaperclipInstanceRoot(), "runtime", @@ -7419,7 +7416,7 @@ async function createRunnerdBackendWithinSessionClaim( turnId: `turn-${input.execution.binding.runId}`, itemId: `item-${input.execution.binding.runId}`, }, - controlPlaneRegistration: (authority) => + controlPlaneRegistration: (authority, attachmentIdentity) => measureNativeRunnerSpan( input.trace, "runner.transport.connect", @@ -7446,7 +7443,9 @@ async function createRunnerdBackendWithinSessionClaim( () => registerRunnerPrpAuthority({ companyId: input.execution.binding.companyId, - runId: input.execution.binding.runId, + runId: + attachmentIdentity?.runId ?? + input.execution.binding.runId, authority, }), ); @@ -7469,7 +7468,9 @@ async function createRunnerdBackendWithinSessionClaim( () => resolvePaperclipRunnerTransport({ target, - runId: input.execution.binding.runId, + runId: + attachmentIdentity?.runId ?? + input.execution.binding.runId, localConnectUrl: "ws://127.0.0.1/unused", runnerPublicUrl: input.runnerPublicUrl, runnerCaBundlePath: input.runnerCaBundlePath, @@ -7505,7 +7506,9 @@ async function createRunnerdBackendWithinSessionClaim( () => resolvePaperclipRunnerTransport({ target, - runId: input.execution.binding.runId, + runId: + attachmentIdentity?.runId ?? + input.execution.binding.runId, localConnectUrl: "ws://127.0.0.1/unused", runnerPublicUrl: input.runnerPublicUrl, runnerCaBundlePath: input.runnerCaBundlePath, @@ -7541,7 +7544,9 @@ async function createRunnerdBackendWithinSessionClaim( () => registerRunnerPrpAuthority({ companyId: input.execution.binding.companyId, - runId: input.execution.binding.runId, + runId: + attachmentIdentity?.runId ?? + input.execution.binding.runId, authority, }), { parentName: "runner.transport.connect" }, diff --git a/server/src/services/native-runtime/native-workspace-finalizer.ts b/server/src/services/native-runtime/native-workspace-finalizer.ts index 49e87e7795..edec11741c 100644 --- a/server/src/services/native-runtime/native-workspace-finalizer.ts +++ b/server/src/services/native-runtime/native-workspace-finalizer.ts @@ -10,6 +10,13 @@ import { } from "@paperclipai/db"; import { workspaceOperationService } from "../workspace-operations.js"; import { inspectManagedGitWorktreeBranch } from "../workspace-runtime.js"; +import { environmentService } from "../environments.js"; +import type { EnvironmentRuntimeService } from "../environment-runtime.js"; +import { resolveEnvironmentExecutionTarget } from "../environment-execution-target.js"; +import { + readNativeWorkspaceSyncReference, + resumeNativeWorkspaceSync, +} from "./native-workspace-sync.js"; function record(value: unknown): Record { return value && typeof value === "object" && !Array.isArray(value) @@ -21,14 +28,27 @@ function readString(value: unknown) { return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; } +function workspaceSyncFailure( + code: "workspace_sync_out_failed" | "workspace_sync_out_unrecoverable", +) { + return { + status: "failed" as const, + exitCode: 1, + stderr: `${code}\n`, + metadata: { workspaceSync: { code } }, + }; +} + /** * Resume the real workspace-finalization action for a result-bearing native - * run. This service observes the workspace; it never fabricates a successful - * marker. The caller decides whether a failed observation is retryable. + * run. Durable sandbox-backed runs export and merge the remote workspace; + * older runs retain the local workspace validation path. The caller owns the + * durable retry and failure policy. */ export async function resumeNativeWorkspaceFinalization(input: { db: Db; runId: string; + environmentRuntime?: EnvironmentRuntimeService; }) { const bound = await input.db.select({ companyId: heartbeatRuns.companyId, @@ -57,7 +77,12 @@ export async function resumeNativeWorkspaceFinalization(input: { eq(workspaceOperations.phase, "workspace_finalize"), )).orderBy(desc(workspaceOperations.createdAt)).limit(1).then((rows) => rows[0] ?? null); - const persistedInput = record(record(bound.runnerProfileJson).nativeExecutionInput); + const persistedInput = record( + record(bound.runnerProfileJson).nativeExecutionInput, + ); + const nativeWorkspaceSync = readNativeWorkspaceSyncReference( + record(bound.runnerProfileJson).nativeWorkspaceSync, + ); const binding = record(persistedInput.binding); const workspaceId = previous?.executionWorkspaceId ?? bound.issueWorkspaceId @@ -87,6 +112,88 @@ export async function resumeNativeWorkspaceFinalization(input: { : "workspace_directory", }, run: async () => { + if (nativeWorkspaceSync) { + if (!input.environmentRuntime) { + return { + status: "failed", + exitCode: 1, + stderr: + "Native workspace finalization cannot access the environment runtime.\n", + }; + } + const environmentsSvc = environmentService(input.db); + const lease = await environmentsSvc.getLeaseById( + nativeWorkspaceSync.leaseId, + ); + const environment = lease?.environmentId + ? await environmentsSvc.getById(lease.environmentId) + : null; + if ( + !lease || + !environment || + lease.companyId !== bound.companyId || + environment.id !== lease.environmentId + ) { + return workspaceSyncFailure("workspace_sync_out_unrecoverable"); + } + if ( + lease.status === "expired" || + lease.status === "failed" || + lease.status === "pending_cleanup" || + !lease.providerLeaseId || + lease.providerLeaseId !== nativeWorkspaceSync.providerLeaseId + ) { + return workspaceSyncFailure("workspace_sync_out_unrecoverable"); + } + const target = await resolveEnvironmentExecutionTarget({ + db: input.db, + companyId: bound.companyId, + adapterType: "paperclip_runner", + environment, + leaseId: lease.id, + leaseMetadata: lease.metadata, + lease, + environmentRuntime: input.environmentRuntime, + }); + if (!target) { + return { + status: "failed", + exitCode: 1, + stderr: "Native workspace finalization target is unavailable.\n", + }; + } + try { + const restored = await resumeNativeWorkspaceSync({ + db: input.db, + runId: input.runId, + target, + }); + if (!restored) { + return workspaceSyncFailure("workspace_sync_out_unrecoverable"); + } + return { + status: "succeeded", + exitCode: 0, + system: + "Native workspace finalization restored the remote workspace.\n", + metadata: { + workspaceSync: { + schema: nativeWorkspaceSync.schema, + workspaceId: nativeWorkspaceSync.workspaceId, + leaseId: nativeWorkspaceSync.leaseId, + }, + }, + }; + } catch (error) { + const code = + error instanceof Error && + (error.message === "workspace_sync_out_unrecoverable" || + error.message.includes("daytona_sandbox_not_found")) + ? "workspace_sync_out_unrecoverable" + : "workspace_sync_out_failed"; + return workspaceSyncFailure(code); + } + } if (!cwd) { return { status: "failed", diff --git a/server/src/services/native-runtime/native-workspace-sync.ts b/server/src/services/native-runtime/native-workspace-sync.ts new file mode 100644 index 0000000000..e8f9f95884 --- /dev/null +++ b/server/src/services/native-runtime/native-workspace-sync.ts @@ -0,0 +1,1004 @@ +import fs from "node:fs/promises"; +import { createReadStream } from "node:fs"; +import path from "node:path"; +import { createHash, randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import { environmentLeases, heartbeatRuns } from "@paperclipai/db"; +import type { EnvironmentLease } from "@paperclipai/shared"; +import { + prepareAdapterExecutionTargetRuntime, + type AdapterExecutionTarget, + type PreparedAdapterExecutionTargetRuntime, +} from "@paperclipai/adapter-utils/execution-target"; +import type { GitWorkspaceSnapshot } from "@paperclipai/adapter-utils/git-workspace-sync"; +import { + directorySnapshotSha256, + parseDirectorySnapshot, + serializeDirectorySnapshot, + type DirectorySnapshot, + type SerializedDirectorySnapshot, +} from "@paperclipai/adapter-utils/workspace-restore-merge"; +import type { + WorkspaceDurableSeedPaths, + WorkspaceInboundMode, +} from "@paperclipai/adapter-utils/sandbox-managed-runtime"; +import { resolvePaperclipInstanceRoot } from "../../home-paths.js"; +import { parseObject } from "../../adapters/utils.js"; +import type { NativeRestartRecoveryClaim } from "./native-restart-recovery.js"; + +const DESCRIPTOR_SCHEMA = "paperclip.native-workspace-sync/v1"; +const STAMP_SCHEMA = "paperclip.native-workspace-stamp/v1"; +const STATE_ROOT_NAME = "native-workspace-sync"; +const DESCRIPTOR_NAME = "descriptor"; +const WORKSPACE_SEED_NAME = "workspace-seed.tar"; +const GIT_SEED_NAME = "git-seed.tar"; +const REMOTE_STAMP_NAME = "workspace-sync-v1.json"; +const MAX_DESCRIPTOR_BYTES = 64 * 1024 * 1024; +const SAFE_SEGMENT_RE = /^[A-Za-z0-9_-]+$/; + +type NativeWorkspaceSyncState = "prepared" | "finalized"; +export type NativeWorkspaceResourceDisposition = + "keep_running" | "stop_and_retain" | "destroy"; + +interface NativeWorkspaceSyncDescriptor { + schema: typeof DESCRIPTOR_SCHEMA; + binding: { + runId: string; + companyId: string; + workspaceId: string; + leaseId: string; + providerLeaseId: string; + localCwd: string; + remoteCwd: string; + }; + state: NativeWorkspaceSyncState; + baselineSha256: string; + baseline: SerializedDirectorySnapshot; + gitSnapshot: GitWorkspaceSnapshot | null; + seed: { + workspaceArchiveSha256: string; + gitArchiveSha256: string | null; + } | null; + createdAt: string; + finalizedAt: string | null; + finalHostSha256: string | null; + resourceDisposition: NativeWorkspaceResourceDisposition | null; +} + +export interface NativeWorkspaceSyncReference { + schema: typeof DESCRIPTOR_SCHEMA; + state: NativeWorkspaceSyncState; + descriptorSha256: string; + baselineSha256: string; + finalHostSha256: string | null; + workspaceId: string; + leaseId: string; + providerLeaseId: string; + remoteCwd: string; + resourceDisposition: NativeWorkspaceResourceDisposition | null; +} + +export interface PreparedNativeWorkspaceSync { + mode: WorkspaceInboundMode; + reference: NativeWorkspaceSyncReference; + restoreWorkspace(): Promise; + cleanup(): Promise; +} + +export type NativeWorkspaceInboundEvidence = + | { + kind: "existing_run"; + restartRecovery: boolean; + sameProviderLease: boolean; + } + | { + kind: "new_run"; + acquisition: "created" | "resumed" | "replacement" | null; + hasPriorStamp: boolean; + }; + +export function classifyNativeWorkspaceInbound( + evidence: NativeWorkspaceInboundEvidence, +): WorkspaceInboundMode { + if (evidence.kind === "existing_run") { + if (!evidence.restartRecovery) { + throw new Error("native_workspace_sync_unexpected_existing_descriptor"); + } + return evidence.sameProviderLease ? "adopt_remote" : "durable_seed"; + } + return evidence.acquisition === "resumed" && evidence.hasPriorStamp + ? "adopt_remote" + : "host_current"; +} + +function canonicalJson(value: unknown): string { + if (Array.isArray(value)) { + return `[${value.map((entry) => canonicalJson(entry)).join(",")}]`; + } + if (value && typeof value === "object") { + const record = value as Record; + return `{${Object.keys(record) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`) + .join(",")}}`; + } + return JSON.stringify(value); +} + +function sha256(value: unknown): string { + return createHash("sha256").update(canonicalJson(value)).digest("hex"); +} + +function requireSafeSegment(value: string, label: string): string { + if (!SAFE_SEGMENT_RE.test(value)) { + throw new Error(`native_workspace_sync_invalid_${label}`); + } + return value; +} + +function descriptorDirectory(runId: string): string { + return path.join( + resolvePaperclipInstanceRoot(), + STATE_ROOT_NAME, + requireSafeSegment(runId, "run_id"), + ); +} + +function descriptorPath(runId: string, descriptorSha256: string): string { + if (!/^[0-9a-f]{64}$/.test(descriptorSha256)) { + throw new Error("native_workspace_sync_descriptor_digest_invalid"); + } + return path.join( + descriptorDirectory(runId), + `${DESCRIPTOR_NAME}.${descriptorSha256}.json`, + ); +} + +function legacyDescriptorPath(runId: string): string { + return path.join(descriptorDirectory(runId), `${DESCRIPTOR_NAME}.json`); +} + +function durableSeedPaths(runId: string): { + workspaceArchivePath: string; + gitArchivePath: string; +} { + const directory = descriptorDirectory(runId); + return { + workspaceArchivePath: path.join(directory, WORKSPACE_SEED_NAME), + gitArchivePath: path.join(directory, GIT_SEED_NAME), + }; +} + +async function sha256File(filePath: string): Promise { + return await new Promise((resolveDigest, rejectDigest) => { + const digest = createHash("sha256"); + const stream = createReadStream(filePath); + stream.on("data", (chunk) => digest.update(chunk)); + stream.on("error", rejectDigest); + stream.on("end", () => resolveDigest(digest.digest("hex"))); + }); +} + +async function verifiedDurableSeed(input: { + runId: string; + seed: NonNullable; + gitSnapshot: GitWorkspaceSnapshot | null; +}): Promise { + try { + const paths = durableSeedPaths(input.runId); + const workspaceStat = await fs.lstat(paths.workspaceArchivePath); + if (workspaceStat.isSymbolicLink() || !workspaceStat.isFile()) { + throw new Error("workspace_sync_out_unrecoverable"); + } + if ( + (await sha256File(paths.workspaceArchivePath)) !== + input.seed.workspaceArchiveSha256 + ) { + throw new Error("workspace_sync_out_unrecoverable"); + } + if (input.gitSnapshot) { + const gitStat = await fs.lstat(paths.gitArchivePath); + if ( + gitStat.isSymbolicLink() || + !gitStat.isFile() || + !input.seed.gitArchiveSha256 || + (await sha256File(paths.gitArchivePath)) !== input.seed.gitArchiveSha256 + ) { + throw new Error("workspace_sync_out_unrecoverable"); + } + } else if (input.seed.gitArchiveSha256 !== null) { + throw new Error("workspace_sync_out_unrecoverable"); + } + return { + workspaceArchivePath: paths.workspaceArchivePath, + workspaceArchiveSha256: input.seed.workspaceArchiveSha256, + gitArchivePath: input.gitSnapshot ? paths.gitArchivePath : null, + gitArchiveSha256: input.seed.gitArchiveSha256, + }; + } catch (error) { + if ( + error instanceof Error && + error.message === "workspace_sync_out_unrecoverable" + ) { + throw error; + } + throw new Error("workspace_sync_out_unrecoverable", { cause: error }); + } +} + +async function ensurePrivateDirectory(dir: string): Promise { + await fs.mkdir(dir, { recursive: true, mode: 0o700 }); + const stat = await fs.lstat(dir); + if (stat.isSymbolicLink() || !stat.isDirectory()) { + throw new Error("native_workspace_sync_state_root_invalid"); + } +} + +async function writeDescriptor( + descriptor: NativeWorkspaceSyncDescriptor, +): Promise { + const dir = descriptorDirectory(descriptor.binding.runId); + await ensurePrivateDirectory(dir); + const body = `${canonicalJson(descriptor)}\n`; + const descriptorSha256 = createHash("sha256").update(body).digest("hex"); + const finalPath = descriptorPath(descriptor.binding.runId, descriptorSha256); + const existingBody = await fs.readFile(finalPath, "utf8").catch(() => null); + if (existingBody !== null && existingBody !== body) { + throw new Error("native_workspace_sync_descriptor_digest_collision"); + } + const tempPath = path.join(dir, `${DESCRIPTOR_NAME}.${randomUUID()}.tmp`); + try { + if (existingBody === null) { + await fs.writeFile(tempPath, body, { encoding: "utf8", mode: 0o600 }); + await fs.rename(tempPath, finalPath); + } + } finally { + await fs.rm(tempPath, { force: true }).catch(() => undefined); + } + return { + schema: DESCRIPTOR_SCHEMA, + state: descriptor.state, + descriptorSha256, + baselineSha256: descriptor.baselineSha256, + finalHostSha256: descriptor.finalHostSha256, + workspaceId: descriptor.binding.workspaceId, + leaseId: descriptor.binding.leaseId, + providerLeaseId: descriptor.binding.providerLeaseId, + remoteCwd: descriptor.binding.remoteCwd, + resourceDisposition: descriptor.resourceDisposition, + }; +} + +export function readNativeWorkspaceSyncReference( + value: unknown, +): NativeWorkspaceSyncReference | null { + const candidate = parseObject(value); + if ( + candidate.schema !== DESCRIPTOR_SCHEMA || + (candidate.state !== "prepared" && candidate.state !== "finalized") || + typeof candidate.descriptorSha256 !== "string" || + !/^[0-9a-f]{64}$/.test(candidate.descriptorSha256) || + typeof candidate.baselineSha256 !== "string" || + !/^[0-9a-f]{64}$/.test(candidate.baselineSha256) || + (candidate.finalHostSha256 !== null && + candidate.finalHostSha256 !== undefined && + (typeof candidate.finalHostSha256 !== "string" || + !/^[0-9a-f]{64}$/.test(candidate.finalHostSha256))) || + typeof candidate.workspaceId !== "string" || + typeof candidate.leaseId !== "string" || + typeof candidate.providerLeaseId !== "string" || + typeof candidate.remoteCwd !== "string" || + (candidate.resourceDisposition !== null && + candidate.resourceDisposition !== undefined && + candidate.resourceDisposition !== "keep_running" && + candidate.resourceDisposition !== "stop_and_retain" && + candidate.resourceDisposition !== "destroy") + ) { + return null; + } + return { + schema: DESCRIPTOR_SCHEMA, + state: candidate.state, + descriptorSha256: candidate.descriptorSha256, + baselineSha256: candidate.baselineSha256, + finalHostSha256: + typeof candidate.finalHostSha256 === "string" + ? candidate.finalHostSha256 + : null, + workspaceId: candidate.workspaceId, + leaseId: candidate.leaseId, + providerLeaseId: candidate.providerLeaseId, + remoteCwd: candidate.remoteCwd, + resourceDisposition: + candidate.resourceDisposition === "keep_running" || + candidate.resourceDisposition === "stop_and_retain" || + candidate.resourceDisposition === "destroy" + ? candidate.resourceDisposition + : null, + }; +} + +function parseGitSnapshot( + value: unknown, +): GitWorkspaceSnapshot | null | undefined { + if (value === null) return null; + const candidate = parseObject(value); + const paths = [ + candidate.overlayPaths, + candidate.deletedPaths, + candidate.ignoredPaths, + ]; + if ( + typeof candidate.headCommit !== "string" || + (candidate.branchName !== null && + typeof candidate.branchName !== "string") || + !paths.every( + (entries) => + Array.isArray(entries) && + entries.every( + (entry) => + typeof entry === "string" && + !path.posix.isAbsolute(entry) && + !path.win32.isAbsolute(entry) && + !entry.split(/[\\/]/).some((segment) => segment === ".."), + ), + ) + ) { + return undefined; + } + return { + headCommit: candidate.headCommit, + branchName: candidate.branchName as string | null, + overlayPaths: [...(candidate.overlayPaths as string[])], + deletedPaths: [...(candidate.deletedPaths as string[])], + ignoredPaths: [...(candidate.ignoredPaths as string[])], + }; +} + +async function readDescriptor(input: { + runId: string; + reference: NativeWorkspaceSyncReference; +}): Promise<{ + descriptor: NativeWorkspaceSyncDescriptor; + baseline: DirectorySnapshot; +}> { + const contentAddressedPath = descriptorPath( + input.runId, + input.reference.descriptorSha256, + ); + // Early v1 writers used descriptor.json. Prefer the immutable, + // content-addressed name, but keep the digest-verified legacy path readable + // so an interrupted upgrade can still finish its already-proposed result. + const filePath = await fs + .lstat(contentAddressedPath) + .then(() => contentAddressedPath) + .catch(async (error: NodeJS.ErrnoException) => { + if (error.code !== "ENOENT") throw error; + const legacyPath = legacyDescriptorPath(input.runId); + await fs.lstat(legacyPath); + return legacyPath; + }); + const stat = await fs.lstat(filePath); + if ( + stat.isSymbolicLink() || + !stat.isFile() || + stat.size > MAX_DESCRIPTOR_BYTES + ) { + throw new Error("native_workspace_sync_descriptor_invalid"); + } + const body = await fs.readFile(filePath, "utf8"); + if ( + createHash("sha256").update(body).digest("hex") !== + input.reference.descriptorSha256 + ) { + throw new Error("native_workspace_sync_descriptor_digest_mismatch"); + } + const parsed = JSON.parse(body) as unknown; + const candidate = parseObject(parsed); + const binding = parseObject(candidate.binding); + const baseline = parseDirectorySnapshot(candidate.baseline); + const gitSnapshot = parseGitSnapshot(candidate.gitSnapshot); + const rawSeed = + candidate.seed === null || candidate.seed === undefined + ? null + : parseObject(candidate.seed); + const seed = + rawSeed === null + ? null + : typeof rawSeed.workspaceArchiveSha256 === "string" && + /^[0-9a-f]{64}$/.test(rawSeed.workspaceArchiveSha256) && + (rawSeed.gitArchiveSha256 === null || + (typeof rawSeed.gitArchiveSha256 === "string" && + /^[0-9a-f]{64}$/.test(rawSeed.gitArchiveSha256))) + ? { + workspaceArchiveSha256: rawSeed.workspaceArchiveSha256, + gitArchiveSha256: rawSeed.gitArchiveSha256 as string | null, + } + : undefined; + const finalHostSha256 = + typeof candidate.finalHostSha256 === "string" && + /^[0-9a-f]{64}$/.test(candidate.finalHostSha256) + ? candidate.finalHostSha256 + : candidate.finalHostSha256 === null + ? null + : undefined; + const resourceDisposition = + candidate.resourceDisposition === "keep_running" || + candidate.resourceDisposition === "stop_and_retain" || + candidate.resourceDisposition === "destroy" + ? candidate.resourceDisposition + : candidate.resourceDisposition === null + ? null + : undefined; + if ( + candidate.schema !== DESCRIPTOR_SCHEMA || + (candidate.state !== "prepared" && candidate.state !== "finalized") || + !baseline || + gitSnapshot === undefined || + seed === undefined || + binding.runId !== input.runId || + binding.workspaceId !== input.reference.workspaceId || + binding.leaseId !== input.reference.leaseId || + binding.providerLeaseId !== input.reference.providerLeaseId || + binding.remoteCwd !== input.reference.remoteCwd || + typeof binding.companyId !== "string" || + typeof binding.localCwd !== "string" || + !path.isAbsolute(binding.localCwd) || + typeof binding.remoteCwd !== "string" || + !path.posix.isAbsolute(binding.remoteCwd) || + typeof candidate.baselineSha256 !== "string" || + candidate.baselineSha256 !== input.reference.baselineSha256 || + directorySnapshotSha256(baseline) !== candidate.baselineSha256 || + typeof candidate.createdAt !== "string" || + !Number.isFinite(Date.parse(candidate.createdAt)) || + candidate.state !== input.reference.state || + finalHostSha256 === undefined || + finalHostSha256 !== input.reference.finalHostSha256 || + resourceDisposition === undefined || + resourceDisposition !== input.reference.resourceDisposition || + (candidate.state === "prepared" && + (candidate.finalizedAt !== null || finalHostSha256 !== null)) || + (candidate.state === "finalized" && + (typeof candidate.finalizedAt !== "string" || + !Number.isFinite(Date.parse(candidate.finalizedAt)) || + finalHostSha256 === null)) + ) { + throw new Error("native_workspace_sync_descriptor_binding_mismatch"); + } + return { + descriptor: { + schema: DESCRIPTOR_SCHEMA, + binding: { + runId: binding.runId as string, + companyId: binding.companyId, + workspaceId: binding.workspaceId as string, + leaseId: binding.leaseId as string, + providerLeaseId: binding.providerLeaseId as string, + localCwd: binding.localCwd, + remoteCwd: binding.remoteCwd as string, + }, + state: candidate.state, + baselineSha256: candidate.baselineSha256, + baseline: candidate.baseline as SerializedDirectorySnapshot, + gitSnapshot, + seed, + createdAt: candidate.createdAt, + finalizedAt: + typeof candidate.finalizedAt === "string" + ? candidate.finalizedAt + : null, + finalHostSha256, + resourceDisposition, + }, + baseline, + }; +} + +async function persistRunReference( + db: Db, + runId: string, + reference: NativeWorkspaceSyncReference, +): Promise { + await db.transaction(async (tx) => { + const run = await tx + .select({ runnerProfileJson: heartbeatRuns.runnerProfileJson }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, runId)) + .for("update") + .limit(1) + .then((rows) => rows[0] ?? null); + if (!run) throw new Error("native_workspace_sync_run_missing"); + await tx + .update(heartbeatRuns) + .set({ + runnerProfileJson: { + ...parseObject(run.runnerProfileJson), + nativeWorkspaceSync: reference, + }, + updatedAt: new Date(), + }) + .where(eq(heartbeatRuns.id, runId)); + }); +} + +async function persistLeaseStamp(input: { + db: Db; + leaseId: string; + stamp: Record; +}): Promise { + await input.db.transaction(async (tx) => { + const lease = await tx + .select({ metadata: environmentLeases.metadata }) + .from(environmentLeases) + .where(eq(environmentLeases.id, input.leaseId)) + .for("update") + .limit(1) + .then((rows) => rows[0] ?? null); + if (!lease) throw new Error("native_workspace_sync_lease_missing"); + await tx + .update(environmentLeases) + .set({ + metadata: { + ...parseObject(lease.metadata), + nativeWorkspaceSync: input.stamp, + }, + updatedAt: new Date(), + }) + .where(eq(environmentLeases.id, input.leaseId)); + }); +} + +function providerLeaseIdFor(input: { + target: Extract; + lease: EnvironmentLease; +}): string { + const providerLeaseId = + input.target.sandboxLeaseAcquisition?.providerLeaseId ?? + input.lease.providerLeaseId; + if (!providerLeaseId) { + throw new Error("native_workspace_sync_provider_lease_missing"); + } + return providerLeaseId; +} + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", "'\\''")}'`; +} + +async function writeRemoteStamp(input: { + target: Extract; + stamp: Record; +}): Promise { + if (!input.target.runner) { + throw new Error("native_workspace_sync_runner_missing"); + } + const runtimeDir = path.posix.join( + input.target.remoteCwd, + ".paperclip-runtime", + "paperclip-runner", + ); + const stampPath = path.posix.join(runtimeDir, REMOTE_STAMP_NAME); + const tempPath = `${stampPath}.tmp`; + const encoded = Buffer.from( + `${canonicalJson(input.stamp)}\n`, + "utf8", + ).toString("base64"); + const result = await input.target.runner.execute({ + command: input.target.shellCommand ?? "sh", + args: [ + "-c", + `umask 077; mkdir -p ${shellQuote(runtimeDir)} && printf %s ${shellQuote(encoded)} | base64 -d > ${shellQuote(tempPath)} && mv ${shellQuote(tempPath)} ${shellQuote(stampPath)}`, + ], + cwd: "/", + timeoutMs: 15_000, + bypassSession: true, + }); + if (result.timedOut || result.exitCode !== 0) { + throw new Error("native_workspace_sync_remote_stamp_failed"); + } +} + +async function remoteStampMatches(input: { + target: Extract; + expected: Record; +}): Promise { + if (!input.target.runner) return false; + const stampPath = path.posix.join( + input.target.remoteCwd, + ".paperclip-runtime", + "paperclip-runner", + REMOTE_STAMP_NAME, + ); + const result = await input.target.runner.execute({ + command: input.target.shellCommand ?? "sh", + args: [ + "-c", + `test -f ${shellQuote(stampPath)} && cat ${shellQuote(stampPath)}`, + ], + cwd: "/", + timeoutMs: 15_000, + bypassSession: true, + }); + if (result.timedOut || result.exitCode !== 0) return false; + try { + return ( + canonicalJson(JSON.parse(result.stdout)) === canonicalJson(input.expected) + ); + } catch { + return false; + } +} + +function leaseStamp(input: { + lease: EnvironmentLease; + workspaceId: string; + providerLeaseId: string; + remoteCwd: string; +}): Record | null { + const stamp = parseObject( + parseObject(input.lease.metadata).nativeWorkspaceSync, + ); + if ( + stamp.schema !== STAMP_SCHEMA || + stamp.workspaceId !== input.workspaceId || + stamp.providerLeaseId !== input.providerLeaseId || + stamp.remoteCwd !== input.remoteCwd || + typeof stamp.hostSha256 !== "string" || + !/^[0-9a-f]{64}$/.test(stamp.hostSha256) + ) { + return null; + } + return stamp; +} + +function finalizedWorkspaceStamp(input: { + descriptor: NativeWorkspaceSyncDescriptor; + hostSha256: string; +}): Record { + return { + schema: STAMP_SCHEMA, + workspaceId: input.descriptor.binding.workspaceId, + providerLeaseId: input.descriptor.binding.providerLeaseId, + remoteCwd: input.descriptor.binding.remoteCwd, + hostSha256: input.hostSha256, + finalizedRunId: input.descriptor.binding.runId, + }; +} + +async function prepareRuntime(input: { + runId: string; + target: Extract; + workspaceLocalDir: string; + mode: WorkspaceInboundMode; + baseline?: DirectorySnapshot; + gitSnapshot?: GitWorkspaceSnapshot | null; + durableSeed?: WorkspaceDurableSeedPaths; +}): Promise { + return prepareAdapterExecutionTargetRuntime({ + runId: input.runId, + target: input.target, + adapterKey: "paperclip-runner", + workspaceLocalDir: input.workspaceLocalDir, + workspaceRemoteDir: input.target.remoteCwd, + workspaceInboundMode: input.mode, + workspaceDurableSeed: input.durableSeed, + workspaceBaseline: input.baseline, + workspaceGitSnapshot: input.gitSnapshot, + }); +} + +async function finalizePreparedRuntime(input: { + db: Db; + runId: string; + target: Extract; + runtime: PreparedAdapterExecutionTargetRuntime; + descriptor: NativeWorkspaceSyncDescriptor; +}): Promise { + await input.runtime.restoreWorkspace(); + const finalSnapshot = + await import("@paperclipai/adapter-utils/workspace-restore-merge").then( + ({ captureDirectorySnapshot }) => + captureDirectorySnapshot(input.descriptor.binding.localCwd, { + exclude: input.runtime.workspaceSyncSnapshot?.baseline.exclude ?? [], + }), + ); + const finalHostSha256 = directorySnapshotSha256(finalSnapshot); + const stamp = finalizedWorkspaceStamp({ + descriptor: input.descriptor, + hostSha256: finalHostSha256, + }); + await writeRemoteStamp({ target: input.target, stamp }); + const finalizedDescriptor: NativeWorkspaceSyncDescriptor = { + ...input.descriptor, + state: "finalized", + finalizedAt: new Date().toISOString(), + finalHostSha256, + }; + const reference = await writeDescriptor(finalizedDescriptor); + await persistRunReference(input.db, input.runId, reference); + await persistLeaseStamp({ + db: input.db, + leaseId: input.descriptor.binding.leaseId, + stamp, + }); + return reference; +} + +export async function prepareNativeWorkspaceSync(input: { + db: Db; + runId: string; + companyId: string; + workspaceId: string; + workspaceLocalDir: string; + target: AdapterExecutionTarget | null; + lease: EnvironmentLease; + restartRecovery?: NativeRestartRecoveryClaim; + resourceDisposition?: NativeWorkspaceResourceDisposition; +}): Promise { + if (input.target?.kind !== "remote" || input.target.transport !== "sandbox") { + return null; + } + const target = input.target; + const providerLeaseId = providerLeaseIdFor({ target, lease: input.lease }); + const run = await input.db + .select({ runnerProfileJson: heartbeatRuns.runnerProfileJson }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, input.runId)) + .limit(1) + .then((rows) => rows[0] ?? null); + if (!run) throw new Error("native_workspace_sync_run_missing"); + const existingReference = readNativeWorkspaceSyncReference( + parseObject(run.runnerProfileJson).nativeWorkspaceSync, + ); + + let runtime: PreparedAdapterExecutionTargetRuntime; + let descriptor: NativeWorkspaceSyncDescriptor; + let mode: WorkspaceInboundMode = "host_current"; + const seedPaths = durableSeedPaths(input.runId); + await ensurePrivateDirectory(descriptorDirectory(input.runId)); + + if (existingReference) { + const existing = await readDescriptor({ + runId: input.runId, + reference: existingReference, + }); + if ( + existing.descriptor.binding.companyId !== input.companyId || + existing.descriptor.binding.workspaceId !== input.workspaceId || + path.resolve(existing.descriptor.binding.localCwd) !== + path.resolve(input.workspaceLocalDir) || + existing.descriptor.binding.remoteCwd !== target.remoteCwd + ) { + throw new Error("native_workspace_sync_descriptor_binding_mismatch"); + } + const sameProviderLease = + existing.descriptor.binding.providerLeaseId === providerLeaseId; + mode = classifyNativeWorkspaceInbound({ + kind: "existing_run", + restartRecovery: Boolean(input.restartRecovery), + sameProviderLease, + }); + const durableSeed = + mode === "durable_seed" + ? existing.descriptor.seed + ? await verifiedDurableSeed({ + runId: input.runId, + seed: existing.descriptor.seed, + gitSnapshot: existing.descriptor.gitSnapshot, + }) + : (() => { + throw new Error("workspace_sync_out_unrecoverable"); + })() + : undefined; + runtime = await prepareRuntime({ + runId: input.runId, + target, + workspaceLocalDir: input.workspaceLocalDir, + mode, + baseline: existing.baseline, + gitSnapshot: existing.descriptor.gitSnapshot, + durableSeed, + }); + descriptor = { + ...existing.descriptor, + binding: { + ...existing.descriptor.binding, + leaseId: target.leaseId ?? input.lease.id, + providerLeaseId, + }, + }; + } else { + const acquisition = target.sandboxLeaseAcquisition?.outcome ?? null; + const priorStamp = leaseStamp({ + lease: input.lease, + workspaceId: input.workspaceId, + providerLeaseId, + remoteCwd: target.remoteCwd, + }); + mode = classifyNativeWorkspaceInbound({ + kind: "new_run", + acquisition, + hasPriorStamp: priorStamp !== null, + }); + runtime = await prepareRuntime({ + runId: input.runId, + target, + workspaceLocalDir: input.workspaceLocalDir, + mode, + durableSeed: { + workspaceArchivePath: seedPaths.workspaceArchivePath, + gitArchivePath: seedPaths.gitArchivePath, + }, + }); + const currentSnapshot = runtime.workspaceSyncSnapshot; + if (!currentSnapshot) { + throw new Error("native_workspace_sync_snapshot_missing"); + } + if (acquisition === "resumed" && priorStamp) { + const currentHostSha256 = directorySnapshotSha256( + currentSnapshot.baseline, + ); + const verifiedWarmAdoption = + priorStamp.hostSha256 === currentHostSha256 && + (await remoteStampMatches({ target, expected: priorStamp })); + if (verifiedWarmAdoption) { + mode = "adopt_remote"; + } else { + mode = "host_current"; + runtime = await prepareRuntime({ + runId: input.runId, + target, + workspaceLocalDir: input.workspaceLocalDir, + mode, + baseline: currentSnapshot.baseline, + gitSnapshot: currentSnapshot.gitSnapshot, + durableSeed: { + workspaceArchivePath: seedPaths.workspaceArchivePath, + gitArchivePath: seedPaths.gitArchivePath, + }, + }); + } + } + const snapshot = runtime.workspaceSyncSnapshot; + if (!snapshot) throw new Error("native_workspace_sync_snapshot_missing"); + const now = new Date().toISOString(); + const workspaceArchiveSha256 = await sha256File( + seedPaths.workspaceArchivePath, + ); + const gitArchiveSha256 = snapshot.gitSnapshot + ? await sha256File(seedPaths.gitArchivePath) + : null; + descriptor = { + schema: DESCRIPTOR_SCHEMA, + binding: { + runId: input.runId, + companyId: input.companyId, + workspaceId: input.workspaceId, + leaseId: target.leaseId ?? input.lease.id, + providerLeaseId, + localCwd: path.resolve(input.workspaceLocalDir), + remoteCwd: target.remoteCwd, + }, + state: "prepared", + baselineSha256: directorySnapshotSha256(snapshot.baseline), + baseline: serializeDirectorySnapshot(snapshot.baseline), + gitSnapshot: snapshot.gitSnapshot, + seed: { workspaceArchiveSha256, gitArchiveSha256 }, + createdAt: now, + finalizedAt: null, + finalHostSha256: null, + resourceDisposition: input.resourceDisposition ?? null, + }; + } + + let reference = await writeDescriptor(descriptor); + await persistRunReference(input.db, input.runId, reference); + let restorePromise: Promise | null = null; + return { + mode, + get reference() { + return reference; + }, + restoreWorkspace: async () => { + if (!restorePromise) { + restorePromise = finalizePreparedRuntime({ + db: input.db, + runId: input.runId, + target, + runtime, + descriptor, + }) + .then((finalizedReference) => { + reference = finalizedReference; + }) + .catch((error) => { + restorePromise = null; + throw error; + }); + } + await restorePromise; + }, + cleanup: () => cleanupNativeWorkspaceSync(input.runId), + }; +} + +export async function resumeNativeWorkspaceSync(input: { + db: Db; + runId: string; + target: AdapterExecutionTarget; +}): Promise { + if (input.target.kind !== "remote" || input.target.transport !== "sandbox") { + throw new Error("workspace_sync_out_unrecoverable"); + } + const run = await input.db + .select({ runnerProfileJson: heartbeatRuns.runnerProfileJson }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, input.runId)) + .limit(1) + .then((rows) => rows[0] ?? null); + const reference = readNativeWorkspaceSyncReference( + parseObject(run?.runnerProfileJson).nativeWorkspaceSync, + ); + if (!reference) return false; + const existing = await readDescriptor({ runId: input.runId, reference }); + const providerLeaseId = + input.target.sandboxLeaseAcquisition?.providerLeaseId ?? + reference.providerLeaseId; + if ( + providerLeaseId !== reference.providerLeaseId || + input.target.remoteCwd !== reference.remoteCwd + ) { + throw new Error("workspace_sync_out_unrecoverable"); + } + if ( + existing.descriptor.state === "finalized" && + existing.descriptor.finalHostSha256 + ) { + const stamp = finalizedWorkspaceStamp({ + descriptor: existing.descriptor, + hostSha256: existing.descriptor.finalHostSha256, + }); + if ( + !(await remoteStampMatches({ target: input.target, expected: stamp })) + ) { + await writeRemoteStamp({ target: input.target, stamp }); + } + await persistLeaseStamp({ + db: input.db, + leaseId: existing.descriptor.binding.leaseId, + stamp, + }); + return true; + } + const runtime = await prepareRuntime({ + runId: input.runId, + target: input.target, + workspaceLocalDir: existing.descriptor.binding.localCwd, + mode: "adopt_remote", + baseline: existing.baseline, + gitSnapshot: existing.descriptor.gitSnapshot, + }); + await finalizePreparedRuntime({ + db: input.db, + runId: input.runId, + target: input.target, + runtime, + descriptor: existing.descriptor, + }); + return true; +} + +export async function cleanupNativeWorkspaceSync(runId: string): Promise { + await fs.rm(descriptorDirectory(runId), { recursive: true, force: true }); +} + +export const nativeWorkspaceSyncInternals = { + descriptorPath, + legacyDescriptorPath, + durableSeedPaths, + writeDescriptor, + readDescriptor, + readReference: readNativeWorkspaceSyncReference, + sha256, +}; diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index 5d533b2d01..01c229242d 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -58,9 +58,13 @@ registry therefore discovers that row through the public environments API. This still provides full isolation because every cell starts a new Paperclip instance and database. -Daytona creates a sandbox environment through the public API. Keep -`reuseLease:false`, `runnerLifecycleMode:"per_turn"`, short provider cleanup -backstops, a Daytona secret reference, and an immutable image digest. Teardown +Daytona creates sandbox environments through the public API. The core fixture +keeps `reuseLease:false` and `runnerLifecycleMode:"per_turn"`. The dedicated +warm-continuity fixture uses `reuseLease:true` and +`runnerLifecycleMode:"warm"`; its distinct `configurationKey` is part of the +suite fingerprint even though both fixtures report `environmentId:"daytona"`. +Keep short provider cleanup backstops, a Daytona secret reference, and an +immutable image digest. Teardown must delete the environment with reusable-lease destruction and must fail the cell if cleanup cannot be confirmed. Keep CPU, memory, and disk explicit: lease metadata and the per-test public-list-price runtime estimate depend on that @@ -93,7 +97,10 @@ marker factories. `question_resume_completion` must define the deterministic browser answer and prove exactly two successful runs with no pending interaction. `plan_approval_completion` must target the exact two-step canonical Plan revision, capture its pending UI, approve in the browser, and -prove exactly two successful runs. +prove exactly two successful runs. `warm_three_turn` provides exactly two +browser follow-up messages, preserves one project/execution-workspace scope, +verifies host file contents after every turn, and finishes within three +ten-minute turn deadlines. Every selected case runs in its own isolated Paperclip process, and independent cases may run concurrently. Follow-up turns inside one case retain their shared diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index ed9fed5caa..b9ffb1dd6c 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -68,10 +68,11 @@ pnpm test:e2e:runner -- --group native --environment local pnpm test:e2e:runner -- --profile runner-codex --case message-marker pnpm test:e2e:runner -- --case plan-revise-accept --group local pnpm test:e2e:runner -- --case ask-question --group native +pnpm test:e2e:runner -- --suite daytona-warm-continuity pnpm test:e2e:runner -- --all ``` -The catalog contains three suites. `core-compatibility` (**Core Runner +The catalog contains four suites. `core-compatibility` (**Core Runner Compatibility**) is seven major runner profiles × local/Daytona × three workflows: 42 cells. Its cases are: @@ -107,7 +108,19 @@ duplicating the final response. The second workflow restarts the isolated Paperclip server while the interaction is waiting, reloads that state, and then resumes it. The suite has no Daytona cells. -The complete catalog is 66 cells (45 local and 21 Daytona) and 114 expected +`daytona-warm-continuity` (**Daytona Warm Continuity**) is exactly two paid +cells: legacy Codex and Runner Codex against one reusable warm Daytona +configuration. Each cell creates a real project with a primary local-path +workspace through the API, selects it in the browser task dialog, and performs +three browser-driven turns on one issue. Every turn reads and extends the same +nonce file, verifies host copy-back, records scheduler/run/end-to-end timing, +and asserts `created`, `resumed`, `resumed` lease acquisition on one sandbox. +Runner Codex additionally proves stable native session, provider session, +runner instance, PID, and process-start identity. Each turn is bounded to ten +minutes, the cell to thirty minutes, and cleanup explicitly deletes the +sandbox rather than waiting for Daytona's idle timeout. + +The complete catalog is 68 cells (45 local and 23 Daytona) and 120 expected paid agent turns. Follow-up steps remain ordered within their cell; all other cells are independent. Narrow selectors are strongly recommended while developing fixtures. @@ -351,7 +364,7 @@ Set `RUNNER_E2E_AWS_ENABLED=true` to route paid cells to the repository-scoped ephemeral AWS RunsOn fleet selected by `runs-on/fleet=paperclip-public-pr-x64/env=public-ci`. Any other value uses the proven GitHub-hosted `ubuntu-latest` target. Set `RUNNER_E2E_MAX_PARALLEL` to an -integer from 1–100 on AWS (default 100); use at least 66 to run the current +integer from 1–100 on AWS (default 100); use at least 68 to run the current complete catalog in one wave. The fallback runner retains its 1–57 limit and default of 32. Multi-turn steps are sequential inside their cell while independent cells overlap. Artifacts and merged HTML/JUnit/normalized reports diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 83ecd13b1f..685c18add8 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -10,7 +10,10 @@ import { runnerProfiles, runnerSuites, runnerTasks, + daytonaWarmContinuityTask, + daytonaWarmEnvironment, isImmutableDaytonaImage, + suiteDefinitionHash, validateRunnerCatalog, } from "./catalog.js"; import { @@ -21,7 +24,7 @@ import { } from "./selectors.js"; describe("runner E2E catalog", () => { - it("validates the core, local-integrity, and breadth suites", () => { + it("validates the core, local-integrity, breadth, and warm suites", () => { expect(runnerProfiles).toHaveLength(7); expect(openRouterBreadthProfiles).toHaveLength(4); expect(runnerEnvironments).toHaveLength(2); @@ -29,10 +32,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 42, 14, 10, + 42, 14, 10, 2, ]); - expect(validateRunnerCatalog()).toHaveLength(66); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(66); + expect(validateRunnerCatalog()).toHaveLength(68); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(68); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(42); @@ -46,18 +49,81 @@ describe("runner E2E catalog", () => { (entry) => entry.suite.id === "openrouter-model-breadth", ), ).toHaveLength(10); + expect( + runnerMatrix.filter( + (entry) => entry.suite.id === "daytona-warm-continuity", + ), + ).toHaveLength(2); expect( runnerMatrix.reduce( (total, execution) => total + execution.task.expectedRunCount, 0, ), - ).toBe(114); + ).toBe(120); expect( runnerTasks.find((task) => task.id === "plan-revise-accept") ?.attemptTimeoutMs, ).toEqual({ local: 8 * 60_000, daytona: 12 * 60_000 }); }); + it("defines the warm Daytona continuity fixture as exactly two Codex cells", () => { + expect(daytonaWarmEnvironment).toMatchObject({ + id: "daytona", + configurationKey: "warm-reuse-v1", + groups: ["daytona", "warm"], + }); + expect( + daytonaWarmEnvironment.buildEnvironment({ + secretRefs: { + DAYTONA_API_KEY: { + type: "secret_ref", + secretId: "22222222-2222-4222-8222-222222222222", + version: "latest", + }, + }, + daytonaImage: `runner@sha256:${"a".repeat(64)}`, + executionId: "warm", + }), + ).toMatchObject({ + config: { + reuseLease: true, + runnerLifecycleMode: "warm", + autoStopInterval: 5, + autoArchiveInterval: 15, + autoDeleteInterval: 60, + }, + }); + expect(daytonaWarmContinuityTask).toMatchObject({ + flow: "warm_three_turn", + expectedRunCount: 3, + turnTimeoutMs: 600_000, + }); + expect( + daytonaWarmContinuityTask.buildFollowupMessages?.("nonce"), + ).toHaveLength(2); + const cells = runnerMatrix.filter( + (entry) => entry.suite.id === "daytona-warm-continuity", + ); + expect(cells.map((entry) => entry.profile.id)).toEqual([ + "legacy-codex", + "runner-codex", + ]); + expect(cells.every((entry) => entry.environment.id === "daytona")).toBe( + true, + ); + const suite = runnerSuites.find( + (candidate) => candidate.id === "daytona-warm-continuity", + )!; + expect( + suiteDefinitionHash({ + ...suite, + environments: [ + { ...daytonaWarmEnvironment, configurationKey: "changed" }, + ], + }), + ).not.toBe(suiteDefinitionHash(suite)); + }); + it("derives the qualified local native OpenCode profiles from the ranked snapshot", () => { expect(openRouterBreadthExcludedModelIds).toEqual(["xiaomi/mimo-v2.5"]); expect(openRouterBreadthExcludedExecutionIds).toEqual([ @@ -181,9 +247,10 @@ describe("runner E2E catalog", () => { question?.buildPrompt("nonce"), ...breadthTasks, ]) { - expect(prompt).toContain("then emit exactly"); + const terminalTextInstruction = prompt?.match(/then emit (?:exactly|only)/)?.[0]; + expect(terminalTextInstruction).toBeDefined(); expect(prompt!.indexOf("paperclip_finish exactly once")).toBeLessThan( - prompt!.indexOf("then emit exactly"), + prompt!.indexOf(terminalTextInstruction!), ); expect(prompt).toContain("Wait for that tool call to succeed"); } @@ -336,7 +403,7 @@ describe("runner E2E catalog", () => { "those two tool calls form one indivisible response sequence", ); expect(task!.buildPrompt("nonce")).toContain( - "Do not emit assistant text, end the heartbeat, or stop after write_document alone", + "Do not emit assistant text, end the response or heartbeat, or stop after write_document alone", ); expect(task!.buildPrompt("nonce")).toContain( "one atomic issue PATCH with status `done` and that exact comment", @@ -433,7 +500,7 @@ describe("runner E2E selectors", () => { "daytona", ]); const selected = selectRunnerExecutions(options); - expect(selected).toHaveLength(12); + expect(selected).toHaveLength(13); expect( selected.every( (entry) => @@ -443,7 +510,7 @@ describe("runner E2E selectors", () => { ).toBe(true); }); - it("rejects groups outside the advertised four", () => { + it("rejects unknown groups", () => { const options = parseRunnerSelectors(["--group", "codex"]); expect(() => selectRunnerExecutions(options)).toThrow("Unknown group"); }); @@ -452,10 +519,10 @@ describe("runner E2E selectors", () => { const jobs = buildMatrixJobs( selectRunnerExecutions(parseRunnerSelectors(["--all"])), ); - expect(jobs).toHaveLength(66); - expect(jobs.filter((job) => job.needsDaytona)).toHaveLength(21); + expect(jobs).toHaveLength(68); + expect(jobs.filter((job) => job.needsDaytona)).toHaveLength(23); expect(jobs.filter((job) => !job.needsDaytona)).toHaveLength(45); - expect(new Set(jobs.map((job) => job.executionId)).size).toBe(66); + expect(new Set(jobs.map((job) => job.executionId)).size).toBe(68); expect( jobs.find( (job) => diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 6b887aaec9..586879eace 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -27,6 +27,7 @@ const SELECTABLE_GROUPS = [ "native", "local", "daytona", + "warm", "core", "breadth", ] as const; @@ -374,6 +375,50 @@ export const runnerEnvironments: readonly EnvironmentFixture[] = [ }, ] as const; +export const daytonaWarmEnvironment: EnvironmentFixture = { + id: "daytona", + configurationKey: "warm-reuse-v1", + label: "Daytona warm reusable sandbox", + groups: ["daytona", "warm"], + driver: "sandbox", + provider: "daytona", + credential: "DAYTONA_API_KEY", + lifecycle: { + setup: "create_via_api", + probe: "run_context_via_api", + cleanup: "delete_via_api_and_destroy_leases", + }, + expectedExecutionTarget: { kind: "remote", transport: "sandbox" }, + buildEnvironment(input) { + if (!isImmutableDaytonaImage(input.daytonaImage)) { + throw new Error( + "PAPERCLIP_E2E_DAYTONA_IMAGE must be an immutable image digest", + ); + } + return { + name: `Runner E2E Daytona warm ${input.executionId}`, + description: "Ephemeral reusable Daytona runner E2E environment", + driver: "sandbox", + config: { + provider: "daytona", + apiKey: requiredDaytonaSecret(input), + image: input.daytonaImage, + cpu: 4, + memory: 4, + disk: 10, + reuseLease: true, + runnerLifecycleMode: "warm", + autoStopInterval: 5, + autoArchiveInterval: 15, + autoDeleteInterval: 60, + timeoutMs: 300_000, + livenessTimeoutMs: 30_000, + }, + envVars: {}, + }; + }, +}; + export const runnerTasks: readonly RunnerTaskFixture[] = [ { id: "message-marker", @@ -719,6 +764,80 @@ const localEnvironment = runnerEnvironments.find( (environment) => environment.id === "local", )!; +function warmTurnMarker(turn: 1 | 2 | 3, nonce: string) { + return `PAPERCLIP_E2E_WARM_T${turn}_${nonce}`; +} + +function warmWorkspaceLine(turn: 1 | 2 | 3, nonce: string) { + return `T${turn}_${nonce}`; +} + +function warmTurnInstructions(turn: 1 | 2 | 3, nonce: string) { + const file = `daytona-warm-${nonce}.txt`; + const lines = Array.from({ length: turn }, (_, index) => + warmWorkspaceLine((index + 1) as 1 | 2 | 3, nonce), + ); + const marker = warmTurnMarker(turn, nonce); + const finalTurn = turn === 3; + return [ + `This is warm Daytona continuity turn ${turn} of 3. Work only in the current execution workspace.`, + turn === 1 + ? `Create ${file} with exactly this one line followed by a newline: ${lines[0]}` + : `Before changing anything, read ${file} and verify its content is exactly ${lines.slice(0, -1).join("\\n")} followed by a newline. Then append exactly ${lines.at(-1)} followed by a newline.`, + `After the write, verify ${file} contains exactly these lines, once each and in order: ${lines.join(" | ")}.`, + `In a native runner, call paperclip_finish exactly once with {reportedWorkDisposition:"${finalTurn ? "done" : "needs_review"}",summary:"${marker}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[{commandOrCheck:"read ${file}",status:"passed"}]}. Wait for that tool call to succeed, then emit exactly ${marker} once as the complete user-facing final response.`, + `In a legacy runner, make exactly one public-API completion write after verification: PATCH /api/issues/$PAPERCLIP_TASK_ID with {"status":"${finalTurn ? "done" : "in_review"}","comment":"${marker}"}. Include Authorization and X-Paperclip-Run-Id. Do not POST a separate comment.`, + `Do not include ${marker} in any other visible response or write. Do not recreate, truncate, reorder, or duplicate prior lines.`, + ].join("\n"); +} + +export const daytonaWarmContinuityTask: RunnerTaskFixture = { + id: "warm-three-turn", + label: "Warm three-turn workspace continuity", + groups: ["warm"], + workMode: "standard", + flow: "warm_three_turn", + expectedRunCount: 3, + attemptTimeoutMs: { local: 30 * 60_000, daytona: 30 * 60_000 }, + turnTimeoutMs: 10 * 60_000, + expectedTerminalState: { issue: "done", run: "succeeded" }, + buildTitle: (nonce) => `Runner E2E warm Daytona continuity ${nonce}`, + buildVisibleMarker: (nonce) => warmTurnMarker(3, nonce), + buildPrompt: (nonce) => warmTurnInstructions(1, nonce), + buildFollowupMessages: (nonce) => [ + warmTurnInstructions(2, nonce), + warmTurnInstructions(3, nonce), + ], + buildMatchers(nonce, execution) { + const markers = ([1, 2, 3] as const).map((turn) => + warmTurnMarker(turn, nonce), + ); + return [ + { kind: "message_exact", expected: markers[2] }, + ...markers.map( + (expected) => + ({ kind: "message_occurrences", expected, count: 1 }) as const, + ), + { kind: "message_ordered", expected: markers }, + { + kind: "file_exact", + path: `daytona-warm-${nonce}.txt`, + expected: `${([1, 2, 3] as const) + .map((turn) => warmWorkspaceLine(turn, nonce)) + .join("\n")}\n`, + }, + { kind: "issue_status", expected: "done" }, + { kind: "run_status", expected: "succeeded" }, + { kind: "runtime_mode", expected: execution.profile.expectedRuntimeMode }, + { kind: "environment", expected: "daytona" }, + ]; + }, +}; + +const codexContinuityProfiles = runnerProfiles.filter((profile) => + ["legacy-codex", "runner-codex"].includes(profile.id), +); + export const runnerSuites: readonly RunnerSuiteFixture[] = [ { id: "core-compatibility", @@ -762,6 +881,17 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ excludedExecutionIds: openRouterBreadthExcludedExecutionIds, }, }, + { + id: "daytona-warm-continuity", + label: "Daytona Warm Continuity", + description: + "Three browser-driven turns on one reusable Daytona sandbox for legacy and native Codex.", + groups: ["daytona", "warm"], + profiles: codexContinuityProfiles, + environments: [daytonaWarmEnvironment], + tasks: [daytonaWarmContinuityTask], + expectedMatrixSize: 2, + }, ] as const; export function suiteDefinitionHash(suite: RunnerSuiteFixture) { @@ -774,7 +904,10 @@ export function suiteDefinitionHash(suite: RunnerSuiteFixture) { model: profile.model, qualification: profile.modelQualification, })), - environments: suite.environments.map((environment) => environment.id), + environments: suite.environments.map((environment) => ({ + id: environment.id, + configurationKey: environment.configurationKey ?? "default", + })), tasks: suite.tasks.map((task) => ({ id: task.id, flow: task.flow, @@ -869,6 +1002,7 @@ export function validateRunnerCatalog(): MatrixExecution[] { ...runnerTasks, ...localIntegrityTasks, ...openRouterBreadthTasks, + daytonaWarmContinuityTask, ]; for (const [label, values] of [ ["suite", runnerSuites], @@ -888,6 +1022,7 @@ export function validateRunnerCatalog(): MatrixExecution[] { ...runnerSuites, ...allProfiles, ...runnerEnvironments, + daytonaWarmEnvironment, ...allTasks, ]) { const unknownGroups = fixture.groups.filter( @@ -916,7 +1051,7 @@ export function validateRunnerCatalog(): MatrixExecution[] { ]), ); - for (const environment of runnerEnvironments) { + for (const environment of [...runnerEnvironments, daytonaWarmEnvironment]) { const payload = environment.buildEnvironment({ secretRefs: sampleRefs, daytonaImage: @@ -968,8 +1103,8 @@ export function validateRunnerCatalog(): MatrixExecution[] { ); } } - if (matrix.length !== 66) - throw new Error(`Expected 66 runner executions; received ${matrix.length}`); + if (matrix.length !== 68) + throw new Error(`Expected 68 runner executions; received ${matrix.length}`); return matrix; } diff --git a/tests/runner-e2e/dashboard.ts b/tests/runner-e2e/dashboard.ts index 820615ff92..6fe3f8d782 100644 --- a/tests/runner-e2e/dashboard.ts +++ b/tests/runner-e2e/dashboard.ts @@ -196,6 +196,18 @@ function renderCase( `, ) .join(""); + const turnTimingRows = (entry?.result.turnTimings ?? []) + .map( + (timing) => ` + ${timing.turn} + ${html(timing.runId)} + ${html(timing.leaseAcquisitionOutcome)} + ${html(timing.schedulerLatencyMs === null ? "unavailable" : durationLabel(timing.schedulerLatencyMs))} + ${html(timing.runDurationMs === null ? "unavailable" : durationLabel(timing.runDurationMs))} + ${html(timing.responseLatencyMs === null ? "unavailable" : durationLabel(timing.responseLatencyMs))} + `, + ) + .join(""); const gallery = screenshots.length ? `