RomanNum3ral
/
paperclip
mirror of https://github.com/paperclipai/paperclip.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix(runner): persist warm Daytona workspaces

This commit is contained in:
Dotta 2026-09-05 10:49:13 -05:00
parent 3da58b185e
commit ee11cd65d9
44 changed files with 4019 additions and 359 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/runner-full-stack-e2e.yml vendored
View File

@ -14,7 +14,7 @@ on:
type: boolean
default: true
group:
description: "Comma-separated groups (AND semantics: legacy,native,local,daytona,core,breadth)"
description: "Comma-separated groups (AND semantics: legacy,native,local,daytona,warm,core,breadth)"
type: string
required: false
suite:

16
packages/adapter-utils/src/command-managed-runtime.ts
View File

@ -12,11 +12,15 @@ import {
type SandboxRemoteExecutionSpec,
type SandboxSyncOperation,
type SandboxSyncResult,
type WorkspaceDurableSeedPaths,
type WorkspaceInboundMode,
} from "./sandbox-managed-runtime.js";
import { preferredShellForSandbox, shellCommandArgs } from "./sandbox-shell.js";
import type { RunProcessResult } from "./server-utils.js";
import type { RuntimeProgressSink, RuntimeStatusSink } from "./runtime-progress.js";
import type { RuntimeSpanRunner } from "./acpx-engine/startup-timing.js";
import type { GitWorkspaceSnapshot } from "./git-workspace-sync.js";
import type { DirectorySnapshot } from "./workspace-restore-merge.js";
/**
* Input for a duplex channel open. The caller supplies only the command argument
@ -516,6 +520,10 @@ export async function prepareCommandManagedRuntime(input: {
workspaceLocalDir: string;
workspaceRemoteDir?: string;
syncWorkspace?: boolean;
workspaceInboundMode?: WorkspaceInboundMode;
workspaceDurableSeed?: WorkspaceDurableSeedPaths;
workspaceBaseline?: DirectorySnapshot;
workspaceGitSnapshot?: GitWorkspaceSnapshot | null;
workspaceExclude?: string[];
preserveAbsentOnRestore?: string[];
assets?: CommandManagedRuntimeAsset[];
@ -577,6 +585,10 @@ export async function prepareCommandManagedRuntime(input: {
workspaceLocalDir: input.workspaceLocalDir,
workspaceRemoteDir,
syncWorkspace: input.syncWorkspace,
workspaceInboundMode: input.workspaceInboundMode,
workspaceDurableSeed: input.workspaceDurableSeed,
workspaceBaseline: input.workspaceBaseline,
workspaceGitSnapshot: input.workspaceGitSnapshot,
workspaceExclude: mergeRuntimeExcludes(input.workspaceExclude),
preserveAbsentOnRestore: input.preserveAbsentOnRestore,
assets: input.assets,
@ -616,6 +628,10 @@ export async function prepareCommandManagedRuntime(input: {
workspaceLocalDir: input.workspaceLocalDir,
workspaceRemoteDir,
syncWorkspace: input.syncWorkspace,
workspaceInboundMode: input.workspaceInboundMode,
workspaceDurableSeed: input.workspaceDurableSeed,
workspaceBaseline: input.workspaceBaseline,
workspaceGitSnapshot: input.workspaceGitSnapshot,
workspaceExclude: mergeRuntimeExcludes(input.workspaceExclude),
preserveAbsentOnRestore: input.preserveAbsentOnRestore,
assets: input.assets,

23
packages/adapter-utils/src/execution-target.ts
View File

@ -18,10 +18,12 @@ import {
import type {
AdditionalSourceStagingFailure,
SandboxAdditionalSource,
WorkspaceDurableSeedPaths,
WorkspaceInboundMode,
} from "./sandbox-managed-runtime.js";
export {
resolveReferencedSourceIgnore,
} from "./sandbox-managed-runtime.js";
import type { GitWorkspaceSnapshot } from "./git-workspace-sync.js";
import type { DirectorySnapshot } from "./workspace-restore-merge.js";
export { resolveReferencedSourceIgnore } from "./sandbox-managed-runtime.js";
export type {
AdditionalSourceStagingFailure,
ReferencedSourceIgnoreResolution,
@ -249,6 +251,10 @@ export interface PreparedAdapterExecutionTargetRuntime {
* stage referenced projects, or when every requested project staged.
*/
additionalSourceFailures: AdditionalSourceStagingFailure[];
workspaceSyncSnapshot: {
baseline: DirectorySnapshot;
gitSnapshot: GitWorkspaceSnapshot | null;
} | null;
restoreWorkspace(onProgress?: RuntimeProgressSink): Promise<void>;
}
@ -1374,6 +1380,10 @@ export async function prepareAdapterExecutionTargetRuntime(input: {
timeoutSec?: number;
workspaceRemoteDir?: string;
syncWorkspace?: boolean;
workspaceInboundMode?: WorkspaceInboundMode;
workspaceDurableSeed?: WorkspaceDurableSeedPaths;
workspaceBaseline?: DirectorySnapshot;
workspaceGitSnapshot?: GitWorkspaceSnapshot | null;
workspaceExclude?: string[];
preserveAbsentOnRestore?: string[];
assets?: AdapterManagedRuntimeAsset[];
@ -1403,6 +1413,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: {
assetDirs: {},
additionalSourceDirs: {},
additionalSourceFailures: [],
workspaceSyncSnapshot: null,
restoreWorkspace: async () => {},
};
}
@ -1428,6 +1439,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: {
// The SSH transport does not stage referenced projects (it is out of scope), so it never
// reports a per-project staging failure.
additionalSourceFailures: [],
workspaceSyncSnapshot: null,
restoreWorkspace: prepared.restoreWorkspace,
};
}
@ -1448,6 +1460,10 @@ export async function prepareAdapterExecutionTargetRuntime(input: {
workspaceLocalDir: input.workspaceLocalDir,
workspaceRemoteDir: input.workspaceRemoteDir,
syncWorkspace: input.syncWorkspace,
workspaceInboundMode: input.workspaceInboundMode,
workspaceDurableSeed: input.workspaceDurableSeed,
workspaceBaseline: input.workspaceBaseline,
workspaceGitSnapshot: input.workspaceGitSnapshot,
workspaceExclude: input.workspaceExclude,
preserveAbsentOnRestore: input.preserveAbsentOnRestore,
assets: input.assets,
@ -1465,6 +1481,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: {
assetDirs: prepared.assetDirs,
additionalSourceDirs: prepared.additionalSourceDirs,
additionalSourceFailures: prepared.additionalSourceFailures,
workspaceSyncSnapshot: prepared.workspaceSyncSnapshot,
restoreWorkspace: prepared.restoreWorkspace,
};
}

123
packages/adapter-utils/src/sandbox-managed-runtime.test.ts
View File

@ -348,6 +348,129 @@ describe("sandbox managed runtime", () => {
}
});
it("adopts a warm remote workspace without inbound overwrite and still merges outbound changes", async () => {
const rootDir = await mkdtemp(
path.join(os.tmpdir(), "paperclip-sandbox-adopt-"),
);
cleanupDirs.push(rootDir);
const localWorkspaceDir = path.join(rootDir, "local-workspace");
const remoteWorkspaceDir = path.join(rootDir, "remote-workspace");
await mkdir(localWorkspaceDir, { recursive: true });
await mkdir(remoteWorkspaceDir, { recursive: true });
await writeFile(
path.join(localWorkspaceDir, "continuity.txt"),
"host baseline\n",
"utf8",
);
await writeFile(
path.join(remoteWorkspaceDir, "continuity.txt"),
"remote retained\n",
"utf8",
);
const client = makeFilesystemClient();
const syncIn = vi.spyOn(client, "syncIn");
const prepared = await prepareSandboxManagedRuntime({
spec: {
transport: "sandbox",
provider: "test",
sandboxId: "sandbox-warm",
remoteCwd: remoteWorkspaceDir,
timeoutMs: 30_000,
apiKey: null,
},
adapterKey: "test-adapter",
client,
workspaceLocalDir: localWorkspaceDir,
workspaceInboundMode: "adopt_remote",
});
expect(syncIn).not.toHaveBeenCalled();
await expect(
readFile(path.join(remoteWorkspaceDir, "continuity.txt"), "utf8"),
).resolves.toBe("remote retained\n");
expect(prepared.workspaceSyncSnapshot).not.toBeNull();
await writeFile(
path.join(remoteWorkspaceDir, "continuity.txt"),
"remote finalized\n",
"utf8",
);
await prepared.restoreWorkspace();
await expect(
readFile(path.join(localWorkspaceDir, "continuity.txt"), "utf8"),
).resolves.toBe("remote finalized\n");
});
it("reconstructs a replacement workspace from the exact durable pre-turn seed", async () => {
const rootDir = await mkdtemp(
path.join(os.tmpdir(), "paperclip-sandbox-durable-seed-"),
);
cleanupDirs.push(rootDir);
const localWorkspaceDir = path.join(rootDir, "local-workspace");
const firstRemoteDir = path.join(rootDir, "first-remote");
const replacementRemoteDir = path.join(rootDir, "replacement-remote");
const durableSeed = {
workspaceArchivePath: path.join(rootDir, "state", "workspace.tar"),
gitArchivePath: path.join(rootDir, "state", "git.tar"),
};
await mkdir(localWorkspaceDir, { recursive: true });
await writeFile(
path.join(localWorkspaceDir, "continuity.txt"),
"durable pre-turn bytes\n",
"utf8",
);
const first = await prepareSandboxManagedRuntime({
spec: {
transport: "sandbox",
provider: "test",
sandboxId: "sandbox-first",
remoteCwd: firstRemoteDir,
timeoutMs: 30_000,
apiKey: null,
},
adapterKey: "test-adapter",
client: makeFilesystemClient(),
workspaceLocalDir: localWorkspaceDir,
workspaceInboundMode: "host_current",
workspaceDurableSeed: durableSeed,
});
expect(first.workspaceSyncSnapshot).not.toBeNull();
await expect(stat(durableSeed.workspaceArchivePath)).resolves.toMatchObject(
{
mode: expect.any(Number),
},
);
await writeFile(
path.join(localWorkspaceDir, "continuity.txt"),
"concurrent host edit must not enter replacement\n",
"utf8",
);
await prepareSandboxManagedRuntime({
spec: {
transport: "sandbox",
provider: "test",
sandboxId: "sandbox-replacement",
remoteCwd: replacementRemoteDir,
timeoutMs: 30_000,
apiKey: null,
},
adapterKey: "test-adapter",
client: makeFilesystemClient(),
workspaceLocalDir: localWorkspaceDir,
workspaceInboundMode: "durable_seed",
workspaceDurableSeed: durableSeed,
workspaceBaseline: first.workspaceSyncSnapshot!.baseline,
workspaceGitSnapshot: first.workspaceSyncSnapshot!.gitSnapshot,
});
await expect(
readFile(path.join(replacementRemoteDir, "continuity.txt"), "utf8"),
).resolves.toBe("durable pre-turn bytes\n");
});
it("preserves excluded local workspace artifacts during restore mirroring", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-sandbox-restore-"));
cleanupDirs.push(rootDir);

228
packages/adapter-utils/src/sandbox-managed-runtime.ts
View File

@ -1,6 +1,10 @@
import { execFile as execFileCallback } from "node:child_process";
import { randomUUID } from "node:crypto";
import { constants as fsConstants, promises as fs } from "node:fs";
import { createHash, randomUUID } from "node:crypto";
import {
constants as fsConstants,
createReadStream,
promises as fs,
} from "node:fs";
import os from "node:os";
import path from "node:path";
import { promisify } from "node:util";
@ -12,6 +16,7 @@ import {
deleteLocalGitRef,
fetchGitBundleIntoLocalRef,
GIT_ARCHIVE_EXCLUDES,
type GitWorkspaceSnapshot,
integrateImportedGitHead,
readGitWorkspaceSnapshot,
ReferencedSourceIgnoreScanLimitExceededError,
@ -20,7 +25,11 @@ import {
withShallowGitWorkspaceClone,
WORKSPACE_GIT_SCAN_SATURATED_CODE,
} from "./git-workspace-sync.js";
import { captureDirectorySnapshot, mergeDirectoryWithBaseline } from "./workspace-restore-merge.js";
import {
captureDirectorySnapshot,
mergeDirectoryWithBaseline,
type DirectorySnapshot,
} from "./workspace-restore-merge.js";
import {
createRuntimeProgressReporter,
type RuntimeProgressDirection,
@ -553,9 +562,29 @@ export interface PreparedSandboxManagedRuntime {
* staged, or when no additional sources were requested.
*/
additionalSourceFailures: AdditionalSourceStagingFailure[];
/** Durable merge inputs used to resume an outbound restore after host restart. */
workspaceSyncSnapshot: {
baseline: DirectorySnapshot;
gitSnapshot: GitWorkspaceSnapshot | null;
} | null;
restoreWorkspace(onProgress?: RuntimeProgressSink): Promise<void>;
}
export type WorkspaceInboundMode =
"host_current" | "durable_seed" | "adopt_remote";
/**
* Controller-owned archives for replaying the exact pre-turn workspace into a
* replacement sandbox. Paths are never sent to the provider as credentials or
* persisted in database metadata.
*/
export interface WorkspaceDurableSeedPaths {
workspaceArchivePath: string;
workspaceArchiveSha256?: string;
gitArchivePath?: string | null;
gitArchiveSha256?: string | null;
}
/** One additional (referenced) project that failed to stage into the sandbox. */
export interface AdditionalSourceStagingFailure {
projectId: string;
@ -719,6 +748,57 @@ async function withTempDir<T>(prefix: string, fn: (dir: string) => Promise<T>):
}
}
async function sha256File(filePath: string): Promise<string> {
return await new Promise((resolveDigest, rejectDigest) => {
const digest = createHash("sha256");
const stream = createReadStream(filePath);
stream.on("data", (chunk) => digest.update(chunk));
stream.on("error", rejectDigest);
stream.on("end", () => resolveDigest(digest.digest("hex")));
});
}
async function copyDurableSeedArchive(input: {
sourcePath: string;
targetPath: string;
expectedSha256?: string | null;
}): Promise<void> {
const source = await fs.lstat(input.sourcePath);
if (source.isSymbolicLink() || !source.isFile()) {
throw new Error("workspace_durable_seed_invalid");
}
if (
input.expectedSha256 &&
(await sha256File(input.sourcePath)) !== input.expectedSha256
) {
throw new Error("workspace_durable_seed_digest_mismatch");
}
await fs.copyFile(input.sourcePath, input.targetPath);
}
async function persistDurableSeedArchive(input: {
sourcePath: string;
targetPath: string;
}): Promise<void> {
const parent = path.dirname(input.targetPath);
await fs.mkdir(parent, { recursive: true, mode: 0o700 });
const parentStat = await fs.lstat(parent);
if (parentStat.isSymbolicLink() || !parentStat.isDirectory()) {
throw new Error("workspace_durable_seed_root_invalid");
}
const temporary = path.join(
parent,
`.${path.basename(input.targetPath)}.${randomUUID()}.tmp`,
);
try {
await fs.copyFile(input.sourcePath, temporary);
await fs.chmod(temporary, 0o600);
await fs.rename(temporary, input.targetPath);
} finally {
await fs.rm(temporary, { force: true }).catch(() => undefined);
}
}
async function execTar(args: string[]): Promise<void> {
await execFile("tar", args, {
env: {
@ -976,6 +1056,12 @@ export async function prepareSandboxManagedRuntime(input: {
workspaceLocalDir: string;
workspaceRemoteDir?: string;
syncWorkspace?: boolean;
/** Selects authoritative host staging, exact durable-seed replay, or no-overwrite adoption. */
workspaceInboundMode?: WorkspaceInboundMode;
workspaceDurableSeed?: WorkspaceDurableSeedPaths;
/** Durable snapshots supplied when reconstructing an interrupted restore. */
workspaceBaseline?: DirectorySnapshot;
workspaceGitSnapshot?: GitWorkspaceSnapshot | null;
workspaceExclude?: string[];
preserveAbsentOnRestore?: string[];
assets?: SandboxManagedRuntimeAsset[];
@ -999,6 +1085,16 @@ export async function prepareSandboxManagedRuntime(input: {
const workspaceRemoteDir = input.workspaceRemoteDir ?? input.spec.remoteCwd;
const runtimeRootDir = path.posix.join(workspaceRemoteDir, ".paperclip-runtime", input.adapterKey);
const syncWorkspace = input.syncWorkspace !== false;
const workspaceInboundMode = input.workspaceInboundMode ?? "host_current";
const stageWorkspace =
syncWorkspace && workspaceInboundMode !== "adopt_remote";
const prepareWorkspaceSeed =
syncWorkspace &&
workspaceInboundMode === "adopt_remote" &&
input.workspaceDurableSeed !== undefined;
if (workspaceInboundMode === "durable_seed" && !input.workspaceDurableSeed) {
throw new Error("workspace_durable_seed_missing");
}
// Reject any unsafe asset key before an archive path or an asset directory is
// built from it. This runs before the git snapshot work so a bad key fails fast.
@ -1025,7 +1121,11 @@ export async function prepareSandboxManagedRuntime(input: {
// It reads git's own bookkeeping to decide what to include/exclude, so it is
// usually fast, but on a large working tree the `--ignored` walk is not free.
const gitSnapshot = syncWorkspace
? await runStepSpan("snapshot.git", () => readGitWorkspaceSnapshot(input.workspaceLocalDir))
? input.workspaceGitSnapshot !== undefined
? input.workspaceGitSnapshot
: await runStepSpan("snapshot.git", () =>
readGitWorkspaceSnapshot(input.workspaceLocalDir),
)
: null;
const gitIgnoredExcludes = gitSnapshot?.ignoredPaths;
const workspaceArchiveExclude = mergeExcludes(
@ -1047,9 +1147,12 @@ export async function prepareSandboxManagedRuntime(input: {
// dominant cost in the pre-`pack` window — it reads the content of every
// non-excluded file, serially — so it earns its own span.
const baselineSnapshot = syncWorkspace
? await runStepSpan("snapshot.baseline", () =>
captureDirectorySnapshot(input.workspaceLocalDir, { exclude: restoreExclude }),
)
? (input.workspaceBaseline ??
(await runStepSpan("snapshot.baseline", () =>
captureDirectorySnapshot(input.workspaceLocalDir, {
exclude: restoreExclude,
}),
)))
: null;
// Every inbound staging step delegates to the provider through `client.syncIn`:
@ -1172,7 +1275,7 @@ export async function prepareSandboxManagedRuntime(input: {
// records its own failure and never rejects.
const inboundTaskIsRequired: boolean[] = [];
if (syncWorkspace) {
if (stageWorkspace || prepareWorkspaceSeed) {
inboundTaskIsRequired.push(true);
inboundTasks.push(() =>
runStepSpan("stage.workspace", async () => {
@ -1203,18 +1306,49 @@ export async function prepareSandboxManagedRuntime(input: {
if (gitSnapshot) {
await emitRuntimeStatus(input.onRuntimeProgress, "git_sync", "Syncing git history to environment");
const gitTarPath = path.join(tempDir, "git-workspace.tar");
const remoteGitTar = path.posix.join(runtimeRootDir, "git-workspace-upload.tar");
await withShallowGitWorkspaceClone({
localDir: input.workspaceLocalDir,
snapshot: gitSnapshot,
}, async (cloneDir) => {
await createTarballFromDirectory({
localDir: cloneDir,
archivePath: gitTarPath,
exclude: [".paperclip-runtime"],
const remoteGitTar = path.posix.join(
runtimeRootDir,
"git-workspace-upload.tar",
);
if (workspaceInboundMode === "durable_seed") {
const durableGitArchive =
input.workspaceDurableSeed?.gitArchivePath;
if (!durableGitArchive) {
throw new Error("workspace_durable_seed_git_missing");
}
await copyDurableSeedArchive({
sourcePath: durableGitArchive,
targetPath: gitTarPath,
expectedSha256: input.workspaceDurableSeed?.gitArchiveSha256,
});
} else {
await withShallowGitWorkspaceClone(
{
localDir: input.workspaceLocalDir,
snapshot: gitSnapshot,
},
async (cloneDir) => {
await createTarballFromDirectory({
localDir: cloneDir,
archivePath: gitTarPath,
exclude: [".paperclip-runtime"],
});
},
);
if (input.workspaceDurableSeed?.gitArchivePath) {
await persistDurableSeedArchive({
sourcePath: gitTarPath,
targetPath: input.workspaceDurableSeed.gitArchivePath,
});
}
}
workspaceFiles.push({
sourcePath: gitTarPath,
targetPath: remoteGitTar,
kind: "file",
access: "rw",
writablePath: workspaceRemoteDir,
});
workspaceFiles.push({ sourcePath: gitTarPath, targetPath: remoteGitTar, kind: "file", access: "rw", writablePath: workspaceRemoteDir });
workspacePostUploadCommands.push({
command: buildWorkspaceTarExtractCommand({
workspaceRemoteDir,
@ -1230,22 +1364,48 @@ export async function prepareSandboxManagedRuntime(input: {
// the preserved names first. The extract runs AFTER the git extract.
await emitRuntimeStatus(input.onRuntimeProgress, "config_sync", "Syncing workspace to environment");
const workspaceTarPath = path.join(tempDir, "workspace.tar");
const workspaceArchiveDir = gitSnapshot ? path.join(tempDir, "workspace-overlay") : input.workspaceLocalDir;
if (gitSnapshot) {
await copySelectedWorkspaceEntries({
sourceDir: input.workspaceLocalDir,
targetDir: workspaceArchiveDir,
relativePaths: gitSnapshot.overlayPaths,
exclude: workspaceArchiveExclude,
if (workspaceInboundMode === "durable_seed") {
await copyDurableSeedArchive({
sourcePath: input.workspaceDurableSeed!.workspaceArchivePath,
targetPath: workspaceTarPath,
expectedSha256:
input.workspaceDurableSeed!.workspaceArchiveSha256,
});
} else {
const workspaceArchiveDir = gitSnapshot
? path.join(tempDir, "workspace-overlay")
: input.workspaceLocalDir;
if (gitSnapshot) {
await copySelectedWorkspaceEntries({
sourceDir: input.workspaceLocalDir,
targetDir: workspaceArchiveDir,
relativePaths: gitSnapshot.overlayPaths,
exclude: workspaceArchiveExclude,
});
}
await createTarballFromDirectory({
localDir: workspaceArchiveDir,
archivePath: workspaceTarPath,
exclude: gitSnapshot ? undefined : workspaceArchiveExclude,
});
if (input.workspaceDurableSeed) {
await persistDurableSeedArchive({
sourcePath: workspaceTarPath,
targetPath: input.workspaceDurableSeed.workspaceArchivePath,
});
}
}
await createTarballFromDirectory({
localDir: workspaceArchiveDir,
archivePath: workspaceTarPath,
exclude: gitSnapshot ? undefined : workspaceArchiveExclude,
const remoteWorkspaceTar = path.posix.join(
runtimeRootDir,
"workspace-upload.tar",
);
workspaceFiles.push({
sourcePath: workspaceTarPath,
targetPath: remoteWorkspaceTar,
kind: "file",
access: "rw",
writablePath: workspaceRemoteDir,
});
const remoteWorkspaceTar = path.posix.join(runtimeRootDir, "workspace-upload.tar");
workspaceFiles.push({ sourcePath: workspaceTarPath, targetPath: remoteWorkspaceTar, kind: "file", access: "rw", writablePath: workspaceRemoteDir });
workspacePostUploadCommands.push({
command: buildWorkspaceTarExtractCommand({
workspaceRemoteDir,
@ -1265,6 +1425,8 @@ export async function prepareSandboxManagedRuntime(input: {
workspaceUploadBytes += (await fs.stat(workspaceTarPath)).size;
});
if (!stageWorkspace) return;
// One confined `syncIn` for the whole merged workspace file set. The confine
// guard covers every mapping BEFORE any bytes upload (fail-closed): a source
// or target escape in EITHER tar mapping stops the upload of both.
@ -1457,6 +1619,10 @@ export async function prepareSandboxManagedRuntime(input: {
assetDirs,
additionalSourceDirs,
additionalSourceFailures,
workspaceSyncSnapshot:
syncWorkspace && baselineSnapshot
? { baseline: baselineSnapshot, gitSnapshot }
: null,
restoreWorkspace: async (onProgress?: RuntimeProgressSink) => {
const restoreSink = onProgress ?? input.onProgress;

33
packages/adapter-utils/src/workspace-restore-merge.test.ts
View File

@ -9,9 +9,12 @@ import { afterEach, describe, expect, it, vi } from "vitest";
import { resolvePaperclipInstanceRootForAdapter } from "./server-utils.js";
import {
captureDirectorySnapshot,
directorySnapshotSha256,
classifyWorkspaceRestoreFailure,
describeWorkspaceRestoreFailure,
mergeDirectoryWithBaseline,
parseDirectorySnapshot,
serializeDirectorySnapshot,
withDirectoryMergeLock,
WORKSPACE_RESTORE_LOCK_TIMEOUT_CODE,
} from "./workspace-restore-merge.js";
@ -27,6 +30,36 @@ describe("workspace restore merge", () => {
}
});
it("round-trips a deterministic durable snapshot and rejects traversal", async () => {
const rootDir = await mkdtemp(
path.join(os.tmpdir(), "paperclip-snapshot-"),
);
cleanupDirs.push(rootDir);
await mkdir(path.join(rootDir, "nested"), { recursive: true });
await writeFile(path.join(rootDir, "b.txt"), "bravo\n", "utf8");
await writeFile(path.join(rootDir, "nested", "a.txt"), "alpha\n", "utf8");
const snapshot = await captureDirectorySnapshot(rootDir, { exclude: [] });
const serialized = serializeDirectorySnapshot(snapshot);
const restored = parseDirectorySnapshot(serialized);
expect(serialized.entries.map(([relativePath]) => relativePath)).toEqual([
"b.txt",
"nested",
"nested/a.txt",
]);
expect(restored).not.toBeNull();
expect(directorySnapshotSha256(restored!)).toBe(
directorySnapshotSha256(snapshot),
);
expect(
parseDirectorySnapshot({
...serialized,
entries: [["../escape", serialized.entries[0]![1]]],
}),
).toBeNull();
});
it("preserves sibling files when sequential stale-baseline restores create the same nested directory tree", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-restore-merge-"));
cleanupDirs.push(rootDir);

83
packages/adapter-utils/src/workspace-restore-merge.ts
View File

@ -5,7 +5,7 @@ import path from "node:path";
import { shouldExcludePath } from "./exclude-patterns.js";
import { resolvePaperclipInstanceRootForAdapter } from "./server-utils.js";
type SnapshotEntry =
export type SnapshotEntry =
| { kind: "dir" }
| { kind: "file"; mode: number; hash: string }
| { kind: "symlink"; target: string };
@ -15,6 +15,87 @@ export interface DirectorySnapshot {
entries: Map<string, SnapshotEntry>;
}
export interface SerializedDirectorySnapshot {
version: 1;
exclude: string[];
entries: Array<[string, SnapshotEntry]>;
}
function isSafeSnapshotRelativePath(value: string): boolean {
if (!value || path.posix.isAbsolute(value) || path.win32.isAbsolute(value)) {
return false;
}
return !value.split(/[\\/]/).some((segment) => segment === "..");
}
function parseSnapshotEntry(value: unknown): SnapshotEntry | null {
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
const candidate = value as Record<string, unknown>;
if (candidate.kind === "dir") return { kind: "dir" };
if (candidate.kind === "symlink" && typeof candidate.target === "string") {
return { kind: "symlink", target: candidate.target };
}
if (
candidate.kind === "file" &&
typeof candidate.mode === "number" &&
Number.isInteger(candidate.mode) &&
candidate.mode >= 0 &&
typeof candidate.hash === "string" &&
/^[0-9a-f]{64}$/.test(candidate.hash)
) {
return { kind: "file", mode: candidate.mode, hash: candidate.hash };
}
return null;
}
export function serializeDirectorySnapshot(
snapshot: DirectorySnapshot,
): SerializedDirectorySnapshot {
return {
version: 1,
exclude: [...snapshot.exclude],
entries: [...snapshot.entries.entries()].sort(([left], [right]) =>
left.localeCompare(right),
),
};
}
export function parseDirectorySnapshot(
value: unknown,
): DirectorySnapshot | null {
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
const candidate = value as Record<string, unknown>;
if (
candidate.version !== 1 ||
!Array.isArray(candidate.exclude) ||
!candidate.exclude.every((entry) => typeof entry === "string") ||
!Array.isArray(candidate.entries)
) {
return null;
}
const entries = new Map<string, SnapshotEntry>();
for (const rawEntry of candidate.entries) {
if (!Array.isArray(rawEntry) || rawEntry.length !== 2) return null;
const [relative, rawSnapshotEntry] = rawEntry;
if (typeof relative !== "string" || !isSafeSnapshotRelativePath(relative)) {
return null;
}
const entry = parseSnapshotEntry(rawSnapshotEntry);
if (!entry || entries.has(relative)) return null;
entries.set(relative, entry);
}
return {
exclude: [...new Set(candidate.exclude as string[])],
entries,
};
}
export function directorySnapshotSha256(snapshot: DirectorySnapshot): string {
return createHash("sha256")
.update(JSON.stringify(serializeDirectorySnapshot(snapshot)))
.digest("hex");
}
async function hashFile(filePath: string): Promise<string> {
return await new Promise((resolve, reject) => {
const hash = createHash("sha256");

58
packages/adapters/codex-local/src/server/execute.test.ts
View File

@ -117,11 +117,10 @@ describe("codex execute — outbound auth copy-back restore contribution", () =>
);
}
async function runTeardown(input: {
sandboxAuth: string;
hostAuth: string;
}): Promise<{ finalHostAuth: string; finalHostMode: number }> {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-codex-copyback-e2e-"));
async function runTeardown(input: { sandboxAuth: string; hostAuth: string }) {
const rootDir = await mkdtemp(
path.join(os.tmpdir(), "paperclip-codex-copyback-e2e-"),
);
cleanupDirs.push(rootDir);
const workspaceDir = path.join(rootDir, "workspace");
// The shared host home is what `resolveSharedCodexHomeDir` returns
@ -137,7 +136,7 @@ describe("codex execute — outbound auth copy-back restore contribution", () =>
process.env.CODEX_HOME = sharedHostHome;
sandboxAuthFixture.bytes = Buffer.from(input.sandboxAuth, "utf8");
await execute({
const executionResult = await execute({
runId: "run-copyback-e2e",
agent: {
id: "agent-1",
@ -178,6 +177,7 @@ describe("codex execute — outbound auth copy-back restore contribution", () =>
return {
finalHostAuth: await readFile(hostAuthPath, "utf8"),
finalHostMode: (await lstat(hostAuthPath)).mode & 0o777,
executionResult,
};
}
@ -231,4 +231,50 @@ describe("codex execute — outbound auth copy-back restore contribution", () =>
expect(result.finalHostMode, entry.name).toBe(0o600);
}
});
it("surfaces workspace restore failure after successful provider execution", async () => {
prepareAdapterExecutionTargetRuntime.mockResolvedValueOnce({
target: { kind: "remote", transport: "ssh" },
workspaceRemoteDir: "/remote/workspace",
runtimeRootDir: REMOTE_RUNTIME_ROOT,
assetDirs: { home: `${REMOTE_RUNTIME_ROOT}/home` },
restoreWorkspace: async () => {
throw new Error("workspace copy-back failed");
},
});
await expect(
runTeardown({
sandboxAuth: subscriptionAuth({ accountId: "acct", marker: "sandbox" }),
hostAuth: subscriptionAuth({ accountId: "acct", marker: "host" }),
}),
).rejects.toThrow("workspace copy-back failed");
});
it("preserves a provider failure when workspace restore also fails", async () => {
runChildProcess.mockResolvedValueOnce({
exitCode: 1,
signal: null,
timedOut: false,
stdout: "",
stderr: "provider failed first",
pid: 321,
startedAt: new Date().toISOString(),
});
prepareAdapterExecutionTargetRuntime.mockResolvedValueOnce({
target: { kind: "remote", transport: "ssh" },
workspaceRemoteDir: "/remote/workspace",
runtimeRootDir: REMOTE_RUNTIME_ROOT,
assetDirs: { home: `${REMOTE_RUNTIME_ROOT}/home` },
restoreWorkspace: async () => {
throw new Error("workspace copy-back failed second");
},
});
const result = await runTeardown({
sandboxAuth: subscriptionAuth({ accountId: "acct", marker: "sandbox" }),
hostAuth: subscriptionAuth({ accountId: "acct", marker: "host" }),
});
expect(result.executionResult.errorMessage).toBe("provider failed first");
});
});

27
packages/adapters/codex-local/src/server/execute.ts
View File

@ -1526,6 +1526,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
};
};
let executionError: unknown = null;
try {
const initial = await runAttempt(sessionId);
if (
@ -1539,22 +1540,26 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
`[paperclip] Codex resume session "${sessionId}" is unavailable; retrying with a fresh session.\n`,
);
const retry = await runAttempt(null);
return toResult(retry, true, true);
const retryResult = toResult(retry, true, true);
if (retryResult.errorMessage) {
executionError = new Error(retryResult.errorMessage);
}
return retryResult;
}
return toResult(initial, false, false);
const result = toResult(initial, false, false);
if (result.errorMessage) {
executionError = new Error(result.errorMessage);
}
return result;
} catch (error) {
executionError = error;
throw error;
} finally {
if (paperclipBridge) {
await paperclipBridge.stop();
}
if (restoreRemoteWorkspace) {
// This teardown runs in a `finally`, so a throw here replaces the
// already-computed run result (`return toResult(...)`) and turns a
// successful Codex run into a failure. The workspace restore — and the
// host credential copy-back inside it — is a best-effort teardown step.
// Keep it rejection-safe: log a fault loudly and keep the pending
// result. The host copy-back installs the credential on disk before any
// diagnostic log runs, so it is already durable when this block returns.
try {
await onLog(
"stdout",
@ -1570,6 +1575,10 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
)}: ${error instanceof Error ? error.message : String(error)}\n`,
),
).catch(() => undefined);
// A provider failure remains the primary outcome. When provider work
// succeeded, however, silently accepting a failed copy-back can lose
// the only workspace edits before a replacement sandbox starts.
if (executionError === null) throw error;
}
}
}

42
packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs
View File

@ -54,7 +54,7 @@ pub(crate) const MAX_SETTLED_PROVIDER_TURN_IDS: usize = 4_096;
type QuestionOptionLabels = BTreeMap<String, BTreeMap<String, String>>;
type QuestionSetMapping = (String, Value, QuestionOptionLabels);
#[derive(Clone)]
#[derive(Clone, PartialEq)]
struct ProviderCompletionContract {
revision: String,
criterion_ids: Vec<String>,
@ -807,6 +807,46 @@ impl CodexProvider {
self.durable_tool_call_replays = true;
}
pub(crate) fn attach_run_in_place(
&mut self,
authorized_tools: impl IntoIterator<Item = AuthorizedTool>,
completion_contract: Option<(&str, &[String])>,
) -> Result<bool, LocalRunnerError> {
let authorized_tools = authorized_tools.into_iter().collect::<Vec<_>>();
let completion_contract =
completion_contract.map(|(revision, criterion_ids)| ProviderCompletionContract {
revision: revision.to_owned(),
criterion_ids: criterion_ids.to_vec(),
});
if authorized_tools != self.authorized_tools
|| completion_contract != self.completion_contract
{
return Ok(false);
}
if self.process.try_wait()?.is_some()
|| self.quarantined
|| self.active_provider_turn_id.is_some()
|| self.ambiguous_turn_start_pending
|| !self.pending_messages.is_empty()
|| !self.deferred_ambiguous_messages.is_empty()
|| !self.pending_tool_requests.is_empty()
|| !self.pending_runtime_requests.is_empty()
{
return Err(LocalRunnerError::invalid(
"Codex warm run attachment requires an idle live provider with no pending work",
));
}
// The provider process and its thread remain authoritative. Exact
// settled-turn identities stay in memory so delayed output from an
// earlier run cannot be accepted as the next turn. A changed semantic
// tool or completion contract returns false so the caller can preserve
// the existing cold-resume behavior for that incompatible boundary.
self.completed_turn_authority = None;
self.completion_reconciliation_pending = false;
self.expected_shutdown = false;
Ok(true)
}
pub(crate) fn restore_completed_turn_authority(
&mut self,
authoritative: bool,

187
packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs
View File

@ -116,6 +116,113 @@ impl CommandLifecycle {
}
}
fn next_authority_config(
command: &Command,
current: &DurableRunnerConfig,
) -> Result<Option<DurableRunnerConfig>, DurableRunnerError> {
if command.command_type != "run.attach" {
return Ok(None);
}
let Some(boundary) = command.payload.get("paperclipNextAuthority") else {
return Ok(None);
};
let identity = boundary
.get("identity")
.and_then(Value::as_object)
.ok_or_else(|| DurableRunnerError::invalid("run.attach authority identity is required"))?;
let read_identity = |key: &str| {
identity
.get(key)
.and_then(Value::as_str)
.filter(|value| !value.is_empty())
.map(str::to_owned)
.ok_or_else(|| {
DurableRunnerError::invalid(format!(
"run.attach authority identity field {key} is required"
))
})
};
let connection = boundary
.get("connection")
.and_then(Value::as_object)
.ok_or_else(|| {
DurableRunnerError::invalid("run.attach authority connection is required")
})?;
let connect_url = match connection.get("mode").and_then(Value::as_str) {
Some("connect") => connection
.get("connectUrl")
.and_then(Value::as_str)
.filter(|value| !value.is_empty())
.map(str::to_owned)
.ok_or_else(|| DurableRunnerError::invalid("run.attach connect URL is required"))?,
Some("listen") => {
let address = connection
.get("listenAddress")
.and_then(Value::as_str)
.ok_or_else(|| {
DurableRunnerError::invalid("run.attach listen address is required")
})?;
let port = connection
.get("listenPort")
.and_then(Value::as_u64)
.ok_or_else(|| DurableRunnerError::invalid("run.attach listen port is required"))?;
let path = connection
.get("listenPath")
.and_then(Value::as_str)
.ok_or_else(|| DurableRunnerError::invalid("run.attach listen path is required"))?;
format!("listen://{address}:{port}{path}")
}
_ => {
return Err(DurableRunnerError::invalid(
"run.attach authority connection mode is invalid",
));
}
};
let mut next = current.clone();
next.connect_url = connect_url;
next.ca_bundle_path = connection
.get("caBundlePath")
.and_then(Value::as_str)
.filter(|value| !value.is_empty())
.map(Into::into);
next.runner_instance_id = read_identity("runnerInstanceId")?;
next.environment_lease_id = read_identity("environmentLeaseId")?;
next.run_id = read_identity("runId")?;
next.normalized_session_id = read_identity("normalizedSessionId")?;
next.turn_id = read_identity("turnId")?;
next.item_id = read_identity("itemId")?;
next.validate()?;
if next.runner_instance_id != current.runner_instance_id
|| next.environment_lease_id != current.environment_lease_id
|| next.normalized_session_id != current.normalized_session_id
|| next.run_id == current.run_id
{
return Err(DurableRunnerError::invalid(
"run.attach authority changed an immutable session binding",
));
}
Ok(Some(next))
}
fn apply_authority_rotation(
state: &mut DurableState,
store: &DurableStateStore,
config: &mut DurableRunnerConfig,
endpoint: &mut RunnerTransportEndpoint,
next: DurableRunnerConfig,
) -> Result<(), DurableRunnerError> {
let reconnect_count = state.reconnect_count.saturating_add(1);
let mut diagnostics = std::mem::take(&mut state.diagnostics);
*endpoint = RunnerTransportEndpoint::new(&next.connect_url, &next.run_id)?;
*config = next;
let mut rotated = DurableState::new(config);
rotated.reconnect_count = reconnect_count;
rotated.diagnostics.append(&mut diagnostics);
rotated.record_diagnostic("runner advanced to a new warm run authority");
*state = rotated;
store.save(state)
}
pub trait CommandExecutor {
fn execute(&mut self, command: &Command) -> Result<CommandExecution, DurableRunnerError>;
@ -136,7 +243,7 @@ pub trait CommandExecutor {
}
pub fn run_durable_runner<E: CommandExecutor>(
config: DurableRunnerConfig,
mut config: DurableRunnerConfig,
bootstrap_ticket: BootstrapTicket,
mut executor: E,
) -> Result<(), DurableRunnerError> {
@ -162,7 +269,7 @@ pub fn run_durable_runner<E: CommandExecutor>(
// Bind listener mode or resolve dial mode before processing commands. Dial
// reconnects retain the same validated addresses so DNS cannot redirect a
// retry after the trust decision.
let endpoint = RunnerTransportEndpoint::new(&config.connect_url, &config.run_id)?;
let mut endpoint = RunnerTransportEndpoint::new(&config.connect_url, &config.run_id)?;
let started = Instant::now();
let mut bootstrap_ticket = Some(bootstrap_ticket);
let mut lease: Option<LeaseCredential> = None;
@ -287,8 +394,10 @@ pub fn run_durable_runner<E: CommandExecutor>(
let mut sent_source_seq = state.acked_source_seq;
let mut lifecycle_after_reply = CommandLifecycle::Continue;
let mut authority_rotation = None;
let mut disconnected = false;
for command in welcome.pending_commands {
let next_authority = next_authority_config(&command, &config)?;
let (result, lifecycle) =
process_command(&mut state, &store, &config, &mut executor, &command)?;
if let Some(durable_lifecycle) = lifecycle.durable_state() {
@ -333,6 +442,15 @@ pub fn run_durable_runner<E: CommandExecutor>(
// then release the executor without observing later commands.
break;
}
if next_authority.is_some() {
authority_rotation = next_authority;
break;
}
}
if let Some(next) = authority_rotation {
apply_authority_rotation(&mut state, &store, &mut config, &mut endpoint, next)?;
disconnected_since = Some(Instant::now());
continue;
}
if !disconnected {
if let Err(error) = send_outbox(&mut transport, &state, &mut sent_source_seq) {
@ -425,6 +543,7 @@ pub fn run_durable_runner<E: CommandExecutor>(
.map_err(|error| {
DurableRunnerError::invalid(format!("command is malformed: {error}"))
})?;
let next_authority = next_authority_config(&command, &config)?;
let (result, lifecycle) =
process_command(&mut state, &store, &config, &mut executor, &command)?;
if let Some(durable_lifecycle) = lifecycle.durable_state() {
@ -468,6 +587,17 @@ pub fn run_durable_runner<E: CommandExecutor>(
);
}
}
if let Some(next) = next_authority {
apply_authority_rotation(
&mut state,
&store,
&mut config,
&mut endpoint,
next,
)?;
disconnected_since = Some(Instant::now());
break;
}
if let Err(error) = send_outbox(&mut transport, &state, &mut sent_source_seq) {
state.record_diagnostic(
"outbox delivery failed; unacknowledged suffix remains durable",
@ -1018,6 +1148,59 @@ mod tests {
}
}
#[test]
fn warm_run_attachment_rotates_only_the_run_authority() {
let directory = std::env::temp_dir().join(format!(
"paperclip-runner-warm-authority-{}",
std::process::id()
));
let _ = fs::remove_dir_all(&directory);
let mut current = config(directory.clone());
current.runner_digest = format!("sha256:{}", "a".repeat(64));
let mut attach = command("run.attach");
attach.payload = json!({
"paperclipNextAuthority": {
"identity": {
"runnerInstanceId": current.runner_instance_id,
"environmentLeaseId": current.environment_lease_id,
"runId": "run_2",
"normalizedSessionId": current.normalized_session_id,
"turnId": "turn_2",
"itemId": "item_2"
},
"connection": {
"mode": "connect",
"connectUrl": "ws://127.0.0.1:3001/path"
}
}
});
let next = next_authority_config(&attach, &current)
.unwrap()
.expect("attachment should carry a new authority");
assert_eq!(next.run_id, "run_2");
assert_eq!(next.connect_url, "ws://127.0.0.1:3001/path");
let store = DurableStateStore::new(&directory).unwrap();
let (mut state, _) = store.load_or_create(&current).unwrap();
state.outbox.push(crate::durable::state::StoredOutboxEvent {
source_seq: 1,
priority: 0,
event_type: "run.attached".to_owned(),
byte_size: 1,
envelope: json!({}),
});
let mut endpoint =
RunnerTransportEndpoint::new(&current.connect_url, &current.run_id).unwrap();
apply_authority_rotation(&mut state, &store, &mut current, &mut endpoint, next).unwrap();
assert_eq!(state.run_id, "run_2");
assert_eq!(state.next_source_seq, 1);
assert!(state.outbox.is_empty());
assert_eq!(current.run_id, "run_2");
fs::remove_dir_all(directory).unwrap();
}
#[test]
fn terminal_lifecycle_is_durable_before_fallible_cleanup() {
let directory = std::env::temp_dir().join(format!(

33
packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs
View File

@ -1675,19 +1675,40 @@ impl CodexCommandExecutor {
next_state.active_provider_result_fingerprint = None;
next_state.active_provider_result_disposition = None;
next_state.last_agent_message = None;
if let Some(provider) = self.provider.as_mut() {
let provider = self.provider.as_mut().ok_or_else(|| {
DurableRunnerError::invalid("run.attach requires the restored Codex provider process")
})?;
let retained_provider = provider
.attach_run_in_place(
next_state.tool_bridge.authorized_tools().cloned(),
next_state.completion_contract.as_ref().map(|contract| {
(
contract.revision.as_str(),
contract.criterion_ids.as_slice(),
)
}),
)
.map_err(|error| {
DurableRunnerError::invalid(format!(
"failed to retain Codex for warm run attachment: {error}"
))
})?;
if !retained_provider {
provider.shutdown().map_err(|error| {
DurableRunnerError::invalid(format!(
"failed to checkpoint Codex before attaching a new run: {error}"
))
})?;
self.provider = None;
}
self.provider = None;
next_state.pending_events.clear();
// Persist the checkpoint as not-open before open_session resumes it for
// the new authority. Otherwise recovery emits a second session.resumed
// notice into the provider queue in addition to the command event.
next_state.lifecycle = "prepared".to_owned();
next_state.lifecycle = if retained_provider {
"session_open".to_owned()
} else {
// The next provider command restores the same checkpointed session
// with the rotated tool/completion authority.
"prepared".to_owned()
};
self.persist_state(&next_state)?;
self.state = Some(next_state);
Ok(())

32
packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts
View File

@ -939,7 +939,7 @@ class AuthorityConnection {
/** Authenticated, replay-safe PRP transport authority. Business operations are caller supplied. */
export class DurablePrpControlPlane {
readonly #identity: DurableRecoveryIdentity;
#identity: DurableRecoveryIdentity;
readonly #store: DurableCoreStore;
#expectedRunnerVersion: string;
#expectedRunnerDigest: string;
@ -1041,6 +1041,36 @@ export class DurablePrpControlPlane {
).length;
}
/**
* Atomically advances a settled reusable runner to a new run authority while
* retaining its existing connection lease secret. The runner performs the
* matching state transition only after acknowledging `run.attach`.
*/
rotateRunIdentity(identity: DurableRecoveryIdentity): void {
if (
!Object.values(identity).every(
(value) => typeof value === "string" && stableIdPattern.test(value),
) ||
identity.runnerInstanceId !== this.#identity.runnerInstanceId ||
identity.environmentLeaseId !== this.#identity.environmentLeaseId ||
identity.normalizedSessionId !== this.#identity.normalizedSessionId ||
identity.runId === this.#identity.runId ||
this.#store.state.commands.some((command) => command.status === "pending")
) {
throw new Error("Durable PRP run identity rotation is invalid.");
}
this.disconnectActiveRunner();
const leases = Object.fromEntries(
Object.entries(this.#store.state.leases).map(([key, lease]) => [
key,
{ ...lease, identity: structuredClone(identity) },
]),
);
Object.assign(this.#store.state, initialCoreState(identity), { leases });
this.#identity = structuredClone(identity);
this.#store.save();
}
issueBootstrapTicket(ttlMs = 5_000): string {
if (!Number.isInteger(ttlMs) || ttlMs < 1_000 || ttlMs > 60_000) {
throw new Error("Durable PRP bootstrap TTL is invalid.");

83
packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts
View File

@ -1965,7 +1965,7 @@ it("steers the active provider turn through the durable PRP command path", async
}
}, 30_000);
it("does not expose cross-run attachment before PRP authority can rotate atomically", async () => {
it("rotates PRP authority in place for a warm cross-run attachment", async () => {
const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-warm-attach-"));
const bundle = createCapabilityRunnerdCodexTransport({
runnerBinary: defaultCapabilityRunnerdBinary(),
@ -1978,22 +1978,36 @@ it("does not expose cross-run attachment before PRP authority can rotate atomica
success: true,
contentItems: [],
}));
const within = async <T>(label: string, promise: Promise<T>) =>
await Promise.race([
promise,
new Promise<never>((_, reject) =>
setTimeout(() => reject(new Error(`${label} timeout`)), 5_000),
),
]);
try {
await bundle.transport.request("initialize", {});
await bundle.transport.request("thread/start", {
cwd: tmpdir(),
dynamicTools: [
{
name: "get_task_context",
description: "Read the active task.",
inputSchema: {
type: "object",
properties: {},
additionalProperties: false,
await within("initialize", bundle.transport.request("initialize", {}));
await within(
"thread start",
bundle.transport.request("thread/start", {
cwd: tmpdir(),
dynamicTools: [
{
name: "get_task_context",
description: "Read the active task.",
inputSchema: {
type: "object",
properties: {},
additionalProperties: false,
},
},
],
completionContract: {
revision: "sha256:warm-three-turn-contract",
criterionIds: ["objective"],
},
],
});
}),
);
const runnerPid = bundle.evidence().runnerPid;
const providerPid = bundle.evidence().codexPid;
const notifications = bundle.transport
@ -2015,12 +2029,45 @@ it("does not expose cross-run attachment before PRP authority can rotate atomica
}
throw new Error(`${label} completion timeout`);
};
await bundle.transport.request("turn/start", {
input: [{ type: "text", text: "first run" }],
});
await within(
"first turn start",
bundle.transport.request("turn/start", {
input: [{ type: "text", text: "first run" }],
}),
);
await waitForCompletion("first run");
expect(bundle.transport.attachRun).toBeUndefined();
await within(
"warm attach",
bundle.transport.attachRun!({
runId: "run-warm-second",
turnId: "turn-warm-second",
itemId: "item-warm-second",
}),
);
await within(
"second turn start",
bundle.transport.request("turn/start", {
input: [{ type: "text", text: "second run" }],
}),
);
await waitForCompletion("second run");
await within(
"second warm attach",
bundle.transport.attachRun!({
runId: "run-warm-third",
turnId: "turn-warm-third",
itemId: "item-warm-third",
}),
);
await within(
"third turn start",
bundle.transport.request("turn/start", {
input: [{ type: "text", text: "third run" }],
}),
);
await waitForCompletion("third run");
expect(bundle.evidence()).toMatchObject({
runnerPid,

113
packages/paperclip-runner/src/live/runnerd-codex-transport.ts
View File

@ -400,7 +400,7 @@ async function rotateExternalAuthorityEpoch(
}
function rotatedRunAttachPayload(
state: Record<string, unknown>,
state: { commands?: unknown },
desired: DurableRecoveryIdentity,
authorizedTools: Record<string, unknown> | null,
completionContract:
@ -418,7 +418,22 @@ function rotatedRunAttachPayload(
);
if (!seed)
throw new Error("native_runner_authority_rotation_seed_unavailable");
const payload = structuredClone(record(seed.payload));
return retargetRunAttachPayload(
record(seed.payload),
desired,
authorizedTools,
completionContract,
);
}
function retargetRunAttachPayload(
seedPayload: Record<string, unknown>,
desired: DurableRecoveryIdentity,
authorizedTools: Record<string, unknown> | null,
completionContract:
{ revision: string; criterionIds: readonly string[] } | undefined,
): Record<string, unknown> {
const payload = structuredClone(seedPayload);
const provider = record(payload.provider);
if (provider.kind === "acpx" || provider.provider === "acpx") {
provider.runId = desired.runId;
@ -958,7 +973,10 @@ export interface CapabilityRunnerdCodexTransportOptions {
itemId: string;
};
/** Registers the run-bound PRP authority on Paperclip's shared HTTP server. */
controlPlaneRegistration?: (authority: DurablePrpControlPlane) => Promise<{
controlPlaneRegistration?: (
authority: DurablePrpControlPlane,
identity?: DurableRecoveryIdentity,
) => Promise<{
connectUrl?: string;
connection?: RunnerProcessConnection;
activate?: () => Promise<void> | void;
@ -1942,6 +1960,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
#expectedProviderTurnId: string | null = null;
#durableTurnId = "";
#authorizedTools: Record<string, unknown> | null = null;
#runAttachTemplate: Record<string, unknown> | null = null;
#closed = false;
#closePromise: Promise<void> | null = null;
#failure: Error | null = null;
@ -2216,6 +2235,79 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
this.#handler = handler;
}
async attachRun(input: {
runId: string;
turnId: string;
itemId: string;
}): Promise<void> {
const core = this.#core;
if (!core || !this.#startupComplete) {
throw new Error("native_runner_prp_run_rotation_unavailable");
}
const prior = core.store.state.identity;
const desired: DurableRecoveryIdentity = {
...prior,
runId: input.runId,
turnId: input.turnId,
itemId: input.itemId,
};
const registration = this.options.controlPlaneRegistration
? await this.options.controlPlaneRegistration(core, desired)
: null;
const connection: RunnerProcessConnection =
registration?.connection ??
(registration?.connectUrl
? { mode: "connect", connectUrl: registration.connectUrl }
: { mode: "connect", connectUrl: core.connectUrl });
const commandId = `command_attach_${createHash("sha256")
.update(`${prior.runId}:${desired.runId}:${desired.turnId}`)
.digest("hex")
.slice(0, 32)}`;
const runAttachTemplate = this.#runAttachTemplate
? retargetRunAttachPayload(
this.#runAttachTemplate,
desired,
this.#authorizedTools,
this.options.resumeCompletionContract,
)
: rotatedRunAttachPayload(
core.store.state,
desired,
this.#authorizedTools,
this.options.resumeCompletionContract,
);
this.#runAttachTemplate = structuredClone(runAttachTemplate);
const payload = {
...runAttachTemplate,
paperclipNextAuthority: { identity: desired, connection },
};
core.queueCommand("run.attach", payload, commandId, true);
await this.#waitCommand("run.attach", commandId);
const attached = core.store.state.commands.find(
(command) => command.commandId === commandId,
);
if (attached?.status !== "completed") {
await Promise.resolve(registration?.release()).catch(() => undefined);
throw new Error("native_runner_prp_run_rotation_failed");
}
const previousRelease = this.#controlPlaneRelease;
core.rotateRunIdentity(desired);
this.#eventIndex = 0;
this.#durableTurnId = desired.turnId;
this.#controlPlaneRelease = registration?.release ?? null;
await registration?.activate?.();
if (registration?.failure) {
void registration.failure.catch((error: unknown) => {
this.#failTransport(
error instanceof Error ? error : new Error(String(error)),
);
});
}
await previousRelease?.();
await this.#awaitRegistrationReady(registration?.ready);
}
async resolveRuntimeRequest(input: {
requestId: string;
turnId: string;
@ -3298,15 +3390,14 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
});
this.#core = core;
if (rotatedAuthority) {
core.queueCommand(
"run.attach",
rotatedRunAttachPayload(
controlPlaneState,
desiredIdentity,
this.#authorizedTools,
this.options.resumeCompletionContract,
),
const runAttachTemplate = rotatedRunAttachPayload(
controlPlaneState,
desiredIdentity,
this.#authorizedTools,
this.options.resumeCompletionContract,
);
this.#runAttachTemplate = structuredClone(runAttachTemplate);
core.queueCommand("run.attach", runAttachTemplate);
}
const committedEvents = core.store.state.committedEvents;
const runAttachment = recoveredRunAttachment(core.store.state);

31
packages/plugins/sandbox-providers/daytona/src/plugin.test.ts
View File

@ -1240,6 +1240,8 @@ describe("Daytona sandbox provider plugin", () => {
providerLeaseId: "sandbox-reuse",
metadata: {
resumedLease: true,
resumedFromState: "stopped",
sandboxState: "started",
workspaceSentinel: {
result: "matched",
token: "sentinel-token",
@ -4060,6 +4062,35 @@ describe("daytona native file-sync hooks", () => {
});
});
it("classifies a deleted sandbox during syncOut with a stable unrecoverable code", async () => {
const hostDir = await makeHostDir();
mockGet.mockRejectedValue(
new MockDaytonaNotFoundError("provider detail must not escape"),
);
await expect(
plugin.definition.onEnvironmentSyncOut?.({
driverKey: "daytona",
companyId: "company-1",
environmentId: "env-1",
config: { timeoutMs: 300000, reuseLease: true },
lease: syncLease(),
operations: [
{
operationId: "sync-op-missing-sandbox",
files: [
{
sourcePath: `${REMOTE_DIR}/out/result.txt`,
targetPath: path.join(hostDir, "result.txt"),
kind: "file",
},
],
},
],
}),
).rejects.toThrow("daytona_sandbox_not_found");
});
it("syncOut snapshot guard re-checks the resolved source is a non-symlink regular file immediately before copying (validation→copy TOCTOU)", async () => {
const hostDir = await makeHostDir();
const sandbox = createMockSandbox();

160
packages/plugins/sandbox-providers/daytona/src/plugin.ts
View File

@ -619,6 +619,8 @@ function leaseMetadata(input: {
shellCommand: "bash" | "sh";
remoteCwd: string;
resumedLease: boolean;
resumedFromState?: string | null;
sandboxState?: string | null;
workspaceSentinel?: WorkspaceSentinelResult;
}) {
return {
@ -626,7 +628,7 @@ function leaseMetadata(input: {
shellCommand: input.shellCommand,
sandboxId: input.sandbox.id,
sandboxName: input.sandbox.name,
sandboxState: input.sandbox.state ?? null,
sandboxState: input.sandboxState ?? input.sandbox.state ?? null,
image: input.config.image,
snapshot: input.config.snapshot,
target: input.sandbox.target,
@ -637,6 +639,9 @@ function leaseMetadata(input: {
...(input.config.archiveOnRelease ? { archiveOnRelease: true } : {}),
remoteCwd: input.remoteCwd,
resumedLease: input.resumedLease,
...(input.resumedLease
? { resumedFromState: input.resumedFromState ?? null }
: {}),
// Record the resources Paperclip attempted to request so future diagnosis
// can compare requested allocation against what Daytona provisioned.
...(input.config.cpu != null ? { cpu: input.config.cpu } : {}),
@ -2187,61 +2192,72 @@ const plugin = definePlugin({
return { providerLeaseId: null, metadata: { expired: true } };
}
// A stopped sandbox loses its session shell, so the stored session id is
// stale after a real restart. Clear the id only when the sandbox is not
// already running, and clear it before the restart. A stopped sandbox has
// no live session, so the clear drops a dead id and a later command opens
// a fresh session. A running sandbox keeps its live session, so the resume
// leaves the id in place; a concurrent command still finds it and teardown
// deletes one session. An unconditional clear would drop the id of a live
// session and leak its shell until sandbox reaping.
if (sandbox.state !== "started") {
sandboxHandleSessionStore.clear(scope);
// A stopped sandbox loses its pseudo-terminals, so a stored duplex channel
// is dead after a real restart. Close and drop every channel on this lease
// before the restart, so no stale channel id survives the resume.
await closeDaytonaDuplexChannelsForLease(params.providerLeaseId);
}
await ensureSandboxStarted(sandbox, toTimeoutSeconds(config.timeoutMs));
try {
const remoteCwd = await resolveSandboxWorkingDirectory(sandbox);
// C3: a resumed lease must clear the workspace sentinel before it is
// trusted, even when the handle came from the cache. On any non-match we
// evict the cached handle and expire the lease so a stale/foreign sandbox
// is never reused on the subsequent (sentinel-skipping) exec path.
const workspaceSentinel = await verifyWorkspaceSentinel({
sandbox,
remoteCwd,
leaseMetadata: params.leaseMetadata,
timeoutSeconds: toTimeoutSeconds(config.timeoutMs),
});
if (workspaceSentinel.result !== "matched") {
evictSandboxHandle(scope);
return { providerLeaseId: null, metadata: { expired: true, workspaceSentinel } };
}
const shellCommand = await detectSandboxShellCommand(sandbox, toTimeoutSeconds(config.timeoutMs));
sandboxHandleCache.markFresh(scope);
sandboxHandleLeaseAdmissionStates.open(scope);
return {
providerLeaseId: sandbox.id,
metadata: leaseMetadata({
config,
sandbox,
shellCommand,
remoteCwd,
resumedLease: true,
workspaceSentinel,
}),
};
} catch (error) {
evictSandboxHandle(scope);
// A timeout, rate limit, or provider 5xx does not prove this sandbox is
// lost. Preserve the exact resource and let the host retry its recorded
// lease; replacement is permitted only after an explicit not-found or
// an immutable workspace identity mismatch.
throw error;
}
}, { allowClosed: true });
// A stopped sandbox loses its session shell, so the stored session id is
// stale after a real restart. Clear the id only when the sandbox is not
// already running, and clear it before the restart. A stopped sandbox has
// no live session, so the clear drops a dead id and a later command opens
// a fresh session. A running sandbox keeps its live session, so the resume
// leaves the id in place; a concurrent command still finds it and teardown
// deletes one session. An unconditional clear would drop the id of a live
// session and leak its shell until sandbox reaping.
if (sandbox.state !== "started") {
sandboxHandleSessionStore.clear(scope);
// A stopped sandbox loses its pseudo-terminals, so a stored duplex channel
// is dead after a real restart. Close and drop every channel on this lease
// before the restart, so no stale channel id survives the resume.
await closeDaytonaDuplexChannelsForLease(params.providerLeaseId);
}
const resumedFromState = sandbox.state ?? null;
await ensureSandboxStarted(sandbox, toTimeoutSeconds(config.timeoutMs));
try {
const remoteCwd = await resolveSandboxWorkingDirectory(sandbox);
// C3: a resumed lease must clear the workspace sentinel before it is
// trusted, even when the handle came from the cache. On any non-match we
// evict the cached handle and expire the lease so a stale/foreign sandbox
// is never reused on the subsequent (sentinel-skipping) exec path.
const workspaceSentinel = await verifyWorkspaceSentinel({
sandbox,
remoteCwd,
leaseMetadata: params.leaseMetadata,
timeoutSeconds: toTimeoutSeconds(config.timeoutMs),
});
if (workspaceSentinel.result !== "matched") {
evictSandboxHandle(scope);
return {
providerLeaseId: null,
metadata: { expired: true, workspaceSentinel },
};
}
const shellCommand = await detectSandboxShellCommand(
sandbox,
toTimeoutSeconds(config.timeoutMs),
);
sandboxHandleCache.markFresh(scope);
sandboxHandleLeaseAdmissionStates.open(scope);
return {
providerLeaseId: sandbox.id,
metadata: leaseMetadata({
config,
sandbox,
shellCommand,
remoteCwd,
resumedLease: true,
resumedFromState,
sandboxState: "started",
workspaceSentinel,
}),
};
} catch (error) {
evictSandboxHandle(scope);
// A timeout, rate limit, or provider 5xx does not prove this sandbox is
// lost. Preserve the exact resource and let the host retry its recorded
// lease; replacement is permitted only after an explicit not-found or
// an immutable workspace identity mismatch.
throw error;
}
},
{ allowClosed: true },
);
},
async onEnvironmentReleaseLease(
@ -2828,18 +2844,28 @@ const plugin = definePlugin({
providerLeaseId: params.lease.providerLeaseId,
config,
};
return await withSandboxActivityGate(scope, async () => {
const sandbox = await getSandbox(scope, { bypassTeardownGate: true });
await ensureSandboxStarted(sandbox, timeoutSeconds);
const result = await performSyncOut({
sandbox,
operations: params.operations,
remoteDir,
timeoutSeconds,
try {
return await withSandboxActivityGate(scope, async () => {
const sandbox = await getSandbox(scope, { bypassTeardownGate: true });
await ensureSandboxStarted(sandbox, timeoutSeconds);
const result = await performSyncOut({
sandbox,
operations: params.operations,
remoteDir,
timeoutSeconds,
});
sandboxHandleCache.markFresh(scope);
return result;
});
sandboxHandleCache.markFresh(scope);
return result;
});
} catch (error) {
// A deleted Daytona sandbox is the one provider failure that proves its
// unexported workspace bytes no longer exist. Convert the SDK class to a
// stable cross-worker message; every other error remains retryable.
if (error instanceof DaytonaNotFoundError) {
throw new Error("daytona_sandbox_not_found");
}
throw error;
}
},
// Open one live login pseudo-terminal. Resolve the cached sandbox by the

26
server/src/__tests__/environment-runtime.test.ts
View File

@ -561,6 +561,18 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
mode: "shared_workspace",
},
});
const workspaceSyncStamp = {
schema: "paperclip.native-workspace-stamp/v1",
workspaceId: seeded.executionWorkspaceId,
providerLeaseId: "sandbox-exact-resume",
remoteCwd: "/workspace",
hostSha256: "a".repeat(64),
finalizedRunId: seeded.runId,
};
await environmentService(db).updateLeaseMetadata(first.lease.id, {
...(first.lease.metadata ?? {}),
nativeWorkspaceSync: workspaceSyncStamp,
});
await runtimeWithPlugin.releaseRunLeases(
seeded.runId,
"released",
@ -592,9 +604,17 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
expect(first.lease.metadata?.sandboxLeaseAcquisition).toEqual({ outcome: "created" });
expect(acquired.lease.providerLeaseId).toBe("sandbox-exact-resume");
expect(acquired.lease.metadata?.sandboxLeaseAcquisition).toEqual({ outcome: "resumed" });
expect(workerManager.call.mock.calls.filter((call) => call[1] === "environmentAcquireLease"))
.toHaveLength(1);
expect(acquired.lease.metadata?.sandboxLeaseAcquisition).toEqual({
outcome: "resumed",
});
expect(acquired.lease.metadata?.nativeWorkspaceSync).toEqual(
workspaceSyncStamp,
);
expect(
workerManager.call.mock.calls.filter(
(call) => call[1] === "environmentAcquireLease",
),
).toHaveLength(1);
});
it("destroys a disposable paperclip_runner sandbox after the turn", async () => {

37
server/src/__tests__/native-sandbox-lifecycle.test.ts
View File

@ -1,5 +1,9 @@
import { describe, expect, it } from "vitest";
import { resolveNativeSandboxLifecycle } from "../services/heartbeat.js";
import {
providerResourceDispositionForTerminalRun,
resolveNativeSandboxLifecycle,
resolveReusableSandboxLifecycle,
} from "../services/heartbeat.js";
const reusableSandbox = {
kind: "remote" as const,
@ -21,6 +25,19 @@ describe("paperclip_runner sandbox lifecycle", () => {
});
});
it("keeps the same warm reusable sandbox for a legacy adapter", () => {
expect(
resolveReusableSandboxLifecycle({
lifecyclePolicy: { mode: "warm", idleTimeoutMs: 300_000 },
target: reusableSandbox,
}),
).toEqual({
runnerProcess: "warm",
sandboxResource: "keep_running",
failoverBackup: "verified",
});
});
it("stops and reuses a per-turn reusable sandbox", () => {
expect(resolveNativeSandboxLifecycle({
adapterType: "paperclip_runner",
@ -71,4 +88,22 @@ describe("paperclip_runner sandbox lifecycle", () => {
target: { kind: "local" },
})).toBeNull();
});
it("keeps a warm sandbox only after a successful turn", () => {
expect(
providerResourceDispositionForTerminalRun("keep_running", "succeeded"),
).toBe("keep_running");
expect(
providerResourceDispositionForTerminalRun("keep_running", "failed"),
).toBe("stop_and_retain");
expect(
providerResourceDispositionForTerminalRun("keep_running", "cancelled"),
).toBe("stop_and_retain");
expect(
providerResourceDispositionForTerminalRun("keep_running", "timed_out"),
).toBe("stop_and_retain");
expect(providerResourceDispositionForTerminalRun("destroy", "failed")).toBe(
"destroy",
);
});
});

189
server/src/__tests__/native-workspace-sync.test.ts Normal file
View File

@ -0,0 +1,189 @@
import { mkdtemp, readdir, rm } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import {
directorySnapshotSha256,
serializeDirectorySnapshot,
} from "@paperclipai/adapter-utils/workspace-restore-merge";
import {
classifyNativeWorkspaceInbound,
nativeWorkspaceSyncInternals,
readNativeWorkspaceSyncReference,
} from "../services/native-runtime/native-workspace-sync.js";
const digest = "a".repeat(64);
describe("native workspace sync durable metadata", () => {
const originalPaperclipHome = process.env.PAPERCLIP_HOME;
const originalPaperclipInstanceId = process.env.PAPERCLIP_INSTANCE_ID;
const cleanupDirs: string[] = [];
afterEach(async () => {
if (originalPaperclipHome === undefined) delete process.env.PAPERCLIP_HOME;
else process.env.PAPERCLIP_HOME = originalPaperclipHome;
if (originalPaperclipInstanceId === undefined)
delete process.env.PAPERCLIP_INSTANCE_ID;
else process.env.PAPERCLIP_INSTANCE_ID = originalPaperclipInstanceId;
await Promise.all(
cleanupDirs.splice(0).map((directory) =>
rm(directory, { recursive: true, force: true }),
),
);
});
it("classifies fresh, warm, replacement, and same-run recovery inputs", () => {
expect(
classifyNativeWorkspaceInbound({
kind: "new_run",
acquisition: "created",
hasPriorStamp: false,
}),
).toBe("host_current");
expect(
classifyNativeWorkspaceInbound({
kind: "new_run",
acquisition: "resumed",
hasPriorStamp: true,
}),
).toBe("adopt_remote");
expect(
classifyNativeWorkspaceInbound({
kind: "new_run",
acquisition: "resumed",
hasPriorStamp: false,
}),
).toBe("host_current");
expect(
classifyNativeWorkspaceInbound({
kind: "new_run",
acquisition: "replacement",
hasPriorStamp: true,
}),
).toBe("host_current");
expect(
classifyNativeWorkspaceInbound({
kind: "existing_run",
restartRecovery: true,
sameProviderLease: true,
}),
).toBe("adopt_remote");
expect(
classifyNativeWorkspaceInbound({
kind: "existing_run",
restartRecovery: true,
sameProviderLease: false,
}),
).toBe("durable_seed");
expect(() =>
classifyNativeWorkspaceInbound({
kind: "existing_run",
restartRecovery: false,
sameProviderLease: true,
}),
).toThrow("native_workspace_sync_unexpected_existing_descriptor");
});
it("reads backward-compatible references and the resource disposition", () => {
const base = {
schema: "paperclip.native-workspace-sync/v1",
state: "prepared",
descriptorSha256: digest,
baselineSha256: digest,
finalHostSha256: null,
workspaceId: "workspace-1",
leaseId: "lease-1",
providerLeaseId: "sandbox-1",
remoteCwd: "/workspace",
};
expect(readNativeWorkspaceSyncReference(base)).toEqual({
...base,
resourceDisposition: null,
});
expect(
readNativeWorkspaceSyncReference({
...base,
resourceDisposition: "keep_running",
}),
).toEqual({ ...base, resourceDisposition: "keep_running" });
expect(
readNativeWorkspaceSyncReference({
...base,
resourceDisposition: "delete_everything",
}),
).toBeNull();
});
it("rejects traversal before constructing a durable state path", () => {
expect(() =>
nativeWorkspaceSyncInternals.descriptorPath("../run", digest),
).toThrow("native_workspace_sync_invalid_run_id");
expect(() =>
nativeWorkspaceSyncInternals.descriptorPath("run-1", "../descriptor"),
).toThrow("native_workspace_sync_descriptor_digest_invalid");
});
it("writes one immutable descriptor when the same state is replayed", async () => {
const paperclipHome = await mkdtemp(
path.join(os.tmpdir(), "paperclip-native-workspace-sync-"),
);
cleanupDirs.push(paperclipHome);
process.env.PAPERCLIP_HOME = paperclipHome;
process.env.PAPERCLIP_INSTANCE_ID = "descriptor-test";
const baseline = {
exclude: [".paperclip-runtime"],
entries: new Map([
["continuity.txt", { kind: "file" as const, mode: 0o644, hash: digest }],
]),
};
const descriptor = {
schema: "paperclip.native-workspace-sync/v1" as const,
binding: {
runId: "run-idempotent",
companyId: "company-1",
workspaceId: "workspace-1",
leaseId: "lease-1",
providerLeaseId: "sandbox-1",
localCwd: path.join(paperclipHome, "workspace"),
remoteCwd: "/workspace",
},
state: "prepared" as const,
baselineSha256: directorySnapshotSha256(baseline),
baseline: serializeDirectorySnapshot(baseline),
gitSnapshot: null,
seed: null,
createdAt: "2026-01-01T00:00:00.000Z",
finalizedAt: null,
finalHostSha256: null,
resourceDisposition: "keep_running" as const,
};
const first = await nativeWorkspaceSyncInternals.writeDescriptor(
descriptor,
);
const second = await nativeWorkspaceSyncInternals.writeDescriptor(
descriptor,
);
expect(second).toEqual(first);
const files = await readdir(
path.dirname(
nativeWorkspaceSyncInternals.descriptorPath(
descriptor.binding.runId,
first.descriptorSha256,
),
),
);
expect(files.filter((file) => file.endsWith(".json"))).toEqual([
`descriptor.${first.descriptorSha256}.json`,
]);
await expect(
nativeWorkspaceSyncInternals.readDescriptor({
runId: descriptor.binding.runId,
reference: first,
}),
).resolves.toMatchObject({ descriptor });
});
});

13
server/src/services/environment-runtime.ts
View File

@ -2060,6 +2060,11 @@ function createSandboxEnvironmentDriver(
...(reusableLease?.metadata?.nativeHarnessBackup
? { nativeHarnessBackup: reusableLease.metadata.nativeHarnessBackup }
: {}),
...(providerLease && reusableLease?.metadata?.nativeWorkspaceSync
? {
nativeWorkspaceSync: reusableLease.metadata.nativeWorkspaceSync,
}
: {}),
...(reusableScope ? { reusableSandboxLease: reusableScope } : {}),
};
try {
@ -2282,6 +2287,13 @@ function createSandboxEnvironmentDriver(
...(reusableLease?.metadata?.nativeHarnessBackup
? { nativeHarnessBackup: reusableLease.metadata.nativeHarnessBackup }
: {}),
...(reusableLease &&
providerLease.providerLeaseId === reusableLease.providerLeaseId &&
reusableLease.metadata?.nativeWorkspaceSync
? {
nativeWorkspaceSync: reusableLease.metadata.nativeWorkspaceSync,
}
: {}),
...(reusableScope ? { reusableSandboxLease: reusableScope } : {}),
};
try {
@ -3154,6 +3166,7 @@ const INTERNAL_PLUGIN_SANDBOX_CONFIG_KEYS = new Set([
"sandboxProviderPlugin",
"sandboxLeaseAcquisition",
"nativeHarnessBackup",
"nativeWorkspaceSync",
]);
// Drop the host-internal and per-lease runtime keys from a sandbox config

270
server/src/services/heartbeat.ts
View File

@ -142,6 +142,9 @@ import {
materializeLegacyQuestionResponseWakeProjection,
materializeNativeInteractionResponses,
NativeCancellationPendingRecoveryError,
prepareNativeWorkspaceSync,
readNativeWorkspaceSyncReference,
recordNativeFinalizationFailure,
type NativeRestartRecoveryClaim,
rebindNativeSessionCheckpoint,
reconcileNativeFinalizations,
@ -1746,12 +1749,53 @@ export function leaseReleaseStatusForRunStatus(
return status === "failed" || status === "timed_out" ? "failed" : "released";
}
export function providerResourceDispositionForTerminalRun(
desired: ProviderResourceDisposition | undefined,
status: string | null | undefined,
): ProviderResourceDisposition | undefined {
return desired === "keep_running" && status !== "succeeded"
? "stop_and_retain"
: desired;
}
export interface NativeSandboxLifecycle {
runnerProcess: "per_turn" | "warm";
sandboxResource: "keep_running" | "stop_and_reuse" | "destroy_after_turn";
failoverBackup: "verified";
}
export function resolveReusableSandboxLifecycle(input: {
lifecyclePolicy:
| { mode: "per_turn"; idleTimeoutMs: null }
| { mode: "warm"; idleTimeoutMs: number };
target: {
kind: "local" | "remote";
transport?: string;
reusableLeaseConfigured?: boolean;
effectiveCapabilities?: { reusableLeases: boolean } | null;
} | null;
}): NativeSandboxLifecycle | null {
if (input.target?.kind !== "remote" || input.target.transport !== "sandbox") {
return null;
}
const reusableLease =
input.target.reusableLeaseConfigured === true &&
input.target.effectiveCapabilities?.reusableLeases === true;
if (input.lifecyclePolicy.mode === "warm" && !reusableLease) {
throw new Error("runner_warm_lifecycle_requires_reusable_provider_lease");
}
return {
runnerProcess: input.lifecyclePolicy.mode,
sandboxResource:
input.lifecyclePolicy.mode === "warm"
? "keep_running"
: reusableLease
? "stop_and_reuse"
: "destroy_after_turn",
failoverBackup: "verified",
};
}
export function resolveNativeSandboxLifecycle(input: {
adapterType: string;
lifecyclePolicy:
@ -1770,22 +1814,7 @@ export function resolveNativeSandboxLifecycle(input: {
input.target.transport !== "sandbox"
)
return null;
const reusableLease =
input.target.reusableLeaseConfigured === true &&
input.target.effectiveCapabilities?.reusableLeases === true;
if (input.lifecyclePolicy.mode === "warm" && !reusableLease) {
throw new Error("runner_warm_lifecycle_requires_reusable_provider_lease");
}
return {
runnerProcess: input.lifecyclePolicy.mode,
sandboxResource:
input.lifecyclePolicy.mode === "warm"
? "keep_running"
: reusableLease
? "stop_and_reuse"
: "destroy_after_turn",
failoverBackup: "verified",
};
return resolveReusableSandboxLifecycle(input);
}
export function applyPersistedExecutionWorkspaceConfig(input: {
@ -8170,6 +8199,18 @@ class NativeSessionResumeScheduledError extends Error {
}
}
class NativeWorkspaceFinalizeScheduledError extends Error {
constructor(
readonly original: unknown,
readonly terminalFailure: boolean,
readonly reasonCode:
"workspace_sync_out_failed" | "workspace_sync_out_unrecoverable",
) {
super("Native workspace finalization recovery has been scheduled.");
this.name = "NativeWorkspaceFinalizeScheduledError";
}
}
type WorkspaceReadyCommentWriter = {
addComment: (
issueId: string,
@ -12945,7 +12986,10 @@ export function heartbeatService(
// A result committed before the old controller stopped outranks process
// recovery. Finish its durable workspace/status suffix before deciding
// whether any provider authority needs to be reopened.
await reconcileNativeFinalizations(db);
await reconcileNativeFinalizations(db, undefined, {
environmentRuntime,
onWorkspaceSettled: settleRecoveredNativeWorkspace,
});
const intent = await readHotRestartIntent().catch((error) => {
logger.warn(
{ err: error },
@ -16854,6 +16898,41 @@ export function heartbeatService(
return blocked;
}
async function settleRecoveredNativeWorkspace(input: {
runId: string;
companyId: string;
agentId: string;
succeeded: boolean;
}) {
const settledRun = await getRun(input.runId);
const workspaceSyncReference = readNativeWorkspaceSyncReference(
parseObject(settledRun?.runnerProfileJson).nativeWorkspaceSync,
);
await releaseEnvironmentLeasesForRun({
runId: input.runId,
companyId: input.companyId,
agentId: input.agentId,
status: settledRun?.status,
failureReason: settledRun?.error ?? undefined,
providerResourceDisposition: input.succeeded
? (workspaceSyncReference?.resourceDisposition ?? "stop_and_retain")
: "stop_and_retain",
});
await releaseRuntimeServicesForRun(input.runId).catch(() => undefined);
await finalizeAgentStatus(
input.agentId,
input.succeeded ? "succeeded" : "failed",
input.succeeded
? null
: (settledRun?.error ?? "native_workspace_sync_out_failed"),
{
wasFirstHeartbeat: settledRun
? timerClaimWasFirstHeartbeat(settledRun)
: undefined,
},
).catch(() => undefined);
}
async function reapOrphanedRuns(opts?: { staleThresholdMs?: number }) {
const staleThresholdMs = opts?.staleThresholdMs ?? 0;
const now = new Date();
@ -16861,7 +16940,10 @@ export function heartbeatService(
// Complete persisted native results before generic orphan recovery. The
// reconciler reads the durable workspace barrier and persisted runtime
// mode, never the current feature flag.
await reconcileNativeFinalizations(db).catch((error) => {
await reconcileNativeFinalizations(db, undefined, {
environmentRuntime,
onWorkspaceSettled: settleRecoveredNativeWorkspace,
}).catch((error) => {
logger.warn(
{ err: error },
"failed to reconcile persisted native finalizations before orphan reaping",
@ -17876,6 +17958,10 @@ export function heartbeatService(
activeRunExecutions.add(run.id);
let runScratch: HeartbeatRunScratch | null = null;
let nativeSessionResumeScheduled = false;
let nativeWorkspaceFinalizeScheduled = false;
let nativeWorkspaceSync: Awaited<
ReturnType<typeof prepareNativeWorkspaceSync>
> = null;
let providerResourceDispositionForRun:
ProviderResourceDisposition | undefined;
let nativeLifecycleTelemetryForRun:
@ -19659,6 +19745,9 @@ export function heartbeatService(
driver: selectedEnvironment.driver,
leaseId: activeEnvironmentLease.lease.id,
workspaceRealization,
sandboxLeaseAcquisition:
activeEnvironmentLease.lease.metadata?.sandboxLeaseAcquisition ??
null,
...(typeof activeEnvironmentLease.lease.metadata?.remoteCwd === "string"
? {
remoteCwd: activeEnvironmentLease.lease.metadata.remoteCwd,
@ -20363,6 +20452,9 @@ export function heartbeatService(
agentId: heartbeatRuns.agentId,
runnerInstanceId: heartbeatRuns.runnerInstanceId,
nativeSessionId: heartbeatRuns.nativeSessionId,
processPid: heartbeatRuns.processPid,
processGroupId: heartbeatRuns.processGroupId,
processStartedAt: heartbeatRuns.processStartedAt,
runnerProfileJson: heartbeatRuns.runnerProfileJson,
})
.from(heartbeatRuns)
@ -20718,6 +20810,21 @@ export function heartbeatService(
? previousNativeRun.runnerInstanceId
: (lockedRun.runnerInstanceId ?? nativeRunnerInstanceId),
nativeSessionId: lockedRun.nativeSessionId ?? nativeSessionId,
processPid:
lockedRun.processPid ??
(previousNativeRun?.nativeSessionId === nativeSessionId
? previousNativeRun.processPid
: null),
processGroupId:
lockedRun.processGroupId ??
(previousNativeRun?.nativeSessionId === nativeSessionId
? previousNativeRun.processGroupId
: null),
processStartedAt:
lockedRun.processStartedAt ??
(previousNativeRun?.nativeSessionId === nativeSessionId
? previousNativeRun.processStartedAt
: null),
nativeIssueId: lockedRun.nativeIssueId ?? issueRef.id,
driverKind:
lockedRun.driverKind ??
@ -20745,7 +20852,30 @@ export function heartbeatService(
})
.onConflictDoNothing();
});
nativeWorkspaceSync = await prepareNativeWorkspaceSync({
db,
runId: run.id,
companyId: agent.companyId,
workspaceId: nativeExecutionWorkspaceId,
workspaceLocalDir: executionWorkspace.cwd,
target: executionTarget,
lease: activeEnvironmentLease.lease,
restartRecovery: runOptions.nativeRestartRecovery,
resourceDisposition: providerResourceDispositionForRun,
});
} else {
const legacyWarmLifecycle =
executionTarget?.kind === "remote" &&
executionTarget.transport === "sandbox" &&
executionTarget.runnerLifecyclePolicy?.mode === "warm"
? resolveReusableSandboxLifecycle({
lifecyclePolicy: executionTarget.runnerLifecyclePolicy,
target: executionTarget,
})
: null;
if (legacyWarmLifecycle?.sandboxResource === "keep_running") {
providerResourceDispositionForRun = "keep_running";
}
await db
.update(heartbeatRuns)
.set({
@ -21315,19 +21445,26 @@ export function heartbeatService(
// If recording the barrier itself fails, propagate as a run failure
// rather than silently leaving dependents stranded behind a missing
// finalize row.
if (nativeWorkspaceSync) {
await nativeWorkspaceSync.restoreWorkspace();
}
await recordWorkspaceFinalize("succeeded");
if (adapterResult.nativeFinalization) {
adapterResult.nativeFinalization.workspaceFinalizeStatus =
"succeeded";
try {
await finalizeNativeRun({
const finalized = await finalizeNativeRun({
db,
runId: run.id,
workspaceFinalizeStatus: "succeeded",
preserveProviderAttempt: Boolean(nativeWorkspaceSync),
});
await dispatchPendingNativeStatusWakeups({
companyId: run.companyId,
});
if (finalized.phase === "committed") {
await nativeWorkspaceSync?.cleanup();
}
} catch (finalizeErr) {
logger.warn(
{ err: finalizeErr, runId: run.id },
@ -21379,6 +21516,40 @@ export function heartbeatService(
);
}
if (nativeRuntimeResolution.kind === "native") {
const proposedResult = await db
.select({ resultId: nativeRunFinalizations.resultId })
.from(nativeRunFinalizations)
.where(eq(nativeRunFinalizations.runId, run.id))
.limit(1)
.then((rows) => rows[0]?.resultId ?? null);
if (proposedResult && nativeWorkspaceSync) {
const workspaceFailureMessage =
adapterErr instanceof Error ? adapterErr.message : "";
const unrecoverable =
workspaceFailureMessage ===
"workspace_sync_out_unrecoverable" ||
workspaceFailureMessage.includes("daytona_sandbox_not_found");
const failure = await recordNativeFinalizationFailure({
db,
runId: run.id,
error: new Error(
unrecoverable
? "native_workspace_sync_out_unrecoverable"
: "native_workspace_sync_out_failed",
),
projectRunStatus: true,
failureScope: "workspace",
permanent: unrecoverable,
});
nativeWorkspaceFinalizeScheduled = true;
throw new NativeWorkspaceFinalizeScheduledError(
adapterErr,
failure.phase === "terminal_failure",
unrecoverable
? "workspace_sync_out_unrecoverable"
: "workspace_sync_out_failed",
);
}
try {
await finalizeNativeRun({
db,
@ -22064,6 +22235,45 @@ export function heartbeatService(
}
return;
}
if (err instanceof NativeWorkspaceFinalizeScheduledError) {
const coordinator = await db
.select({
nextAttemptAt: nativeRunFinalizations.nextAttemptAt,
attempt: nativeRunFinalizations.attempt,
})
.from(nativeRunFinalizations)
.where(eq(nativeRunFinalizations.runId, run.id))
.limit(1)
.then((rows) => rows[0] ?? null);
await appendRunEvent(run, {
eventType: "lifecycle",
stream: "system",
level: err.terminalFailure ? "error" : "warn",
message: err.terminalFailure
? "native result is durable, but the sandbox containing unexported workspace changes is unrecoverable"
: "native result is durable; workspace copy-back will retry without another provider turn",
payload: {
attempt: coordinator?.attempt ?? null,
nextAttemptAt: coordinator?.nextAttemptAt?.toISOString() ?? null,
fallbackSuppressed: true,
retryReasonCode: err.reasonCode,
},
}).catch(() => undefined);
if (err.terminalFailure) {
// The durable coordinator already failed the run, blocked the
// issue, and cleared its execution lock. Let ordinary teardown
// release the now-useless lease and return the agent to service.
nativeWorkspaceFinalizeScheduled = false;
providerResourceDispositionForRun = "stop_and_retain";
await finalizeAgentStatus(
run.agentId,
"failed",
"native_workspace_sync_out_unrecoverable",
{ wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run) },
).catch(() => undefined);
}
return;
}
const message = redactCurrentUserText(
err instanceof Error ? err.message : "Unknown adapter failure",
await getCurrentUserRedactionOptions(),
@ -22456,7 +22666,11 @@ export function heartbeatService(
// terminal". When the teardown reaches this point with the run still
// running or queued, force a terminal status before the lease is
// released, so the UI never shows a finished task as "Live".
if (latestRun && !nativeSessionResumeScheduled) {
if (
latestRun &&
!nativeSessionResumeScheduled &&
!nativeWorkspaceFinalizeScheduled
) {
latestRun = await terminalizeRunOnLeaseRelease(latestRun).catch(
(terminalizeErr) => {
logger.error(
@ -22467,7 +22681,15 @@ export function heartbeatService(
},
);
}
if (!nativeSessionResumeScheduled) {
// Warm retention is earned only by a fully successful turn. A failed,
// cancelled, or timed-out run stops the reusable sandbox so the next
// acquisition must revalidate and explicitly resume it.
providerResourceDispositionForRun =
providerResourceDispositionForTerminalRun(
providerResourceDispositionForRun,
latestRun?.status,
);
if (!nativeSessionResumeScheduled && !nativeWorkspaceFinalizeScheduled) {
await releaseEnvironmentLeasesForRun({
runId: run.id,
companyId: run.companyId,
@ -22539,7 +22761,11 @@ export function heartbeatService(
}
}
activeRunExecutions.delete(run.id);
if (!nativeSessionResumeScheduled && !shutdownInProgress) {
if (
!nativeSessionResumeScheduled &&
!nativeWorkspaceFinalizeScheduled &&
!shutdownInProgress
) {
await startNextQueuedRunForAgent(run.agentId);
}
}

1
server/src/services/native-runtime/index.ts
View File

@ -9,4 +9,5 @@ export * from "./paperclip-control-plane-port.js";
export * from "./native-run-finalizer.js";
export * from "./native-finalization-reconciler.js";
export * from "./native-restart-recovery.js";
export * from "./native-workspace-sync.js";
export * from "./status-arbiter.js";

129
server/src/services/native-runtime/native-finalization-reconciler.ts
View File

@ -24,6 +24,11 @@ import { issueRecoveryActionService } from "../issue-recovery-actions.js";
import { issueService } from "../issues.js";
import { emitAgentTaskRun } from "../agent-task-run-telemetry.js";
import { resumeNativeWorkspaceFinalization } from "./native-workspace-finalizer.js";
import {
cleanupNativeWorkspaceSync,
readNativeWorkspaceSyncReference,
} from "./native-workspace-sync.js";
import type { EnvironmentRuntimeService } from "../environment-runtime.js";
import { classifyNativeEvidence } from "./evidence-classifier.js";
import { recordNativeWorkAssessment } from "./work-assessments.js";
import {
@ -373,19 +378,33 @@ export async function claimNativeSessionResumptions(input: {
}
/** Recovery is keyed only by persisted mode/coordinator state, never the live flag. */
export async function reconcileNativeFinalizations(db: Db, runIds?: string[]) {
const rows = await db.select({
runId: heartbeatRuns.id,
companyId: heartbeatRuns.companyId,
agentId: heartbeatRuns.agentId,
issueId: nativeRunFinalizations.issueId,
issueStatus: issues.status,
issueStatusVersion: issues.statusVersion,
issueDecisionId: issues.lastStatusDecisionId,
coordinatorPhase: nativeRunFinalizations.phase,
assessmentId: nativeRunFinalizations.assessmentId,
decisionId: nativeRunFinalizations.decisionId,
})
export async function reconcileNativeFinalizations(
db: Db,
runIds?: string[],
options: {
environmentRuntime?: EnvironmentRuntimeService;
onWorkspaceSettled?: (input: {
runId: string;
companyId: string;
agentId: string;
succeeded: boolean;
}) => Promise<void>;
} = {},
) {
const rows = await db
.select({
runId: heartbeatRuns.id,
companyId: heartbeatRuns.companyId,
agentId: heartbeatRuns.agentId,
issueId: nativeRunFinalizations.issueId,
issueStatus: issues.status,
issueStatusVersion: issues.statusVersion,
issueDecisionId: issues.lastStatusDecisionId,
coordinatorPhase: nativeRunFinalizations.phase,
assessmentId: nativeRunFinalizations.assessmentId,
decisionId: nativeRunFinalizations.decisionId,
runnerProfileJson: heartbeatRuns.runnerProfileJson,
})
.from(heartbeatRuns)
.innerJoin(nativeRunFinalizations, eq(nativeRunFinalizations.runId, heartbeatRuns.id))
.innerJoin(issues, and(
@ -617,14 +636,59 @@ export async function reconcileNativeFinalizations(db: Db, runIds?: string[]) {
if (!recoveryDecision.effects.some((effect) => effect.kind === "resume_workspace_operation")) {
throw new Error("native_reconciliation_workspace_resume_policy_missing");
}
const operation = await resumeNativeWorkspaceFinalization({ db, runId: row.runId });
const workspaceFinalizeStatus = operation.status === "succeeded" ? "succeeded" : "failed";
const operation = await resumeNativeWorkspaceFinalization({
db,
runId: row.runId,
environmentRuntime: options.environmentRuntime,
});
const workspaceFinalizeStatus =
operation.status === "succeeded" ? "succeeded" : "failed";
if (workspaceFinalizeStatus === "failed") {
const unrecoverable = operation.stderrExcerpt?.includes(
"workspace_sync_out_unrecoverable",
);
const failure = await recordNativeFinalizationFailure({
db,
runId: row.runId,
error: new Error(
unrecoverable
? "native_workspace_sync_out_unrecoverable"
: "native_workspace_sync_out_failed",
),
projectRunStatus: true,
failureScope: "workspace",
permanent: unrecoverable,
});
results.push({
...failure,
reconciliationAction: "resume_workspace_operation" as const,
workspaceOperationId: operation.id,
workspaceFinalizeStatus,
reconciliationDecision: recoveryDecision,
});
if (failure.phase === "terminal_failure") {
await options
.onWorkspaceSettled?.({
runId: row.runId,
companyId: row.companyId,
agentId: row.agentId,
succeeded: false,
})
.catch(() => undefined);
}
continue;
}
try {
const finalized = await finalizeNativeRun({
db,
runId: row.runId,
workspaceFinalizeStatus,
projectRunStatus: true,
preserveProviderAttempt: Boolean(
readNativeWorkspaceSyncReference(
record(row.runnerProfileJson).nativeWorkspaceSync,
),
),
});
results.push({
...finalized,
@ -633,6 +697,20 @@ export async function reconcileNativeFinalizations(db: Db, runIds?: string[]) {
workspaceFinalizeStatus,
reconciliationDecision: recoveryDecision,
});
if (
workspaceFinalizeStatus === "succeeded" &&
finalized.phase === "committed"
) {
await cleanupNativeWorkspaceSync(row.runId).catch(() => undefined);
await options
.onWorkspaceSettled?.({
runId: row.runId,
companyId: row.companyId,
agentId: row.agentId,
succeeded: true,
})
.catch(() => undefined);
}
} catch (error) {
const failure = await recordNativeFinalizationFailure({
db,
@ -659,12 +737,29 @@ export async function reconcileNativeFinalizations(db: Db, runIds?: string[]) {
throw new Error("native_reconciliation_replay_policy_invalid");
}
try {
results.push(await finalizeNativeRun({
const finalized = await finalizeNativeRun({
db,
runId: row.runId,
workspaceFinalizeStatus: barrier.status as "succeeded" | "failed",
projectRunStatus: true,
}));
preserveProviderAttempt: Boolean(
readNativeWorkspaceSyncReference(
record(row.runnerProfileJson).nativeWorkspaceSync,
),
),
});
results.push(finalized);
if (finalized.phase === "committed") {
await cleanupNativeWorkspaceSync(row.runId).catch(() => undefined);
await options
.onWorkspaceSettled?.({
runId: row.runId,
companyId: row.companyId,
agentId: row.agentId,
succeeded: true,
})
.catch(() => undefined);
}
} catch (error) {
results.push(await recordNativeFinalizationFailure({
db,

92
server/src/services/native-runtime/native-run-finalizer-telemetry.test.ts
View File

@ -279,6 +279,93 @@ describeEmbeddedPostgres("native run finalizer / status decision committer — a
expect(run?.status).toBe("running");
});
it("bounds workspace-only retries without consuming the provider attempt", async () => {
const fixture = await seedNativeRun();
await db.insert(nativeRunFinalizations).values({
runId: fixture.runId,
companyId,
issueId: fixture.issueId,
phase: "observed",
attempt: 1,
});
for (
let expectedAttempt = 1;
expectedAttempt <= 3;
expectedAttempt += 1
) {
await recordNativeFinalizationFailure({
db,
runId: fixture.runId,
error: new Error("native_workspace_sync_out_failed"),
projectRunStatus: true,
failureScope: "workspace",
});
const coordinator = await db
.select()
.from(nativeRunFinalizations)
.where(eq(nativeRunFinalizations.runId, fixture.runId))
.then((rows) => rows[0]!);
expect(coordinator.attempt).toBe(1);
expect(coordinator.failureDetail).toMatchObject({
workspaceFinalizeAttempt: expectedAttempt,
});
expect(coordinator.phase).toBe(
expectedAttempt === 3 ? "terminal_failure" : "retryable_failure",
);
}
await expect(
db
.select({ status: heartbeatRuns.status })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, fixture.runId))
.then((rows) => rows[0]?.status),
).resolves.toBe("failed");
});
it("blocks immediately when the sandbox with unexported changes is gone", async () => {
const fixture = await seedNativeRun();
await db.insert(nativeRunFinalizations).values({
runId: fixture.runId,
companyId,
issueId: fixture.issueId,
phase: "workspace_finalizing",
attempt: 1,
resultId: null,
});
const failure = await recordNativeFinalizationFailure({
db,
runId: fixture.runId,
error: new Error("native_workspace_sync_out_unrecoverable"),
projectRunStatus: true,
failureScope: "workspace",
permanent: true,
});
expect(failure).toMatchObject({
phase: "terminal_failure",
failureCode: "native_workspace_sync_out_unrecoverable",
nextAttemptAt: null,
attempt: 1,
});
await expect(
db
.select({ status: heartbeatRuns.status })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, fixture.runId))
.then((rows) => rows[0]?.status),
).resolves.toBe("failed");
await expect(
db
.select({ status: issues.status })
.from(issues)
.where(eq(issues.id, fixture.issueId))
.then((rows) => rows[0]?.status),
).resolves.toBe("blocked");
});
it("emits exactly one event for a cancel_continuations write (trap 2: :685/:518 overlap)", async () => {
const fixture = await seedNativeRun();
// Build the minimal real rows commitNativeStatusDecision's foreign keys
@ -329,7 +416,10 @@ describeEmbeddedPostgres("native run finalizer / status decision committer — a
toStatus: "cancelled",
reasonCode: "cancellation_issue_authorized",
unblockDescriptor: null,
effects: [{ kind: "release_checkout" }, { kind: "cancel_continuations" }],
effects: [
{ kind: "release_checkout" },
{ kind: "cancel_continuations" },
],
};
const callsBefore = mockTelemetryClient.track.mock.calls.length;

221
server/src/services/native-runtime/native-run-finalizer.ts
View File

@ -126,7 +126,11 @@ async function acceptedInteractionFromRun(input: {
.then((rows) => rows[0] ?? null);
}
async function claimCoordinator(input: { db: Db; runId: string }) {
async function claimCoordinator(input: {
db: Db;
runId: string;
preserveProviderAttempt?: boolean;
}) {
const leaseOwner = `native-finalizer:${randomUUID()}`;
const now = new Date();
const claimed = await input.db.transaction(async (tx) => {
@ -152,23 +156,33 @@ async function claimCoordinator(input: { db: Db; runId: string }) {
}
if (coordinator.phase === "terminal_failure") throw new Error("native_finalization_terminal_failure");
if (
coordinator.leaseOwner
&& coordinator.leaseExpiresAt
&& coordinator.leaseExpiresAt > now
&& coordinator.leaseOwner !== leaseOwner
) throw new Error("native_finalization_lease_busy");
const [updated] = await tx.update(nativeRunFinalizations).set({
leaseOwner,
leaseExpiresAt: new Date(now.getTime() + 5 * 60_000),
attempt: coordinator.attempt + 1,
phase: coordinator.phase === "retryable_failure"
? coordinator.assessmentId ? "arbitrating" : "workspace_finalizing"
: coordinator.phase,
failureCode: null,
failureDetail: null,
nextAttemptAt: null,
updatedAt: now,
}).where(eq(nativeRunFinalizations.runId, input.runId)).returning();
coordinator.leaseOwner &&
coordinator.leaseExpiresAt &&
coordinator.leaseExpiresAt > now &&
coordinator.leaseOwner !== leaseOwner
)
throw new Error("native_finalization_lease_busy");
const [updated] = await tx
.update(nativeRunFinalizations)
.set({
leaseOwner,
leaseExpiresAt: new Date(now.getTime() + 5 * 60_000),
attempt: input.preserveProviderAttempt
? coordinator.attempt
: coordinator.attempt + 1,
phase:
coordinator.phase === "retryable_failure"
? coordinator.assessmentId
? "arbitrating"
: "workspace_finalizing"
: coordinator.phase,
failureCode: null,
failureDetail: null,
nextAttemptAt: null,
updatedAt: now,
})
.where(eq(nativeRunFinalizations.runId, input.runId))
.returning();
if (!updated) throw new Error("native_finalization_claim_failed");
return { coordinator: updated, leaseOwner };
});
@ -183,6 +197,8 @@ async function recordRetryableFailure(input: {
message: string;
nextAction: string;
projectRunStatus?: boolean;
failureScope?: "provider" | "workspace";
permanent?: boolean;
}) {
const now = new Date();
const nextAttemptAt = new Date(now.getTime() + 30_000);
@ -228,50 +244,87 @@ async function recordRetryableFailure(input: {
const supersededByNewerRun = Boolean(
latestDecisionRun && latestDecisionRun.createdAt > input.run.createdAt,
);
const exhausted = input.coordinator.attempt >= 3;
const phase = supersededByNewerRun || exhausted
? "terminal_failure" as const
: "retryable_failure" as const;
const priorFailureDetail = record(input.coordinator.failureDetail);
const workspaceFinalizeAttempt =
input.failureScope === "workspace"
? (typeof priorFailureDetail.workspaceFinalizeAttempt === "number" &&
Number.isInteger(priorFailureDetail.workspaceFinalizeAttempt) &&
priorFailureDetail.workspaceFinalizeAttempt >= 0
? priorFailureDetail.workspaceFinalizeAttempt
: 0) + 1
: null;
const exhausted =
input.permanent === true ||
(workspaceFinalizeAttempt !== null
? workspaceFinalizeAttempt >= 3
: input.coordinator.attempt >= 3);
const phase =
supersededByNewerRun || exhausted
? ("terminal_failure" as const)
: ("retryable_failure" as const);
const failureCode = supersededByNewerRun
? "native_finalization_superseded"
: exhausted
? "native_finalization_retry_exhausted"
: input.failureCode;
await tx.update(nativeRunFinalizations).set({
phase,
leaseOwner: null,
leaseExpiresAt: null,
failureCode,
failureDetail: {
message: input.message.slice(0, 2_000),
originalFailureCode: input.failureCode,
recoveryOwner: supersededByNewerRun
? { kind: "none", reason: "newer_native_decision" }
: exhausted
? { kind: "board" }
: { kind: "agent", agentId: input.run.agentId },
nextAction: input.nextAction,
},
nextAttemptAt: supersededByNewerRun || exhausted ? null : nextAttemptAt,
updatedAt: now,
}).where(eq(nativeRunFinalizations.runId, input.run.id));
const projectsTerminalStatus = exhausted && !supersededByNewerRun && input.projectRunStatus;
const [updatedRun] = await tx.update(heartbeatRuns).set({
...(projectsTerminalStatus ? {
status: exhaustedRunStatus,
finishedAt: input.run.finishedAt ?? now,
} : {}),
nativePhase: phase,
nativePhaseUpdatedAt: now,
resultJson: {
...record(input.run.resultJson),
finalizationPhase: phase,
: input.permanent
? input.failureCode
: exhausted
? input.failureScope === "workspace"
? "native_workspace_sync_out_retry_exhausted"
: "native_finalization_retry_exhausted"
: input.failureCode;
await tx
.update(nativeRunFinalizations)
.set({
phase,
leaseOwner: null,
leaseExpiresAt: null,
failureCode,
originalFailureCode: input.failureCode,
nextAttemptAt: supersededByNewerRun || exhausted ? null : nextAttemptAt.toISOString(),
},
updatedAt: now,
}).where(eq(heartbeatRuns.id, input.run.id)).returning();
failureDetail: {
message: input.message.slice(0, 2_000),
originalFailureCode: input.failureCode,
...(workspaceFinalizeAttempt === null
? {}
: { workspaceFinalizeAttempt }),
recoveryOwner: supersededByNewerRun
? { kind: "none", reason: "newer_native_decision" }
: exhausted
? { kind: "board" }
: { kind: "agent", agentId: input.run.agentId },
nextAction: input.nextAction,
},
nextAttemptAt: supersededByNewerRun || exhausted ? null : nextAttemptAt,
updatedAt: now,
})
.where(eq(nativeRunFinalizations.runId, input.run.id));
const projectsTerminalStatus =
exhausted && !supersededByNewerRun && input.projectRunStatus;
const [updatedRun] = await tx
.update(heartbeatRuns)
.set({
...(projectsTerminalStatus
? {
status:
input.failureScope === "workspace"
? "failed"
: exhaustedRunStatus,
finishedAt: input.run.finishedAt ?? now,
}
: {}),
nativePhase: phase,
nativePhaseUpdatedAt: now,
resultJson: {
...record(input.run.resultJson),
finalizationPhase: phase,
failureCode,
originalFailureCode: input.failureCode,
nextAttemptAt:
supersededByNewerRun || exhausted
? null
: nextAttemptAt.toISOString(),
},
updatedAt: now,
})
.where(eq(heartbeatRuns.id, input.run.id))
.returning();
if (projectsTerminalStatus) terminalRunToEmit = updatedRun ?? null;
if (supersededByNewerRun) {
await issueRecoveryActionService(tx as unknown as Db).resolveActiveForIssue({
@ -284,9 +337,16 @@ async function recordRetryableFailure(input: {
resolutionNote: "A newer native run already committed the authoritative issue decision; the stale finalizer was retired without changing issue state.",
}, tx);
} else if (exhausted) {
await issueService(tx as unknown as Db).update(input.coordinator.issueId, {
status: "in_review",
}, tx);
await issueService(tx as unknown as Db).update(
input.coordinator.issueId,
{
status:
input.permanent && input.failureScope === "workspace"
? "blocked"
: "in_review",
},
tx,
);
}
if (!supersededByNewerRun) {
await issueRecoveryActionService(tx as unknown as Db).upsertSourceScoped({
@ -301,10 +361,15 @@ async function recordRetryableFailure(input: {
evidence: {
runId: input.run.id,
coordinatorAttempt: input.coordinator.attempt,
...(workspaceFinalizeAttempt === null
? {}
: { workspaceFinalizeAttempt }),
originalFailureCode: input.failureCode,
},
nextAction: exhausted
? `Finalization retry budget exhausted. ${input.nextAction}`
? input.permanent
? input.nextAction
: `Finalization retry budget exhausted. ${input.nextAction}`
: input.nextAction,
wakePolicy: exhausted
? null
@ -328,6 +393,8 @@ export async function recordNativeFinalizationFailure(input: {
runId: string;
error: unknown;
projectRunStatus?: boolean;
failureScope?: "provider" | "workspace";
permanent?: boolean;
}) {
const [run, coordinator] = await Promise.all([
input.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId))
@ -344,8 +411,15 @@ export async function recordNativeFinalizationFailure(input: {
coordinator,
failureCode,
message,
nextAction: "Repair the persisted native result or contract discriminator, then resume finalization from the coordinator.",
nextAction:
input.failureScope === "workspace"
? input.permanent
? "Restore the exact sandbox containing the unexported workspace changes, or resolve the task manually from durable evidence."
: "Retry workspace export and merge from the retained sandbox; do not submit another provider turn."
: "Repair the persisted native result or contract discriminator, then resume finalization from the coordinator.",
projectRunStatus: input.projectRunStatus,
failureScope: input.failureScope,
permanent: input.permanent,
});
}
@ -395,12 +469,23 @@ export async function finalizeNativeRun(input: {
workspaceFinalizeStatus: "succeeded" | "failed";
/** Reconciliation owns terminal run projection; the live heartbeat does it afterward. */
projectRunStatus?: boolean;
/** Workspace-only replay must not consume the provider recovery budget. */
preserveProviderAttempt?: boolean;
failpoint?: NativeStatusCommitFailpoint;
}) {
const run = await input.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId))
.limit(1).then((rows) => rows[0] ?? null);
if (!run || run.runtimeMode !== "native") throw new Error("native_finalization_run_missing");
const claim = await claimCoordinator({ db: input.db, runId: input.runId });
const run = await input.db
.select()
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, input.runId))
.limit(1)
.then((rows) => rows[0] ?? null);
if (!run || run.runtimeMode !== "native")
throw new Error("native_finalization_run_missing");
const claim = await claimCoordinator({
db: input.db,
runId: input.runId,
preserveProviderAttempt: input.preserveProviderAttempt,
});
const coordinator = claim.coordinator;
if (!claim.leaseOwner && coordinator.phase === "committed") {
if (input.projectRunStatus) await projectCommittedRun({ db: input.db, run, coordinator });

36
server/src/services/native-runtime/native-session-executor.test.ts
View File

@ -2880,7 +2880,7 @@ describe("native warm session supervision", () => {
);
});
it("rehydrates a runnerd warm session from its checkpoint under a fresh run authority", async () => {
it("reattaches a live runnerd warm session under a fresh run authority", async () => {
const stateBase = await mkdtemp(
join(tmpdir(), "paperclip-runnerd-warm-authority-"),
);
@ -2889,9 +2889,7 @@ describe("native warm session supervision", () => {
process.env.PAPERCLIP_RUNNER_STATE_DIR = stateBase;
process.env.PAPERCLIP_HOME = stateBase;
const firstClose = vi.fn(async () => undefined);
const secondClose = vi.fn(async () => undefined);
const firstSession = { close: firstClose };
const secondSession = { close: secondClose };
const first = {
...execution,
binding: {
@ -2947,18 +2945,8 @@ describe("native warm session supervision", () => {
return result;
})
.mockImplementationOnce(async (options) => {
expect(options.existingSession).toBeUndefined();
expect(options.persistedSession).toEqual(
expect.objectContaining({
identity: expect.objectContaining({
runId: second.binding.runId,
sessionId: second.session.normalizedSessionId,
}),
providerSessionId: "provider-runnerd-warm",
activeTurnId: null,
}),
);
options.onSession?.(secondSession);
expect(options.existingSession).toBe(firstSession);
expect(options.persistedSession).toBeUndefined();
return result;
});
@ -3014,10 +3002,10 @@ describe("native warm session supervision", () => {
runnerInstanceId: "runner-runnerd-warm",
useRunnerd: true,
});
expect(firstClose).toHaveBeenCalledWith({
reason: "warm native session authority epoch rotated",
});
await vi.waitFor(() => expect(secondClose).toHaveBeenCalled(), {
expect(firstClose).not.toHaveBeenCalled();
await vi.waitFor(() => expect(firstClose).toHaveBeenCalledWith({
reason: "warm native session idle timeout",
}), {
timeout: 500,
});
} finally {
@ -5392,9 +5380,13 @@ describe("runnerd provider runtime wiring", () => {
const firstOptions = state.createBackend.mock.calls[0]![1];
const continuationOptions = state.createBackend.mock.calls[1]![1];
await expect(firstOptions.dynamicToolHandler!({})).rejects.toThrow(
"native_tool_authority_epoch_revoked",
);
// The retained runner backend owns one stable callback. After run.attach,
// that callback routes through the session-scope authority registry to
// the new run; stale provider calls are rejected earlier by runnerd's
// turn identity boundary.
await expect(firstOptions.dynamicToolHandler!({})).resolves.toEqual({
runId: continuation.binding.runId,
});
await expect(
continuationOptions.dynamicToolHandler!({}),
).resolves.toEqual({ runId: continuation.binding.runId });

55
server/src/services/native-runtime/native-session-executor.ts
View File

@ -2623,7 +2623,13 @@ export async function nativeProviderRecoveryEvidence(input: {
const durableEvents = await input.db
.select({ eventType: heartbeatRunEvents.eventType })
.from(heartbeatRunEvents)
.where(eq(heartbeatRunEvents.runId, input.runId));
.where(
and(
eq(heartbeatRunEvents.runId, input.runId),
inArray(heartbeatRunEvents.eventType, [...PROVIDER_DURABLE_EVENT_TYPES]),
),
)
.limit(1);
const providerEventsExist = durableEvents.some((event) =>
PROVIDER_DURABLE_EVENT_TYPES.has(event.eventType),
);
@ -4171,23 +4177,7 @@ async function executePaperclipNativeSessionWithinScope(
entry.busy = true;
if (entry.idleTimer !== null) clearTimeout(entry.idleTimer);
entry.idleTimer = null;
if (input.useRunnerd) {
// A retained driver closes over the tool authority from the run that
// created it. Preserve provider continuity through its durable
// checkpoint, but rebuild the runnerd backend so the next run gets a
// fresh, immutable authority epoch. Reusing the live driver would
// either retain stale authority or require rebinding old callbacks.
warmNativeSessions.delete(warmSessionId);
await entry.session.close({
reason: "warm native session authority epoch rotated",
});
persistedWarmSession = loadWarmNativeCheckpoint(
input.execution,
warmConfigDigest,
);
} else {
existingWarmSession = entry.session;
}
existingWarmSession = entry.session;
}
} else {
persistedWarmSession = loadWarmNativeCheckpoint(
@ -7235,6 +7225,13 @@ async function createRunnerdBackendWithinSessionClaim(
input.restartRecovery?.kind === "reattach_existing_runner"
? input.restartRecovery.process
: null;
const executeCurrentToolAuthority = (
call: Parameters<SessionToolAuthorityEpoch["execute"]>[0],
) => {
const current = sessionToolAuthorityEpochs.get(sessionScopeId);
if (!current) throw new Error("native_session_tool_authority_unavailable");
return current.execute(call);
};
const backend = createNativeSessionBackend(runnerExecution, {
runnerInstanceId: input.runnerInstanceId,
environment: effectiveRunnerEnvironment,
@ -7243,8 +7240,8 @@ async function createRunnerdBackendWithinSessionClaim(
: "local_filesystem",
onSpawn: input.onSpawn,
dynamicTools,
dynamicToolHandler: (call) => authorityEpoch.execute(call),
acpxDynamicToolHandler: (call) => authorityEpoch.execute(call),
dynamicToolHandler: executeCurrentToolAuthority,
acpxDynamicToolHandler: executeCurrentToolAuthority,
opencodeRuntimeDirectory: resolve(
resolvePaperclipInstanceRoot(),
"runtime",
@ -7419,7 +7416,7 @@ async function createRunnerdBackendWithinSessionClaim(
turnId: `turn-${input.execution.binding.runId}`,
itemId: `item-${input.execution.binding.runId}`,
},
controlPlaneRegistration: (authority) =>
controlPlaneRegistration: (authority, attachmentIdentity) =>
measureNativeRunnerSpan(
input.trace,
"runner.transport.connect",
@ -7446,7 +7443,9 @@ async function createRunnerdBackendWithinSessionClaim(
() =>
registerRunnerPrpAuthority({
companyId: input.execution.binding.companyId,
runId: input.execution.binding.runId,
runId:
attachmentIdentity?.runId ??
input.execution.binding.runId,
authority,
}),
);
@ -7469,7 +7468,9 @@ async function createRunnerdBackendWithinSessionClaim(
() =>
resolvePaperclipRunnerTransport({
target,
runId: input.execution.binding.runId,
runId:
attachmentIdentity?.runId ??
input.execution.binding.runId,
localConnectUrl: "ws://127.0.0.1/unused",
runnerPublicUrl: input.runnerPublicUrl,
runnerCaBundlePath: input.runnerCaBundlePath,
@ -7505,7 +7506,9 @@ async function createRunnerdBackendWithinSessionClaim(
() =>
resolvePaperclipRunnerTransport({
target,
runId: input.execution.binding.runId,
runId:
attachmentIdentity?.runId ??
input.execution.binding.runId,
localConnectUrl: "ws://127.0.0.1/unused",
runnerPublicUrl: input.runnerPublicUrl,
runnerCaBundlePath: input.runnerCaBundlePath,
@ -7541,7 +7544,9 @@ async function createRunnerdBackendWithinSessionClaim(
() =>
registerRunnerPrpAuthority({
companyId: input.execution.binding.companyId,
runId: input.execution.binding.runId,
runId:
attachmentIdentity?.runId ??
input.execution.binding.runId,
authority,
}),
{ parentName: "runner.transport.connect" },

113
server/src/services/native-runtime/native-workspace-finalizer.ts
View File

@ -10,6 +10,13 @@ import {
} from "@paperclipai/db";
import { workspaceOperationService } from "../workspace-operations.js";
import { inspectManagedGitWorktreeBranch } from "../workspace-runtime.js";
import { environmentService } from "../environments.js";
import type { EnvironmentRuntimeService } from "../environment-runtime.js";
import { resolveEnvironmentExecutionTarget } from "../environment-execution-target.js";
import {
readNativeWorkspaceSyncReference,
resumeNativeWorkspaceSync,
} from "./native-workspace-sync.js";
function record(value: unknown): Record<string, unknown> {
return value && typeof value === "object" && !Array.isArray(value)
@ -21,14 +28,27 @@ function readString(value: unknown) {
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
}
function workspaceSyncFailure(
code: "workspace_sync_out_failed" | "workspace_sync_out_unrecoverable",
) {
return {
status: "failed" as const,
exitCode: 1,
stderr: `${code}\n`,
metadata: { workspaceSync: { code } },
};
}
/**
* Resume the real workspace-finalization action for a result-bearing native
* run. This service observes the workspace; it never fabricates a successful
* marker. The caller decides whether a failed observation is retryable.
* run. Durable sandbox-backed runs export and merge the remote workspace;
* older runs retain the local workspace validation path. The caller owns the
* durable retry and failure policy.
*/
export async function resumeNativeWorkspaceFinalization(input: {
db: Db;
runId: string;
environmentRuntime?: EnvironmentRuntimeService;
}) {
const bound = await input.db.select({
companyId: heartbeatRuns.companyId,
@ -57,7 +77,12 @@ export async function resumeNativeWorkspaceFinalization(input: {
eq(workspaceOperations.phase, "workspace_finalize"),
)).orderBy(desc(workspaceOperations.createdAt)).limit(1).then((rows) => rows[0] ?? null);
const persistedInput = record(record(bound.runnerProfileJson).nativeExecutionInput);
const persistedInput = record(
record(bound.runnerProfileJson).nativeExecutionInput,
);
const nativeWorkspaceSync = readNativeWorkspaceSyncReference(
record(bound.runnerProfileJson).nativeWorkspaceSync,
);
const binding = record(persistedInput.binding);
const workspaceId = previous?.executionWorkspaceId
?? bound.issueWorkspaceId
@ -87,6 +112,88 @@ export async function resumeNativeWorkspaceFinalization(input: {
: "workspace_directory",
},
run: async () => {
if (nativeWorkspaceSync) {
if (!input.environmentRuntime) {
return {
status: "failed",
exitCode: 1,
stderr:
"Native workspace finalization cannot access the environment runtime.\n",
};
}
const environmentsSvc = environmentService(input.db);
const lease = await environmentsSvc.getLeaseById(
nativeWorkspaceSync.leaseId,
);
const environment = lease?.environmentId
? await environmentsSvc.getById(lease.environmentId)
: null;
if (
!lease ||
!environment ||
lease.companyId !== bound.companyId ||
environment.id !== lease.environmentId
) {
return workspaceSyncFailure("workspace_sync_out_unrecoverable");
}
if (
lease.status === "expired" ||
lease.status === "failed" ||
lease.status === "pending_cleanup" ||
!lease.providerLeaseId ||
lease.providerLeaseId !== nativeWorkspaceSync.providerLeaseId
) {
return workspaceSyncFailure("workspace_sync_out_unrecoverable");
}
const target = await resolveEnvironmentExecutionTarget({
db: input.db,
companyId: bound.companyId,
adapterType: "paperclip_runner",
environment,
leaseId: lease.id,
leaseMetadata: lease.metadata,
lease,
environmentRuntime: input.environmentRuntime,
});
if (!target) {
return {
status: "failed",
exitCode: 1,
stderr: "Native workspace finalization target is unavailable.\n",
};
}
try {
const restored = await resumeNativeWorkspaceSync({
db: input.db,
runId: input.runId,
target,
});
if (!restored) {
return workspaceSyncFailure("workspace_sync_out_unrecoverable");
}
return {
status: "succeeded",
exitCode: 0,
system:
"Native workspace finalization restored the remote workspace.\n",
metadata: {
workspaceSync: {
schema: nativeWorkspaceSync.schema,
workspaceId: nativeWorkspaceSync.workspaceId,
leaseId: nativeWorkspaceSync.leaseId,
},
},
};
} catch (error) {
const code =
error instanceof Error &&
(error.message === "workspace_sync_out_unrecoverable" ||
error.message.includes("daytona_sandbox_not_found"))
? "workspace_sync_out_unrecoverable"
: "workspace_sync_out_failed";
return workspaceSyncFailure(code);
}
}
if (!cwd) {
return {
status: "failed",

980
server/src/services/native-runtime/native-workspace-sync.ts Normal file
View File

@ -0,0 +1,980 @@
import fs from "node:fs/promises";
import { createReadStream } from "node:fs";
import path from "node:path";
import { createHash, randomUUID } from "node:crypto";
import { eq } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import { environmentLeases, heartbeatRuns } from "@paperclipai/db";
import type { EnvironmentLease } from "@paperclipai/shared";
import {
prepareAdapterExecutionTargetRuntime,
type AdapterExecutionTarget,
type PreparedAdapterExecutionTargetRuntime,
} from "@paperclipai/adapter-utils/execution-target";
import type { GitWorkspaceSnapshot } from "@paperclipai/adapter-utils/git-workspace-sync";
import {
directorySnapshotSha256,
parseDirectorySnapshot,
serializeDirectorySnapshot,
type DirectorySnapshot,
type SerializedDirectorySnapshot,
} from "@paperclipai/adapter-utils/workspace-restore-merge";
import type {
WorkspaceDurableSeedPaths,
WorkspaceInboundMode,
} from "@paperclipai/adapter-utils/sandbox-managed-runtime";
import { resolvePaperclipInstanceRoot } from "../../home-paths.js";
import { parseObject } from "../../adapters/utils.js";
import type { NativeRestartRecoveryClaim } from "./native-restart-recovery.js";
const DESCRIPTOR_SCHEMA = "paperclip.native-workspace-sync/v1";
const STAMP_SCHEMA = "paperclip.native-workspace-stamp/v1";
const STATE_ROOT_NAME = "native-workspace-sync";
const DESCRIPTOR_NAME = "descriptor";
const WORKSPACE_SEED_NAME = "workspace-seed.tar";
const GIT_SEED_NAME = "git-seed.tar";
const REMOTE_STAMP_NAME = "workspace-sync-v1.json";
const MAX_DESCRIPTOR_BYTES = 64 * 1024 * 1024;
const SAFE_SEGMENT_RE = /^[A-Za-z0-9_-]+$/;
type NativeWorkspaceSyncState = "prepared" | "finalized";
export type NativeWorkspaceResourceDisposition =
"keep_running" | "stop_and_retain" | "destroy";
interface NativeWorkspaceSyncDescriptor {
schema: typeof DESCRIPTOR_SCHEMA;
binding: {
runId: string;
companyId: string;
workspaceId: string;
leaseId: string;
providerLeaseId: string;
localCwd: string;
remoteCwd: string;
};
state: NativeWorkspaceSyncState;
baselineSha256: string;
baseline: SerializedDirectorySnapshot;
gitSnapshot: GitWorkspaceSnapshot | null;
seed: {
workspaceArchiveSha256: string;
gitArchiveSha256: string | null;
} | null;
createdAt: string;
finalizedAt: string | null;
finalHostSha256: string | null;
resourceDisposition: NativeWorkspaceResourceDisposition | null;
}
export interface NativeWorkspaceSyncReference {
schema: typeof DESCRIPTOR_SCHEMA;
state: NativeWorkspaceSyncState;
descriptorSha256: string;
baselineSha256: string;
finalHostSha256: string | null;
workspaceId: string;
leaseId: string;
providerLeaseId: string;
remoteCwd: string;
resourceDisposition: NativeWorkspaceResourceDisposition | null;
}
export interface PreparedNativeWorkspaceSync {
mode: WorkspaceInboundMode;
reference: NativeWorkspaceSyncReference;
restoreWorkspace(): Promise<void>;
cleanup(): Promise<void>;
}
export type NativeWorkspaceInboundEvidence =
| {
kind: "existing_run";
restartRecovery: boolean;
sameProviderLease: boolean;
}
| {
kind: "new_run";
acquisition: "created" | "resumed" | "replacement" | null;
hasPriorStamp: boolean;
};
export function classifyNativeWorkspaceInbound(
evidence: NativeWorkspaceInboundEvidence,
): WorkspaceInboundMode {
if (evidence.kind === "existing_run") {
if (!evidence.restartRecovery) {
throw new Error("native_workspace_sync_unexpected_existing_descriptor");
}
return evidence.sameProviderLease ? "adopt_remote" : "durable_seed";
}
return evidence.acquisition === "resumed" && evidence.hasPriorStamp
? "adopt_remote"
: "host_current";
}
function canonicalJson(value: unknown): string {
if (Array.isArray(value)) {
return `[${value.map((entry) => canonicalJson(entry)).join(",")}]`;
}
if (value && typeof value === "object") {
const record = value as Record<string, unknown>;
return `{${Object.keys(record)
.sort()
.map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`)
.join(",")}}`;
}
return JSON.stringify(value);
}
function sha256(value: unknown): string {
return createHash("sha256").update(canonicalJson(value)).digest("hex");
}
function requireSafeSegment(value: string, label: string): string {
if (!SAFE_SEGMENT_RE.test(value)) {
throw new Error(`native_workspace_sync_invalid_${label}`);
}
return value;
}
function descriptorDirectory(runId: string): string {
return path.join(
resolvePaperclipInstanceRoot(),
STATE_ROOT_NAME,
requireSafeSegment(runId, "run_id"),
);
}
function descriptorPath(runId: string, descriptorSha256: string): string {
if (!/^[0-9a-f]{64}$/.test(descriptorSha256)) {
throw new Error("native_workspace_sync_descriptor_digest_invalid");
}
return path.join(
descriptorDirectory(runId),
`${DESCRIPTOR_NAME}.${descriptorSha256}.json`,
);
}
function legacyDescriptorPath(runId: string): string {
return path.join(descriptorDirectory(runId), `${DESCRIPTOR_NAME}.json`);
}
function durableSeedPaths(runId: string): {
workspaceArchivePath: string;
gitArchivePath: string;
} {
const directory = descriptorDirectory(runId);
return {
workspaceArchivePath: path.join(directory, WORKSPACE_SEED_NAME),
gitArchivePath: path.join(directory, GIT_SEED_NAME),
};
}
async function sha256File(filePath: string): Promise<string> {
return await new Promise((resolveDigest, rejectDigest) => {
const digest = createHash("sha256");
const stream = createReadStream(filePath);
stream.on("data", (chunk) => digest.update(chunk));
stream.on("error", rejectDigest);
stream.on("end", () => resolveDigest(digest.digest("hex")));
});
}
async function verifiedDurableSeed(input: {
runId: string;
seed: NonNullable<NativeWorkspaceSyncDescriptor["seed"]>;
gitSnapshot: GitWorkspaceSnapshot | null;
}): Promise<WorkspaceDurableSeedPaths> {
try {
const paths = durableSeedPaths(input.runId);
const workspaceStat = await fs.lstat(paths.workspaceArchivePath);
if (workspaceStat.isSymbolicLink() || !workspaceStat.isFile()) {
throw new Error("workspace_sync_out_unrecoverable");
}
if (
(await sha256File(paths.workspaceArchivePath)) !==
input.seed.workspaceArchiveSha256
) {
throw new Error("workspace_sync_out_unrecoverable");
}
if (input.gitSnapshot) {
const gitStat = await fs.lstat(paths.gitArchivePath);
if (
gitStat.isSymbolicLink() ||
!gitStat.isFile() ||
!input.seed.gitArchiveSha256 ||
(await sha256File(paths.gitArchivePath)) !== input.seed.gitArchiveSha256
) {
throw new Error("workspace_sync_out_unrecoverable");
}
} else if (input.seed.gitArchiveSha256 !== null) {
throw new Error("workspace_sync_out_unrecoverable");
}
return {
workspaceArchivePath: paths.workspaceArchivePath,
workspaceArchiveSha256: input.seed.workspaceArchiveSha256,
gitArchivePath: input.gitSnapshot ? paths.gitArchivePath : null,
gitArchiveSha256: input.seed.gitArchiveSha256,
};
} catch (error) {
if (
error instanceof Error &&
error.message === "workspace_sync_out_unrecoverable"
) {
throw error;
}
throw new Error("workspace_sync_out_unrecoverable", { cause: error });
}
}
async function ensurePrivateDirectory(dir: string): Promise<void> {
await fs.mkdir(dir, { recursive: true, mode: 0o700 });
const stat = await fs.lstat(dir);
if (stat.isSymbolicLink() || !stat.isDirectory()) {
throw new Error("native_workspace_sync_state_root_invalid");
}
}
async function writeDescriptor(
descriptor: NativeWorkspaceSyncDescriptor,
): Promise<NativeWorkspaceSyncReference> {
const dir = descriptorDirectory(descriptor.binding.runId);
await ensurePrivateDirectory(dir);
const body = `${canonicalJson(descriptor)}\n`;
const descriptorSha256 = createHash("sha256").update(body).digest("hex");
const finalPath = descriptorPath(descriptor.binding.runId, descriptorSha256);
const existingBody = await fs.readFile(finalPath, "utf8").catch(() => null);
if (existingBody !== null && existingBody !== body) {
throw new Error("native_workspace_sync_descriptor_digest_collision");
}
const tempPath = path.join(dir, `${DESCRIPTOR_NAME}.${randomUUID()}.tmp`);
try {
if (existingBody === null) {
await fs.writeFile(tempPath, body, { encoding: "utf8", mode: 0o600 });
await fs.rename(tempPath, finalPath);
}
} finally {
await fs.rm(tempPath, { force: true }).catch(() => undefined);
}
return {
schema: DESCRIPTOR_SCHEMA,
state: descriptor.state,
descriptorSha256,
baselineSha256: descriptor.baselineSha256,
finalHostSha256: descriptor.finalHostSha256,
workspaceId: descriptor.binding.workspaceId,
leaseId: descriptor.binding.leaseId,
providerLeaseId: descriptor.binding.providerLeaseId,
remoteCwd: descriptor.binding.remoteCwd,
resourceDisposition: descriptor.resourceDisposition,
};
}
export function readNativeWorkspaceSyncReference(
value: unknown,
): NativeWorkspaceSyncReference | null {
const candidate = parseObject(value);
if (
candidate.schema !== DESCRIPTOR_SCHEMA ||
(candidate.state !== "prepared" && candidate.state !== "finalized") ||
typeof candidate.descriptorSha256 !== "string" ||
!/^[0-9a-f]{64}$/.test(candidate.descriptorSha256) ||
typeof candidate.baselineSha256 !== "string" ||
!/^[0-9a-f]{64}$/.test(candidate.baselineSha256) ||
(candidate.finalHostSha256 !== null &&
candidate.finalHostSha256 !== undefined &&
(typeof candidate.finalHostSha256 !== "string" ||
!/^[0-9a-f]{64}$/.test(candidate.finalHostSha256))) ||
typeof candidate.workspaceId !== "string" ||
typeof candidate.leaseId !== "string" ||
typeof candidate.providerLeaseId !== "string" ||
typeof candidate.remoteCwd !== "string" ||
(candidate.resourceDisposition !== null &&
candidate.resourceDisposition !== undefined &&
candidate.resourceDisposition !== "keep_running" &&
candidate.resourceDisposition !== "stop_and_retain" &&
candidate.resourceDisposition !== "destroy")
) {
return null;
}
return {
schema: DESCRIPTOR_SCHEMA,
state: candidate.state,
descriptorSha256: candidate.descriptorSha256,
baselineSha256: candidate.baselineSha256,
finalHostSha256:
typeof candidate.finalHostSha256 === "string"
? candidate.finalHostSha256
: null,
workspaceId: candidate.workspaceId,
leaseId: candidate.leaseId,
providerLeaseId: candidate.providerLeaseId,
remoteCwd: candidate.remoteCwd,
resourceDisposition:
candidate.resourceDisposition === "keep_running" ||
candidate.resourceDisposition === "stop_and_retain" ||
candidate.resourceDisposition === "destroy"
? candidate.resourceDisposition
: null,
};
}
function parseGitSnapshot(
value: unknown,
): GitWorkspaceSnapshot | null | undefined {
if (value === null) return null;
const candidate = parseObject(value);
const paths = [
candidate.overlayPaths,
candidate.deletedPaths,
candidate.ignoredPaths,
];
if (
typeof candidate.headCommit !== "string" ||
(candidate.branchName !== null &&
typeof candidate.branchName !== "string") ||
!paths.every(
(entries) =>
Array.isArray(entries) &&
entries.every(
(entry) =>
typeof entry === "string" &&
!path.posix.isAbsolute(entry) &&
!path.win32.isAbsolute(entry) &&
!entry.split(/[\\/]/).some((segment) => segment === ".."),
),
)
) {
return undefined;
}
return {
headCommit: candidate.headCommit,
branchName: candidate.branchName as string | null,
overlayPaths: [...(candidate.overlayPaths as string[])],
deletedPaths: [...(candidate.deletedPaths as string[])],
ignoredPaths: [...(candidate.ignoredPaths as string[])],
};
}
async function readDescriptor(input: {
runId: string;
reference: NativeWorkspaceSyncReference;
}): Promise<{
descriptor: NativeWorkspaceSyncDescriptor;
baseline: DirectorySnapshot;
}> {
const contentAddressedPath = descriptorPath(
input.runId,
input.reference.descriptorSha256,
);
// Early v1 writers used descriptor.json. Prefer the immutable,
// content-addressed name, but keep the digest-verified legacy path readable
// so an interrupted upgrade can still finish its already-proposed result.
const filePath = await fs
.lstat(contentAddressedPath)
.then(() => contentAddressedPath)
.catch(async (error: NodeJS.ErrnoException) => {
if (error.code !== "ENOENT") throw error;
const legacyPath = legacyDescriptorPath(input.runId);
await fs.lstat(legacyPath);
return legacyPath;
});
const stat = await fs.lstat(filePath);
if (
stat.isSymbolicLink() ||
!stat.isFile() ||
stat.size > MAX_DESCRIPTOR_BYTES
) {
throw new Error("native_workspace_sync_descriptor_invalid");
}
const body = await fs.readFile(filePath, "utf8");
if (
createHash("sha256").update(body).digest("hex") !==
input.reference.descriptorSha256
) {
throw new Error("native_workspace_sync_descriptor_digest_mismatch");
}
const parsed = JSON.parse(body) as unknown;
const candidate = parseObject(parsed);
const binding = parseObject(candidate.binding);
const baseline = parseDirectorySnapshot(candidate.baseline);
const gitSnapshot = parseGitSnapshot(candidate.gitSnapshot);
const rawSeed =
candidate.seed === null || candidate.seed === undefined
? null
: parseObject(candidate.seed);
const seed =
rawSeed === null
? null
: typeof rawSeed.workspaceArchiveSha256 === "string" &&
/^[0-9a-f]{64}$/.test(rawSeed.workspaceArchiveSha256) &&
(rawSeed.gitArchiveSha256 === null ||
(typeof rawSeed.gitArchiveSha256 === "string" &&
/^[0-9a-f]{64}$/.test(rawSeed.gitArchiveSha256)))
? {
workspaceArchiveSha256: rawSeed.workspaceArchiveSha256,
gitArchiveSha256: rawSeed.gitArchiveSha256 as string | null,
}
: undefined;
const finalHostSha256 =
typeof candidate.finalHostSha256 === "string" &&
/^[0-9a-f]{64}$/.test(candidate.finalHostSha256)
? candidate.finalHostSha256
: candidate.finalHostSha256 === null
? null
: undefined;
const resourceDisposition =
candidate.resourceDisposition === "keep_running" ||
candidate.resourceDisposition === "stop_and_retain" ||
candidate.resourceDisposition === "destroy"
? candidate.resourceDisposition
: candidate.resourceDisposition === null
? null
: undefined;
if (
candidate.schema !== DESCRIPTOR_SCHEMA ||
(candidate.state !== "prepared" && candidate.state !== "finalized") ||
!baseline ||
gitSnapshot === undefined ||
seed === undefined ||
binding.runId !== input.runId ||
binding.workspaceId !== input.reference.workspaceId ||
binding.leaseId !== input.reference.leaseId ||
binding.providerLeaseId !== input.reference.providerLeaseId ||
binding.remoteCwd !== input.reference.remoteCwd ||
typeof binding.companyId !== "string" ||
typeof binding.localCwd !== "string" ||
!path.isAbsolute(binding.localCwd) ||
typeof binding.remoteCwd !== "string" ||
!path.posix.isAbsolute(binding.remoteCwd) ||
typeof candidate.baselineSha256 !== "string" ||
candidate.baselineSha256 !== input.reference.baselineSha256 ||
directorySnapshotSha256(baseline) !== candidate.baselineSha256 ||
typeof candidate.createdAt !== "string" ||
!Number.isFinite(Date.parse(candidate.createdAt)) ||
candidate.state !== input.reference.state ||
finalHostSha256 === undefined ||
finalHostSha256 !== input.reference.finalHostSha256 ||
resourceDisposition === undefined ||
resourceDisposition !== input.reference.resourceDisposition ||
(candidate.state === "prepared" &&
(candidate.finalizedAt !== null || finalHostSha256 !== null)) ||
(candidate.state === "finalized" &&
(typeof candidate.finalizedAt !== "string" ||
!Number.isFinite(Date.parse(candidate.finalizedAt)) ||
finalHostSha256 === null))
) {
throw new Error("native_workspace_sync_descriptor_binding_mismatch");
}
return {
descriptor: {
schema: DESCRIPTOR_SCHEMA,
binding: {
runId: binding.runId as string,
companyId: binding.companyId,
workspaceId: binding.workspaceId as string,
leaseId: binding.leaseId as string,
providerLeaseId: binding.providerLeaseId as string,
localCwd: binding.localCwd,
remoteCwd: binding.remoteCwd as string,
},
state: candidate.state,
baselineSha256: candidate.baselineSha256,
baseline: candidate.baseline as SerializedDirectorySnapshot,
gitSnapshot,
seed,
createdAt: candidate.createdAt,
finalizedAt:
typeof candidate.finalizedAt === "string"
? candidate.finalizedAt
: null,
finalHostSha256,
resourceDisposition,
},
baseline,
};
}
async function persistRunReference(
db: Db,
runId: string,
reference: NativeWorkspaceSyncReference,
): Promise<void> {
await db.transaction(async (tx) => {
const run = await tx
.select({ runnerProfileJson: heartbeatRuns.runnerProfileJson })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, runId))
.for("update")
.limit(1)
.then((rows) => rows[0] ?? null);
if (!run) throw new Error("native_workspace_sync_run_missing");
await tx
.update(heartbeatRuns)
.set({
runnerProfileJson: {
...parseObject(run.runnerProfileJson),
nativeWorkspaceSync: reference,
},
updatedAt: new Date(),
})
.where(eq(heartbeatRuns.id, runId));
});
}
async function persistLeaseStamp(input: {
db: Db;
leaseId: string;
stamp: Record<string, unknown>;
}): Promise<void> {
await input.db.transaction(async (tx) => {
const lease = await tx
.select({ metadata: environmentLeases.metadata })
.from(environmentLeases)
.where(eq(environmentLeases.id, input.leaseId))
.for("update")
.limit(1)
.then((rows) => rows[0] ?? null);
if (!lease) throw new Error("native_workspace_sync_lease_missing");
await tx
.update(environmentLeases)
.set({
metadata: {
...parseObject(lease.metadata),
nativeWorkspaceSync: input.stamp,
},
updatedAt: new Date(),
})
.where(eq(environmentLeases.id, input.leaseId));
});
}
function providerLeaseIdFor(input: {
target: Extract<AdapterExecutionTarget, { transport: "sandbox" }>;
lease: EnvironmentLease;
}): string {
const providerLeaseId =
input.target.sandboxLeaseAcquisition?.providerLeaseId ??
input.lease.providerLeaseId;
if (!providerLeaseId) {
throw new Error("native_workspace_sync_provider_lease_missing");
}
return providerLeaseId;
}
function shellQuote(value: string): string {
return `'${value.replaceAll("'", "'\\''")}'`;
}
async function writeRemoteStamp(input: {
target: Extract<AdapterExecutionTarget, { transport: "sandbox" }>;
stamp: Record<string, unknown>;
}): Promise<void> {
if (!input.target.runner) {
throw new Error("native_workspace_sync_runner_missing");
}
const runtimeDir = path.posix.join(
input.target.remoteCwd,
".paperclip-runtime",
"paperclip-runner",
);
const stampPath = path.posix.join(runtimeDir, REMOTE_STAMP_NAME);
const tempPath = `${stampPath}.tmp`;
const encoded = Buffer.from(
`${canonicalJson(input.stamp)}\n`,
"utf8",
).toString("base64");
const result = await input.target.runner.execute({
command: input.target.shellCommand ?? "sh",
args: [
"-c",
`umask 077; mkdir -p ${shellQuote(runtimeDir)} && printf %s ${shellQuote(encoded)} | base64 -d > ${shellQuote(tempPath)} && mv ${shellQuote(tempPath)} ${shellQuote(stampPath)}`,
],
cwd: "/",
timeoutMs: 15_000,
bypassSession: true,
});
if (result.timedOut || result.exitCode !== 0) {
throw new Error("native_workspace_sync_remote_stamp_failed");
}
}
async function remoteStampMatches(input: {
target: Extract<AdapterExecutionTarget, { transport: "sandbox" }>;
expected: Record<string, unknown>;
}): Promise<boolean> {
if (!input.target.runner) return false;
const stampPath = path.posix.join(
input.target.remoteCwd,
".paperclip-runtime",
"paperclip-runner",
REMOTE_STAMP_NAME,
);
const result = await input.target.runner.execute({
command: input.target.shellCommand ?? "sh",
args: [
"-c",
`test -f ${shellQuote(stampPath)} && cat ${shellQuote(stampPath)}`,
],
cwd: "/",
timeoutMs: 15_000,
bypassSession: true,
});
if (result.timedOut || result.exitCode !== 0) return false;
try {
return (
canonicalJson(JSON.parse(result.stdout)) === canonicalJson(input.expected)
);
} catch {
return false;
}
}
function leaseStamp(input: {
lease: EnvironmentLease;
workspaceId: string;
providerLeaseId: string;
remoteCwd: string;
}): Record<string, unknown> | null {
const stamp = parseObject(
parseObject(input.lease.metadata).nativeWorkspaceSync,
);
if (
stamp.schema !== STAMP_SCHEMA ||
stamp.workspaceId !== input.workspaceId ||
stamp.providerLeaseId !== input.providerLeaseId ||
stamp.remoteCwd !== input.remoteCwd ||
typeof stamp.hostSha256 !== "string" ||
!/^[0-9a-f]{64}$/.test(stamp.hostSha256)
) {
return null;
}
return stamp;
}
async function prepareRuntime(input: {
runId: string;
target: Extract<AdapterExecutionTarget, { transport: "sandbox" }>;
workspaceLocalDir: string;
mode: WorkspaceInboundMode;
baseline?: DirectorySnapshot;
gitSnapshot?: GitWorkspaceSnapshot | null;
durableSeed?: WorkspaceDurableSeedPaths;
}): Promise<PreparedAdapterExecutionTargetRuntime> {
return prepareAdapterExecutionTargetRuntime({
runId: input.runId,
target: input.target,
adapterKey: "paperclip-runner",
workspaceLocalDir: input.workspaceLocalDir,
workspaceRemoteDir: input.target.remoteCwd,
workspaceInboundMode: input.mode,
workspaceDurableSeed: input.durableSeed,
workspaceBaseline: input.baseline,
workspaceGitSnapshot: input.gitSnapshot,
});
}
async function finalizePreparedRuntime(input: {
db: Db;
runId: string;
target: Extract<AdapterExecutionTarget, { transport: "sandbox" }>;
runtime: PreparedAdapterExecutionTargetRuntime;
descriptor: NativeWorkspaceSyncDescriptor;
}): Promise<NativeWorkspaceSyncReference> {
await input.runtime.restoreWorkspace();
const finalSnapshot =
await import("@paperclipai/adapter-utils/workspace-restore-merge").then(
({ captureDirectorySnapshot }) =>
captureDirectorySnapshot(input.descriptor.binding.localCwd, {
exclude: input.runtime.workspaceSyncSnapshot?.baseline.exclude ?? [],
}),
);
const finalHostSha256 = directorySnapshotSha256(finalSnapshot);
const stamp = {
schema: STAMP_SCHEMA,
workspaceId: input.descriptor.binding.workspaceId,
providerLeaseId: input.descriptor.binding.providerLeaseId,
remoteCwd: input.descriptor.binding.remoteCwd,
hostSha256: finalHostSha256,
finalizedRunId: input.runId,
};
await writeRemoteStamp({ target: input.target, stamp });
const finalizedDescriptor: NativeWorkspaceSyncDescriptor = {
...input.descriptor,
state: "finalized",
finalizedAt: new Date().toISOString(),
finalHostSha256,
};
const reference = await writeDescriptor(finalizedDescriptor);
await persistRunReference(input.db, input.runId, reference);
await persistLeaseStamp({
db: input.db,
leaseId: input.descriptor.binding.leaseId,
stamp,
});
return reference;
}
export async function prepareNativeWorkspaceSync(input: {
db: Db;
runId: string;
companyId: string;
workspaceId: string;
workspaceLocalDir: string;
target: AdapterExecutionTarget | null;
lease: EnvironmentLease;
restartRecovery?: NativeRestartRecoveryClaim;
resourceDisposition?: NativeWorkspaceResourceDisposition;
}): Promise<PreparedNativeWorkspaceSync | null> {
if (input.target?.kind !== "remote" || input.target.transport !== "sandbox") {
return null;
}
const target = input.target;
const providerLeaseId = providerLeaseIdFor({ target, lease: input.lease });
const run = await input.db
.select({ runnerProfileJson: heartbeatRuns.runnerProfileJson })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, input.runId))
.limit(1)
.then((rows) => rows[0] ?? null);
if (!run) throw new Error("native_workspace_sync_run_missing");
const existingReference = readNativeWorkspaceSyncReference(
parseObject(run.runnerProfileJson).nativeWorkspaceSync,
);
let runtime: PreparedAdapterExecutionTargetRuntime;
let descriptor: NativeWorkspaceSyncDescriptor;
let mode: WorkspaceInboundMode = "host_current";
const seedPaths = durableSeedPaths(input.runId);
await ensurePrivateDirectory(descriptorDirectory(input.runId));
if (existingReference) {
const existing = await readDescriptor({
runId: input.runId,
reference: existingReference,
});
if (
existing.descriptor.binding.companyId !== input.companyId ||
existing.descriptor.binding.workspaceId !== input.workspaceId ||
path.resolve(existing.descriptor.binding.localCwd) !==
path.resolve(input.workspaceLocalDir) ||
existing.descriptor.binding.remoteCwd !== target.remoteCwd
) {
throw new Error("native_workspace_sync_descriptor_binding_mismatch");
}
const sameProviderLease =
existing.descriptor.binding.providerLeaseId === providerLeaseId;
mode = classifyNativeWorkspaceInbound({
kind: "existing_run",
restartRecovery: Boolean(input.restartRecovery),
sameProviderLease,
});
const durableSeed =
mode === "durable_seed"
? existing.descriptor.seed
? await verifiedDurableSeed({
runId: input.runId,
seed: existing.descriptor.seed,
gitSnapshot: existing.descriptor.gitSnapshot,
})
: (() => {
throw new Error("workspace_sync_out_unrecoverable");
})()
: undefined;
runtime = await prepareRuntime({
runId: input.runId,
target,
workspaceLocalDir: input.workspaceLocalDir,
mode,
baseline: existing.baseline,
gitSnapshot: existing.descriptor.gitSnapshot,
durableSeed,
});
descriptor = {
...existing.descriptor,
binding: {
...existing.descriptor.binding,
leaseId: target.leaseId ?? input.lease.id,
providerLeaseId,
},
};
} else {
const acquisition = target.sandboxLeaseAcquisition?.outcome ?? null;
const priorStamp = leaseStamp({
lease: input.lease,
workspaceId: input.workspaceId,
providerLeaseId,
remoteCwd: target.remoteCwd,
});
mode = classifyNativeWorkspaceInbound({
kind: "new_run",
acquisition,
hasPriorStamp: priorStamp !== null,
});
runtime = await prepareRuntime({
runId: input.runId,
target,
workspaceLocalDir: input.workspaceLocalDir,
mode,
durableSeed: {
workspaceArchivePath: seedPaths.workspaceArchivePath,
gitArchivePath: seedPaths.gitArchivePath,
},
});
const currentSnapshot = runtime.workspaceSyncSnapshot;
if (!currentSnapshot) {
throw new Error("native_workspace_sync_snapshot_missing");
}
if (acquisition === "resumed" && priorStamp) {
const currentHostSha256 = directorySnapshotSha256(
currentSnapshot.baseline,
);
const verifiedWarmAdoption =
priorStamp.hostSha256 === currentHostSha256 &&
(await remoteStampMatches({ target, expected: priorStamp }));
if (verifiedWarmAdoption) {
mode = "adopt_remote";
} else {
mode = "host_current";
runtime = await prepareRuntime({
runId: input.runId,
target,
workspaceLocalDir: input.workspaceLocalDir,
mode,
baseline: currentSnapshot.baseline,
gitSnapshot: currentSnapshot.gitSnapshot,
durableSeed: {
workspaceArchivePath: seedPaths.workspaceArchivePath,
gitArchivePath: seedPaths.gitArchivePath,
},
});
}
}
const snapshot = runtime.workspaceSyncSnapshot;
if (!snapshot) throw new Error("native_workspace_sync_snapshot_missing");
const now = new Date().toISOString();
const workspaceArchiveSha256 = await sha256File(
seedPaths.workspaceArchivePath,
);
const gitArchiveSha256 = snapshot.gitSnapshot
? await sha256File(seedPaths.gitArchivePath)
: null;
descriptor = {
schema: DESCRIPTOR_SCHEMA,
binding: {
runId: input.runId,
companyId: input.companyId,
workspaceId: input.workspaceId,
leaseId: target.leaseId ?? input.lease.id,
providerLeaseId,
localCwd: path.resolve(input.workspaceLocalDir),
remoteCwd: target.remoteCwd,
},
state: "prepared",
baselineSha256: directorySnapshotSha256(snapshot.baseline),
baseline: serializeDirectorySnapshot(snapshot.baseline),
gitSnapshot: snapshot.gitSnapshot,
seed: { workspaceArchiveSha256, gitArchiveSha256 },
createdAt: now,
finalizedAt: null,
finalHostSha256: null,
resourceDisposition: input.resourceDisposition ?? null,
};
}
let reference = await writeDescriptor(descriptor);
await persistRunReference(input.db, input.runId, reference);
let restorePromise: Promise<void> | null = null;
return {
mode,
get reference() {
return reference;
},
restoreWorkspace: async () => {
if (!restorePromise) {
restorePromise = finalizePreparedRuntime({
db: input.db,
runId: input.runId,
target,
runtime,
descriptor,
})
.then((finalizedReference) => {
reference = finalizedReference;
})
.catch((error) => {
restorePromise = null;
throw error;
});
}
await restorePromise;
},
cleanup: () => cleanupNativeWorkspaceSync(input.runId),
};
}
export async function resumeNativeWorkspaceSync(input: {
db: Db;
runId: string;
target: AdapterExecutionTarget;
}): Promise<boolean> {
if (input.target.kind !== "remote" || input.target.transport !== "sandbox") {
throw new Error("workspace_sync_out_unrecoverable");
}
const run = await input.db
.select({ runnerProfileJson: heartbeatRuns.runnerProfileJson })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, input.runId))
.limit(1)
.then((rows) => rows[0] ?? null);
const reference = readNativeWorkspaceSyncReference(
parseObject(run?.runnerProfileJson).nativeWorkspaceSync,
);
if (!reference) return false;
const existing = await readDescriptor({ runId: input.runId, reference });
if (
existing.descriptor.state === "finalized" &&
existing.descriptor.finalHostSha256
) {
return true;
}
const providerLeaseId =
input.target.sandboxLeaseAcquisition?.providerLeaseId ??
reference.providerLeaseId;
if (
providerLeaseId !== reference.providerLeaseId ||
input.target.remoteCwd !== reference.remoteCwd
) {
throw new Error("workspace_sync_out_unrecoverable");
}
const runtime = await prepareRuntime({
runId: input.runId,
target: input.target,
workspaceLocalDir: existing.descriptor.binding.localCwd,
mode: "adopt_remote",
baseline: existing.baseline,
gitSnapshot: existing.descriptor.gitSnapshot,
});
await finalizePreparedRuntime({
db: input.db,
runId: input.runId,
target: input.target,
runtime,
descriptor: existing.descriptor,
});
return true;
}
export async function cleanupNativeWorkspaceSync(runId: string): Promise<void> {
await fs.rm(descriptorDirectory(runId), { recursive: true, force: true });
}
export const nativeWorkspaceSyncInternals = {
descriptorPath,
legacyDescriptorPath,
durableSeedPaths,
writeDescriptor,
readDescriptor,
readReference: readNativeWorkspaceSyncReference,
sha256,
};

15
tests/runner-e2e/FIXTURES.md
View File

@ -58,9 +58,13 @@ registry therefore discovers that row through the public environments API.
This still provides full isolation because every cell starts a new Paperclip
instance and database.
Daytona creates a sandbox environment through the public API. Keep
`reuseLease:false`, `runnerLifecycleMode:"per_turn"`, short provider cleanup
backstops, a Daytona secret reference, and an immutable image digest. Teardown
Daytona creates sandbox environments through the public API. The core fixture
keeps `reuseLease:false` and `runnerLifecycleMode:"per_turn"`. The dedicated
warm-continuity fixture uses `reuseLease:true` and
`runnerLifecycleMode:"warm"`; its distinct `configurationKey` is part of the
suite fingerprint even though both fixtures report `environmentId:"daytona"`.
Keep short provider cleanup backstops, a Daytona secret reference, and an
immutable image digest. Teardown
must delete the environment with reusable-lease destruction and must fail the
cell if cleanup cannot be confirmed. Keep CPU, memory, and disk explicit: lease
metadata and the per-test public-list-price runtime estimate depend on that
@ -93,7 +97,10 @@ marker factories. `question_resume_completion` must define the deterministic
browser answer and prove exactly two successful runs with no pending
interaction. `plan_approval_completion` must target the exact two-step
canonical Plan revision, capture its pending UI, approve in the browser, and
prove exactly two successful runs.
prove exactly two successful runs. `warm_three_turn` provides exactly two
browser follow-up messages, preserves one project/execution-workspace scope,
verifies host file contents after every turn, and finishes within three
ten-minute turn deadlines.
Every selected case runs in its own isolated Paperclip process, and independent
cases may run concurrently. Follow-up turns inside one case retain their shared

19
tests/runner-e2e/README.md
View File

@ -68,10 +68,11 @@ pnpm test:e2e:runner -- --group native --environment local
pnpm test:e2e:runner -- --profile runner-codex --case message-marker
pnpm test:e2e:runner -- --case plan-revise-accept --group local
pnpm test:e2e:runner -- --case ask-question --group native
pnpm test:e2e:runner -- --suite daytona-warm-continuity
pnpm test:e2e:runner -- --all
```
The catalog contains three suites. `core-compatibility` (**Core Runner
The catalog contains four suites. `core-compatibility` (**Core Runner
Compatibility**) is seven major runner profiles × local/Daytona × three
workflows: 42 cells. Its cases are:
@ -107,7 +108,19 @@ duplicating the final response. The second workflow restarts the isolated
Paperclip server while the interaction is waiting, reloads that state, and
then resumes it. The suite has no Daytona cells.
The complete catalog is 66 cells (45 local and 21 Daytona) and 114 expected
`daytona-warm-continuity` (**Daytona Warm Continuity**) is exactly two paid
cells: legacy Codex and Runner Codex against one reusable warm Daytona
configuration. Each cell creates a real project with a primary local-path
workspace through the API, selects it in the browser task dialog, and performs
three browser-driven turns on one issue. Every turn reads and extends the same
nonce file, verifies host copy-back, records scheduler/run/end-to-end timing,
and asserts `created`, `resumed`, `resumed` lease acquisition on one sandbox.
Runner Codex additionally proves stable native session, provider session,
runner instance, PID, and process-start identity. Each turn is bounded to ten
minutes, the cell to thirty minutes, and cleanup explicitly deletes the
sandbox rather than waiting for Daytona's idle timeout.
The complete catalog is 68 cells (45 local and 23 Daytona) and 120 expected
paid agent turns. Follow-up steps remain ordered within their cell; all other
cells are independent. Narrow selectors are strongly recommended while
developing fixtures.
@ -351,7 +364,7 @@ Set `RUNNER_E2E_AWS_ENABLED=true` to route paid cells to the repository-scoped
ephemeral AWS RunsOn fleet selected by
`runs-on/fleet=paperclip-public-pr-x64/env=public-ci`. Any other value uses the
proven GitHub-hosted `ubuntu-latest` target. Set `RUNNER_E2E_MAX_PARALLEL` to an
integer from 1–100 on AWS (default 100); use at least 66 to run the current
integer from 1–100 on AWS (default 100); use at least 68 to run the current
complete catalog in one wave. The fallback runner retains its 1–57 limit and
default of 32. Multi-turn steps are sequential inside their cell while
independent cells overlap. Artifacts and merged HTML/JUnit/normalized reports

93
tests/runner-e2e/catalog.test.ts
View File

@ -10,7 +10,10 @@ import {
runnerProfiles,
runnerSuites,
runnerTasks,
daytonaWarmContinuityTask,
daytonaWarmEnvironment,
isImmutableDaytonaImage,
suiteDefinitionHash,
validateRunnerCatalog,
} from "./catalog.js";
import {
@ -21,7 +24,7 @@ import {
} from "./selectors.js";
describe("runner E2E catalog", () => {
it("validates the core, local-integrity, and breadth suites", () => {
it("validates the core, local-integrity, breadth, and warm suites", () => {
expect(runnerProfiles).toHaveLength(7);
expect(openRouterBreadthProfiles).toHaveLength(4);
expect(runnerEnvironments).toHaveLength(2);
@ -29,10 +32,10 @@ describe("runner E2E catalog", () => {
expect(localIntegrityTasks).toHaveLength(2);
expect(openRouterBreadthTasks).toHaveLength(3);
expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([
42, 14, 10,
42, 14, 10, 2,
]);
expect(validateRunnerCatalog()).toHaveLength(66);
expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(66);
expect(validateRunnerCatalog()).toHaveLength(68);
expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(68);
expect(
runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"),
).toHaveLength(42);
@ -46,18 +49,81 @@ describe("runner E2E catalog", () => {
(entry) => entry.suite.id === "openrouter-model-breadth",
),
).toHaveLength(10);
expect(
runnerMatrix.filter(
(entry) => entry.suite.id === "daytona-warm-continuity",
),
).toHaveLength(2);
expect(
runnerMatrix.reduce(
(total, execution) => total + execution.task.expectedRunCount,
0,
),
).toBe(114);
).toBe(120);
expect(
runnerTasks.find((task) => task.id === "plan-revise-accept")
?.attemptTimeoutMs,
).toEqual({ local: 8 * 60_000, daytona: 12 * 60_000 });
});
it("defines the warm Daytona continuity fixture as exactly two Codex cells", () => {
expect(daytonaWarmEnvironment).toMatchObject({
id: "daytona",
configurationKey: "warm-reuse-v1",
groups: ["daytona", "warm"],
});
expect(
daytonaWarmEnvironment.buildEnvironment({
secretRefs: {
DAYTONA_API_KEY: {
type: "secret_ref",
secretId: "22222222-2222-4222-8222-222222222222",
version: "latest",
},
},
daytonaImage: `runner@sha256:${"a".repeat(64)}`,
executionId: "warm",
}),
).toMatchObject({
config: {
reuseLease: true,
runnerLifecycleMode: "warm",
autoStopInterval: 5,
autoArchiveInterval: 15,
autoDeleteInterval: 60,
},
});
expect(daytonaWarmContinuityTask).toMatchObject({
flow: "warm_three_turn",
expectedRunCount: 3,
turnTimeoutMs: 600_000,
});
expect(
daytonaWarmContinuityTask.buildFollowupMessages?.("nonce"),
).toHaveLength(2);
const cells = runnerMatrix.filter(
(entry) => entry.suite.id === "daytona-warm-continuity",
);
expect(cells.map((entry) => entry.profile.id)).toEqual([
"legacy-codex",
"runner-codex",
]);
expect(cells.every((entry) => entry.environment.id === "daytona")).toBe(
true,
);
const suite = runnerSuites.find(
(candidate) => candidate.id === "daytona-warm-continuity",
)!;
expect(
suiteDefinitionHash({
...suite,
environments: [
{ ...daytonaWarmEnvironment, configurationKey: "changed" },
],
}),
).not.toBe(suiteDefinitionHash(suite));
});
it("derives the qualified local native OpenCode profiles from the ranked snapshot", () => {
expect(openRouterBreadthExcludedModelIds).toEqual(["xiaomi/mimo-v2.5"]);
expect(openRouterBreadthExcludedExecutionIds).toEqual([
@ -181,9 +247,10 @@ describe("runner E2E catalog", () => {
question?.buildPrompt("nonce"),
...breadthTasks,
]) {
expect(prompt).toContain("then emit exactly");
const terminalTextInstruction = prompt?.match(/then emit (?:exactly|only)/)?.[0];
expect(terminalTextInstruction).toBeDefined();
expect(prompt!.indexOf("paperclip_finish exactly once")).toBeLessThan(
prompt!.indexOf("then emit exactly"),
prompt!.indexOf(terminalTextInstruction!),
);
expect(prompt).toContain("Wait for that tool call to succeed");
}
@ -336,7 +403,7 @@ describe("runner E2E catalog", () => {
"those two tool calls form one indivisible response sequence",
);
expect(task!.buildPrompt("nonce")).toContain(
"Do not emit assistant text, end the heartbeat, or stop after write_document alone",
"Do not emit assistant text, end the response or heartbeat, or stop after write_document alone",
);
expect(task!.buildPrompt("nonce")).toContain(
"one atomic issue PATCH with status `done` and that exact comment",
@ -433,7 +500,7 @@ describe("runner E2E selectors", () => {
"daytona",
]);
const selected = selectRunnerExecutions(options);
expect(selected).toHaveLength(12);
expect(selected).toHaveLength(13);
expect(
selected.every(
(entry) =>
@ -443,7 +510,7 @@ describe("runner E2E selectors", () => {
).toBe(true);
});
it("rejects groups outside the advertised four", () => {
it("rejects unknown groups", () => {
const options = parseRunnerSelectors(["--group", "codex"]);
expect(() => selectRunnerExecutions(options)).toThrow("Unknown group");
});
@ -452,10 +519,10 @@ describe("runner E2E selectors", () => {
const jobs = buildMatrixJobs(
selectRunnerExecutions(parseRunnerSelectors(["--all"])),
);
expect(jobs).toHaveLength(66);
expect(jobs.filter((job) => job.needsDaytona)).toHaveLength(21);
expect(jobs).toHaveLength(68);
expect(jobs.filter((job) => job.needsDaytona)).toHaveLength(23);
expect(jobs.filter((job) => !job.needsDaytona)).toHaveLength(45);
expect(new Set(jobs.map((job) => job.executionId)).size).toBe(66);
expect(new Set(jobs.map((job) => job.executionId)).size).toBe(68);
expect(
jobs.find(
(job) =>

143
tests/runner-e2e/catalog.ts
View File

@ -27,6 +27,7 @@ const SELECTABLE_GROUPS = [
"native",
"local",
"daytona",
"warm",
"core",
"breadth",
] as const;
@ -374,6 +375,50 @@ export const runnerEnvironments: readonly EnvironmentFixture[] = [
},
] as const;
export const daytonaWarmEnvironment: EnvironmentFixture = {
id: "daytona",
configurationKey: "warm-reuse-v1",
label: "Daytona warm reusable sandbox",
groups: ["daytona", "warm"],
driver: "sandbox",
provider: "daytona",
credential: "DAYTONA_API_KEY",
lifecycle: {
setup: "create_via_api",
probe: "run_context_via_api",
cleanup: "delete_via_api_and_destroy_leases",
},
expectedExecutionTarget: { kind: "remote", transport: "sandbox" },
buildEnvironment(input) {
if (!isImmutableDaytonaImage(input.daytonaImage)) {
throw new Error(
"PAPERCLIP_E2E_DAYTONA_IMAGE must be an immutable image digest",
);
}
return {
name: `Runner E2E Daytona warm ${input.executionId}`,
description: "Ephemeral reusable Daytona runner E2E environment",
driver: "sandbox",
config: {
provider: "daytona",
apiKey: requiredDaytonaSecret(input),
image: input.daytonaImage,
cpu: 4,
memory: 4,
disk: 10,
reuseLease: true,
runnerLifecycleMode: "warm",
autoStopInterval: 5,
autoArchiveInterval: 15,
autoDeleteInterval: 60,
timeoutMs: 300_000,
livenessTimeoutMs: 30_000,
},
envVars: {},
};
},
};
export const runnerTasks: readonly RunnerTaskFixture[] = [
{
id: "message-marker",
@ -719,6 +764,80 @@ const localEnvironment = runnerEnvironments.find(
(environment) => environment.id === "local",
)!;
function warmTurnMarker(turn: 1 | 2 | 3, nonce: string) {
return `PAPERCLIP_E2E_WARM_T${turn}_${nonce}`;
}
function warmWorkspaceLine(turn: 1 | 2 | 3, nonce: string) {
return `T${turn}_${nonce}`;
}
function warmTurnInstructions(turn: 1 | 2 | 3, nonce: string) {
const file = `daytona-warm-${nonce}.txt`;
const lines = Array.from({ length: turn }, (_, index) =>
warmWorkspaceLine((index + 1) as 1 | 2 | 3, nonce),
);
const marker = warmTurnMarker(turn, nonce);
const finalTurn = turn === 3;
return [
`This is warm Daytona continuity turn ${turn} of 3. Work only in the current execution workspace.`,
turn === 1
? `Create ${file} with exactly this one line followed by a newline: ${lines[0]}`
: `Before changing anything, read ${file} and verify its content is exactly ${lines.slice(0, -1).join("\\n")} followed by a newline. Then append exactly ${lines.at(-1)} followed by a newline.`,
`After the write, verify ${file} contains exactly these lines, once each and in order: ${lines.join(" | ")}.`,
`In a native runner, call paperclip_finish exactly once with {reportedWorkDisposition:"${finalTurn ? "done" : "needs_review"}",summary:"${marker}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[{commandOrCheck:"read ${file}",status:"passed"}]}. Wait for that tool call to succeed, then emit exactly ${marker} once as the complete user-facing final response.`,
`In a legacy runner, make exactly one public-API completion write after verification: PATCH /api/issues/$PAPERCLIP_TASK_ID with {"status":"${finalTurn ? "done" : "in_review"}","comment":"${marker}"}. Include Authorization and X-Paperclip-Run-Id. Do not POST a separate comment.`,
`Do not include ${marker} in any other visible response or write. Do not recreate, truncate, reorder, or duplicate prior lines.`,
].join("\n");
}
export const daytonaWarmContinuityTask: RunnerTaskFixture = {
id: "warm-three-turn",
label: "Warm three-turn workspace continuity",
groups: ["warm"],
workMode: "standard",
flow: "warm_three_turn",
expectedRunCount: 3,
attemptTimeoutMs: { local: 30 * 60_000, daytona: 30 * 60_000 },
turnTimeoutMs: 10 * 60_000,
expectedTerminalState: { issue: "done", run: "succeeded" },
buildTitle: (nonce) => `Runner E2E warm Daytona continuity ${nonce}`,
buildVisibleMarker: (nonce) => warmTurnMarker(3, nonce),
buildPrompt: (nonce) => warmTurnInstructions(1, nonce),
buildFollowupMessages: (nonce) => [
warmTurnInstructions(2, nonce),
warmTurnInstructions(3, nonce),
],
buildMatchers(nonce, execution) {
const markers = ([1, 2, 3] as const).map((turn) =>
warmTurnMarker(turn, nonce),
);
return [
{ kind: "message_exact", expected: markers[2] },
...markers.map(
(expected) =>
({ kind: "message_occurrences", expected, count: 1 }) as const,
),
{ kind: "message_ordered", expected: markers },
{
kind: "file_exact",
path: `daytona-warm-${nonce}.txt`,
expected: `${([1, 2, 3] as const)
.map((turn) => warmWorkspaceLine(turn, nonce))
.join("\n")}\n`,
},
{ kind: "issue_status", expected: "done" },
{ kind: "run_status", expected: "succeeded" },
{ kind: "runtime_mode", expected: execution.profile.expectedRuntimeMode },
{ kind: "environment", expected: "daytona" },
];
},
};
const codexContinuityProfiles = runnerProfiles.filter((profile) =>
["legacy-codex", "runner-codex"].includes(profile.id),
);
export const runnerSuites: readonly RunnerSuiteFixture[] = [
{
id: "core-compatibility",
@ -762,6 +881,17 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
excludedExecutionIds: openRouterBreadthExcludedExecutionIds,
},
},
{
id: "daytona-warm-continuity",
label: "Daytona Warm Continuity",
description:
"Three browser-driven turns on one reusable Daytona sandbox for legacy and native Codex.",
groups: ["daytona", "warm"],
profiles: codexContinuityProfiles,
environments: [daytonaWarmEnvironment],
tasks: [daytonaWarmContinuityTask],
expectedMatrixSize: 2,
},
] as const;
export function suiteDefinitionHash(suite: RunnerSuiteFixture) {
@ -774,7 +904,10 @@ export function suiteDefinitionHash(suite: RunnerSuiteFixture) {
model: profile.model,
qualification: profile.modelQualification,
})),
environments: suite.environments.map((environment) => environment.id),
environments: suite.environments.map((environment) => ({
id: environment.id,
configurationKey: environment.configurationKey ?? "default",
})),
tasks: suite.tasks.map((task) => ({
id: task.id,
flow: task.flow,
@ -869,6 +1002,7 @@ export function validateRunnerCatalog(): MatrixExecution[] {
...runnerTasks,
...localIntegrityTasks,
...openRouterBreadthTasks,
daytonaWarmContinuityTask,
];
for (const [label, values] of [
["suite", runnerSuites],
@ -888,6 +1022,7 @@ export function validateRunnerCatalog(): MatrixExecution[] {
...runnerSuites,
...allProfiles,
...runnerEnvironments,
daytonaWarmEnvironment,
...allTasks,
]) {
const unknownGroups = fixture.groups.filter(
@ -916,7 +1051,7 @@ export function validateRunnerCatalog(): MatrixExecution[] {
]),
);
for (const environment of runnerEnvironments) {
for (const environment of [...runnerEnvironments, daytonaWarmEnvironment]) {
const payload = environment.buildEnvironment({
secretRefs: sampleRefs,
daytonaImage:
@ -968,8 +1103,8 @@ export function validateRunnerCatalog(): MatrixExecution[] {
);
}
}
if (matrix.length !== 66)
throw new Error(`Expected 66 runner executions; received ${matrix.length}`);
if (matrix.length !== 68)
throw new Error(`Expected 68 runner executions; received ${matrix.length}`);
return matrix;
}

13
tests/runner-e2e/dashboard.ts
View File

@ -196,6 +196,18 @@ function renderCase(
</tr>`,
)
.join("");
const turnTimingRows = (entry?.result.turnTimings ?? [])
.map(
(timing) => `<tr>
<td>${timing.turn}</td>
<td><code>${html(timing.runId)}</code></td>
<td>${html(timing.leaseAcquisitionOutcome)}</td>
<td>${html(timing.schedulerLatencyMs === null ? "unavailable" : durationLabel(timing.schedulerLatencyMs))}</td>
<td>${html(timing.runDurationMs === null ? "unavailable" : durationLabel(timing.runDurationMs))}</td>
<td>${html(timing.responseLatencyMs === null ? "unavailable" : durationLabel(timing.responseLatencyMs))}</td>
</tr>`,
)
.join("");
const gallery = screenshots.length
? `<div class="gallery" aria-label="Screenshots for ${html(execution.id)}">${screenshots
.map(
@ -268,6 +280,7 @@ function renderCase(
: ""
}
<p class="detail">${html(detail)}</p>
${turnTimingRows ? `<div class="matcher-wrap"><table class="matchers"><thead><tr><th>Turn</th><th>Run</th><th>Lease</th><th>Scheduler</th><th>Run duration</th><th>Response</th></tr></thead><tbody>${turnTimingRows}</tbody></table></div>` : ""}
${matcherRows ? `<div class="matcher-wrap"><table class="matchers"><thead><tr><th>Result</th><th>Matcher</th><th>Expectation</th><th>Detail</th></tr></thead><tbody>${matcherRows}</tbody></table></div>` : `<p class="detail">No matcher result was recorded.</p>`}
${entry ? `<details class="usage"><summary>Usage and billing metadata</summary><pre>${html(JSON.stringify({ billing, rawUsage: entry.result.usage ?? null }, null, 2))}</pre></details>` : ""}
${links}

4
tests/runner-e2e/history.test.ts
View File

@ -195,8 +195,8 @@ describe("runner E2E campaign history", () => {
expect(index).toContain("Runner E2E campaigns");
expect(index).toContain("complete-green");
expect(index).toContain("complete-red");
expect(index).toContain("66/66 passed");
expect(index).toContain("65/66 passed");
expect(index).toContain("68/68 passed");
expect(index).toContain("67/68 passed");
expect(index).toContain("Open report&nbsp;→");
expect(index).toContain(
"campaigns/complete-red/public-images/campaign-summary.png",

76
tests/runner-e2e/live-fixtures.test.ts
View File

@ -70,4 +70,80 @@ describe("live runner fixtures", () => {
"DELETE /api/environments/environment-1?destroyReusableSandboxLeases=true",
);
});
it("creates a primary project workspace for reusable Daytona scope", async () => {
const calls: string[] = [];
const api = {
async post(path: string, data?: Record<string, unknown>) {
calls.push(`POST ${path}`);
if (path === "/api/plugins/install") {
return {
id: "plugin-daytona",
pluginKey: "paperclip.daytona-sandbox-provider",
status: "ready",
};
}
if (path === "/api/companies") {
return { id: "company-1", name: "Runner E2E" };
}
if (path.endsWith("/environments")) {
return { id: "environment-1", driver: "sandbox" };
}
if (path.endsWith("/agents")) {
return { id: "agent-1", name: "Agent", companyId: "company-1" };
}
if (path.endsWith("/projects")) {
expect(data).toMatchObject({
executionWorkspacePolicy: {
enabled: true,
defaultMode: "shared_workspace",
environmentId: "environment-1",
},
workspace: {
sourceType: "local_path",
cwd: "/tmp/workspace",
isPrimary: true,
},
});
return {
id: "project-1",
name: data?.name,
primaryWorkspace: {
id: "project-workspace-1",
cwd: "/tmp/workspace",
},
};
}
throw new Error(`Unexpected POST ${path}`);
},
async postSensitive(_path: string, data?: Record<string, unknown>) {
return { id: `secret-${String(data?.key).toLowerCase()}` };
},
async delete() {},
} as unknown as RunnerApi;
const execution = runnerMatrix.find(
(candidate) =>
candidate.id ===
"daytona-warm-continuity.runner-codex.daytona.warm-three-turn",
)!;
const fixtures = await setupLiveFixtures({
api,
execution,
executionNonce: "nonce",
workspacePath: "/tmp/workspace",
credentials: {
OPENAI_API_KEY: "openai-test-value",
DAYTONA_API_KEY: "daytona-test-value",
},
daytonaImage:
"ghcr.io/paperclip/image@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
});
expect(fixtures.project?.primaryWorkspace?.id).toBe("project-workspace-1");
expect(
calls.indexOf("POST /api/companies/company-1/environments"),
).toBeLessThan(calls.indexOf("POST /api/companies/company-1/projects"));
await fixtures.teardown();
});
});

48
tests/runner-e2e/live-fixtures.ts
View File

@ -32,12 +32,21 @@ interface AgentRecord {
name: string;
companyId: string;
}
interface ProjectRecord {
id: string;
name: string;
primaryWorkspace?: {
id: string;
cwd?: string | null;
} | null;
}
export interface LiveFixtureValues {
company: CompanyRecord;
secretRefs: SecretReferenceMap;
environment: EnvironmentRecord;
agent: AgentRecord;
project?: ProjectRecord;
teardown(): Promise<void>;
}
@ -210,12 +219,51 @@ export async function setupLiveFixtures(input: {
},
});
if (execution.environment.configurationKey === "warm-reuse-v1") {
registry.register<ProjectRecord>({
id: "project",
dependencies: ["company", "environment"],
async setup(resolved) {
const company = value<CompanyRecord>(resolved, "company");
const environment = value<EnvironmentRecord>(resolved, "environment");
return api.post<ProjectRecord>(
`/api/companies/${company.id}/projects`,
{
name: `Runner E2E warm project ${input.executionNonce}`,
description:
"Ephemeral project anchoring a reusable Daytona execution workspace",
executionWorkspacePolicy: {
enabled: true,
defaultMode: "shared_workspace",
sharedWorkspaceConcurrency: "serialize",
allowIssueOverride: false,
environmentId: environment.id,
workspaceStrategy: { type: "project_primary" },
},
workspace: {
name: "Primary",
sourceType: "local_path",
cwd: input.workspacePath,
isPrimary: true,
},
},
);
},
async teardown() {
// The isolated instance is deleted after provider resources are gone.
},
});
}
const setup = await registry.setupAll();
return {
company: value<CompanyRecord>(setup.values, "company"),
secretRefs: value<SecretReferenceMap>(setup.values, "secrets"),
environment: value<EnvironmentRecord>(setup.values, "environment"),
agent: value<AgentRecord>(setup.values, "agent"),
...(setup.values.has("project")
? { project: value<ProjectRecord>(setup.values, "project") }
: {}),
teardown: setup.teardown,
};
}

5
tests/runner-e2e/matchers.ts
View File

@ -116,6 +116,7 @@ export async function evaluateMatcher(
passed = actual === matcher.expected;
} else if (
matcher.kind === "file_exists" ||
matcher.kind === "file_exact" ||
matcher.kind === "file_contains"
) {
try {
@ -124,7 +125,9 @@ export async function evaluateMatcher(
(await readFile(matcher.path, "utf8"));
passed =
matcher.kind === "file_exists" ||
String(actual).includes(matcher.expected);
(matcher.kind === "file_exact"
? String(actual) === matcher.expected
: String(actual).includes(matcher.expected));
} catch {
actual = undefined;
passed = false;

16
tests/runner-e2e/report.test.ts
View File

@ -44,6 +44,19 @@ describe("runner E2E report aggregation", () => {
workflowRunUrl: "https://example.test/actions/runs/forged",
},
runIds: ["run-2"],
turnTimings: [
{
turn: 1,
submittedAt: "2026-08-26T00:00:00.000Z",
runStartedAt: "2026-08-26T00:00:00.100Z",
runFinishedAt: "2026-08-26T00:00:01.000Z",
schedulerLatencyMs: 100,
runDurationMs: 900,
responseLatencyMs: 1_000,
runId: "run-2",
leaseAcquisitionOutcome: "created",
},
],
usage: {
inputTokens: 1_250,
outputTokens: 75,
@ -201,6 +214,9 @@ describe("runner E2E report aggregation", () => {
);
expect(dashboard).toContain("message_contains");
expect(dashboard).toContain("Matchers and test context");
expect(dashboard).toContain("Scheduler");
expect(dashboard).toContain("Run duration");
expect(dashboard).toContain("100ms");
expect(dashboard).toContain("Campaign billing summary");
expect(dashboard).toContain("LLM reported subtotal");
expect(dashboard).toContain("Agent execution time");

472
tests/runner-e2e/runner.spec.ts
View File

@ -44,6 +44,9 @@ interface IssueRecord {
status: string;
workMode?: string;
assigneeAgentId?: string | null;
projectId?: string | null;
projectWorkspaceId?: string | null;
executionWorkspaceId?: string | null;
executionRunId?: string | null;
checkoutRunId?: string | null;
}
@ -66,6 +69,9 @@ interface RunRecord {
continuationAttempt?: number;
retryOfRunId?: string | null;
runnerInstanceId?: string | null;
nativeSessionId?: string | null;
processPid?: number | null;
processStartedAt?: string | null;
contextSnapshot?: Record<string, unknown> | null;
runnerProfileJson?: Record<string, unknown> | null;
usageJson?: Record<string, unknown> | null;
@ -80,13 +86,17 @@ interface RunRecord {
interface EnvironmentLeaseRecord {
id: string;
status?: string;
leasePolicy?: string;
providerLeaseId?: string | null;
executionWorkspaceId?: string | null;
metadata?: Record<string, unknown> | null;
issueId?: string | null;
heartbeatRunId?: string | null;
provider?: string | null;
acquiredAt?: string | null;
releasedAt?: string | null;
updatedAt?: string | null;
metadata?: Record<string, unknown> | null;
}
interface InteractionRecord {
@ -290,6 +300,7 @@ async function createTaskThroughUi(input: {
title: string;
prompt: string;
workMode: "standard" | "planning" | "ask";
projectName?: string;
}) {
const issuesUrl = `/${encodeURIComponent(input.issuePrefix)}/issues`;
const newTask = input.page.getByRole("button", { name: "New Task" }).first();
@ -336,9 +347,29 @@ async function createTaskThroughUi(input: {
.getByPlaceholder("Search assignees...")
.fill(input.agentName);
await input.page.getByText(input.agentName, { exact: true }).last().click();
if (input.projectName) {
const dialog = input.page.getByRole("dialog");
await dialog.getByRole("button", { name: "Project", exact: true }).click();
await dialog.getByPlaceholder("Search projects...").fill(input.projectName);
await dialog.getByText(input.projectName, { exact: true }).last().click();
}
const submittedAtMs = Date.now();
await input.page
.getByRole("button", { name: "Create Task", exact: true })
.click();
return submittedAtMs;
}
async function submitTaskReply(page: Page, body: string): Promise<number> {
const composer = page.getByTestId("task-chat-composer-input").last();
await expect(composer).toBeVisible({ timeout: 30_000 });
await composer
.locator('[contenteditable="true"], textarea')
.first()
.fill(body);
const submittedAtMs = Date.now();
await page.getByTestId("task-chat-composer-send").last().click();
return submittedAtMs;
}
function matchingRuns(runs: RunRecord[], issue: IssueRecord) {
@ -524,7 +555,11 @@ for (const execution of executions) {
page,
request,
}, testInfo) => {
test.setTimeout(deadlineMs + 90_000);
test.setTimeout(
execution.task.flow === "warm_three_turn"
? deadlineMs
: deadlineMs + 90_000,
);
const startedAtMs = Date.now();
const startedAt = new Date(startedAtMs).toISOString();
const nonce = `${randomBytes(6).toString("hex")}-${attempt}`;
@ -541,6 +576,8 @@ for (const execution of executions) {
let selectedRuns: RunRecord[] = [];
let runtimeLeases: EnvironmentLeaseRecord[] = [];
let matcherResults: MatcherResult[] = [];
let turnTimings: NonNullable<RunnerE2EResult["turnTimings"]> | undefined;
const turnSubmissionTimesMs: number[] = [];
const screenshots: NonNullable<RunnerE2EResult["screenshots"]> = [];
let primaryError: unknown;
let failureClassOverride: FailureClass | undefined;
@ -746,14 +783,17 @@ for (const execution of executions) {
throw new Error(
"Created fixture company did not return an issue prefix",
);
await createTaskThroughUi({
page,
issuePrefix,
agentName: fixtures.agent.name,
title,
prompt,
workMode: execution.task.workMode,
});
turnSubmissionTimesMs.push(
await createTaskThroughUi({
page,
issuePrefix,
agentName: fixtures.agent.name,
title,
prompt,
workMode: execution.task.workMode,
projectName: fixtures.project?.name,
}),
);
const deadlineAt = startedAtMs + deadlineMs;
issue = await pollUntil({
@ -781,6 +821,16 @@ for (const execution of executions) {
`UI-created issue work mode was ${String(issue.workMode)}; expected ${execution.task.workMode}`,
);
}
if (fixtures.project) {
if (
issue.projectId !== fixtures.project.id ||
issue.projectWorkspaceId !== fixtures.project.primaryWorkspace?.id
) {
throw new Error(
`Warm task did not retain its project/workspace scope: ${JSON.stringify({ projectId: issue.projectId, projectWorkspaceId: issue.projectWorkspaceId })}`,
);
}
}
await page.goto(
`/${encodeURIComponent(issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`,
@ -827,6 +877,7 @@ for (const execution of executions) {
let planLifecycleEvidence: Record<string, unknown> | null = null;
let questionLifecycleEvidence: Record<string, unknown> | null = null;
let warmLifecycleEvidence: Record<string, unknown> | null = null;
let expectedQuestionResolution: {
interactionId: string;
optionId: string;
@ -1214,12 +1265,165 @@ for (const execution of executions) {
.last()
.click();
planLifecycleEvidence = { interaction, plan };
} else if (execution.task.flow === "warm_three_turn") {
const followups = execution.task.buildFollowupMessages?.(nonce);
if (!followups || !fixtures.project?.primaryWorkspace?.id) {
throw new Error(
`Warm fixture ${execution.task.id} is missing its project or follow-up messages`,
);
}
const workspaceFile = path.join(
workspacePath,
`daytona-warm-${nonce}.txt`,
);
const turnEvidence: Array<Record<string, unknown>> = [];
for (const completedTurn of [1, 2] as const) {
const turnDeadlineAt = Math.min(
deadlineAt,
turnSubmissionTimesMs[completedTurn - 1]! +
(execution.task.turnTimeoutMs ?? 10 * 60_000),
);
const waitingState = await pollUntil({
label: `warm Daytona turn ${completedTurn} review state for issue ${issue.id}`,
deadlineAt: turnDeadlineAt,
load: loadTaskState,
accept: ({ currentIssue, taskRuns }) =>
currentIssue.status === "in_review" &&
taskRuns.length === completedTurn &&
taskRuns.every((run) => run.status === "succeeded"),
reject: ({ taskRuns }) =>
definitiveRunFailure(taskRuns) ??
(taskRuns.length > completedTurn
? `warm turn ${completedTurn} dispatched duplicate runs`
: undefined),
});
const expectedPrefix = `${Array.from(
{ length: completedTurn },
(_, index) => `T${index + 1}_${nonce}`,
).join("\n")}\n`;
const hostContent = await readFile(workspaceFile, "utf8");
if (hostContent !== expectedPrefix) {
throw new Error(
`Host workspace was not finalized after warm turn ${completedTurn}: expected ${JSON.stringify(expectedPrefix)}, observed ${JSON.stringify(hostContent)}`,
);
}
if (
waitingState.currentIssue.projectId !== fixtures.project.id ||
waitingState.currentIssue.projectWorkspaceId !==
fixtures.project.primaryWorkspace.id ||
!waitingState.currentIssue.executionWorkspaceId
) {
throw new Error(
`Warm turn ${completedTurn} lost its project execution-workspace scope`,
);
}
const completedRunIds = new Set(
waitingState.taskRuns.map((candidate) => candidate.id),
);
const retainedTurnLeases = await pollUntil({
label: `retained Daytona leases after warm turn ${completedTurn}`,
deadlineAt: Math.min(turnDeadlineAt, Date.now() + 30_000),
intervalMs: 500,
load: () =>
api.get<EnvironmentLeaseRecord[]>(
`/api/environments/${fixtures!.environment.id}/leases`,
),
accept: (leases) => {
const runOrder = new Map(
waitingState.taskRuns.map((candidate, index) => [
candidate.id,
index,
]),
);
const completed = leases
.filter(
(lease) =>
lease.heartbeatRunId &&
completedRunIds.has(lease.heartbeatRunId),
)
.sort(
(left, right) =>
(runOrder.get(left.heartbeatRunId ?? "") ?? 0) -
(runOrder.get(right.heartbeatRunId ?? "") ?? 0),
);
return (
completed.length === completedTurn &&
completed.every(
(lease) =>
lease.status === "retained" &&
lease.leasePolicy === "reuse_by_environment" &&
typeof lease.providerLeaseId === "string" &&
record(lease.metadata).sandboxState === "started",
) &&
completed
.slice(1)
.every(
(lease) =>
record(lease.metadata).resumedFromState === "started",
)
);
},
});
const turnRunOrder = new Map(
waitingState.taskRuns.map((candidate, index) => [
candidate.id,
index,
]),
);
const completedLeases = retainedTurnLeases
.filter(
(lease) =>
lease.heartbeatRunId &&
completedRunIds.has(lease.heartbeatRunId),
)
.sort(
(left, right) =>
(turnRunOrder.get(left.heartbeatRunId ?? "") ?? 0) -
(turnRunOrder.get(right.heartbeatRunId ?? "") ?? 0),
);
if (
new Set(completedLeases.map((lease) => lease.providerLeaseId))
.size !== 1
) {
throw new Error(
`Warm turn ${completedTurn} replaced its Daytona sandbox`,
);
}
turnEvidence.push({
turn: completedTurn,
issue: waitingState.currentIssue,
run: waitingState.taskRuns.at(-1),
hostContent,
leases: completedLeases,
});
await page.goto(
`/${encodeURIComponent(issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`,
{ waitUntil: "domcontentloaded" },
);
await captureScreenshot(
`warm-turn-${completedTurn}`,
`Warm Daytona turn ${completedTurn} awaiting review`,
`warm-turn-${completedTurn}.png`,
);
turnSubmissionTimesMs.push(
await submitTaskReply(page, followups[completedTurn - 1]),
);
}
warmLifecycleEvidence = { turns: turnEvidence };
}
const taskMatchers = execution.task.buildMatchers(nonce, execution);
const terminalDeadlineAt =
execution.task.flow === "warm_three_turn"
? Math.min(
deadlineAt,
turnSubmissionTimesMs.at(-1)! +
(execution.task.turnTimeoutMs ?? 10 * 60_000),
)
: deadlineAt;
let terminal = await pollUntil({
label: `issue ${issue.id} and heartbeat run terminal state`,
deadlineAt,
deadlineAt: terminalDeadlineAt,
load: loadTaskState,
accept: ({ currentIssue, taskRuns }) =>
currentIssue.status === execution.task.expectedTerminalState.issue &&
@ -1289,6 +1493,45 @@ for (const execution of executions) {
),
);
selectedRuns = sortRunsChronologically(selectedRuns);
if (execution.task.flow === "warm_three_turn") {
turnTimings = selectedRuns.map((candidate, index) => {
const submittedAtMs = turnSubmissionTimesMs[index]!;
const runStartedAtMs = candidate.startedAt
? Date.parse(candidate.startedAt)
: Number.NaN;
const runFinishedAtMs = candidate.finishedAt
? Date.parse(candidate.finishedAt)
: Number.NaN;
const acquisition = record(
record(candidate.contextSnapshot).paperclipEnvironment,
).sandboxLeaseAcquisition;
const acquisitionOutcome = record(acquisition).outcome;
return {
turn: index + 1,
submittedAt: new Date(submittedAtMs).toISOString(),
runStartedAt: candidate.startedAt ?? null,
runFinishedAt: candidate.finishedAt ?? null,
schedulerLatencyMs: Number.isFinite(runStartedAtMs)
? Math.max(0, runStartedAtMs - submittedAtMs)
: null,
runDurationMs:
Number.isFinite(runStartedAtMs) &&
Number.isFinite(runFinishedAtMs)
? Math.max(0, runFinishedAtMs - runStartedAtMs)
: null,
responseLatencyMs: Number.isFinite(runFinishedAtMs)
? Math.max(0, runFinishedAtMs - submittedAtMs)
: null,
runId: candidate.id,
leaseAcquisitionOutcome:
acquisitionOutcome === "created" ||
acquisitionOutcome === "resumed" ||
acquisitionOutcome === "replacement"
? acquisitionOutcome
: "unknown",
};
});
}
const finalRun = selectedRuns.at(-1)!;
const run =
selectedRuns.find(
@ -1500,10 +1743,31 @@ for (const execution of executions) {
interactions: terminal.interactions,
},
};
const fileObservations = Object.fromEntries(
await Promise.all(
taskMatchers
.filter(
(matcher) =>
matcher.kind === "file_exists" ||
matcher.kind === "file_exact" ||
matcher.kind === "file_contains",
)
.map(async (matcher) => [
matcher.path,
await readFile(
path.isAbsolute(matcher.path)
? matcher.path
: path.join(workspacePath, matcher.path),
"utf8",
).catch(() => undefined),
]),
),
);
matcherResults = await Promise.all(
taskMatchers.map((matcher) =>
evaluateMatcher(matcher, {
...matcherObservation,
files: fileObservations,
// Multi-run tasks intentionally retain earlier waiting/revision
// replies. Exact completion text belongs to the chronological
// final run, while occurrence checks still span every agent
@ -1571,6 +1835,190 @@ for (const execution of executions) {
"expected a Daytona sandbox lease on the run context",
);
}
if (execution.task.flow === "warm_three_turn") {
const runEnvironmentContexts = selectedRuns.map((candidate) =>
record(record(candidate.contextSnapshot).paperclipEnvironment),
);
const leaseIds = runEnvironmentContexts.map((entry) => entry.leaseId);
const acquisitionOutcomes = runEnvironmentContexts.map(
(entry) => record(entry.sandboxLeaseAcquisition).outcome,
);
const projectWorkspaceIds = selectedRuns.map(
(candidate) =>
record(record(candidate.contextSnapshot).paperclipWorkspace)
.workspaceId,
);
const executionWorkspaceIds = selectedRuns.map(
(candidate) => record(candidate.contextSnapshot).executionWorkspaceId,
);
if (
leaseIds.some(
(leaseId) => typeof leaseId !== "string" || leaseId.length === 0,
)
) {
invariantFailures.push(
`expected a persisted Daytona lease row for every warm turn; observed ${JSON.stringify(leaseIds)}`,
);
}
if (
JSON.stringify(acquisitionOutcomes) !==
JSON.stringify(["created", "resumed", "resumed"])
) {
invariantFailures.push(
`expected warm lease outcomes created,resumed,resumed; observed ${JSON.stringify(acquisitionOutcomes)}`,
);
}
if (
!fixtures.project?.primaryWorkspace?.id ||
projectWorkspaceIds.some(
(workspaceId) =>
workspaceId !== fixtures!.project!.primaryWorkspace!.id,
) ||
new Set(executionWorkspaceIds).size !== 1 ||
executionWorkspaceIds[0] !== issue.executionWorkspaceId ||
issue.projectId !== fixtures.project.id ||
issue.projectWorkspaceId !== fixtures.project.primaryWorkspace.id ||
!issue.executionWorkspaceId
) {
invariantFailures.push(
`warm task did not preserve project, project-workspace, and execution-workspace identity: ${JSON.stringify({ projectWorkspaceIds, executionWorkspaceIds, projectId: issue.projectId, projectWorkspaceId: issue.projectWorkspaceId, executionWorkspaceId: issue.executionWorkspaceId })}`,
);
}
if (execution.profile.generation === "native") {
const stableIdentityFields: Array<{
label: string;
values: unknown[];
}> = [
{
label: "native session",
values: selectedRuns.map(
(candidate) => candidate.nativeSessionId,
),
},
{
label: "runner instance",
values: selectedRuns.map(
(candidate) => candidate.runnerInstanceId,
),
},
{
label: "provider session",
values: selectedRuns.map((candidate) => candidate.sessionIdAfter),
},
{
label: "runner pid",
values: selectedRuns.map((candidate) => candidate.processPid),
},
{
label: "runner process fingerprint",
values: selectedRuns.map(
(candidate) => candidate.processStartedAt,
),
},
];
for (const { label, values } of stableIdentityFields) {
if (
values.some(
(value) =>
value === null ||
value === undefined ||
String(value).length === 0,
) ||
new Set(values).size !== 1
) {
invariantFailures.push(
`expected one stable ${label} across native warm turns; observed ${JSON.stringify(values)}`,
);
}
}
}
if (
turnTimings?.length !== 3 ||
turnTimings.some(
(timing) =>
timing.runStartedAt === null ||
timing.runFinishedAt === null ||
timing.schedulerLatencyMs === null ||
timing.runDurationMs === null ||
timing.responseLatencyMs === null ||
timing.responseLatencyMs >
(execution.task.turnTimeoutMs ?? 10 * 60_000),
)
) {
invariantFailures.push(
`warm turn timing data was incomplete or exceeded its structural deadline: ${JSON.stringify(turnTimings)}`,
);
}
const retainedLeases = await pollUntil({
label: `retained warm Daytona lease for issue ${issue.id}`,
deadlineAt: Math.min(deadlineAt, Date.now() + 30_000),
intervalMs: 500,
load: () =>
api.get<EnvironmentLeaseRecord[]>(
`/api/environments/${fixtures!.environment.id}/leases`,
),
accept: (leases) => {
const warmLeases = leases.filter(
(lease) =>
lease.issueId === issue!.id &&
selectedRuns.some(
(candidate) => candidate.id === lease.heartbeatRunId,
),
);
return (
warmLeases.length === 3 &&
warmLeases.every(
(lease) =>
lease.status === "retained" &&
lease.leasePolicy === "reuse_by_environment" &&
typeof lease.providerLeaseId === "string" &&
record(lease.metadata).sandboxState === "started",
)
);
},
});
const selectedRunOrder = new Map(
selectedRuns.map((candidate, index) => [candidate.id, index]),
);
const warmLeases = retainedLeases
.filter(
(lease) =>
lease.issueId === issue!.id &&
selectedRunOrder.has(lease.heartbeatRunId ?? ""),
)
.sort(
(left, right) =>
(selectedRunOrder.get(left.heartbeatRunId ?? "") ?? 0) -
(selectedRunOrder.get(right.heartbeatRunId ?? "") ?? 0),
);
const providerLeaseIds = warmLeases.map(
(lease) => lease.providerLeaseId,
);
const resumedFromStates = warmLeases
.slice(1)
.map((lease) => record(lease.metadata).resumedFromState);
if (
new Set(providerLeaseIds).size !== 1 ||
typeof providerLeaseIds[0] !== "string" ||
JSON.stringify(resumedFromStates) !==
JSON.stringify(["started", "started"])
) {
invariantFailures.push(
`expected one continuously-started Daytona sandbox; observed ${JSON.stringify({ providerLeaseIds, resumedFromStates })}`,
);
}
warmLifecycleEvidence = {
...(warmLifecycleEvidence ?? {}),
leaseIds,
acquisitionOutcomes,
projectWorkspaceIds,
executionWorkspaceIds,
providerLeaseIds,
resumedFromStates,
retainedLeases: warmLeases,
turnTimings,
};
}
if (execution.environment.id === "local") {
const runLogContent = String(record(runLog).content ?? "");
// The log endpoint returns NDJSON, so quotes inside each `chunk` are
@ -1723,6 +2171,7 @@ for (const execution of executions) {
interactions: terminal.interactions,
planLifecycleEvidence,
questionLifecycleEvidence,
warmLifecycleEvidence,
matcherResults,
invariantFailures,
runEvents,
@ -1940,6 +2389,7 @@ for (const execution of executions) {
issueId: issue?.id,
issueIdentifier: issue?.identifier ?? null,
runIds: selectedRuns.map((run) => run.id),
...(turnTimings ? { turnTimings } : {}),
startedAt,
finishedAt: new Date(finishedAtMs).toISOString(),
durationMs: finishedAtMs - startedAtMs,

17
tests/runner-e2e/support.test.ts
View File

@ -341,6 +341,23 @@ describe("runner E2E matchers", () => {
expect(result?.detail).toContain("observed 2");
});
it("matches finalized workspace files byte-for-byte", async () => {
const [matched, extraLine] = await evaluateMatchers(
[
{ kind: "file_exact", path: "continuity.txt", expected: "T1\nT2\n" },
{ kind: "file_exact", path: "duplicate.txt", expected: "T1\nT2\n" },
],
{
files: {
"continuity.txt": "T1\nT2\n",
"duplicate.txt": "T1\nT2\nT2\n",
},
},
);
expect(matched?.passed).toBe(true);
expect(extraLine?.passed).toBe(false);
});
it("normalizes ordered fragments and evaluates nested JSON Schema", async () => {
const results = await evaluateMatchers(
[

19
tests/runner-e2e/types.ts
View File

@ -13,7 +13,8 @@ export type RunnerTaskFlow =
| "single_turn"
| "plan_revision_acceptance"
| "question_resume_completion"
| "plan_approval_completion";
| "plan_approval_completion"
| "warm_three_turn";
export interface SecretReference {
type: "secret_ref";
@ -73,6 +74,8 @@ export interface RunnerProfileFixture {
export interface EnvironmentFixture {
id: RunnerEnvironmentId;
/** Distinguishes materially different configurations that share a provider ID. */
configurationKey?: string;
label: string;
groups: readonly string[];
driver: "local" | "sandbox";
@ -103,6 +106,7 @@ export type Matcher =
| { kind: "runtime_mode"; expected: RunnerGeneration }
| { kind: "environment"; expected: RunnerEnvironmentId }
| { kind: "file_exists"; path: string }
| { kind: "file_exact"; path: string; expected: string }
| { kind: "file_contains"; path: string; expected: string }
| { kind: "artifact_exists"; name: string; mimeType?: string }
| { kind: "json_path"; path: string; expected: unknown }
@ -124,6 +128,8 @@ export interface RunnerTaskFixture {
buildPrompt(nonce: string): string;
buildVisibleMarker(nonce: string): string;
buildRevisionRequest?(nonce: string): string;
buildFollowupMessages?(nonce: string): readonly [string, string];
turnTimeoutMs?: number;
buildQuestionAnswer?(nonce: string): {
optionLabel: string;
expectedMarker: string;
@ -260,6 +266,17 @@ export interface RunnerE2EResult {
issueId?: string;
issueIdentifier?: string | null;
runIds?: string[];
turnTimings?: Array<{
turn: number;
submittedAt: string;
runStartedAt: string | null;
runFinishedAt: string | null;
schedulerLatencyMs: number | null;
runDurationMs: number | null;
responseLatencyMs: number | null;
runId: string;
leaseAcquisitionOutcome: "created" | "resumed" | "replacement" | "unknown";
}>;
startedAt: string;
finishedAt: string;
durationMs: number;

2
tests/runner-e2e/workflow-security.test.ts
View File

@ -4,7 +4,7 @@ import { describe, expect, it } from "vitest";
const repositoryRoot = path.resolve(import.meta.dirname, "../..");
const ordinaryPrTrustedWorkflowRevision =
"a0a78ee60946a5f79f85b2bd0584fc766fae43bb";
"03609aa6ecc9a047ed53d6b6469d8be554fbc46d";
const fullStackTestNeeds =
/needs:\s*\[\s*authorize,\s*target_lock,\s*catalog,\s*daytona_image,\s*build_runner_artifacts,\s*build_remote_provider_pack,?\s*\]/u;
const buildRunnerNeeds =