From a4c655183d96ee928103c52ec441fc5449e38cd3 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 7 Sep 2026 13:47:12 -0500 Subject: [PATCH 1/4] fix(adapters): leave sandbox work folders to the remote runner Co-Authored-By: Paperclip --- .../claude-local/src/server/execute.ts | 4 ++- .../opencode-local/src/server/execute.ts | 4 ++- .../adapters/pi-local/src/server/execute.ts | 4 ++- ...legacy-sandbox-work-folder-startup.test.ts | 36 +++++++++++++++++++ 4 files changed, 45 insertions(+), 3 deletions(-) create mode 100644 server/src/__tests__/legacy-sandbox-work-folder-startup.test.ts diff --git a/packages/adapters/claude-local/src/server/execute.ts b/packages/adapters/claude-local/src/server/execute.ts index 0f24419c9f..1d25cc9313 100644 --- a/packages/adapters/claude-local/src/server/execute.ts +++ b/packages/adapters/claude-local/src/server/execute.ts @@ -212,7 +212,9 @@ async function buildClaudeRuntimeConfig(input: ClaudeExecutionInput): Promise = { ...buildPaperclipEnv(agent) }; diff --git a/packages/adapters/opencode-local/src/server/execute.ts b/packages/adapters/opencode-local/src/server/execute.ts index dba49d84c8..c56b7f01c0 100644 --- a/packages/adapters/opencode-local/src/server/execute.ts +++ b/packages/adapters/opencode-local/src/server/execute.ts @@ -252,7 +252,9 @@ export async function execute(ctx: AdapterExecutionContext): Promise { + it.each([["claude_local", claude], ["opencode_local", opencode], ["pi_local", pi]] as const)( + "%s leaves sandbox paths to the remote runner", + async (adapterType, execute) => { + const root = await mkdtemp(join(tmpdir(), "remote-work-folder-")); + const hostSentinel = join(root, "host-file"); + await writeFile(hostSentinel, "host-owned"); + // This sandbox path cannot be mkdir'd on the app host. Reaching the + // remote command probe proves startup did not claim the path locally. + const cwd = join(hostSentinel, "repos", "project"); + const probe = vi.fn(async () => { throw new Error("remote-probe-reached"); }); + try { + await expect(execute({ + runId: "remote-work-folder-run", + agent: { id: "agent", companyId: "company", name: "test", adapterType, adapterConfig: {} }, + config: { engine: "cli", cwd, command: "qualified-provider", env: {} }, + context: { paperclipWorkspace: { cwd } }, + runtime: { sessionId: null, sessionParams: null, sessionDisplayId: null, taskKey: null }, + executionTarget: { kind: "remote", transport: "sandbox", providerKey: "daytona", + remoteCwd: cwd, workFolderHome: root, runner: { execute: probe } }, + onLog: async () => {}, + })).rejects.toThrow("remote-probe-reached"); + expect(probe).toHaveBeenCalled(); + expect(await readFile(hostSentinel, "utf8")).toBe("host-owned"); + } finally { await rm(root, { recursive: true, force: true }); } + }, + ); +}); From 82b19d50915fc0ba44a600a1bb556ead5074f21d Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 7 Sep 2026 13:53:39 -0500 Subject: [PATCH 2/4] fix(work-folders): preserve save status when activity logging fails Co-Authored-By: Paperclip --- .../src/__tests__/sandbox-work-folders.test.ts | 17 ++++++++++++++++- server/src/services/sandbox-work-folders.ts | 4 +++- tests/runner-e2e/deployed-work-folders.spec.ts | 10 +++++++--- tests/runner-e2e/playwright.deployed.config.ts | 2 +- 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/server/src/__tests__/sandbox-work-folders.test.ts b/server/src/__tests__/sandbox-work-folders.test.ts index aa2cd53327..ee6bdf98ae 100644 --- a/server/src/__tests__/sandbox-work-folders.test.ts +++ b/server/src/__tests__/sandbox-work-folders.test.ts @@ -6,10 +6,11 @@ import { eq } from "drizzle-orm"; import fs from "node:fs/promises"; import path from "node:path"; import os from "node:os"; -import { agents, assets, companyMemberships, issueAttachments, companies, createDb, heartbeatRuns, issues, environments, environmentLeases, projects, projectWorkspaces, taskRepositoryBindings, workFolderObjects, startEmbeddedPostgresTestDatabase, type Db } from "@paperclipai/db"; +import { agents, assets, companyMemberships, issueAttachments, companies, createDb, heartbeatRuns, issues, environments, environmentLeases, projects, projectWorkspaces, taskRepositoryBindings, workFolderObjects, workFolderRuns, startEmbeddedPostgresTestDatabase, type Db } from "@paperclipai/db"; import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js"; import { prepareSandboxWorkFolders } from "../services/sandbox-work-folders.js"; import { retainUnsavedWorkFolderLease, workFolderSandboxKey } from "../services/work-folder-retention.js"; +import * as activityLog from "../services/activity-log.js"; import { workFolderService } from "../services/work-folders.js"; import { collectWorkFolderGarbage } from "../services/work-folder-garbage.js"; import { localTestWorkFolderRunner } from "./helpers/work-folder-runner.js"; @@ -61,6 +62,20 @@ describe("shared sandbox work-folder lifecycle", () => { runner: { execute: (input) => localTestWorkFolderRunner.execute({ ...input, env: { ...input.env, HOME: home } }) } } }); active.push(run); return run; } + it("keeps successful checkpoints saved when activity logging fails", async () => { + const activity = vi.spyOn(activityLog, "logActivity").mockRejectedValue(new Error("activity unavailable")); + try { + const run = await prepare(path.join(root, "activity-failure"), randomUUID()); + await fs.writeFile(path.join(run.home, "task/activity-proof.txt"), "saved despite logging failure"); + await run.stop(); active.splice(active.indexOf(run), 1); + const [state] = await db.select().from(workFolderRuns).where(eq(workFolderRuns.runId, run.manifest.runId)); + expect(state?.state).toBe("saved"); + expect(state?.lastSavedAt).not.toBeNull(); + const folder = await workFolderService(db, storage).ensure({ companyId, scope: "task", ownerId: taskId }); + expect((await workFolderService(db, storage).list(folder)).files.some((file) => file.path === "activity-proof.txt")).toBe(true); + } finally { activity.mockRestore(); } + }, 120_000); + it("reuses clones and restores saved unpushed work, staged changes, and task files after losing the sandbox", async () => { const home = path.join(root, "sandbox"); const leaseId = randomUUID(); diff --git a/server/src/services/sandbox-work-folders.ts b/server/src/services/sandbox-work-folders.ts index 34af680892..5c25f848c0 100644 --- a/server/src/services/sandbox-work-folders.ts +++ b/server/src/services/sandbox-work-folders.ts @@ -16,6 +16,7 @@ import { workFolderRepositoryService } from "./work-folder-repositories.js"; import { startWorkFolderCheckpointer } from "./work-folder-checkpointer.js"; import { logActivity } from "./activity-log.js"; import { assertWorkFolderAccess } from "./work-folder-access.js"; +import { logger } from "../middleware/logger.js"; function signature(entry: WorkTreeEntry | undefined) { return entry ? JSON.stringify([entry.kind, entry.sha256, entry.executable]) : "missing"; @@ -284,7 +285,8 @@ export async function prepareSandboxWorkFolders(input: { await logActivity(db, { companyId: input.companyId, actorType: "agent", actorId: input.agentId, agentId: input.agentId, runId: input.runId, issueId: input.taskId, responsibleUserIdOverride: input.responsibleUserId, action, entityType: "heartbeat_run", entityId: input.runId, - details: { scopes: WORK_FOLDER_SCOPES.filter((scope) => Boolean(folders[scope])), repositories: bindings.length } }); + details: { scopes: WORK_FOLDER_SCOPES.filter((scope) => Boolean(folders[scope])), repositories: bindings.length } }) + .catch((error) => logger.warn({ err: error, runId: input.runId }, "Work-folder activity could not be recorded")); } try { await seedAttachments(); diff --git a/tests/runner-e2e/deployed-work-folders.spec.ts b/tests/runner-e2e/deployed-work-folders.spec.ts index 5cd752d069..4e2daace43 100644 --- a/tests/runner-e2e/deployed-work-folders.spec.ts +++ b/tests/runner-e2e/deployed-work-folders.spec.ts @@ -9,7 +9,10 @@ import { DeployedStackApi, loadDeployedStack } from "./deployed-stack.js"; const stack = loadDeployedStack(); const api = new DeployedStackApi(stack); const folder = (scope: string, ownerId: string) => `/api/companies/${stack.companyId}/work-folders/${scope}/${encodeURIComponent(ownerId)}`; -test.describe.configure({ mode: "serial" }); +test.beforeAll(async () => { + const health = await api.json<{ commit: string }>("/api/health"); + expect(health.commit, "Only exercise the declared deployed candidate").toBe(stack.commit); +}); test("deployed candidate and complete supported adapter inventory", async ({}, info) => { const health = await api.json<{ commit: string }>("/api/health"); @@ -77,16 +80,17 @@ for (const profile of stack.profiles) { "Then complete this task successfully. Do not print credentials or modify unrelated files.", ].join("\n"), }); + await info.attach("task", { contentType: "application/json", body: Buffer.from(JSON.stringify({ profile: profile.id, ...issue })) }); const base = folder("task", issue.id); await pollUntil({ label: `${profile.id} completed run and durable task file`, deadlineAt: Date.now() + 840_000, intervalMs: 5_000, load: async () => ({ issue: await api.json<{ status: string }>(`/api/issues/${issue.id}`), saves: await api.json(`${base}/sync`) }), accept: (state) => state.issue.status === "done" && state.saves.some((save) => !save.active && save.state === "saved" && save.lastSavedAt !== null), - reject: (state) => state.saves.some((save) => save.state === "failed") ? "Work-folder save failed" : undefined, + reject: (state) => state.saves.some((save) => save.state === "failed") ? "Work-folder save failed" + : state.issue.status === "blocked" || state.issue.status === "cancelled" ? `Task ${issue.identifier} ended ${state.issue.status}` : undefined, }); const content = await api.request(`${base}/content?path=acceptance.txt`); expect(content.status).toBe(200); expect(await content.text()).toBe(nonce); - await info.attach("task", { contentType: "application/json", body: Buffer.from(JSON.stringify({ profile: profile.id, ...issue })) }); }); } diff --git a/tests/runner-e2e/playwright.deployed.config.ts b/tests/runner-e2e/playwright.deployed.config.ts index eba8da98f8..b9e6742bbf 100644 --- a/tests/runner-e2e/playwright.deployed.config.ts +++ b/tests/runner-e2e/playwright.deployed.config.ts @@ -8,7 +8,7 @@ if (!output || !path.isAbsolute(output)) throw new Error("PAPERCLIP_DEPLOYED_STA export default defineConfig({ testDir: ".", testMatch: "deployed-work-folders.spec.ts", - fullyParallel: false, workers: 1, retries: 0, timeout: 900_000, + fullyParallel: true, workers: 2, retries: 0, timeout: 900_000, use: { baseURL: stack.baseURL, trace: "off", video: "off" }, // No webServer: every operation reaches the deployed tenant and database. outputDir: path.join(output, "results"), From 2e810127c4fe6d504b1ffa17366415b58f36a2bf Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 7 Sep 2026 14:24:57 -0500 Subject: [PATCH 3/4] fix: preserve restored setup and bind native sync to sandbox home Record checkpoint intent before mutations and permit CI-owned lock resolution when building a staging migrator. Co-Authored-By: Paperclip --- scripts/build-preview-migrator.mjs | 9 +++++- scripts/build-preview-migrator.test.mjs | 24 ++++++++++++++- ...ment-execution-target-capabilities.test.ts | 21 ++++++++++++-- .../__tests__/sandbox-work-folders.test.ts | 29 ++++++++++++++----- .../services/environment-execution-target.ts | 14 +++++++-- server/src/services/sandbox-work-folders.ts | 17 +++++++---- 6 files changed, 93 insertions(+), 21 deletions(-) diff --git a/scripts/build-preview-migrator.mjs b/scripts/build-preview-migrator.mjs index 3d920c5419..f9cc502f20 100644 --- a/scripts/build-preview-migrator.mjs +++ b/scripts/build-preview-migrator.mjs @@ -17,12 +17,19 @@ export function previewIdentity(sha, date, artifactBaseUrl) { baseUrl: `${base.href.replace(/\/$/, "")}/${sha}` }; } +export function assertPreviewSourceClean(repo) { + // Repository policy regenerates the lock in CI for manifest-only branches. + // That generated input is allowed; every tracked source input must still + // match the commit identifying both the app image and migrator artifact. + execFileSync("git", ["diff", "--quiet", "HEAD", "--", ".", ":(exclude)pnpm-lock.yaml"], { cwd: repo }); +} + export function buildPreviewMigrator(outputDirectory, artifactBaseUrl) { const repo = path.resolve(fileURLToPath(new URL("..", import.meta.url))); const git = (...args) => execFileSync("git", args, { cwd: repo, encoding: "utf8" }).trim(); const sha = git("rev-parse", "HEAD"); if (process.env.GITHUB_SHA && process.env.GITHUB_SHA !== sha) throw new Error("Preview checkout differs from the workflow commit"); - git("diff", "--quiet", "HEAD"); + assertPreviewSourceClean(repo); const identity = previewIdentity(sha, new Date(git("show", "-s", "--format=%cI", "HEAD")), artifactBaseUrl); execFileSync("pnpm", ["--filter", "@paperclipai/db...", "build"], { cwd: repo, stdio: "inherit" }); const output = path.resolve(outputDirectory); diff --git a/scripts/build-preview-migrator.test.mjs b/scripts/build-preview-migrator.test.mjs index ac3a0cccbd..d0a737d5ab 100644 --- a/scripts/build-preview-migrator.test.mjs +++ b/scripts/build-preview-migrator.test.mjs @@ -1,6 +1,28 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { previewIdentity } from "./build-preview-migrator.mjs"; +import { execFileSync } from "node:child_process"; +import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { assertPreviewSourceClean, previewIdentity } from "./build-preview-migrator.mjs"; + +test("preview source identity permits CI lock resolution but rejects staged and unstaged source drift", () => { + const repo = mkdtempSync(path.join(os.tmpdir(), "preview-source-")); + const git = (...args) => execFileSync("git", args, { cwd: repo, stdio: "pipe" }); + try { + git("init"); + for (const file of ["pnpm-lock.yaml", "package.json"]) writeFileSync(path.join(repo, file), "initial\n"); + git("add", "."); + git("-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-m", "initial"); + assertPreviewSourceClean(repo); + writeFileSync(path.join(repo, "pnpm-lock.yaml"), "CI resolved\n"); + assertPreviewSourceClean(repo); + writeFileSync(path.join(repo, "package.json"), "source drift\n"); + assert.throws(() => assertPreviewSourceClean(repo)); + git("add", "package.json"); + assert.throws(() => assertPreviewSourceClean(repo)); + } finally { rmSync(repo, { recursive: true, force: true }); } +}); test("preview artifact identity is immutable, namespaced, and ordered by commit time", () => { const sha = "2f42a4968d5761fd62172e35ecf8188195b8d431"; diff --git a/server/src/__tests__/environment-execution-target-capabilities.test.ts b/server/src/__tests__/environment-execution-target-capabilities.test.ts index 5a35dec846..29a660e0a1 100644 --- a/server/src/__tests__/environment-execution-target-capabilities.test.ts +++ b/server/src/__tests__/environment-execution-target-capabilities.test.ts @@ -83,14 +83,14 @@ async function buildSandboxTarget(input: { environment: { id: "env-1", driver: "sandbox", config: { provider: "daytona" } }, leaseId: "lease-1", leaseMetadata: { remoteCwd: "/work" }, - lease: { id: "lease-1", leasePolicy: "reuse_by_environment" } as never, + lease: { id: "lease-1", leasePolicy: "reuse_by_environment", metadata: { remoteCwd: "/work", marker: "preserved" } } as never, environmentRuntime, }); if (target?.kind !== "remote" || target.transport !== "sandbox") { throw new Error("expected a sandbox target"); } - return { target, execute }; + return { target, execute, environmentRuntime }; } describe("resolveEnvironmentExecutionTarget effective capability snapshot", () => { @@ -268,6 +268,23 @@ describe("effective snapshot gates the sync decision", () => { expect(target.runner?.syncOut).toBeTypeOf("function"); }); + it("uses the host-bound home for sync after work folders are prepared without changing the primary workspace", async () => { + const { target, environmentRuntime } = await buildSandboxTarget({ snapshot: FULL_GRANT, supportsSync: true }); + await target.runner!.syncOut!([]); + expect(environmentRuntime.syncOut).toHaveBeenLastCalledWith(expect.objectContaining({ + lease: expect.objectContaining({ metadata: { remoteCwd: "/work", marker: "preserved" } }), + })); + target.workFolderHome = "/home/daytona"; + await target.runner!.syncIn!([]); + await target.runner!.syncOut!([]); + for (const sync of [environmentRuntime.syncIn, environmentRuntime.syncOut]) { + expect(sync).toHaveBeenLastCalledWith(expect.objectContaining({ + lease: expect.objectContaining({ metadata: { remoteCwd: "/home/daytona", marker: "preserved" } }), + })); + } + expect(target.remoteCwd).toBe("/work"); + }); + it("omits the native sync hooks when the snapshot removes a sync verb", async () => { // The snapshot verified inbound sync but not outbound sync. The runner // exposes the sync hooks both-or-neither, so it keeps the base64 fallback. diff --git a/server/src/__tests__/sandbox-work-folders.test.ts b/server/src/__tests__/sandbox-work-folders.test.ts index ee6bdf98ae..cc515fe64a 100644 --- a/server/src/__tests__/sandbox-work-folders.test.ts +++ b/server/src/__tests__/sandbox-work-folders.test.ts @@ -40,7 +40,8 @@ describe("shared sandbox work-folder lifecycle", () => { await fs.symlink("tracked", path.join(source, "link")); await exec("git", ["-C", source, "add", "."]); await exec("git", ["-C", source, "-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-m", "initial"]); - await db.insert(projectWorkspaces).values({ companyId, projectId, name, repoUrl: source, sourceType: "git_repo", isPrimary: name === "repo-one" }); + await db.insert(projectWorkspaces).values({ companyId, projectId, name, repoUrl: source, sourceType: "git_repo", isPrimary: name === "repo-one", + setupCommand: "printf 'initialized\\n' >> .setup-count" }); } }, 60_000); afterAll(async () => { @@ -62,18 +63,27 @@ describe("shared sandbox work-folder lifecycle", () => { runner: { execute: (input) => localTestWorkFolderRunner.execute({ ...input, env: { ...input.env, HOME: home } }) } } }); active.push(run); return run; } - it("keeps successful checkpoints saved when activity logging fails", async () => { + it("retains unaudited edits and retries without misreporting a completed checkpoint", async () => { + const run = await prepare(path.join(root, "activity-failure"), randomUUID()); + await run.flush(); + const [before] = await db.select().from(workFolderRuns).where(eq(workFolderRuns.runId, run.manifest.runId)); + await fs.writeFile(path.join(run.home, "task/activity-proof.txt"), "saved after logging recovers"); const activity = vi.spyOn(activityLog, "logActivity").mockRejectedValue(new Error("activity unavailable")); try { - const run = await prepare(path.join(root, "activity-failure"), randomUUID()); - await fs.writeFile(path.join(run.home, "task/activity-proof.txt"), "saved despite logging failure"); - await run.stop(); active.splice(active.indexOf(run), 1); + await expect(run.stop()).rejects.toThrow("activity unavailable"); const [state] = await db.select().from(workFolderRuns).where(eq(workFolderRuns.runId, run.manifest.runId)); - expect(state?.state).toBe("saved"); - expect(state?.lastSavedAt).not.toBeNull(); + expect(state?.state).toBe("failed"); + expect(state?.lastSavedAt).toEqual(before?.lastSavedAt); const folder = await workFolderService(db, storage).ensure({ companyId, scope: "task", ownerId: taskId }); - expect((await workFolderService(db, storage).list(folder)).files.some((file) => file.path === "activity-proof.txt")).toBe(true); + expect((await workFolderService(db, storage).list(folder)).files.some((file) => file.path === "activity-proof.txt")).toBe(false); + expect(await fs.readFile(path.join(run.home, "task/activity-proof.txt"), "utf8")).toBe("saved after logging recovers"); } finally { activity.mockRestore(); } + await run.stop(); active.splice(active.indexOf(run), 1); + const [saved] = await db.select().from(workFolderRuns).where(eq(workFolderRuns.runId, run.manifest.runId)); + expect(saved?.state).toBe("saved"); + expect(saved!.lastSavedAt!.getTime()).toBeGreaterThan(before!.lastSavedAt!.getTime()); + const folder = await workFolderService(db, storage).ensure({ companyId, scope: "task", ownerId: taskId }); + expect((await workFolderService(db, storage).list(folder)).files.some((file) => file.path === "activity-proof.txt")).toBe(true); }, 120_000); it("reuses clones and restores saved unpushed work, staged changes, and task files after losing the sandbox", async () => { @@ -85,6 +95,7 @@ describe("shared sandbox work-folder lifecycle", () => { expect(first.primaryRepo).toBe(path.join(home, "repos/repo-one")); await fs.writeFile(path.join(home, "task/report.md"), "durable task file"); const repo = first.primaryRepo; + expect(await fs.readFile(path.join(repo, ".setup-count"), "utf8")).toBe("initialized\n"); await fs.writeFile(path.join(repo, "tracked"), "committed\n"); await exec("git", ["-C", repo, "add", "."]); await exec("git", ["-C", repo, "-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-m", "unpushed"]); @@ -95,10 +106,12 @@ describe("shared sandbox work-folder lifecycle", () => { await fs.writeFile(path.join(repo, "untracked"), "untracked\n"); await first.stop(); active.splice(active.indexOf(first), 1); const warm = await prepare(home, randomUUID(), leaseId); + expect(await fs.readFile(path.join(repo, ".setup-count"), "utf8")).toBe("initialized\n"); expect(await fs.readFile(path.join(repo, "tracked"), "utf8")).toBe("unstaged\n"); await warm.stop(); active.splice(active.indexOf(warm), 1); await fs.rm(home, { recursive: true }); const restored = await prepare(path.join(root, "replacement"), randomUUID()); + expect(await fs.readFile(path.join(restored.primaryRepo, ".setup-count"), "utf8")).toBe("initialized\n"); expect(await fs.readFile(path.join(restored.home, "task/report.md"), "utf8")).toBe("durable task file"); expect((await exec("git", ["-C", restored.primaryRepo, "rev-parse", "HEAD"])).stdout.trim()).toBe(expectedHead); expect((await exec("git", ["-C", restored.primaryRepo, "show", ":tracked"])).stdout).toBe("staged\n"); diff --git a/server/src/services/environment-execution-target.ts b/server/src/services/environment-execution-target.ts index d1aefd34f2..8166e5f925 100644 --- a/server/src/services/environment-execution-target.ts +++ b/server/src/services/environment-execution-target.ts @@ -4,6 +4,7 @@ import { adapterSupportsRemoteManagedEnvironments } from "@paperclipai/shared"; import { adapterExecutionTargetToRemoteSpec, type AdapterExecutionTarget, + type AdapterSandboxExecutionTarget, type SandboxLeaseAcquisition, } from "@paperclipai/adapter-utils/execution-target"; import type { DuplexObservabilityRecorder } from "@paperclipai/adapter-utils/duplex-observability"; @@ -359,7 +360,13 @@ export async function resolveEnvironmentExecutionTarget(input: { } } - return { + // The coordinator binds the natural home after acquiring this target. Read + // that host-owned binding at sync time instead of capturing the old cwd. + // Provider path/symlink confinement still applies to every transfer. + const syncLease = () => target.workFolderHome + ? { ...input.lease!, metadata: { ...input.lease!.metadata, remoteCwd: target.workFolderHome } } + : input.lease!; + const target: AdapterSandboxExecutionTarget = { kind: "remote", transport: "sandbox", providerKey: parsed.config.provider, @@ -595,13 +602,13 @@ export async function resolveEnvironmentExecutionTarget(input: { syncIn: (operations) => input.environmentRuntime!.syncIn({ environment: input.environment as Environment, - lease: input.lease!, + lease: syncLease(), operations, }), syncOut: (operations) => input.environmentRuntime!.syncOut({ environment: input.environment as Environment, - lease: input.lease!, + lease: syncLease(), operations, }), } @@ -625,6 +632,7 @@ export async function resolveEnvironmentExecutionTarget(input: { } : undefined, }; + return target; } if ( diff --git a/server/src/services/sandbox-work-folders.ts b/server/src/services/sandbox-work-folders.ts index 5c25f848c0..a583e9ac0b 100644 --- a/server/src/services/sandbox-work-folders.ts +++ b/server/src/services/sandbox-work-folders.ts @@ -16,7 +16,6 @@ import { workFolderRepositoryService } from "./work-folder-repositories.js"; import { startWorkFolderCheckpointer } from "./work-folder-checkpointer.js"; import { logActivity } from "./activity-log.js"; import { assertWorkFolderAccess } from "./work-folder-access.js"; -import { logger } from "../middleware/logger.js"; function signature(entry: WorkTreeEntry | undefined) { return entry ? JSON.stringify([entry.kind, entry.sha256, entry.executable]) : "missing"; @@ -228,12 +227,14 @@ export async function prepareSandboxWorkFolders(input: { const root = path.posix.join(paths.repos!, binding.name); const probe = await target.runner!.execute({ command: "git", args: ["-C", root, "rev-parse", "--git-dir"], bypassSession: true, timeoutMs: 10_000 }); const freshCheckout = probe.exitCode !== 0; + let restoredCheckout = false; if (freshCheckout) { // Publish the checkout directory only after every restore object or // clone step completes. An interrupted attempt cannot masquerade as a // reusable checkout merely because it contains a .git directory. const temporary = path.posix.join(staging, `repo-${binding.id}-${randomUUID()}`); const restored = await repositories.restore(binding, temporary, staging); + restoredCheckout = restored; if (!restored) { const auth = await resolveGitAuth(workspace.repoUrl!); const result = await target.runner!.execute({ command: "git", args: [...(auth?.configArgs ?? []), "clone", "--no-hardlinks", @@ -264,7 +265,9 @@ export async function prepareSandboxWorkFolders(input: { } await transport.moveRoot(temporary, root); } - if ((!binding.setupComplete || freshCheckout) && workspace.setupCommand) { + // A complete checkpoint already contains the setup's durable outputs. + // Replacing the sandbox must not repeat completed project setup. + if ((!binding.setupComplete || (freshCheckout && !restoredCheckout)) && workspace.setupCommand) { const setup = await target.runner!.execute({ command: "sh", args: ["-c", workspace.setupCommand], cwd: root, bypassSession: true, timeoutMs: 300_000 }); if (setup.exitCode !== 0 || setup.timedOut) throw new Error(`Repository ${binding.name} setup failed`); } @@ -285,17 +288,19 @@ export async function prepareSandboxWorkFolders(input: { await logActivity(db, { companyId: input.companyId, actorType: "agent", actorId: input.agentId, agentId: input.agentId, runId: input.runId, issueId: input.taskId, responsibleUserIdOverride: input.responsibleUserId, action, entityType: "heartbeat_run", entityId: input.runId, - details: { scopes: WORK_FOLDER_SCOPES.filter((scope) => Boolean(folders[scope])), repositories: bindings.length } }) - .catch((error) => logger.warn({ err: error, runId: input.runId }, "Work-folder activity could not be recorded")); + details: { phase: "started", scopes: WORK_FOLDER_SCOPES.filter((scope) => Boolean(folders[scope])), repositories: bindings.length } }); } try { + // Record intent before mutations. An unavailable audit store blocks new + // work instead of turning an already completed save into a false failure. + // The run's persisted state/lastSavedAt records checkpoint completion. + await recordCheckpoint("work_folder.prepared"); await seedAttachments(); await importAgentFiles(); // A resumed sandbox can hold edits newer than its last completed checkpoint. if (previous) for (const scope of WORK_FOLDER_SCOPES) await outgoing(scope); for (const scope of WORK_FOLDER_SCOPES) await incoming(scope); await prepareRepositories(); - await recordCheckpoint("work_folder.prepared"); await saveState("starting"); if (previous?.refreshRequested) await db.update(workFolderRuns).set({ refreshRequested: false }) .where(eq(workFolderRuns.runId, previous.runId)); @@ -306,10 +311,10 @@ export async function prepareSandboxWorkFolders(input: { const checkpointer = startWorkFolderCheckpointer({ async checkpoint() { await assertBindings(); + await recordCheckpoint("work_folder.checkpoint"); await saveState("saving"); for (const scope of WORK_FOLDER_SCOPES) await outgoing(scope); for (const { binding, root } of bindings) await repositories.checkpoint(binding, root); - await recordCheckpoint("work_folder.checkpoint"); await saveState("saved"); }, async onError() { await saveState("failed", "Files could not be saved; the sandbox must be retained for recovery"); }, From 6651964a4c6f756f1b585a19af0df693318b56bc Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 7 Sep 2026 14:25:53 -0500 Subject: [PATCH 4/4] test: verify warm repo state and real staging checkpoint intervals Co-Authored-By: Paperclip --- .../runner-e2e/deployed-work-folders.spec.ts | 63 ++++++++++++++++++- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/tests/runner-e2e/deployed-work-folders.spec.ts b/tests/runner-e2e/deployed-work-folders.spec.ts index 4e2daace43..3a381595bf 100644 --- a/tests/runner-e2e/deployed-work-folders.spec.ts +++ b/tests/runner-e2e/deployed-work-folders.spec.ts @@ -67,7 +67,8 @@ for (const [scope, owner] of [["task", stack.taskId], ["agent", stack.agentId], } for (const profile of stack.profiles) { - test(`${profile.id} creates durable work from the actual sandbox home`, async ({}, info) => { + test(`${profile.id} preserves task-specific repo and file state across cold and warm runs`, async ({}, info) => { + test.setTimeout(1_800_000); const nonce = randomUUID(); const issue = await api.json<{ id: string; identifier: string }>(`/api/companies/${stack.companyId}/issues`, "POST", { title: `Work folder acceptance ${profile.id} ${nonce}`, projectId: stack.projectId, @@ -76,13 +77,18 @@ for (const profile of stack.profiles) { "Perform this sandbox acceptance task using real filesystem tools.", "Verify cwd equals the operating-system HOME and task, agent, user, project, repos, .codex, .cache are directories beneath it.", "Verify repos contains at least two independent Git checkouts. Fail the task with the actual error if either assertion fails.", + "In each repo, assert .acceptance-owner does not exist (another task must not share this checkout).", + `In each repo write '${nonce}' without a newline to .acceptance-owner, git add ONLY that file, and create a local commit using git -c user.name=Acceptance -c user.email=acceptance@example.invalid commit -m acceptance. Do not push.`, + "Save each repo's HEAD to $HOME/task/head-.txt.", + "In each repo write 'staged' without newline to .acceptance-state, git add ONLY that file, then replace its working-tree content with 'unstaged' without newline. Write 'untracked' without newline to .acceptance-untracked and leave it untracked.", + "If .acceptance-setup-count exists, assert it has exactly one line. Never run setup yourself.", `Write exactly '${nonce}' without a newline into $HOME/task/acceptance.txt and $HOME/agent/acceptance-${nonce}.txt.`, "Then complete this task successfully. Do not print credentials or modify unrelated files.", ].join("\n"), }); await info.attach("task", { contentType: "application/json", body: Buffer.from(JSON.stringify({ profile: profile.id, ...issue })) }); const base = folder("task", issue.id); - await pollUntil({ label: `${profile.id} completed run and durable task file`, deadlineAt: Date.now() + 840_000, + const cold = await pollUntil({ label: `${profile.id} completed run and durable task file`, deadlineAt: Date.now() + 840_000, intervalMs: 5_000, load: async () => ({ issue: await api.json<{ status: string }>(`/api/issues/${issue.id}`), saves: await api.json(`${base}/sync`) }), @@ -92,5 +98,58 @@ for (const profile of stack.profiles) { }); const content = await api.request(`${base}/content?path=acceptance.txt`); expect(content.status).toBe(200); expect(await content.text()).toBe(nonce); + const coldRunIds = new Set(cold.saves.map((save) => save.runId)); + await api.json(`/api/issues/${issue.id}`, "PATCH", { status: "todo", description: [ + "Continue this sandbox acceptance task. This is a warm run; inspect the existing work without repairing it.", + "Verify cwd equals HOME and all seven directories still exist.", + `Assert $HOME/task/acceptance.txt and every repo's committed HEAD:.acceptance-owner equal '${nonce}'.`, + "For each repo assert HEAD equals the saved task/head-.txt, index :.acceptance-state equals 'staged', working .acceptance-state equals 'unstaged', and .acceptance-untracked equals 'untracked' and remains untracked.", + "If .acceptance-setup-count exists, assert exactly one line. Fail with the actual discrepancy; do not recreate missing state or rerun setup.", + `Write exactly '${nonce}' without a newline to $HOME/task/warm.txt, then complete the task.`, + ].join("\n") }); + const warm = await pollUntil({ label: `${profile.id} warm run preserves saved work`, deadlineAt: Date.now() + 840_000, + intervalMs: 5_000, + load: async () => ({ issue: await api.json<{ status: string }>(`/api/issues/${issue.id}`), + saves: (await api.json(`${base}/sync`)).filter((save) => !coldRunIds.has(save.runId)) }), + accept: (state) => state.issue.status === "done" && state.saves.some((save) => !save.active && save.state === "saved" && save.lastSavedAt !== null), + reject: (state) => state.saves.some((save) => save.state === "failed") ? "Warm save failed" + : ["blocked", "cancelled"].includes(state.issue.status) ? `Warm task ended ${state.issue.status}` : undefined, + }); + const warmContent = await api.request(`${base}/content?path=warm.txt`); + expect(warmContent.status).toBe(200); expect(await warmContent.text()).toBe(nonce); + await info.attach("cold-and-warm-checkpoints", { contentType: "application/json", body: Buffer.from(JSON.stringify({ cold: cold.saves, warm: warm.saves })) }); }); } + +test("saves during two real 180-second intervals and flushes the final edit", async ({}, info) => { + const profile = stack.profiles.find((entry) => entry.id === "legacy-codex")!; + const nonce = randomUUID(); + const issue = await api.json<{ id: string; identifier: string }>(`/api/companies/${stack.companyId}/issues`, "POST", { + title: `Real checkpoint intervals ${nonce}`, projectId: stack.projectId, assigneeAgentId: profile.agentId, status: "todo", + description: [ + "Run a real timed persistence acceptance test. Execute the following shell sequence and wait for it to finish, keeping this task in progress throughout both sleeps. Use a tool timeout of at least 420 seconds, or poll its session until it exits. Do not shorten either sleep or mark the task complete early.", + `printf '${nonce}:one' > "$HOME/task/interval.txt"; sleep 190; printf '${nonce}:two' > "$HOME/task/interval.txt"; sleep 190; printf '${nonce}:final' > "$HOME/task/interval.txt"`, + "After the command exits successfully, complete the task. Do not print credentials.", + ].join("\n"), + }); + await info.attach("task", { contentType: "application/json", body: Buffer.from(JSON.stringify(issue)) }); + const base = folder("task", issue.id); + const observations: Array<{ observedAt: string; phase: string; saves: WorkFolderSyncStatus[] }> = []; + for (const phase of ["one", "two", "final"]) { + const snapshot = await pollUntil({ label: `durable interval ${phase}`, deadlineAt: Date.now() + (phase === "one" ? 840_000 : 300_000), intervalMs: 3_000, + load: async () => { + const response = await api.request(`${base}/content?path=interval.txt`); + return { content: response.ok ? await response.text() : null, saves: await api.json(`${base}/sync`), + issue: await api.json<{ status: string }>(`/api/issues/${issue.id}`) }; + }, + accept: (state) => state.content === `${nonce}:${phase}` && state.saves.some((save) => save.state === "saved" && save.lastSavedAt !== null && save.active === (phase !== "final")), + reject: (state) => state.saves.some((save) => save.state === "failed") ? "Timed checkpoint failed" + : ["blocked", "cancelled"].includes(state.issue.status) ? `Timed task ended ${state.issue.status}` : undefined, + }); + observations.push({ observedAt: new Date().toISOString(), phase, saves: snapshot.saves }); + } + const first = observations[0]!.saves.find((save) => save.active)!; + const second = observations[1]!.saves.find((save) => save.runId === first.runId)!; + expect(Date.parse(second.lastSavedAt!) - Date.parse(first.lastSavedAt!)).toBeGreaterThanOrEqual(170_000); + await info.attach("real-checkpoint-intervals", { contentType: "application/json", body: Buffer.from(JSON.stringify(observations, null, 2)) }); +});