diff --git a/server/src/__tests__/sandbox-work-folders.test.ts b/server/src/__tests__/sandbox-work-folders.test.ts index 593c3531a5..69a38b5637 100644 --- a/server/src/__tests__/sandbox-work-folders.test.ts +++ b/server/src/__tests__/sandbox-work-folders.test.ts @@ -111,9 +111,17 @@ describe("shared sandbox work-folder lifecycle", () => { await run.flush(); const [before] = await db.select().from(workFolderRuns).where(eq(workFolderRuns.runId, run.manifest.runId)); await fs.writeFile(path.join(run.home, "task/activity-proof.txt"), "saved after logging recovers"); + const beforeCompletion = vi.fn(async () => { + const [state] = await db.select().from(workFolderRuns).where(eq(workFolderRuns.runId, run.manifest.runId)); + expect(state?.state).toBe("saved"); + expect(state?.manifest.finalCheckpointAt).toBeUndefined(); + const folder = await workFolderService(db, storage).ensure({ companyId, scope: "task", ownerId: taskId }); + expect((await workFolderService(db, storage).list(folder)).files.some((file) => file.path === "activity-proof.txt")).toBe(true); + }); const activity = vi.spyOn(activityLog, "logActivity").mockRejectedValue(new Error("activity unavailable")); try { - await expect(run.stop()).rejects.toThrow("activity unavailable"); + await expect(run.stop(beforeCompletion)).rejects.toThrow("activity unavailable"); + expect(beforeCompletion).not.toHaveBeenCalled(); const [state] = await db.select().from(workFolderRuns).where(eq(workFolderRuns.runId, run.manifest.runId)); expect(state?.state).toBe("failed"); expect(state?.lastSavedAt).toEqual(before?.lastSavedAt); @@ -121,9 +129,11 @@ describe("shared sandbox work-folder lifecycle", () => { expect((await workFolderService(db, storage).list(folder)).files.some((file) => file.path === "activity-proof.txt")).toBe(false); expect(await fs.readFile(path.join(run.home, "task/activity-proof.txt"), "utf8")).toBe("saved after logging recovers"); } finally { activity.mockRestore(); } - await run.stop(); active.splice(active.indexOf(run), 1); + await run.stop(beforeCompletion); active.splice(active.indexOf(run), 1); + expect(beforeCompletion).toHaveBeenCalledTimes(1); const [saved] = await db.select().from(workFolderRuns).where(eq(workFolderRuns.runId, run.manifest.runId)); expect(saved?.state).toBe("saved"); + expect(saved?.manifest.finalCheckpointAt).toBeTruthy(); expect(saved!.lastSavedAt!.getTime()).toBeGreaterThan(before!.lastSavedAt!.getTime()); const folder = await workFolderService(db, storage).ensure({ companyId, scope: "task", ownerId: taskId }); expect((await workFolderService(db, storage).list(folder)).files.some((file) => file.path === "activity-proof.txt")).toBe(true); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 4a51d18b3a..deabf14739 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -18040,6 +18040,7 @@ export function heartbeatService( let sandboxWorkFolders: Awaited> | null = null; let workFolderSaveFailed = false; let nativeTaskSessionPersisted = false; + let beforeWorkFolderCompletion: (() => Promise) | undefined; let workFolderLeaseId: string | null = null; let nativeSessionResumeScheduled = false; let nativeWorkspaceFinalizeScheduled = false; @@ -21612,35 +21613,38 @@ export function heartbeatService( // finalize row. if (sandboxWorkFolders) { if (adapterResult.nativeFinalization && taskKey) { - // Publish the resumable identity before the final file-save - // barrier lets native reconciliation expose terminal status. - // A user can start the next turn as soon as completion appears. - const sessionState = resolveNextSessionState({ - adapterType: agent.adapterType, - codec: sessionCodec, - adapterResult, - outcome: adapterResult.nativeFinalization.terminal.runTerminalState === "succeeded" - ? "succeeded" : "failed", - previousParams: previousSessionParams, - previousDisplayId: runtimeForAdapter.sessionDisplayId, - previousLegacySessionId: runtimeForAdapter.sessionId, - }); - await upsertTaskSession({ - companyId: agent.companyId, - agentId: agent.id, - adapterType: agent.adapterType, - taskKey, - sessionParamsJson: attachPaperclipSessionMetadataToSessionParams( - sessionState.params, configuredModel, sessionConfigMetadata, - ), - sessionDisplayId: sessionState.displayId, - lastRunId: run.id, - lastError: adapterResult.errorMessage ?? null, - }); - nativeTaskSessionPersisted = true; + const nativeTerminal = adapterResult.nativeFinalization.terminal.runTerminalState; + beforeWorkFolderCompletion = async () => { + if (nativeTaskSessionPersisted) return; + // The coordinator invokes this after saving data and before + // publishing the barrier that allows terminal reconciliation. + const sessionState = resolveNextSessionState({ + adapterType: agent.adapterType, + codec: sessionCodec, + adapterResult, + outcome: nativeTerminal === "succeeded" + ? "succeeded" : "failed", + previousParams: previousSessionParams, + previousDisplayId: runtimeForAdapter.sessionDisplayId, + previousLegacySessionId: runtimeForAdapter.sessionId, + }); + await upsertTaskSession({ + companyId: agent.companyId, + agentId: agent.id, + adapterType: agent.adapterType, + taskKey, + sessionParamsJson: attachPaperclipSessionMetadataToSessionParams( + sessionState.params, configuredModel, sessionConfigMetadata, + ), + sessionDisplayId: sessionState.displayId, + lastRunId: run.id, + lastError: adapterResult.errorMessage ?? null, + }); + nativeTaskSessionPersisted = true; + }; } workFolderSaveFailed = true; - await sandboxWorkFolders.stop(); + await sandboxWorkFolders.stop(beforeWorkFolderCompletion); sandboxWorkFolders = null; workFolderSaveFailed = false; } @@ -22854,7 +22858,7 @@ export function heartbeatService( } } finally { if (sandboxWorkFolders) { - try { await sandboxWorkFolders.stop(); workFolderSaveFailed = false; } + try { await sandboxWorkFolders.stop(beforeWorkFolderCompletion); workFolderSaveFailed = false; } catch (error) { workFolderSaveFailed = true; logger.error({ err: error, runId: run.id }, "Work folder save failed; retaining sandbox for recovery"); } } let latestRun = await getRun(run.id).catch(() => null); diff --git a/server/src/services/sandbox-work-folders.ts b/server/src/services/sandbox-work-folders.ts index 8d007e025b..be1c31f96e 100644 --- a/server/src/services/sandbox-work-folders.ts +++ b/server/src/services/sandbox-work-folders.ts @@ -321,7 +321,7 @@ export async function prepareSandboxWorkFolders(input: { return { manifest, home, identityChanged, primaryRepo: bindings.find(({ binding }) => manifest.repositories.some((repo) => repo.bindingId === binding.id && repo.primary))?.root ?? bindings[0]?.root ?? paths.task!, env: { HOME: home, AGENT_HOME: paths.agent!, PAPERCLIP_PRIMARY_REPO: bindings.find(({ binding }) => manifest.repositories.some((repo) => repo.bindingId === binding.id && repo.primary))?.root ?? bindings[0]?.root ?? paths.task!, PAPERCLIP_TASK_DIR: paths.task!, PAPERCLIP_AGENT_DIR: paths.agent!, PAPERCLIP_USER_DIR: paths.user!, PAPERCLIP_PROJECT_DIR: paths.project!, PAPERCLIP_REPOS_DIR: paths.repos! }, - flush: checkpointer.flush, stop: async () => { + flush: checkpointer.flush, stop: async (beforeCompletion?: () => Promise) => { await checkpointer.stop(); const [run] = await db.select({ refreshRequested: workFolderRuns.refreshRequested }).from(workFolderRuns) .where(eq(workFolderRuns.runId, input.runId)); @@ -333,6 +333,9 @@ export async function prepareSandboxWorkFolders(input: { await db.update(workFolderRuns).set({ refreshRequested: false, baselines, updatedAt: new Date() }) .where(eq(workFolderRuns.runId, input.runId)); } + // Native resume identity must be published after the data is durable, + // but before completion can release a new turn onto this sandbox. + await beforeCompletion?.(); manifest.finalCheckpointAt = new Date().toISOString(); await saveState("saved"); } }; diff --git a/tests/runner-e2e/work-folder-acceptance-prompts.ts b/tests/runner-e2e/work-folder-acceptance-prompts.ts index 522266e0cf..98e2bd08f8 100644 --- a/tests/runner-e2e/work-folder-acceptance-prompts.ts +++ b/tests/runner-e2e/work-folder-acceptance-prompts.ts @@ -41,7 +41,7 @@ export function repoAcceptancePrompt(nonce: string, warm: boolean): string { return [ "Execute this exact acceptance shell script from your initial working directory in one tool call. Use real filesystem tools; do not simulate its result.", warm ? "This must reuse the same warm sandbox. Do not repair, recreate, or reset missing state." : "This creates disposable local commits and staged, unstaged, and untracked test files. Do not push.", - 'If any assertion fails, stop and PATCH the task with status "blocked" and unblockDescriptor {"owner":"board","action":"Investigate the failed acceptance assertion"}, including the actual error in your comment. Otherwise mark the Paperclip task done after the script succeeds.', + 'If any assertion fails, stop and PATCH the task with status "blocked" and unblockDescriptor {"owner":{"agentId":""},"action":"Investigate the failed acceptance assertion"}, replacing the placeholder with your agent ID and including the actual error in your comment. Otherwise mark the Paperclip task done after the script succeeds.', "```sh", repoAcceptanceScript(nonce, warm), "```", ].join("\n"); }