From 5a2ed9196306fa96782146f19679d47b809187ae Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 7 Sep 2026 16:07:46 -0500 Subject: [PATCH] Publish warm session before native sandbox completion Wait for the final work-folder checkpoint before background reconciliation can finalize a sandbox run. Persist its resumable task identity before exposing completion, and avoid late cleanup overwriting a newer turn. Co-Authored-By: Paperclip --- .../native-finalization-recovery.test.ts | 24 ++++++++++++++ server/src/services/heartbeat.ts | 32 ++++++++++++++++++- .../native-finalization-reconciler.ts | 12 +++++++ .../work-folder-acceptance-prompts.ts | 2 +- 4 files changed, 68 insertions(+), 2 deletions(-) diff --git a/server/src/__tests__/native-finalization-recovery.test.ts b/server/src/__tests__/native-finalization-recovery.test.ts index 0f3ef6a0d6..3c688f9119 100644 --- a/server/src/__tests__/native-finalization-recovery.test.ts +++ b/server/src/__tests__/native-finalization-recovery.test.ts @@ -12,6 +12,7 @@ import { nativeRunResults, statusDecisions, workAssessments, + workFolderRuns, workspaceOperations, } from "@paperclipai/db"; import { @@ -208,6 +209,29 @@ describe("P6-16/P6-25/P6-28 native finalization recovery", () => { await temporary.cleanup(); }); + it("does not publish native completion before the sandbox's final durable save", async () => { + const manifest = { version: 1 as const, companyId, runId, agentId, taskId: issueId, + responsibleUserId: null, projectId: null, leaseId: runId, sandboxKey: runId, + home: "/home/sandbox", folders: {}, repositories: [] }; + await db.insert(workFolderRuns).values({ runId, companyId, manifest }); + try { + for (const state of ["starting", "saving", "saved", "failed"] as const) { + await db.update(workFolderRuns).set({ state, lastSavedAt: new Date() }) + .where(eq(workFolderRuns.runId, runId)); + await expect(reconcileNativeFinalizations(db, [runId])).resolves.toEqual([]); + } + // A failed refresh/save cannot reuse an older final-save marker. + await db.update(workFolderRuns).set({ manifest: { ...manifest, finalCheckpointAt: new Date().toISOString() } }) + .where(eq(workFolderRuns.runId, runId)); + await expect(reconcileNativeFinalizations(db, [runId])).resolves.toEqual([]); + const run = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + expect(run[0]?.status).toBe("running"); + await expect(db.select().from(statusDecisions).where(eq(statusDecisions.issueId, issueId))).resolves.toHaveLength(0); + } finally { + await db.delete(workFolderRuns).where(eq(workFolderRuns.runId, runId)); + } + }); + it("fails closed into bounded named recovery without consulting the live flag or falling back", async () => { await expect(reconcileNativeFinalizations(db, [runId])).resolves.toEqual([ expect.objectContaining({ phase: "retryable_failure", failureCode: "native_finalization_invalid" }), diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index f9750de3cb..4a51d18b3a 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -18039,6 +18039,7 @@ export function heartbeatService( let runScratch: HeartbeatRunScratch | null = null; let sandboxWorkFolders: Awaited> | null = null; let workFolderSaveFailed = false; + let nativeTaskSessionPersisted = false; let workFolderLeaseId: string | null = null; let nativeSessionResumeScheduled = false; let nativeWorkspaceFinalizeScheduled = false; @@ -21610,6 +21611,34 @@ export function heartbeatService( // rather than silently leaving dependents stranded behind a missing // finalize row. if (sandboxWorkFolders) { + if (adapterResult.nativeFinalization && taskKey) { + // Publish the resumable identity before the final file-save + // barrier lets native reconciliation expose terminal status. + // A user can start the next turn as soon as completion appears. + const sessionState = resolveNextSessionState({ + adapterType: agent.adapterType, + codec: sessionCodec, + adapterResult, + outcome: adapterResult.nativeFinalization.terminal.runTerminalState === "succeeded" + ? "succeeded" : "failed", + previousParams: previousSessionParams, + previousDisplayId: runtimeForAdapter.sessionDisplayId, + previousLegacySessionId: runtimeForAdapter.sessionId, + }); + await upsertTaskSession({ + companyId: agent.companyId, + agentId: agent.id, + adapterType: agent.adapterType, + taskKey, + sessionParamsJson: attachPaperclipSessionMetadataToSessionParams( + sessionState.params, configuredModel, sessionConfigMetadata, + ), + sessionDisplayId: sessionState.displayId, + lastRunId: run.id, + lastError: adapterResult.errorMessage ?? null, + }); + nativeTaskSessionPersisted = true; + } workFolderSaveFailed = true; await sandboxWorkFolders.stop(); sandboxWorkFolders = null; @@ -22318,7 +22347,7 @@ export function heartbeatService( }, normalizedUsage, ); - if (taskKey) { + if (taskKey && !nativeTaskSessionPersisted) { if ( adapterResult.clearSession || (!nextSessionState.params && !nextSessionState.displayId) @@ -22617,6 +22646,7 @@ export function heartbeatService( if ( taskKey && + !nativeTaskSessionPersisted && (previousSessionParams || previousSessionDisplayId || taskSession) ) { await upsertTaskSession({ diff --git a/server/src/services/native-runtime/native-finalization-reconciler.ts b/server/src/services/native-runtime/native-finalization-reconciler.ts index 294a92b985..cc3251bc0e 100644 --- a/server/src/services/native-runtime/native-finalization-reconciler.ts +++ b/server/src/services/native-runtime/native-finalization-reconciler.ts @@ -12,6 +12,7 @@ import { statusDecisionEffects, statusDecisions, workAssessments, + workFolderRuns, workspaceOperations, } from "@paperclipai/db"; import { finalizeNativeRun, recordNativeFinalizationFailure } from "./native-run-finalizer.js"; @@ -404,6 +405,8 @@ export async function reconcileNativeFinalizations( assessmentId: nativeRunFinalizations.assessmentId, decisionId: nativeRunFinalizations.decisionId, runnerProfileJson: heartbeatRuns.runnerProfileJson, + workFolderState: workFolderRuns.state, + workFolderManifest: workFolderRuns.manifest, }) .from(heartbeatRuns) .innerJoin(nativeRunFinalizations, eq(nativeRunFinalizations.runId, heartbeatRuns.id)) @@ -411,6 +414,10 @@ export async function reconcileNativeFinalizations( eq(issues.id, nativeRunFinalizations.issueId), eq(issues.companyId, heartbeatRuns.companyId), )) + .leftJoin(workFolderRuns, and( + eq(workFolderRuns.runId, heartbeatRuns.id), + eq(workFolderRuns.companyId, heartbeatRuns.companyId), + )) .where(and( eq(heartbeatRuns.runtimeMode, "native"), isNotNull(nativeRunFinalizations.resultId), @@ -428,6 +435,11 @@ export async function reconcileNativeFinalizations( )); const results = []; for (const row of rows) { + // A scoped sandbox's final flush is its durability barrier. A periodic + // save or the old host workspace directory cannot substitute for it. + // The live executor also publishes its next-turn session before this + // barrier, so completion cannot race a warm restart with a fresh identity. + if (row.workFolderManifest && (row.workFolderState !== "saved" || !row.workFolderManifest.finalCheckpointAt)) continue; const pendingEffects = row.decisionId ? await db.select({ id: statusDecisionEffects.id }).from(statusDecisionEffects).where(and( eq(statusDecisionEffects.companyId, row.companyId), diff --git a/tests/runner-e2e/work-folder-acceptance-prompts.ts b/tests/runner-e2e/work-folder-acceptance-prompts.ts index 593a7ce50d..522266e0cf 100644 --- a/tests/runner-e2e/work-folder-acceptance-prompts.ts +++ b/tests/runner-e2e/work-folder-acceptance-prompts.ts @@ -41,7 +41,7 @@ export function repoAcceptancePrompt(nonce: string, warm: boolean): string { return [ "Execute this exact acceptance shell script from your initial working directory in one tool call. Use real filesystem tools; do not simulate its result.", warm ? "This must reuse the same warm sandbox. Do not repair, recreate, or reset missing state." : "This creates disposable local commits and staged, unstaged, and untracked test files. Do not push.", - "If any assertion fails, mark the task blocked with the actual error. Otherwise mark the Paperclip task done after the script succeeds.", + 'If any assertion fails, stop and PATCH the task with status "blocked" and unblockDescriptor {"owner":"board","action":"Investigate the failed acceptance assertion"}, including the actual error in your comment. Otherwise mark the Paperclip task done after the script succeeds.', "```sh", repoAcceptanceScript(nonce, warm), "```", ].join("\n"); }