From a24d331c414a28f8d19627295d96d680dbd81848 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 9 Sep 2026 00:06:05 -0500 Subject: [PATCH] test: require explicit final checkpoints in warm qualification Expose the existing finalization timestamp through the scoped sync API and reject periodic-only or out-of-run saves. Co-Authored-By: Paperclip --- doc/sandbox-work-folders.md | 5 +- packages/shared/src/work-folders.ts | 2 + .../work-folder-route-storage.test.ts | 18 +++++++ server/src/routes/work-folders.ts | 1 + tests/runner-e2e/warm-workspace.test.ts | 48 +++++++++++++++++++ tests/runner-e2e/warm-workspace.ts | 18 +++++++ 6 files changed, 91 insertions(+), 1 deletion(-) diff --git a/doc/sandbox-work-folders.md b/doc/sandbox-work-folders.md index 63e64a4872..3772c0c1a1 100644 --- a/doc/sandbox-work-folders.md +++ b/doc/sandbox-work-folders.md @@ -361,7 +361,10 @@ at version 2 so existing native session backups remain restorable. Warm continuity qualification reads the full run record before selecting the persistence contract; company run listings omit the scoped-folder manifest. Each completed scoped turn must have its own successful final save before the -harness reads cached bytes. Legacy host-workspace fallback is used only when +harness reads cached bytes. The sync API exposes `finalCheckpointAt` from the +run manifest, distinct from periodic `lastSavedAt`. Qualification requires a +successful terminal run and finalization/save timestamps within that run; a +periodic save cannot substitute for completed finalization. Legacy host-workspace fallback is used only when the full run record has no scoped manifest. The three-turn fixture supplies a shell script that compares exact bytes before diff --git a/packages/shared/src/work-folders.ts b/packages/shared/src/work-folders.ts index 17cfd0e4f4..9343fb7e68 100644 --- a/packages/shared/src/work-folders.ts +++ b/packages/shared/src/work-folders.ts @@ -37,6 +37,8 @@ export interface WorkFolderSyncStatus { agentId?: string; state: "starting" | "saved" | "saving" | "failed"; lastSavedAt: string | null; + /** Successful run finalization; absent on older servers and periodic-only saves. */ + finalCheckpointAt?: string | null; error: string | null; refreshRequested: boolean; active: boolean; diff --git a/server/src/__tests__/work-folder-route-storage.test.ts b/server/src/__tests__/work-folder-route-storage.test.ts index fd4ed7a296..00dfe38d8c 100644 --- a/server/src/__tests__/work-folder-route-storage.test.ts +++ b/server/src/__tests__/work-folder-route-storage.test.ts @@ -66,3 +66,21 @@ it.each([ expect.objectContaining({ runId: "saved-run", state: priorState, lastSavedAt: "2026-09-08T12:00:00.000Z" }), ]); }); + +it.each([undefined, "2026-09-09T04:00:00.000Z"])( + "distinguishes periodic saves from explicit finalization (%s)", + async (finalCheckpointAt) => { + const saved = { + folderRun: { runId: "run", manifest: { agentId: "agent", sandboxKey: "sandbox", finalCheckpointAt }, + state: "saved", lastSavedAt: new Date("2026-09-09T04:00:00Z"), error: null, refreshRequested: false }, + status: "succeeded", + }; + const query = { from: () => query, innerJoin: () => query, where: () => query, + orderBy: () => query, limit: async () => [saved] }; + const app = express(); + app.use("/api", workFolderRoutes({ select: () => query } as unknown as Db)); + const response = await request(app).get("/api/companies/11111111-1111-4111-8111-111111111111/work-folders/task/22222222-2222-4222-8222-222222222222/sync").expect(200); + expect(response.body).toEqual([expect.objectContaining({ runId: "run", state: "saved", active: false, + lastSavedAt: "2026-09-09T04:00:00.000Z", finalCheckpointAt: finalCheckpointAt ?? null })]); + }, +); diff --git a/server/src/routes/work-folders.ts b/server/src/routes/work-folders.ts index 5440ffe820..d0b24bb4a1 100644 --- a/server/src/routes/work-folders.ts +++ b/server/src/routes/work-folders.ts @@ -95,6 +95,7 @@ export function workFolderRoutes(db: Db, provider?: StorageProvider) { const interrupted = !active && (row.state === "starting" || row.state === "saving"); return { runId: row.runId, agentId: row.manifest.agentId, state: interrupted ? "failed" : row.state, lastSavedAt: row.lastSavedAt, error: interrupted ? row.error ?? "Run ended before its final file save completed." : row.error, + finalCheckpointAt: row.manifest.finalCheckpointAt ?? null, refreshRequested: row.refreshRequested, active }; }; const leases = new Set(); diff --git a/tests/runner-e2e/warm-workspace.test.ts b/tests/runner-e2e/warm-workspace.test.ts index cd3f7e435e..e7674a4e03 100644 --- a/tests/runner-e2e/warm-workspace.test.ts +++ b/tests/runner-e2e/warm-workspace.test.ts @@ -34,6 +34,7 @@ async function fixture() { state: "saved", active: false, lastSavedAt: "2026-09-09T04:00:00Z", + finalCheckpointAt: "2026-09-09T04:00:00Z" as string | null | undefined, }; const response = { ok: () => true, @@ -44,6 +45,9 @@ async function fixture() { id: "run", companyId: "company", agentId: "agent", + status: "succeeded", + startedAt: "2026-09-09T03:59:00Z" as string | null, + finishedAt: "2026-09-09T04:00:01Z" as string | null, contextSnapshot: { paperclipWorkFolders: binding } as Record< string, unknown @@ -200,6 +204,50 @@ describe("warm workspace persistence observation", () => { expect(input.api.request.get).not.toHaveBeenCalled(); }); + it.each(["failed", "cancelled", "timed_out", "running"])( + "rejects a %s run even when its periodic checkpoint is saved", + async (status) => { + const { input, fullRun } = await fixture(); + fullRun.status = status; + await expect(readWarmWorkspaceFile(input)).rejects.toThrow( + "Warm turn must succeed", + ); + expect(input.api.request.get).not.toHaveBeenCalled(); + }, + ); + + it.each([ + { finalCheckpointAt: undefined }, + { finalCheckpointAt: null }, + { finalCheckpointAt: "invalid" }, + { finalCheckpointAt: "2026-09-09T03:58:59Z" }, + { finalCheckpointAt: "2026-09-09T04:00:02Z" }, + { lastSavedAt: "2026-09-09T03:59:59Z" }, + { lastSavedAt: "2026-09-09T04:00:02Z" }, + ])( + "rejects periodic-only or unrelated finalization evidence: %j", + async (override) => { + const { input, saved } = await fixture(); + Object.assign(saved, override); + await expect(readWarmWorkspaceFile(input)).rejects.toThrow( + "explicit finalization", + ); + expect(input.api.request.get).not.toHaveBeenCalled(); + }, + ); + + it.each(["startedAt", "finishedAt"] as const)( + "rejects missing run boundary %s", + async (field) => { + const { input, fullRun } = await fixture(); + fullRun[field] = null; + await expect(readWarmWorkspaceFile(input)).rejects.toThrow( + "explicit finalization", + ); + expect(input.api.request.get).not.toHaveBeenCalled(); + }, + ); + it.each([ { runId: "older-run" }, { state: "failed" }, diff --git a/tests/runner-e2e/warm-workspace.ts b/tests/runner-e2e/warm-workspace.ts index 542bd6574d..31b49a6c5c 100644 --- a/tests/runner-e2e/warm-workspace.ts +++ b/tests/runner-e2e/warm-workspace.ts @@ -31,6 +31,9 @@ export async function readWarmWorkspaceFile(input: { const fullRun = await api.get< typeof run & { contextSnapshot: Record | null; + status: string; + startedAt: string | null; + finishedAt: string | null; } >(`/api/heartbeat-runs/${encodeURIComponent(run.id)}`); assert.equal(fullRun.id, run.id); @@ -46,6 +49,7 @@ export async function readWarmWorkspaceFile(input: { !Array.isArray(fullRun.contextSnapshot)), "Invalid full run context", ); + assert.equal(fullRun.status, "succeeded", "Warm turn must succeed"); const manifest = fullRun.contextSnapshot?.paperclipWorkFolders; if (manifest === undefined || manifest === null) { return { @@ -71,6 +75,20 @@ export async function readWarmWorkspaceFile(input: { saved && saved.state === "saved" && !saved.active && saved.lastSavedAt, "The completed warm turn must have a successful final file save", ); + const startedAt = Date.parse(fullRun.startedAt ?? ""); + const finishedAt = Date.parse(fullRun.finishedAt ?? ""); + const finalizedAt = Date.parse(saved.finalCheckpointAt ?? ""); + const savedAt = Date.parse(saved.lastSavedAt); + assert( + Number.isFinite(startedAt) && + Number.isFinite(finishedAt) && + Number.isFinite(finalizedAt) && + Number.isFinite(savedAt) && + startedAt <= finalizedAt && + finalizedAt <= savedAt && + savedAt <= finishedAt, + "Warm turn requires explicit finalization and save timestamps within this run", + ); const response = await api.request.get( `${base}/content?path=${encodeURIComponent(filename)}`, );