diff --git a/server/src/__tests__/sandbox-work-folders.test.ts b/server/src/__tests__/sandbox-work-folders.test.ts index aa2cd53327..ee6bdf98ae 100644 --- a/server/src/__tests__/sandbox-work-folders.test.ts +++ b/server/src/__tests__/sandbox-work-folders.test.ts @@ -6,10 +6,11 @@ import { eq } from "drizzle-orm"; import fs from "node:fs/promises"; import path from "node:path"; import os from "node:os"; -import { agents, assets, companyMemberships, issueAttachments, companies, createDb, heartbeatRuns, issues, environments, environmentLeases, projects, projectWorkspaces, taskRepositoryBindings, workFolderObjects, startEmbeddedPostgresTestDatabase, type Db } from "@paperclipai/db"; +import { agents, assets, companyMemberships, issueAttachments, companies, createDb, heartbeatRuns, issues, environments, environmentLeases, projects, projectWorkspaces, taskRepositoryBindings, workFolderObjects, workFolderRuns, startEmbeddedPostgresTestDatabase, type Db } from "@paperclipai/db"; import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js"; import { prepareSandboxWorkFolders } from "../services/sandbox-work-folders.js"; import { retainUnsavedWorkFolderLease, workFolderSandboxKey } from "../services/work-folder-retention.js"; +import * as activityLog from "../services/activity-log.js"; import { workFolderService } from "../services/work-folders.js"; import { collectWorkFolderGarbage } from "../services/work-folder-garbage.js"; import { localTestWorkFolderRunner } from "./helpers/work-folder-runner.js"; @@ -61,6 +62,20 @@ describe("shared sandbox work-folder lifecycle", () => { runner: { execute: (input) => localTestWorkFolderRunner.execute({ ...input, env: { ...input.env, HOME: home } }) } } }); active.push(run); return run; } + it("keeps successful checkpoints saved when activity logging fails", async () => { + const activity = vi.spyOn(activityLog, "logActivity").mockRejectedValue(new Error("activity unavailable")); + try { + const run = await prepare(path.join(root, "activity-failure"), randomUUID()); + await fs.writeFile(path.join(run.home, "task/activity-proof.txt"), "saved despite logging failure"); + await run.stop(); active.splice(active.indexOf(run), 1); + const [state] = await db.select().from(workFolderRuns).where(eq(workFolderRuns.runId, run.manifest.runId)); + expect(state?.state).toBe("saved"); + expect(state?.lastSavedAt).not.toBeNull(); + const folder = await workFolderService(db, storage).ensure({ companyId, scope: "task", ownerId: taskId }); + expect((await workFolderService(db, storage).list(folder)).files.some((file) => file.path === "activity-proof.txt")).toBe(true); + } finally { activity.mockRestore(); } + }, 120_000); + it("reuses clones and restores saved unpushed work, staged changes, and task files after losing the sandbox", async () => { const home = path.join(root, "sandbox"); const leaseId = randomUUID(); diff --git a/server/src/services/sandbox-work-folders.ts b/server/src/services/sandbox-work-folders.ts index 34af680892..5c25f848c0 100644 --- a/server/src/services/sandbox-work-folders.ts +++ b/server/src/services/sandbox-work-folders.ts @@ -16,6 +16,7 @@ import { workFolderRepositoryService } from "./work-folder-repositories.js"; import { startWorkFolderCheckpointer } from "./work-folder-checkpointer.js"; import { logActivity } from "./activity-log.js"; import { assertWorkFolderAccess } from "./work-folder-access.js"; +import { logger } from "../middleware/logger.js"; function signature(entry: WorkTreeEntry | undefined) { return entry ? JSON.stringify([entry.kind, entry.sha256, entry.executable]) : "missing"; @@ -284,7 +285,8 @@ export async function prepareSandboxWorkFolders(input: { await logActivity(db, { companyId: input.companyId, actorType: "agent", actorId: input.agentId, agentId: input.agentId, runId: input.runId, issueId: input.taskId, responsibleUserIdOverride: input.responsibleUserId, action, entityType: "heartbeat_run", entityId: input.runId, - details: { scopes: WORK_FOLDER_SCOPES.filter((scope) => Boolean(folders[scope])), repositories: bindings.length } }); + details: { scopes: WORK_FOLDER_SCOPES.filter((scope) => Boolean(folders[scope])), repositories: bindings.length } }) + .catch((error) => logger.warn({ err: error, runId: input.runId }, "Work-folder activity could not be recorded")); } try { await seedAttachments(); diff --git a/tests/runner-e2e/deployed-work-folders.spec.ts b/tests/runner-e2e/deployed-work-folders.spec.ts index 5cd752d069..4e2daace43 100644 --- a/tests/runner-e2e/deployed-work-folders.spec.ts +++ b/tests/runner-e2e/deployed-work-folders.spec.ts @@ -9,7 +9,10 @@ import { DeployedStackApi, loadDeployedStack } from "./deployed-stack.js"; const stack = loadDeployedStack(); const api = new DeployedStackApi(stack); const folder = (scope: string, ownerId: string) => `/api/companies/${stack.companyId}/work-folders/${scope}/${encodeURIComponent(ownerId)}`; -test.describe.configure({ mode: "serial" }); +test.beforeAll(async () => { + const health = await api.json<{ commit: string }>("/api/health"); + expect(health.commit, "Only exercise the declared deployed candidate").toBe(stack.commit); +}); test("deployed candidate and complete supported adapter inventory", async ({}, info) => { const health = await api.json<{ commit: string }>("/api/health"); @@ -77,16 +80,17 @@ for (const profile of stack.profiles) { "Then complete this task successfully. Do not print credentials or modify unrelated files.", ].join("\n"), }); + await info.attach("task", { contentType: "application/json", body: Buffer.from(JSON.stringify({ profile: profile.id, ...issue })) }); const base = folder("task", issue.id); await pollUntil({ label: `${profile.id} completed run and durable task file`, deadlineAt: Date.now() + 840_000, intervalMs: 5_000, load: async () => ({ issue: await api.json<{ status: string }>(`/api/issues/${issue.id}`), saves: await api.json(`${base}/sync`) }), accept: (state) => state.issue.status === "done" && state.saves.some((save) => !save.active && save.state === "saved" && save.lastSavedAt !== null), - reject: (state) => state.saves.some((save) => save.state === "failed") ? "Work-folder save failed" : undefined, + reject: (state) => state.saves.some((save) => save.state === "failed") ? "Work-folder save failed" + : state.issue.status === "blocked" || state.issue.status === "cancelled" ? `Task ${issue.identifier} ended ${state.issue.status}` : undefined, }); const content = await api.request(`${base}/content?path=acceptance.txt`); expect(content.status).toBe(200); expect(await content.text()).toBe(nonce); - await info.attach("task", { contentType: "application/json", body: Buffer.from(JSON.stringify({ profile: profile.id, ...issue })) }); }); } diff --git a/tests/runner-e2e/playwright.deployed.config.ts b/tests/runner-e2e/playwright.deployed.config.ts index eba8da98f8..b9e6742bbf 100644 --- a/tests/runner-e2e/playwright.deployed.config.ts +++ b/tests/runner-e2e/playwright.deployed.config.ts @@ -8,7 +8,7 @@ if (!output || !path.isAbsolute(output)) throw new Error("PAPERCLIP_DEPLOYED_STA export default defineConfig({ testDir: ".", testMatch: "deployed-work-folders.spec.ts", - fullyParallel: false, workers: 1, retries: 0, timeout: 900_000, + fullyParallel: true, workers: 2, retries: 0, timeout: 900_000, use: { baseURL: stack.baseURL, trace: "off", video: "off" }, // No webServer: every operation reaches the deployed tenant and database. outputDir: path.join(output, "results"),