import { test, expect } from "@playwright/test"; import { execFileSync, spawn, type ChildProcess } from "node:child_process"; import { createServer } from "node:http"; import { resolve } from "node:path"; import { writeFile } from "node:fs/promises"; import { listenOnFetchAllowedPort } from "../fetch-allowed-port"; // Each attempt starts the source CLI's test-drive without --data-dir. The model // and MCP provider are deterministic fixtures; authorization/cards/wakes are real. for (const journey of [ "safe", "uncertain", "safe_restart", "ceo_lineage", "legacy_unknown", ] as const) test(`fresh execution recovery and current-request journey: ${journey}`, async ({ page, }, info) => { const root = resolve(import.meta.dirname, "../../.."); let processHandle: ChildProcess | undefined; let logs = ""; let diagnosticState = async () => ({}); async function stopDrive() { if (processHandle?.pid) { const child = processHandle; const stopped = new Promise((done) => child.once("exit", () => done()), ); try { process.kill(-child.pid!, "SIGTERM"); } catch {} if (child.exitCode === null && child.signalCode === null) { let timeout: ReturnType | undefined; await Promise.race([ stopped, new Promise((done) => { timeout = setTimeout(() => { try { process.kill(-child.pid!, "SIGKILL"); } catch {} done(); }, 15_000); }), ]); clearTimeout(timeout); } } } const calls: string[] = []; let gmailReadCount = 0; const fixture = createServer(async (req, res) => { const chunks: Buffer[] = []; for await (const part of req) chunks.push(Buffer.from(part)); const message = JSON.parse(Buffer.concat(chunks).toString() || "{}"); const gmail = req.url === "/gmail"; calls.push(`${gmail ? "gmail" : "notion"}:${message.method}`); if (message.id === undefined) { res.writeHead(202); res.end(); return; } const result = message.method === "initialize" ? { protocolVersion: "2024-11-05", capabilities: { tools: {} }, serverInfo: { name: gmail ? "gmail-fixture" : "heliotrope", version: "1", }, } : message.method === "tools/list" ? { tools: [ { name: gmail ? "gmail_read" : "archive_read", description: gmail ? "Read gmail-fixture launch email decisions" : "Read heliotrope launch decisions", annotations: { readOnlyHint: true, destructiveHint: false }, inputSchema: { type: "object", properties: {}, additionalProperties: false, }, }, ], } : message.method === "tools/call" ? { content: [ { type: "text", text: gmail ? ++gmailReadCount === 1 && journey !== "ceo_lineage" ? journey !== "uncertain" ? "RECOVERY-INJECT-SAFE" : "RECOVERY-INJECT-UNKNOWN-WRITE" : "GMAIL-73: The launch email confirms Friday approval. Source: https://example.invalid/mail/gmail-73" : "HELIOTROPE-42: Launch in two stages; support handoff belongs to Mira. Source: https://example.invalid/launch/heliotrope-42", }, ], } : {}; res.writeHead(200, { "content-type": "application/json" }); res.end(JSON.stringify({ jsonrpc: "2.0", id: message.id, result })); }); try { const port = await listenOnFetchAllowedPort(fixture); const env = { ...process.env, PAPERCLIP_RECOVERY_CEO_LINEAGE: journey === "ceo_lineage" ? "1" : "0", IN_FEED_FIXTURE_KEY: "not-a-real-model-key", NODE_ENV: "test", PATH: `${root}/tests/e2e/fixtures/recovery-bin:${process.env.PATH}`, }; delete env.DATABASE_URL; delete env.DATABASE_MIGRATION_URL; processHandle = spawn( process.execPath, [ "cli/node_modules/tsx/dist/cli.mjs", "cli/src/index.ts", "test-drive", "--harness", "codex", "--api-key-env", "IN_FEED_FIXTURE_KEY", "--company-name", "Execution recovery fixture", "--no-browser", ], { cwd: root, env, detached: true, stdio: ["ignore", "pipe", "pipe"] }, ); processHandle.stdout!.on("data", (chunk) => { logs += chunk.toString(); }); processHandle.stderr!.on("data", (chunk) => { logs += chunk.toString(); }); await expect .poll( () => logs.match( /Paperclip is ready at (http:\/\/127\.0\.0\.1:\d+)/, )?.[1], { timeout: 100_000 }, ) .toBeTruthy(); let base = logs.match( /Paperclip is ready at (http:\/\/127\.0\.0\.1:\d+)/, )![1]!; const api = async (path: string, method = "GET", data?: unknown) => { const response = await page.request.fetch(`${base}/api${path}`, { method, data, }); expect(response.ok(), await response.text()).toBeTruthy(); return response.json(); }; const isSettledRun = (run: { status: string; errorCode?: string }) => run.status === "succeeded" || (run.status === "cancelled" && run.errorCode === "issue_not_in_progress"); const health = await api("/health"); expect(health).toMatchObject({ status: "ok", deploymentMode: "local_trusted", bootstrapStatus: "ready", serverInfo: { git: { branchName: execFileSync("git", ["branch", "--show-current"], { cwd: root, encoding: "utf8", }).trim(), }, }, }); const [company] = await api("/companies"); let [agent] = await api(`/companies/${company.id}/agents`); expect(agent.adapterType).toBe("codex_local"); expect(await api(`/companies/${company.id}/issues`)).toEqual([]); expect(await api(`/companies/${company.id}/heartbeat-runs`)).toEqual([]); expect( (await api(`/companies/${company.id}/tools/connections`)).connections, ).toEqual([]); diagnosticState = async () => ({ base, companyId: company.id, agentId: agent.id, tasks: (await api(`/companies/${company.id}/issues`)).map( (task: Record) => ({ id: task.id, status: task.status, assigneeAgentId: task.assigneeAgentId, }), ), runs: (await api(`/companies/${company.id}/heartbeat-runs`)).map( (run: Record) => ({ id: run.id, status: run.status, runtimeMode: run.runtimeMode, error: run.error, errorCode: run.errorCode, }), ), }); const dataDir = logs .match(/Data directory: ([^\n\r]+)/)![1]! .replace(/\u001b\[[0-9;]*m/g, "") .trim(); await writeFile( info.outputPath("running-instance.json"), JSON.stringify( { base, dataDir, companyId: company.id, agentId: agent.id }, null, 2, ), ); const prefix = `/${company.issuePrefix}`; await page.goto(base + prefix + "/dashboard"); await expect(page.getByText("No runs yet").first()).toBeVisible({ timeout: 30_000 }); if (journey === "legacy_unknown") { agent = await api(`/companies/${company.id}/agents`, "POST", { name: "Legacy executor", role: "engineer", adapterType: "process", adapterConfig: { command: process.execPath, args: ["-e", "console.error('Deterministic provider failure; no recoverable session contract'); process.exit(1)"] }, }); await page.getByRole("link", { name: "Tasks", exact: true }).click(); await page.getByRole("button", { name: "New Task", exact: true }).last().click(); await page.getByPlaceholder("Task title").fill("Read the legacy fixture report"); await page.getByRole("button", { name: "Assignee", exact: true }).click(); await page.getByRole("button", { name: agent.name, exact: true }).click(); await page.getByRole("button", { name: "Create Task", exact: true }).click(); await page.getByRole("complementary").getByRole("link", { name: /Read the legacy fixture report/ }).click(); await expect.poll(async () => { const tasks = await api(`/companies/${company.id}/issues`); return tasks.find((issue: { title: string }) => issue.title === "Read the legacy fixture report")?.status; }, { timeout: 60_000 }).toBe("blocked"); await expect(page.getByText("Blocked", { exact: true }).first()).toBeVisible({ timeout: 30_000 }); await expect(page.getByRole("button", { name: "Reconcile and continue" })).toHaveCount(0); await expect(page.getByRole("button", { name: "Try again", exact: true })).toHaveCount(0); const composer = page.locator('[contenteditable="true"]').last(); await composer.fill("Independent draft remains usable"); await expect(composer).toContainText("Independent draft remains usable"); await page.screenshot({ path: info.outputPath("legacy-recovery-needed.png"), fullPage: true }); // Observe past the shared retry delay; absence of a successor is part // of the fail-closed contract, not merely a momentary UI state. await page.waitForTimeout(35_000); const runs = await api(`/companies/${company.id}/heartbeat-runs`); expect(runs).toHaveLength(1); expect(await api(`/heartbeat-runs/${runs[0].id}`)).toMatchObject({ runtimeMode: "legacy", status: "failed" }); await page.reload(); await expect(page.getByRole("button", { name: "Reconcile and continue" })).toHaveCount(0); await writeFile(info.outputPath("instance-and-runs.json"), JSON.stringify({ base, dataDir, health, dependency: "deterministic legacy process fixture", companyId: company.id, agentId: agent.id, runs }, null, 2)); return; } await page.goto( base + prefix + "/company/settings/instance/experimental", ); const nativeRunnerToggle = page.getByRole("switch", { name: "Toggle Paperclip Runner experimental setting", }); // Fresh instances may already enable the native runner. Configure the // desired state instead of blindly toggling the current default off. if (await nativeRunnerToggle.getAttribute("aria-checked") !== "true") { await nativeRunnerToggle.click(); } await expect(nativeRunnerToggle).toHaveAttribute("aria-checked", "true"); await expect .poll( async () => (await api("/instance/settings/experimental")).enableNativeRunner, ) .toBe(true); await page.goto(base + prefix + `/agents/${agent.id}/configuration`); await page.getByRole("button", { name: "Codex", exact: true }).click(); await page.getByRole("button", { name: /Paperclip Runner/ }).click(); await page .getByRole("button", { name: /^Save(?: changes)?$/ }) .first() .click(); await expect .poll(async () => (await api(`/agents/${agent.id}`)).adapterType) .toBe("paperclip_runner"); const nativeAgent = await api(`/agents/${agent.id}`); expect(nativeAgent.adapterConfig.env).toEqual(agent.adapterConfig.env); if (journey !== "ceo_lineage") { const { instructionsFilePath: _instructions, ...executorConfig } = nativeAgent.adapterConfig; agent = await api(`/companies/${company.id}/agents`, "POST", { name: "Executor", role: "engineer", adapterType: "paperclip_runner", adapterConfig: executorConfig, }); } const holder = await api(`/companies/${company.id}/agents`, "POST", { name: "Archive holder", role: "qa", adapterType: "process", adapterConfig: { command: process.execPath, args: ["-e", "process.exit(0)"], }, }); const connections: Record = {}; for (const service of ["notion", "gmail"]) { await page.goto(base + prefix + "/apps"); const custom = page .getByRole("list", { name: "Connector list" }) .getByRole("listitem") .filter({ hasText: "Connect your own tool" }); await custom .getByRole("button", { name: "Connect", exact: true }) .click(); await custom .getByRole("button", { name: "Connect your own MCP server" }) .click(); await page .getByPlaceholder("https://example.com/actions") .fill(`http://127.0.0.1:${port}/${service}`); await page .getByRole("button", { name: "Continue", exact: true }) .click(); await page.getByRole("radio", { name: "Just agents I pick" }).click(); await page.getByRole("button", { name: /Select agents/ }).click(); await page.getByRole("checkbox", { name: /Archive holder/ }).check(); await page.keyboard.press("Escape"); await page.getByRole("button", { name: "Save and continue" }).click(); await page.getByRole("button", { name: /Check link/i }).click(); await expect( page.getByRole("heading", { name: /is ready/i }), ).toBeVisible({ timeout: 30_000 }); const connection = ( await api(`/companies/${company.id}/tools/connections`) ).connections.find( (row: { id: string }) => !Object.values(connections).some( (existing) => existing.id === row.id, ), ); connections[service] = connection; const installs = ( await api(`/tool-connections/${connection.id}/installs`) ).installs; expect(installs).toEqual([ expect.objectContaining({ targetId: holder.id }), ]); } await page.getByRole("link", { name: "Tasks", exact: true }).click(); await page .getByRole("button", { name: "New Task", exact: true }) .last() .click(); await page .getByPlaceholder("Task title") .fill( `${journey === "ceo_lineage" ? "CEO descendant fixture: " : ""}Find the heliotrope launch notes and summarize the decisions with a source link`, ); await page.getByRole("button", { name: "Assignee", exact: true }).click(); await page.getByRole("button", { name: agent.name, exact: true }).click(); await page .getByRole("button", { name: "Create Task", exact: true }) .click(); await page .getByRole("complementary") .getByRole("link", { name: /Find the heliotrope launch notes/ }) .click(); await expect(page).toHaveURL(/issues\/(?:[a-f0-9-]+|[A-Z]+-\d+)/); const connectPending = async (service: "notion" | "gmail") => { const connection = connections[service]!; const card = page .getByTestId("connection-intent-focus-target") .filter({ hasText: connection.name }); await expect( card.getByRole("button", { name: /Connect \/ Use existing/ }), ).toBeVisible({ timeout: 60_000 }); await page.screenshot({ path: info.outputPath(`${service}-pending.png`), fullPage: true, }); await card .getByRole("button", { name: /Connect \/ Use existing/ }) .click(); await expect(page.getByRole("dialog")).toBeVisible(); await page .getByRole("dialog") .getByRole("button", { name: new RegExp(connection.name) }) .click(); await expect( card.getByText(`${connection.name} connected`, { exact: true }), ).toBeVisible({ timeout: 30_000 }); }; if (journey === "ceo_lineage") await expect( page .getByText("Provider child finished; root execution continues.", { exact: true, }) .first(), ).toBeVisible({ timeout: 60_000 }); await connectPending("notion"); await expect( page.getByText(/HELIOTROPE-42: Launch in two stages/).first(), ).toBeVisible({ timeout: 60_000 }); await expect .poll( async () => (await api(`/companies/${company.id}/heartbeat-runs`)).every( isSettledRun, ), { timeout: 60_000 }, ) .toBe(true); const [task] = await api(`/companies/${company.id}/issues`); const originalTitle = task.title; await page .getByRole("textbox") .last() .fill( "Now summarize my Gmail emails about launch decisions, with the source link.", ); await page.getByRole("button", { name: "Send", exact: true }).click(); await connectPending("gmail"); if (journey === "safe_restart") { await expect .poll( async () => (await api(`/companies/${company.id}/heartbeat-runs`)).some( (run: { status: string; scheduledRetryReason?: string }) => run.status === "scheduled_retry" && run.scheduledRetryReason === "native_safe_replacement", ), { timeout: 45_000 }, ) .toBe(true); await page.screenshot({ path: info.outputPath("retry-before-restart.png"), fullPage: true, }); await stopDrive(); const startOffset = logs.length; processHandle = spawn( process.execPath, [ "cli/node_modules/tsx/dist/cli.mjs", "cli/src/index.ts", "test-drive", "--data-dir", dataDir, "--harness", "codex", "--api-key-env", "IN_FEED_FIXTURE_KEY", "--no-browser", ], { cwd: root, env, detached: true, stdio: ["ignore", "pipe", "pipe"] }, ); processHandle.stdout!.on("data", (chunk) => { logs += chunk.toString(); }); processHandle.stderr!.on("data", (chunk) => { logs += chunk.toString(); }); await expect .poll( () => logs .slice(startOffset) .match( /Paperclip is ready at (http:\/\/127\.0\.0\.1:\d+)/, )?.[1], { timeout: 100_000 }, ) .toBeTruthy(); base = logs .slice(startOffset) .match(/Paperclip is ready at (http:\/\/127\.0\.0\.1:\d+)/)![1]!; expect((await api("/health")).serverInfo.git.branchName).toBe( health.serverInfo.git.branchName, ); expect((await api("/companies"))[0].id).toBe(company.id); await page.goto(base + prefix + `/issues/${task.id}`); } const composer = page.getByRole("textbox").last(); await composer.fill("An unsent draft stays available during recovery."); await expect(composer).toBeEditable(); if (journey !== "uncertain") { await expect( page .getByText(/GMAIL-73: The launch email confirms Friday approval/) .first(), ).toBeVisible({ timeout: 100_000 }); expect(gmailReadCount).toBe(journey === "ceo_lineage" ? 1 : 2); } else { await expect.poll(async () => (await api(`/issues/${task.id}`)).status, { timeout: 100_000 }).toBe("blocked"); await expect(page.getByText("Blocked", { exact: true }).first()).toBeVisible({ timeout: 30_000 }); expect(gmailReadCount).toBe(1); await expect(page.getByText(/GMAIL-73/)).toHaveCount(0); await expect(page.getByRole("button", { name: /Reconcile and continue|Try again/ })).toHaveCount(0); await expect(page.getByRole("dialog", { name: "Reconcile execution" })).toHaveCount(0); const recovery = (await api(`/issues/${task.id}`)).activeRecoveryAction; expect(recovery).toBeNull(); await page.screenshot({ path: info.outputPath("uncertain-automatic-no-replay.png"), fullPage: true }); // Past the retry delay, unknown effects still cannot be replayed. await page.waitForTimeout(35_000); expect(gmailReadCount).toBe(1); } await expect(composer).toHaveText( "An unsent draft stays available during recovery.", ); await expect .poll( async () => (await api(`/companies/${company.id}/heartbeat-runs`)).every( (run: { status: string }) => !["running", "queued", "scheduled_retry"].includes(run.status), ), { timeout: 60_000 }, ) .toBe(true); if (journey !== "uncertain") await expect(async () => { const answer = page.getByText(/GMAIL-73: The launch email confirms Friday approval/).first(); // Refresh can replace the streamed row with its persisted transcript. // Re-resolve the locator if that handoff detaches it during scrolling. await answer.scrollIntoViewIfNeeded(); await expect(answer).toBeInViewport(); }).toPass({ timeout: 10_000 }); await page.screenshot({ path: info.outputPath(`${journey}-outcome.png`), fullPage: true, }); const finalRuns = await Promise.all( (await api(`/companies/${company.id}/heartbeat-runs`)).map( (run: { id: string }) => api(`/heartbeat-runs/${run.id}`), ), ); const replacements = finalRuns.filter( (run: { scheduledRetryReason?: string }) => run.scheduledRetryReason === "native_safe_replacement", ); expect(replacements).toHaveLength( journey === "safe" || journey === "safe_restart" ? 1 : 0, ); expect( finalRuns.every( (run: { runtimeMode: string }) => run.runtimeMode === "native", ), ).toBe(true); expect( finalRuns.some((run: { status: string }) => run.status === "running"), ).toBe(false); expect((await api(`/issues/${task.id}`)).title).toBe(originalTitle); await composer.fill(""); await page.reload(); await expect(page.getByText(/Due now/, { exact: true })).toHaveCount(0); if (journey !== "uncertain") await expect( page .getByText(/GMAIL-73: The launch email confirms Friday approval/) .first(), ).toBeVisible(); else { expect((await api(`/issues/${task.id}`)).status).toBe("blocked"); await expect(page.getByRole("button", { name: "Reconcile and continue" })).toHaveCount(0); } if (journey !== "uncertain") await expect(async () => { const answer = page.getByText(/GMAIL-73: The launch email confirms Friday approval/).first(); // Refresh can replace the streamed row with its persisted transcript. // Re-resolve the locator if that handoff detaches it during scrolling. await answer.scrollIntoViewIfNeeded(); await expect(answer).toBeInViewport(); }).toPass({ timeout: 10_000 }); await page.screenshot({ path: info.outputPath(`${journey}-after-refresh.png`), fullPage: true, }); const evidence = JSON.stringify( { base, dataDir, health, journey, companyId: company.id, agentId: agent.id, taskId: task.id, connections, runs: finalRuns.map((run: Record) => ({ id: run.id, runtimeMode: run.runtimeMode, status: run.status, retryOfRunId: run.retryOfRunId, errorCode: run.errorCode, execution: run.execution, })), dependency: "fixture", provider: "codex", fixtureModel: "in-feed-fixture", calls, }, null, 2, ); await writeFile(info.outputPath("instance-and-runs.json"), evidence); await info.attach("instance-and-runs", { body: evidence, contentType: "application/json", }); } finally { await writeFile( info.outputPath("diagnostics.json"), JSON.stringify(await diagnosticState().catch(() => ({})), null, 2), ); await writeFile(info.outputPath("test-drive.log"), logs); await stopDrive(); fixture.closeAllConnections(); await new Promise((done) => fixture.close(() => done())); } });