import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { eq } from "drizzle-orm"; import { agents, companies, completionContracts, createDb, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, nativeRunResults, statusDecisions, workAssessments, workspaceOperations, } from "@paperclipai/db"; import { CONTROL_PLANE_CONFORMANCE_OPEN, CONTROL_PLANE_CONFORMANCE_RESULT, CONTROL_PLANE_CONFORMANCE_TERMINAL, } from "../vendor/paperclip-runner/testing.js"; import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { reconcileNativeFinalizations } from "../services/native-runtime/native-finalization-reconciler.js"; import { PaperclipControlPlanePort } from "../services/native-runtime/paperclip-control-plane-port.js"; describe("P6-16/P6-25/P6-28 native finalization recovery", () => { let temporary: Awaited>; let db: ReturnType; const companyId = "72000000-0000-4000-8000-000000000001"; const agentId = "72000000-0000-4000-8000-000000000002"; const issueId = "72000000-0000-4000-8000-000000000003"; const contractId = "72000000-0000-4000-8000-000000000004"; const runId = "72000000-0000-4000-8000-000000000005"; const staleIssueId = "72000000-0000-4000-8000-000000000013"; const staleContractId = "72000000-0000-4000-8000-000000000014"; const staleRunId = "72000000-0000-4000-8000-000000000015"; beforeAll(async () => { temporary = await startEmbeddedPostgresTestDatabase("paperclip-native-recovery-"); db = createDb(temporary.connectionString); await db.insert(companies).values({ id: companyId, name: "Native recovery", issuePrefix: "NRC" }); await db.insert(agents).values({ id: agentId, companyId, name: "Recovery agent", adapterType: "codex_local", status: "running", }); await db.insert(issues).values({ id: issueId, companyId, title: "Recover invalid native finalization", status: "in_progress", assigneeAgentId: agentId, workMode: "standard", }); await db.insert(completionContracts).values({ id: contractId, companyId, issueId, revision: 1, schemaVersion: "paperclip.completion-contract.v1", policyVersion: "phase6-v1", risk: "standard", completionAuthority: "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal", contractJson: { revision: "phase6-v1", objective: "Recover invalid native finalization", criteria: [{ id: "objective", requirement: "Recovery remains live" }], }, canonicalSha256: "native-recovery-contract", createdByActorType: "system", createdByActorId: "test", }); await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", runtimeMode: "native", runtimeModeReason: "persisted_before_kill_switch", nativeIssueId: issueId, nativeSessionId: runId, runnerInstanceId: contractId, completionContractId: contractId, completionContractSha256: "native-recovery-contract", contextSnapshot: { issueId }, }); const port = new PaperclipControlPlanePort(db, { companyId, issueId, runId, agentId, sessionId: runId, completionContractId: contractId, completionContractSha256: "native-recovery-contract", sourceInstanceId: contractId, controlPlaneSourceInstanceId: "recovery-control", }); await port.openRun({ ...CONTROL_PLANE_CONFORMANCE_OPEN, identity: { companyId, issueId, runId, agentId, sessionId: runId }, sourceInstanceId: contractId, }); await port.completeRun({ result: CONTROL_PLANE_CONFORMANCE_RESULT, terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, callerResultId: "recovery-result", }); const stored = await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, runId)) .limit(1).then((rows) => rows[0]!); await db.update(nativeRunResults).set({ resultJson: { ...(stored.resultJson as Record), terminal: { ...CONTROL_PLANE_CONFORMANCE_TERMINAL, runTerminalState: "unknown" }, }, }).where(eq(nativeRunResults.id, stored.id)); await db.insert(workspaceOperations).values({ companyId, heartbeatRunId: runId, issueId, phase: "workspace_finalize", status: "succeeded", }); await db.insert(issues).values({ id: staleIssueId, companyId, title: "Retire a stale invalid finalizer", status: "in_progress", assigneeAgentId: agentId, workMode: "standard", }); await db.insert(completionContracts).values({ id: staleContractId, companyId, issueId: staleIssueId, revision: 1, schemaVersion: "paperclip.completion-contract.v1", policyVersion: "phase6-v1", risk: "standard", completionAuthority: "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal", contractJson: { revision: "phase6-v1", objective: "Retire a stale invalid finalizer", criteria: [{ id: "objective", requirement: "Keep the newer decision" }], }, canonicalSha256: "stale-finalizer-contract", createdByActorType: "system", createdByActorId: "test", }); await db.insert(heartbeatRuns).values({ id: staleRunId, companyId, agentId, status: "running", runtimeMode: "native", runtimeModeReason: "persisted_before_kill_switch", nativeIssueId: staleIssueId, nativeSessionId: staleRunId, runnerInstanceId: staleContractId, completionContractId: staleContractId, completionContractSha256: "stale-finalizer-contract", contextSnapshot: { issueId: staleIssueId }, }); const stalePort = new PaperclipControlPlanePort(db, { companyId, issueId: staleIssueId, runId: staleRunId, agentId, sessionId: staleRunId, completionContractId: staleContractId, completionContractSha256: "stale-finalizer-contract", sourceInstanceId: staleContractId, controlPlaneSourceInstanceId: "stale-control", }); await stalePort.openRun({ ...CONTROL_PLANE_CONFORMANCE_OPEN, identity: { companyId, issueId: staleIssueId, runId: staleRunId, agentId, sessionId: staleRunId }, sourceInstanceId: staleContractId, }); await stalePort.completeRun({ result: CONTROL_PLANE_CONFORMANCE_RESULT, terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, callerResultId: "stale-result", }); const staleStored = await db.select().from(nativeRunResults) .where(eq(nativeRunResults.runId, staleRunId)) .limit(1).then((rows) => rows[0]!); await db.update(nativeRunResults).set({ resultJson: { ...(staleStored.resultJson as Record), terminal: { ...CONTROL_PLANE_CONFORMANCE_TERMINAL, runTerminalState: "unknown" }, }, }).where(eq(nativeRunResults.id, staleStored.id)); await db.insert(workspaceOperations).values({ companyId, heartbeatRunId: staleRunId, issueId: staleIssueId, phase: "workspace_finalize", status: "succeeded", }); }, 30_000); afterAll(async () => { await temporary.cleanup(); }); it("fails closed into bounded named recovery without consulting the live flag or falling back", async () => { await expect(reconcileNativeFinalizations(db, [runId])).resolves.toEqual([ expect.objectContaining({ phase: "retryable_failure", failureCode: "native_finalization_invalid" }), ]); await expect(db.select().from(issues).where(eq(issues.id, issueId))).resolves.toEqual([ expect.objectContaining({ status: "in_progress", statusVersion: 0, lastStatusDecisionId: null }), ]); await expect(db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId))).resolves.toEqual([ expect.objectContaining({ status: "active", ownerAgentId: agentId, cause: "native_finalization_invalid" }), ]); await expect(db.select().from(statusDecisions).where(eq(statusDecisions.issueId, issueId))).resolves.toHaveLength(0); for (let retry = 0; retry < 2; retry += 1) { await db.update(nativeRunFinalizations).set({ nextAttemptAt: new Date(0) }) .where(eq(nativeRunFinalizations.runId, runId)); await reconcileNativeFinalizations(db, [runId]); } await expect(db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId))).resolves.toEqual([ expect.objectContaining({ phase: "terminal_failure", attempt: 3, failureCode: "native_finalization_retry_exhausted", nextAttemptAt: null, }), ]); await expect(db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId))).resolves.toEqual([ expect.objectContaining({ runtimeMode: "native", status: "succeeded", nativePhase: "terminal_failure", resultJson: expect.objectContaining({ prpRunTerminalState: "succeeded" }), }), ]); await expect(reconcileNativeFinalizations(db, [runId])).resolves.toEqual([]); }); it("retires an older failed finalizer when a newer run already committed the issue", async () => { await expect(reconcileNativeFinalizations(db, [staleRunId])).resolves.toEqual([ expect.objectContaining({ phase: "retryable_failure", failureCode: "native_finalization_invalid" }), ]); const newerRunId = "72000000-0000-4000-8000-000000000016"; await db.insert(heartbeatRuns).values({ id: newerRunId, companyId, agentId, status: "succeeded", runtimeMode: "native", nativeIssueId: staleIssueId, completionContractId: staleContractId, completionContractSha256: "stale-finalizer-contract", contextSnapshot: { issueId: staleIssueId }, }); const [newerResult] = await db.insert(nativeRunResults).values({ companyId, issueId: staleIssueId, runId: newerRunId, completionContractId: staleContractId, callerResultId: "newer-result", serverFingerprint: "newer-result-fingerprint", schemaStatus: "accepted", resultJson: {}, canonicalSha256: "newer-result-sha", }).returning(); const [newerAssessment] = await db.insert(workAssessments).values({ companyId, issueId: staleIssueId, runId: newerRunId, contractId: staleContractId, resultId: newerResult!.id, triggerKind: "native_result", triggerRef: newerResult!.id, triggerCapability: "server_native_finalizer", triggerActorCompanyId: companyId, priorIssueStatus: "in_progress", priorStatusVersion: 0, policyVersion: "phase6-v3", assessmentJson: {}, inputDigest: "newer-assessment-digest", }).returning(); const [newerDecision] = await db.insert(statusDecisions).values({ companyId, issueId: staleIssueId, runId: newerRunId, assessmentId: newerAssessment!.id, decisionVersion: 1, policyVersion: "phase6-v3", fromStatus: "in_progress", toStatus: "done", reasonCode: "completion_claim_policy_accepted", decisionJson: {}, decisionDigest: "newer-decision-digest", applicationState: "applied", appliedAt: new Date(), }).returning(); await db.update(issues).set({ status: "done", statusVersion: 1, lastStatusDecisionId: newerDecision!.id, }).where(eq(issues.id, staleIssueId)); await db.update(nativeRunFinalizations).set({ nextAttemptAt: new Date(0) }) .where(eq(nativeRunFinalizations.runId, staleRunId)); await expect(reconcileNativeFinalizations(db, [staleRunId])).resolves.toEqual([ expect.objectContaining({ phase: "terminal_failure", failureCode: "native_finalization_superseded" }), ]); await expect(db.select().from(issues).where(eq(issues.id, staleIssueId))).resolves.toEqual([ expect.objectContaining({ status: "done", lastStatusDecisionId: newerDecision!.id }), ]); await expect(db.select().from(issueRecoveryActions) .where(eq(issueRecoveryActions.sourceIssueId, staleIssueId))).resolves.toEqual([ expect.objectContaining({ status: "resolved", outcome: "false_positive" }), ]); }); });