diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index a155634c6c..2daf6fb37f 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -519,6 +519,11 @@ Recovery rule: This is a dispatch recovery, not a continuation recovery. +Recovery hand-back is covered by the same liveness guarantee: + +- an `issue_recovery_action_restored` wake requested while the resolving recovery run is still active is persisted as a follow-up and dispatched only after that run exits, so it cannot be coalesced into the run that requested it +- if that follow-up is nevertheless lost, the stranded-work backstop treats an assigned `todo` issue with a resolved `handed_back` recovery action from during or after its latest successful run as stranded and queues the bounded assignment recovery wake; the successful resolving run is not, by itself, evidence that the handed-back source work is live + ### 9.2 Stranded assigned `in_progress` Example: diff --git a/server/src/__tests__/heartbeat-comment-wake-batching.test.ts b/server/src/__tests__/heartbeat-comment-wake-batching.test.ts index 3e6d30fe8c..4a07e57c82 100644 --- a/server/src/__tests__/heartbeat-comment-wake-batching.test.ts +++ b/server/src/__tests__/heartbeat-comment-wake-batching.test.ts @@ -295,6 +295,125 @@ describeEmbeddedPostgres("heartbeat comment wake batching", () => { expect(runs[0]?.id).toBe(runId); }); + it("defers recovery hand-back wakes until the resolving run exits", async () => { + const companyId = randomUUID(); + const agentId = randomUUID(); + const issueId = randomUUID(); + const runId = randomUUID(); + const recoveryActionId = randomUUID(); + const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`; + const heartbeat = heartbeatService(db); + + await db.insert(companies).values({ + id: companyId, + name: "Paperclip", + issuePrefix, + requireBoardApprovalForNewAgents: false, + defaultResponsibleUserId: "responsible-user", + }); + + await db.insert(agents).values({ + id: agentId, + companyId, + name: "Recovery owner", + role: "engineer", + status: "running", + adapterType: "process", + adapterConfig: {}, + runtimeConfig: {}, + permissions: {}, + }); + + await db.insert(heartbeatRuns).values({ + id: runId, + companyId, + agentId, + invocationSource: "assignment", + triggerDetail: "system", + status: "running", + contextSnapshot: { + issueId, + taskId: issueId, + wakeReason: "source_scoped_recovery_action", + }, + }); + runningProcesses.set(runId, { + child: {} as never, + graceSec: 0, + processGroupId: null, + }); + + await db.insert(issues).values({ + id: issueId, + companyId, + title: "Resume handed-back work", + status: "todo", + priority: "medium", + responsibleUserId: "responsible-user", + assigneeAgentId: agentId, + executionRunId: runId, + executionAgentNameKey: "recovery-owner", + executionLockedAt: new Date(), + issueNumber: 1, + identifier: `${issuePrefix}-1`, + }); + + const followupRun = await heartbeat.wakeup(agentId, { + source: "automation", + triggerDetail: "system", + reason: "issue_recovery_action_restored", + payload: { + issueId, + recoveryActionId, + mutation: "recovery_action_resolution", + }, + contextSnapshot: { + issueId, + taskId: issueId, + recoveryActionId, + wakeReason: "issue_recovery_action_restored", + source: "issue.recovery_action_resolution", + }, + requestedByActorType: "agent", + requestedByActorId: agentId, + }); + + expect(followupRun).toBeNull(); + + const deferred = await db + .select() + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.agentId, agentId), + eq(agentWakeupRequests.status, "deferred_issue_execution"), + ), + ) + .then((rows) => rows[0] ?? null); + + expect(deferred).toMatchObject({ + reason: "issue_execution_deferred", + runId: null, + payload: expect.objectContaining({ + issueId, + recoveryActionId, + mutation: "recovery_action_resolution", + }), + }); + expect((deferred?.payload as Record)._paperclipWakeContext).toMatchObject({ + issueId, + taskId: issueId, + recoveryActionId, + wakeReason: "issue_recovery_action_restored", + source: "issue.recovery_action_resolution", + }); + + const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.agentId, agentId)); + expect(runs).toHaveLength(1); + expect(runs[0]?.id).toBe(runId); + }); + it("batches deferred comment wakes and forwards the ordered batch to the next run", async () => { const gateway = await createControlledGatewayServer(); const companyId = randomUUID(); diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index 1ce6c93858..fd23f53a49 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -3751,6 +3751,60 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { } }); + it("re-enqueues handed-back todo work when its resolving run succeeded but the wake was lost", async () => { + const { companyId, agentId, issueId, runId } = await seedStrandedIssueFixture({ + status: "todo", + runStatus: "succeeded", + }); + const resolvedAt = new Date("2026-03-19T00:04:00.000Z"); + await db.insert(issueRecoveryActions).values({ + companyId, + sourceIssueId: issueId, + kind: "stranded_assigned_issue", + status: "resolved", + ownerType: "agent", + ownerAgentId: agentId, + previousOwnerAgentId: agentId, + returnOwnerAgentId: agentId, + cause: "stranded_assigned_issue", + fingerprint: `handed-back:${issueId}`, + nextAction: "Resume source work", + outcome: "handed_back", + resolutionNote: "Returned source work to the original owner", + resolvedAt, + createdAt: new Date("2026-03-19T00:01:00.000Z"), + updatedAt: resolvedAt, + }); + const heartbeat = heartbeatService(db); + + const result = await heartbeat.reconcileStrandedAssignedIssues(); + expect(result.assignmentDispatched).toBe(0); + expect(result.dispatchRequeued).toBe(1); + expect(result.continuationRequeued).toBe(0); + expect(result.escalated).toBe(0); + expect(result.issueIds).toEqual([issueId]); + + const runs = await db + .select() + .from(heartbeatRuns) + .where(eq(heartbeatRuns.agentId, agentId)); + expect(runs).toHaveLength(2); + + const retryRun = runs.find((row) => row.id !== runId); + expect(retryRun?.contextSnapshot).toMatchObject({ + issueId, + taskId: issueId, + wakeReason: "issue_assignment_recovery", + retryReason: "assignment_recovery", + source: "issue.assignment_recovery", + retryOfRunId: runId, + }); + expect(retryRun?.contextSnapshot as Record).not.toHaveProperty("modelProfile"); + if (retryRun) { + await waitForRunToSettle(heartbeat, retryRun.id); + } + }); + it("re-enqueues an already stranded execution-review participant during reconciliation", async () => { const { agentId, issueId, runId, wakeupRequestId, stageId } = await seedInReviewParticipantRunFixture(); const finishedAt = new Date("2026-03-19T00:05:00.000Z"); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 8f94007c56..bcc54e70c3 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -544,6 +544,7 @@ function mergeAdapterRecoveryMetadata(input: { const RUNNING_ISSUE_WAKE_REASONS_REQUIRING_FOLLOWUP = new Set([ "approval_approved", ISSUE_BLOCKERS_RESOLVED_WAKE_REASON, + "issue_recovery_action_restored", ]); const ISSUE_RESPONSIBLE_USER_WAKE_REASONS = new Set([ "issue_assigned", diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index d3af94c6f3..0610362e3a 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -140,7 +140,15 @@ type ResolvedDependencyWakeBackstopOptions = { type LatestIssueRun = Pick< typeof heartbeatRuns.$inferSelect, - "id" | "agentId" | "status" | "error" | "errorCode" | "contextSnapshot" | "livenessState" + | "id" + | "agentId" + | "status" + | "error" + | "errorCode" + | "contextSnapshot" + | "livenessState" + | "startedAt" + | "createdAt" > & { resultJson?: unknown; } | null; @@ -784,6 +792,8 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) contextSnapshot: heartbeatRuns.contextSnapshot, livenessState: heartbeatRuns.livenessState, resultJson: heartbeatRuns.resultJson, + startedAt: heartbeatRuns.startedAt, + createdAt: heartbeatRuns.createdAt, }) .from(heartbeatRuns) .where( @@ -812,6 +822,8 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) contextSnapshot: heartbeatRuns.contextSnapshot, livenessState: heartbeatRuns.livenessState, resultJson: heartbeatRuns.resultJson, + startedAt: heartbeatRuns.startedAt, + createdAt: heartbeatRuns.createdAt, }) .from(heartbeatRuns) .where( @@ -948,6 +960,29 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) .then((rows) => Boolean(rows[0])); } + async function wasTodoHandedBackDuringOrAfterLatestRun( + issue: typeof issues.$inferSelect, + latestRun: LatestIssueRun, + ) { + if (issue.status !== "todo" || latestRun?.status !== "succeeded") return false; + const runBeganAt = latestRun.startedAt ?? latestRun.createdAt; + + return db + .select({ id: issueRecoveryActions.id }) + .from(issueRecoveryActions) + .where( + and( + eq(issueRecoveryActions.companyId, issue.companyId), + eq(issueRecoveryActions.sourceIssueId, issue.id), + eq(issueRecoveryActions.status, "resolved"), + eq(issueRecoveryActions.outcome, "handed_back"), + gte(issueRecoveryActions.resolvedAt, runBeganAt), + ), + ) + .limit(1) + .then((rows) => Boolean(rows[0])); + } + async function hasQueuedIssueWake(companyId: string, issueId: string, agentId?: string | null) { return db .select({ id: agentWakeupRequests.id }) @@ -1026,6 +1061,8 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) contextSnapshot: heartbeatRuns.contextSnapshot, livenessState: heartbeatRuns.livenessState, resultJson: heartbeatRuns.resultJson, + startedAt: heartbeatRuns.startedAt, + createdAt: heartbeatRuns.createdAt, }) .from(heartbeatRuns) .where( @@ -4001,7 +4038,10 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) continue; } - if (latestRun.status === "succeeded") { + if ( + latestRun.status === "succeeded" && + !(await wasTodoHandedBackDuringOrAfterLatestRun(issue, latestRun)) + ) { result.skipped += 1; continue; }