From c1eafb8d9b68931650c36395389a9053e704ba82 Mon Sep 17 00:00:00 2001 From: nicls Date: Sat, 12 Sep 2026 09:52:15 +0200 Subject: [PATCH 1/2] fix(recovery): block routine disposition continuation --- .../heartbeat-process-recovery.test.ts | 106 ++++++++++++++++++ server/src/services/recovery/service.ts | 53 ++++++++- 2 files changed, 158 insertions(+), 1 deletion(-) diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index f77f756285..2960c6db06 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -5775,6 +5775,112 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { }); }); + it("blocks routine execution after successful missing-disposition recovery instead of starting productive continuation", async () => { + const { companyId, agentId, runId, issueId } = + await seedStrandedIssueFixture({ + status: "in_progress", + runStatus: "succeeded", + livenessState: "advanced", + }); + const sourceRunId = randomUUID(); + await db + .update(issues) + .set({ + originKind: "routine_execution", + originId: randomUUID(), + }) + .where(eq(issues.id, issueId)); + await db + .update(heartbeatRuns) + .set({ + contextSnapshot: { + issueId, + taskId: issueId, + wakeReason: "source_scoped_recovery_action", + recoveryActionId: randomUUID(), + recoveryCause: SUCCESSFUL_RUN_MISSING_STATE_REASON, + sourceRunId, + recoveryIntent: "status_only", + allowDeliverableWork: false, + allowDocumentUpdates: false, + resumeRequiresNormalModel: true, + }, + }) + .where(eq(heartbeatRuns.id, runId)); + + const result = + await heartbeatService(db).reconcileStrandedAssignedIssues(); + + expect(result.continuationRequeued).toBe(0); + expect(result.successfulRunHandoffEscalated).toBe(1); + expect(result.issueIds).toEqual([issueId]); + expect( + await db + .select() + .from(issues) + .where(eq(issues.id, issueId)) + .then((rows) => rows[0]?.status), + ).toBe("blocked"); + await expectSourceScopedStrandedRecoveryAction({ + companyId, + agentId, + issueId, + runId, + previousStatus: "in_progress", + retryReason: null, + cause: SUCCESSFUL_RUN_MISSING_STATE_REASON, + kind: "missing_disposition", + }); + const runs = await db + .select() + .from(heartbeatRuns) + .where(eq(heartbeatRuns.agentId, agentId)); + expect(runs).toHaveLength(1); + }); + + it("preserves productive continuation for a routine execution after fresh owner work", async () => { + const { agentId, runId, issueId } = await seedStrandedIssueFixture({ + status: "in_progress", + runStatus: "succeeded", + livenessState: "advanced", + }); + await db + .update(issues) + .set({ + originKind: "routine_execution", + originId: randomUUID(), + }) + .where(eq(issues.id, issueId)); + await db + .update(heartbeatRuns) + .set({ + contextSnapshot: { + issueId, + taskId: issueId, + wakeReason: "issue_commented", + source: "issue.comment", + }, + }) + .where(eq(heartbeatRuns.id, runId)); + + const result = + await heartbeatService(db).reconcileStrandedAssignedIssues(); + + expect(result.continuationRequeued).toBe(1); + expect(result.successfulRunHandoffEscalated).toBe(0); + const runs = await db + .select() + .from(heartbeatRuns) + .where(eq(heartbeatRuns.agentId, agentId)); + expect(runs).toHaveLength(2); + expect( + runs.find((run) => run.id !== runId)?.contextSnapshot, + ).toMatchObject({ + issueId, + source: "issue.productive_terminal_continuation_recovery", + }); + }); + it("converts a continuation parked for review into a dependency wait on its open sub-tasks", async () => { const { companyId, agentId, issueId } = await seedStrandedIssueFixture({ status: "in_progress", diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 0361621ec1..46070204f8 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -752,6 +752,55 @@ function isExhaustedSuccessfulRunHandoff(latestRun: LatestIssueRun) { return { ...evidence, exhausted: true }; } +function routineMissingDispositionRecoveryEvidence( + issue: Pick, + latestRun: LatestIssueRun, +) { + // A status-only recovery may succeed without resolving the routine item. + // Treat that lineage as exhausted so it cannot become productive work. + if ( + issue.originKind !== "routine_execution" || + latestRun?.status !== "succeeded" + ) + return null; + + const context = parseObject(latestRun.contextSnapshot); + const paperclipWake = parseObject(context.paperclipWake); + const recovery = parseObject(paperclipWake.recovery); + const wakeReason = + readNonEmptyString(context.wakeReason) ?? + readNonEmptyString(paperclipWake.reason); + const recoveryCause = + readNonEmptyString(context.recoveryCause) ?? + readNonEmptyString(recovery.cause); + const isRecoveryActionRun = + wakeReason === "source_scoped_recovery_action" || + readNonEmptyString(context.recoveryActionId) !== null; + const isMissingDispositionRecovery = + recoveryCause === SUCCESSFUL_RUN_MISSING_STATE_REASON || + recoveryCause === "successful_run_missing_issue_disposition"; + if (!isRecoveryActionRun || !isMissingDispositionRecovery) return null; + + return { + sourceRunId: + readNonEmptyString(context.sourceRunId) ?? + readNonEmptyString(context.resumeFromRunId) ?? + readNonEmptyString(context.retryOfRunId), + correctiveRunId: latestRun.id, + missingDisposition: + readNonEmptyString(context.missingDisposition) ?? "clear_next_step", + handoffAttempt: Math.max(1, asNumber(context.handoffAttempt, 1)), + maxHandoffAttempts: Math.max( + 1, + asNumber( + context.maxHandoffAttempts, + DEFAULT_MAX_SUCCESSFUL_RUN_HANDOFF_ATTEMPTS, + ), + ), + exhausted: true, + }; +} + function issueIdFromRunContext(contextSnapshot: unknown) { const context = parseObject(contextSnapshot); return ( @@ -4938,7 +4987,9 @@ export function recoveryService( } continue; } - const handoffEvidence = isExhaustedSuccessfulRunHandoff(latestRun); + const handoffEvidence = + isExhaustedSuccessfulRunHandoff(latestRun) ?? + routineMissingDispositionRecoveryEvidence(issue, latestRun); if (handoffEvidence) { if (isPluginManagedIssueLifecycle(issue)) { result.skipped += 1; From 646625790853090c247e6e81ed74e94070ea9e79 Mon Sep 17 00:00:00 2001 From: nicls Date: Sat, 12 Sep 2026 10:02:44 +0200 Subject: [PATCH 2/2] fix(recovery): preserve fresh routine owner direction --- .../heartbeat-process-recovery.test.ts | 32 ++++++++++++----- server/src/services/recovery/service.ts | 35 +++++++++++++++++-- 2 files changed, 57 insertions(+), 10 deletions(-) diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index 2960c6db06..7fc0be4c81 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -5838,12 +5838,14 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { expect(runs).toHaveLength(1); }); - it("preserves productive continuation for a routine execution after fresh owner work", async () => { - const { agentId, runId, issueId } = await seedStrandedIssueFixture({ - status: "in_progress", - runStatus: "succeeded", - livenessState: "advanced", - }); + it("preserves productive continuation when fresh owner direction precedes its asynchronous wake", async () => { + const { companyId, agentId, runId, issueId } = + await seedStrandedIssueFixture({ + status: "in_progress", + runStatus: "succeeded", + livenessState: "advanced", + }); + const recoveryRunAt = new Date(Date.now() - 1_000); await db .update(issues) .set({ @@ -5857,11 +5859,25 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { contextSnapshot: { issueId, taskId: issueId, - wakeReason: "issue_commented", - source: "issue.comment", + wakeReason: "source_scoped_recovery_action", + recoveryActionId: randomUUID(), + recoveryCause: SUCCESSFUL_RUN_MISSING_STATE_REASON, + recoveryIntent: "status_only", + allowDeliverableWork: false, + allowDocumentUpdates: false, + resumeRequiresNormalModel: true, }, + createdAt: recoveryRunAt, }) .where(eq(heartbeatRuns.id, runId)); + await db.insert(issueComments).values({ + companyId, + issueId, + authorType: "user", + authorUserId: "local-board", + body: "Continue with this new owner instruction.", + createdAt: new Date(recoveryRunAt.getTime() + 500), + }); const result = await heartbeatService(db).reconcileStrandedAssignedIssues(); diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 46070204f8..81278a2973 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -6,6 +6,7 @@ import { gt, gte, inArray, + isNotNull, isNull, not, notInArray, @@ -1022,6 +1023,28 @@ export function recoveryService( .then((rows) => rows[0] ?? null); } + async function hasFreshUserDirectionAfterRun( + issue: Pick, + latestRun: NonNullable, + ) { + return db + .select({ id: issueComments.id }) + .from(issueComments) + .where( + and( + eq(issueComments.companyId, issue.companyId), + eq(issueComments.issueId, issue.id), + isNotNull(issueComments.authorUserId), + isNull(issueComments.authorAgentId), + isNull(issueComments.createdByRunId), + isNull(issueComments.deletedAt), + gt(issueComments.createdAt, latestRun.createdAt), + ), + ) + .limit(1) + .then((rows) => Boolean(rows[0])); + } + async function summarizeRecentContinuationRetries( companyId: string, issueId: string, @@ -4987,9 +5010,17 @@ export function recoveryService( } continue; } - const handoffEvidence = - isExhaustedSuccessfulRunHandoff(latestRun) ?? + const exhaustedHandoffEvidence = + isExhaustedSuccessfulRunHandoff(latestRun); + const routineRecoveryEvidence = routineMissingDispositionRecoveryEvidence(issue, latestRun); + const hasFreshUserDirection = + routineRecoveryEvidence && latestRun + ? await hasFreshUserDirectionAfterRun(issue, latestRun) + : false; + const handoffEvidence = + exhaustedHandoffEvidence ?? + (hasFreshUserDirection ? null : routineRecoveryEvidence); if (handoffEvidence) { if (isPluginManagedIssueLifecycle(issue)) { result.skipped += 1;