diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index 4f1048318f..c7ad11a150 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -6707,6 +6707,73 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { }); }); + // SPC-21314 deficiency #3 / SPC-37112 / SPC-39089: without this gate the + // exact same fixture (paused, non-invokable source owner) escalates every + // reconciler tick via the branch exercised above — even when the issue has + // a legitimate monitor wake armed days out. That flap burned an in_progress + // issue with a 9-day-out monitor with 10+ issue_continuation_needed wakes + // in ~20 minutes on SPC-37112. + it("does not escalate a stranded-looking issue with a monitor wake armed far in the future", async () => { + const { companyId, agentId, issueId } = await seedStrandedIssueFixture({ + status: "in_progress", + runStatus: "cancelled", + retryReason: "issue_continuation_needed", + runErrorCode: "issue_continuation_waiting_on_review", + // The guard compares against wall-clock `new Date()`, not the fixture's + // fixed 2026-03-19 fixture timestamps, so this must be in the real + // future regardless of when the suite runs. + monitorNextCheckAt: new Date("2099-03-19T00:00:00.000Z"), + }); + await db + .update(agents) + .set({ status: "paused" }) + .where(eq(agents.id, agentId)); + + const result = await heartbeatService(db).reconcileStrandedAssignedIssues(); + expect(result.escalated).toBe(0); + expect(result.armedMonitorExempted).toBe(1); + expect(result.issueIds).toEqual([]); + + const sourceIssue = await db + .select() + .from(issues) + .where(eq(issues.id, issueId)) + .then((rows) => rows[0] ?? null); + expect(sourceIssue).toMatchObject({ + status: "in_progress", + assigneeAgentId: agentId, + }); + + const actions = await db + .select() + .from(issueRecoveryActions) + .where( + and( + eq(issueRecoveryActions.companyId, companyId), + eq(issueRecoveryActions.sourceIssueId, issueId), + ), + ); + expect(actions).toHaveLength(0); + + const wakeups = await db + .select() + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.agentId, agentId), + ), + ); + expect( + wakeups.some( + (wake) => + wake.reason === "issue_continuation_needed" || + (wake.payload as { retryReason?: string } | null)?.retryReason === + "issue_continuation_needed", + ), + ).toBe(false); + }); + it("keeps a legacy agent-owned recovery action readable without scheduling another takeover wake", async () => { const { companyId, agentId, issueId } = await seedStrandedIssueFixture({ status: "in_progress", diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 961d17e110..61f266ad0c 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -220,6 +220,30 @@ export function resolveSuccessfulRunMissingStateMaxAttempts( return SUCCESSFUL_RUN_MISSING_STATE_MAX_ATTEMPTS; } +// SPC-21314 deficiency #3 / SPC-37112 / SPC-39089: an issue with an armed, +// unexpired `executionState.monitor` (persisted denormalized as +// `issue.monitorNextCheckAt`) owns its own wake cadence via +// `tickDueIssueMonitors`. Treating it as "stranded" in +// `reconcileStrandedAssignedIssues` duplicates that cadence and can +// re-escalate on every reconciler tick: SPC-37112 saw an in_progress issue +// with a monitor armed 9 days out rewoken via `issue_continuation_needed` +// 10+ times in ~20 minutes (~1-2 min cadence) before the assignee worked +// around it by force-setting status to `blocked` — itself a known-bad move +// because it silently clears the monitor. `monitorNextCheckAt` is only +// non-null while the monitor is "scheduled" (armed); it is nulled out on +// trigger/clear/exhaustion, so a future value here is a reliable proxy for +// "not cleared, not expired." +export function hasArmedMonitorWake( + issue: { status: string; monitorNextCheckAt: Date | null }, + now: Date, +): boolean { + return ( + (issue.status === "in_progress" || issue.status === "in_review") && + !!issue.monitorNextCheckAt && + issue.monitorNextCheckAt.getTime() > now.getTime() + ); +} + type RecoveryWakeupOptions = { source?: "timer" | "assignment" | "on_demand" | "automation"; triggerDetail?: "manual" | "ping" | "callback" | "system"; @@ -4216,6 +4240,7 @@ export function recoveryService( recentProgressExempted: 0, operatorCancelExempted: 0, onboardingFirstTaskExempted: 0, + armedMonitorExempted: 0, skipped: 0, issueIds: [] as string[], }; @@ -4278,6 +4303,12 @@ export function recoveryService( continue; } + if (hasArmedMonitorWake(issue, new Date())) { + result.armedMonitorExempted += 1; + result.skipped += 1; + continue; + } + let latestRun = await getLatestIssueRun(issue.companyId, issue.id); const agent = await getAgent(agentId); diff --git a/server/src/services/recovery/successful-run-missing-state-cap.test.ts b/server/src/services/recovery/successful-run-missing-state-cap.test.ts index 197d1dbbce..2496e2e7e6 100644 --- a/server/src/services/recovery/successful-run-missing-state-cap.test.ts +++ b/server/src/services/recovery/successful-run-missing-state-cap.test.ts @@ -3,6 +3,7 @@ import { SUCCESSFUL_RUN_MISSING_STATE_MAX_ATTEMPTS, SUCCESSFUL_RUN_MISSING_STATE_MAX_ATTEMPTS_CEILING, SUCCESSFUL_RUN_MISSING_STATE_MAX_ATTEMPTS_DEFAULT, + hasArmedMonitorWake, parseSuccessfulRunMissingStateMaxAttempts, resolveSuccessfulRunMissingStateMaxAttempts, } from "./service.js"; @@ -74,3 +75,58 @@ describe("resolveSuccessfulRunMissingStateMaxAttempts", () => { ); }); }); + +// SPC-21314 deficiency #3 / SPC-37112 / SPC-39089: reconcileStrandedAssignedIssues +// must not treat an issue as stranded while it has a legitimately armed, +// unexpired monitor wake — that cadence is owned by tickDueIssueMonitors, not +// the stranded-issue reconciler. +describe("hasArmedMonitorWake", () => { + const now = new Date("2026-09-09T00:00:00.000Z"); + const nineDaysOut = new Date("2026-09-18T00:00:00.000Z"); + const oneMinuteAgo = new Date("2026-09-08T23:59:00.000Z"); + + it("is true for an in_progress issue with a future monitor wake", () => { + expect( + hasArmedMonitorWake( + { status: "in_progress", monitorNextCheckAt: nineDaysOut }, + now, + ), + ).toBe(true); + }); + + it("is true for an in_review issue with a future monitor wake", () => { + expect( + hasArmedMonitorWake( + { status: "in_review", monitorNextCheckAt: nineDaysOut }, + now, + ), + ).toBe(true); + }); + + it("is false once the monitor wake is in the past (due, not armed)", () => { + expect( + hasArmedMonitorWake( + { status: "in_progress", monitorNextCheckAt: oneMinuteAgo }, + now, + ), + ).toBe(false); + }); + + it("is false when there is no scheduled monitor", () => { + expect( + hasArmedMonitorWake({ status: "in_progress", monitorNextCheckAt: null }, now), + ).toBe(false); + }); + + it("is false for statuses the monitor cannot be scheduled on", () => { + expect( + hasArmedMonitorWake({ status: "todo", monitorNextCheckAt: nineDaysOut }, now), + ).toBe(false); + expect( + hasArmedMonitorWake({ status: "blocked", monitorNextCheckAt: nineDaysOut }, now), + ).toBe(false); + expect( + hasArmedMonitorWake({ status: "done", monitorNextCheckAt: nineDaysOut }, now), + ).toBe(false); + }); +});