From e9b1ef846c6b916a13dd765ea7b26a30e99d34a5 Mon Sep 17 00:00:00 2001 From: Builder Date: Mon, 27 Jul 2026 02:32:52 +0000 Subject: [PATCH 1/3] fix(recovery): skip resolved-dependency backstop candidates with non-invokable assignees The `reconcileResolvedDependencyWakeBackstop` sweep listed every blocked issue whose blockers had resolved and enqueued a wake for its assignee. It did not check whether the assignee was actually invokable, so an issue assigned to a paused, terminated, or pending_approval agent triggered a fresh wake on every heartbeat. Each wake failed with `wake_target_not_invokable` (409) and the same issue re-entered the candidate set the next tick, producing a steady stream of warnings until an operator intervened. Add `agents.status NOT IN (paused, terminated, pending_approval)` to the candidate query, mirroring the atomic guard already used when the heartbeat flips an agent to `running` (see `services/heartbeat.ts` around the pause durability update). Both branches of `queryCandidates` (with and without `blockerIssueId`) now `innerJoin(agents)` and share the filter. Org-chain invalidity is still evaluated per-candidate at the enqueue path, so no behavior change for the org-chain case. --- ...eartbeat-issue-liveness-escalation.test.ts | 30 +++++++++++++++++++ server/src/services/recovery/service.ts | 14 ++++++++- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/server/src/__tests__/heartbeat-issue-liveness-escalation.test.ts b/server/src/__tests__/heartbeat-issue-liveness-escalation.test.ts index 1f6d5b72a8..506705f05b 100644 --- a/server/src/__tests__/heartbeat-issue-liveness-escalation.test.ts +++ b/server/src/__tests__/heartbeat-issue-liveness-escalation.test.ts @@ -630,6 +630,36 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => { expect(wakes.some((wake) => ["queued", "claimed", "completed"].includes(wake.status))).toBe(true); }); + it.each(["paused", "terminated", "pending_approval"] as const)( + "skips resolved-dependency backstop candidates whose assignee is %s", + async (assigneeStatus) => { + await enableAutoRecovery(); + const { companyId, agentId, blockedIssueId } = + await seedResolvedDependencyBackstopFixture({ workspaceState: "none" }); + // Flip the assignee into a directly non-invokable state. The backstop + // must drop the candidate at the SQL layer — otherwise it enqueues a + // wake that always fails with 409 and loops on every heartbeat. + await db.update(agents).set({ status: assigneeStatus }).where(eq(agents.id, agentId)); + + const result = await heartbeatService(db).reconcileIssueGraphLiveness(); + + expect(result.dependencyWakeBackstopChecked).toBe(0); + expect(result.dependencyWakesHealed).toBe(0); + expect(result.dependencyWakeIssueIds).not.toContain(blockedIssueId); + + const wakes = await db + .select({ id: agentWakeupRequests.id }) + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.reason, "issue_blockers_resolved"), + ), + ); + expect(wakes).toHaveLength(0); + }, + ); + it("waits for workspace finalize before healing a resolved blocked dependent", async () => { await enableAutoRecovery(); const { companyId, agentId, blockedIssueId, blockerIssueId, executionWorkspaceId } = diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 8b00c7d07b..67ff74e0f6 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -53,7 +53,7 @@ import { buildIssueBlockersResolvedWakeStateKey, findExistingIssueBlockersResolvedWakeForReadyState, } from "../issue-dependency-wakeups.js"; -import { evaluateAgentInvokabilityFromDb } from "../agent-invokability.js"; +import { DIRECT_NON_INVOKABLE_STATUSES, evaluateAgentInvokabilityFromDb } from "../agent-invokability.js"; import { isHeartbeatWakeOnDemandEnabled } from "../heartbeat-policy.js"; import { getRunLogStore } from "../run-log-store.js"; import { @@ -6303,6 +6303,16 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) eq(issues.status, "blocked"), visibleIssueCondition(), sql`${issues.assigneeAgentId} is not null`, + // Skip candidates whose assignee is directly non-invokable (paused, + // terminated, pending_approval). Without this filter the backstop + // repeatedly enqueues wakes that always fail with 409, producing a + // steady stream of `wake_target_not_invokable` warnings for the same + // issue every heartbeat until an operator intervenes. Matches the + // atomic guard in the heartbeat's `agents.status -> running` update. + // Org-chain invalidity is not covered here for cost reasons; the + // enqueue path still filters those cases via + // `evaluateAgentInvokabilityFromDb`. + notInArray(agents.status, [...DIRECT_NON_INVOKABLE_STATUSES]), ]; if (opts?.companyId) filters.push(eq(issues.companyId, opts.companyId)); if (afterIssueId) filters.push(gt(issues.id, afterIssueId)); @@ -6324,6 +6334,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) }) .from(issueRelations) .innerJoin(issues, eq(issueRelations.relatedIssueId, issues.id)) + .innerJoin(agents, eq(agents.id, issues.assigneeAgentId)) .where(and(...filters)) .orderBy(asc(issues.id)) .limit(RESOLVED_DEPENDENCY_WAKE_BACKSTOP_CANDIDATE_LIMIT); @@ -6338,6 +6349,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) totalCount: sql`count(*) over()::int`, }) .from(issues) + .innerJoin(agents, eq(agents.id, issues.assigneeAgentId)) .where(and(...filters)) .orderBy(asc(issues.id)) .limit(RESOLVED_DEPENDENCY_WAKE_BACKSTOP_CANDIDATE_LIMIT);