From f572e08678d57de7afcf3dcb7788f792190a3291 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sat, 22 Aug 2026 11:41:24 -0500 Subject: [PATCH] fix(recovery): stop automatic stranded-task takeovers (#11961) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - The recovery service restores execution when a task loses its live path. > - The service retries the original agent for a limited number of attempts. > - The old fallback could select a manager or an executive and wake that agent. > - That fallback changed the effective recovery owner without a board decision. > - This pull request keeps the source owner and gives the exhausted recovery decision to the board. > - The benefit is a clear ownership rule with no automatic task takeover. ## Linked Issues or Issue Description Refs: #11807 Refs: #11817 **What existing behavior does this improve?** This improves stranded-task recovery in the server and the recovery action card in the board UI. **Subsystem affected** Cross-cutting: server recovery orchestration, recovery observability, board UI, and execution documentation. **Current behavior** Paperclip retries the original agent for a limited number of attempts. After the retry limit, it can select a manager, task creator, CTO, or CEO as a recovery owner. It can then wake that substitute agent. The source task keeps its assignee, but the automatic substitute wake creates an implicit takeover path. **Proposed behavior** Paperclip keeps the limited retry path for the original agent. If recovery is exhausted or unsafe, Paperclip creates one board-owned source recovery action. It keeps both source assignee fields. It does not wake a substitute agent. The board can repair, retry the original owner, explicitly reassign, or resolve the task. **Reason and benefit** Source task ownership must remain stable until a person or an approved policy changes it. The new rule removes implicit manager and executive takeover. It also gives operators clear evidence through the `board_escalation_no_takeover_v1` routing marker. **Breaking changes** Automatic recovery no longer wakes a manager or executive after the original-agent retry limit. Existing active agent-owned recovery actions remain visible and can resolve. Paperclip does not schedule a new takeover wake for those legacy actions. ## What Changed - Route exhausted and unsafe stranded recovery to a board-owned source action. - Preserve agent and user assignee fields during automatic escalation. - Keep limited same-agent continuity repair and provider quota monitoring. - Stop new manager, creator, CTO, and CEO recovery wakes. - Keep legacy agent-owned recovery actions readable and resolvable. - Add the routing marker to new board escalation evidence and observability. - Update recovery notices, the board UI card, tests, and execution documentation. ## Verification - Run `pnpm -r typecheck`. - Run `pnpm build`. - Run `pnpm check:token-gates`. - Run `pnpm --filter @paperclipai/server exec vitest run src/__tests__/heartbeat-process-recovery.test.ts`. - Run `pnpm --filter @paperclipai/server exec vitest run src/__tests__/heartbeat-workspace-branch-containment.test.ts`. - Run the focused recovery and UI Vitest files changed by this pull request. - Confirm that a paused or over-budget source owner creates one board action, keeps the source assignee, and creates no substitute wake. ## Risks - Operators must now make the final recovery decision after the original-agent limit. - Legacy agent-owned actions use their stored contract. This avoids a rollout-time ownership rewrite. - No database migration or API response shape changes are included. - The tests cover concurrent escalation, paused and over-budget owners, legacy actions, provider quota monitoring, and UI presentation. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used OpenAI Codex with GPT-5. The hosted exact model revision and context window are not exposed. Reasoning, tool use, and code execution were enabled. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge Co-authored-by: Paperclip --- doc/SPEC-implementation.md | 8 +- doc/SPEC.md | 1 + doc/execution-semantics.md | 16 +- .../src/__tests__/disposition-repair.test.ts | 21 - .../heartbeat-process-recovery.test.ts | 423 ++++----- ...tbeat-workspace-branch-containment.test.ts | 14 +- .../__tests__/issue-recovery-actions.test.ts | 156 +-- .../__tests__/recovery-observability.test.ts | 24 + server/src/services/heartbeat.ts | 2 - server/src/services/issue-recovery-actions.ts | 78 +- server/src/services/recovery-observability.ts | 6 +- .../services/recovery/disposition-repair.ts | 11 - server/src/services/recovery/service.ts | 896 +++--------------- .../services/recovery/stranded-notice.test.ts | 2 +- .../src/services/recovery/stranded-notice.ts | 8 +- .../recovery/successful-run-handoff.test.ts | 2 +- .../recovery/successful-run-handoff.ts | 10 +- .../IssueRecoveryActionCard.test.tsx | 21 +- ui/src/components/IssueRecoveryActionCard.tsx | 2 +- ui/src/fixtures/systemNoticeFixtures.ts | 4 +- ui/src/lib/successful-run-handoff.ts | 2 +- ui/src/pages/SystemNoticeUxLab.tsx | 2 +- 22 files changed, 569 insertions(+), 1140 deletions(-) diff --git a/doc/SPEC-implementation.md b/doc/SPEC-implementation.md index bb672aca60..5a4984b1bb 100644 --- a/doc/SPEC-implementation.md +++ b/doc/SPEC-implementation.md @@ -514,13 +514,13 @@ V1 non-terminal liveness rule: - disposition repair revalidates blockers, children, interactions, approvals, monitors, execution stages, queued wakes, active runs, work products, owner invokability, budgets, and governance before every attempt; the attempt bound is keyed by durable source state, so comments or equivalent parked prose do not reset it while durable source-state changes may establish a new fingerprint - backwards-compatible upgrades count consecutive historical `issue_continuation_waiting_on_review` cancellations for the unchanged accepted-interaction source state against the same five-attempt disposition-repair ceiling; missing pre-upgrade recovery-action rows do not reset the budget - the source fingerprint, source-attempt count, next due time, source owner, and return owner persist in the recovery action; startup and periodic reconciliation resume that exact lineage without duplicate wakes, fold it when a current typed wait appears, and reschedule or escalate an expired action that has no live scheduled run -- source-attempt exhaustion opens one source-scoped manager recovery action without changing the source assignee; manager runs get five separate persisted attempts on the same immediate, 60, 120, 240, and 480 second bounded-delay schedule, then escalate visibly to the board when exhausted +- source-attempt exhaustion opens one board-owned source-scoped recovery action without changing the source assignee and without waking a substitute agent; the board explicitly chooses whether to repair, retry the original owner, reassign, or resolve - an active recovery action counts as a live source or blocker-chain path only while its owner has a live run, queued wake, scheduled retry, typed wait, or explicit board escalation; source and blocker projections consume the same nested recovery-path result - when Paperclip cannot safely infer the next action, it surfaces the problem through visible blocked/recovery work instead of silently completing or reassigning work - explicit recovery actions are the liveness primitive; source-scoped actions are the default form, issue-backed recovery is a fallback for independent repair work or safety boundaries, and comments alone are evidence rather than a healthy liveness path -- recovery-action ownership is separate from source-task ownership: automatic repair and manager escalation preserve the source `assigneeAgentId`; reassignment requires an explicit decision or a policy-defined serious failure -- source-scoped recovery routing is cause-keyed: lost processes, missing successful-run dispositions, and output-inactivity terminations retry the original agent when invokable; provider-quota failures create/reuse a scheduled wait-recovery monitor without a takeover wake; workspace validation and unknown causes route to the manager ladder -- recovery-scoped wakes replace the normal deliverable execution contract with a cause-specific recovery contract, and successful repair returns the issue to the recorded original owner by default while recording `handed_back` versus `owner_completed` +- recovery-action ownership is separate from source-task ownership: automatic repair and board escalation preserve both source assignee fields; reassignment requires an explicit board decision or a policy-defined serious failure +- source-scoped recovery routing is cause-keyed: bounded continuity and disposition repair may retry only the original agent; provider-quota failures create/reuse a scheduled wait-recovery monitor; every other exhausted or unsafe path creates/reuses a board-owned recovery action with `routingPolicy: board_escalation_no_takeover_v1` and no substitute-agent wake +- legacy active agent-owned recovery actions remain readable, resolvable, and API-compatible after upgrade, but reconciliation does not enqueue another takeover wake for them Detailed ownership, execution, blocker, active-run watchdog, crash-recovery, and non-terminal liveness semantics are documented in `doc/execution-semantics.md`. diff --git a/doc/SPEC.md b/doc/SPEC.md index 95b632d492..aaa91cc9c4 100644 --- a/doc/SPEC.md +++ b/doc/SPEC.md @@ -424,6 +424,7 @@ Paperclip manages task-linked work artifacts: issue documents (rich-text plans, When an agent crashes or disappears mid-task, Paperclip does **not** auto-reassign or auto-release the task. Instead: - Paperclip surfaces stale tasks (tasks in `in_progress` with no recent activity) through dashboards and reporting +- Paperclip may perform bounded continuity repair with the same assigned agent; when that is exhausted or unsafe, it opens a board-owned recovery action without waking a substitute agent - Paperclip does not fail silently — the auditing and visibility tools make problems obvious - Recovery is handled by humans or by emergent processes (e.g. a project manager agent whose job is to monitor for stale work and surface it) diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 9be7afd89a..9964cdea4c 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -519,7 +519,7 @@ Example: Recovery rule: - if the latest issue-linked run failed/timed out/cancelled and no live execution path remains, Paperclip queues one automatic assignment recovery wake -- if that recovery wake also finishes and the issue is still stranded, Paperclip moves the issue to `blocked` and opens or updates an explicit recovery action when a bounded owner/action is known; the visible comment is evidence, not the recovery path by itself +- if that recovery wake also finishes and the issue is still stranded, Paperclip moves the issue to `blocked` and opens or updates a board-owned recovery action without changing the source assignee or waking a substitute agent; the visible comment is evidence, not the recovery path by itself This is a dispatch recovery, not a continuation recovery. @@ -540,7 +540,7 @@ Example: Recovery rule: - Paperclip queues one automatic continuation wake -- if that continuation wake also finishes and the issue is still stranded, Paperclip moves the issue to `blocked` and opens or updates an explicit recovery action when a bounded owner/action is known; the visible comment is evidence, not the recovery path by itself +- if that continuation wake also finishes and the issue is still stranded, Paperclip moves the issue to `blocked` and opens or updates a board-owned recovery action without changing the source assignee or waking a substitute agent; the visible comment is evidence, not the recovery path by itself This is an active-work continuity recovery. @@ -558,7 +558,7 @@ Recovery rule for a parked-for-review continuation: - the retry bound is keyed by an idempotent durable source-state fingerprint. Comments, repeated parked summaries, and equivalent prose do not reset it. Durable changes such as source status or assignee changes, dependency or interaction changes, approval or execution-policy changes, monitor changes, or work-product changes may create a new fingerprint - on upgrade, consecutive historical `issue_continuation_waiting_on_review` cancellations for the same accepted interaction and still-unchanged durable source state seed this same counter. Five applicable pre-upgrade parks therefore exhaust the ceiling immediately; the absence of a historical `deliberate_wait_without_target` recovery-action row does not grant five new attempts - the action persists the unchanged fingerprint, source-attempt count, due time, source owner, and return owner. Startup and periodic reconciliation reuse that state, fold the action when a current typed wait appears, and reschedule or escalate an expired attempt that has no live scheduled run. Idempotency keys prevent a restart from creating duplicate wakes or scheduled runs -- after five attempts with the same fingerprint, Paperclip opens one separate source-scoped manager recovery action. The manager owns only path repair; the source assignee remains unchanged. The manager gets five separate attempts on the same immediate, 60, 120, 240, and 480 second bounded-delay schedule before exhaustion escalates visibly to the board +- after five attempts with the same fingerprint, Paperclip opens one board-owned source-scoped recovery action. The source assignee remains unchanged, no manager/creator/executive substitute is woken, and the board chooses whether to repair, retry the original owner, explicitly reassign, or resolve - a recovery action is a healthy wait only while its owner has a live run, queued wake, scheduled retry, typed wait, or explicit board escalation. Source liveness and every blocker-chain projection use that same nested result An accepted interaction supersedes a continuation park recorded before that acceptance. A queued continuation carrying a parseable `interactionResolvedAt` must not be cancelled solely because an older continuation summary says to wait for review or approval. Interaction-continuation recovery is bounded: after three consecutive continuation wakes are cancelled without a run starting, recovery converts a real dependency wait when one exists or escalates the missing execution path visibly instead of requeueing forever. @@ -788,9 +788,11 @@ Examples: - a dependency graph has an invalid/uninvokable owner, unassigned blocker, or invalid review participant - an active run is silent past the watchdog threshold -The recovery action stays source-scoped by default. The source issue should show the recovery owner, cause, evidence, next action, and wake or monitor policy in its own thread/detail surface. +The recovery action stays source-scoped by default. Stranded-task escalation is board-owned and records the cause, evidence, next action, source and return owner, `routingPolicy: board_escalation_no_takeover_v1`, and wake or monitor policy in the source thread/detail surface. -The recovery owner owns the repair action, not the source deliverable. Manager escalation must preserve the source issue assignee unless an operator makes an explicit reassignment decision or an applicable serious-failure policy authorizes transfer. +The board owns the recovery decision, not the source deliverable. Automatic recovery must preserve both source assignee fields. Only an explicit operator decision or applicable serious-failure policy may transfer the deliverable. + +An upgrade may encounter an already-active agent-owned recovery action. Paperclip keeps that record readable and resolvable for compatibility, but periodic reconciliation does not enqueue another takeover wake from it. Create an issue-backed recovery action only when a separate issue is the right execution object. In that fallback form, the source issue remains visible and is blocked on the recovery issue when blocking is necessary for correctness. The recovery owner must restore a live path, resolve the source issue manually, delegate real follow-up work, or record the reason the signal is a false positive. @@ -802,7 +804,7 @@ Human escalation is required when the next safe action depends on board judgment Examples: -- all candidate recovery owners are paused, terminated, pending approval, or budget-blocked +- the original owner is paused, terminated, pending approval, or budget-blocked - the issue is human-owned rather than agent-owned - the run is intentionally quiet but needs an operator decision before cancellation or continuation @@ -822,7 +824,7 @@ The recovery model is intentionally conservative: - preserve ownership - use the cause-specific bound when the control plane lost execution continuity; deliberate waits without a target use five fingerprinted original-owner disposition repairs -- open an explicit recovery action when the system can identify a bounded recovery owner/action +- open a board-owned recovery action when the original-owner bound is exhausted or unsafe - escalate visibly when the system cannot safely keep going ## 15. Practical Interpretation diff --git a/server/src/__tests__/disposition-repair.test.ts b/server/src/__tests__/disposition-repair.test.ts index 6d0b3ad583..c53f303668 100644 --- a/server/src/__tests__/disposition-repair.test.ts +++ b/server/src/__tests__/disposition-repair.test.ts @@ -4,10 +4,7 @@ import { classifyContinuationFailure } from "../services/recovery/service.ts"; import { DISPOSITION_REPAIR_BASE_DELAYS_MS, DISPOSITION_REPAIR_MAX_ATTEMPTS, - RECOVERY_OWNER_BASE_DELAYS_MS, - RECOVERY_OWNER_MAX_ATTEMPTS, dispositionRepairDelayMs, - recoveryOwnerDelayMs, } from "../services/recovery/disposition-repair.ts"; const deliberateWaitRun = { @@ -53,22 +50,4 @@ describe("owner-sticky disposition repair", () => { expect(dispositionRepairDelayMs(2, fingerprint)).toEqual(timings[1]); expect(() => dispositionRepairDelayMs(6, fingerprint)).toThrow(/Invalid disposition repair attempt/); }); - - it("keeps recovery-owner retries separate and bounded", () => { - const fingerprint = "disposition_repair:v1:manager"; - const timings = [1, 2, 3, 4, 5].map((attempt) => recoveryOwnerDelayMs(attempt, fingerprint)); - - expect(RECOVERY_OWNER_MAX_ATTEMPTS).toBe(5); - expect(RECOVERY_OWNER_BASE_DELAYS_MS).toEqual([0, 60_000, 120_000, 240_000, 480_000]); - expect(timings[0]).toEqual({ baseDelayMs: 0, jitterMs: 0, delayMs: 0 }); - expect(timings[1]?.baseDelayMs).toBe(60_000); - expect(timings[1]?.jitterMs).toBeLessThanOrEqual(6_000); - expect(timings[2]?.baseDelayMs).toBe(120_000); - expect(timings[2]?.jitterMs).toBeLessThanOrEqual(12_000); - expect(timings[3]?.baseDelayMs).toBe(240_000); - expect(timings[3]?.jitterMs).toBeLessThanOrEqual(24_000); - expect(timings[4]?.baseDelayMs).toBe(480_000); - expect(timings[4]?.jitterMs).toBeLessThanOrEqual(48_000); - expect(() => recoveryOwnerDelayMs(6, fingerprint)).toThrow(/Invalid recovery owner attempt/); - }); }); diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index 25f031c945..bd1136990d 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -1022,8 +1022,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { recoveryIssueId: null, kind: input.kind ?? "stranded_assigned_issue", status: "active", - ownerType: "agent", - ownerAgentId: input.agentId, + ownerType: "board", + ownerAgentId: null, previousOwnerAgentId: input.previousOwnerAgentId ?? input.agentId, returnOwnerAgentId: input.returnOwnerAgentId ?? input.agentId, cause: input.cause ?? "stranded_assigned_issue", @@ -1035,14 +1035,15 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { previousStatus: input.previousStatus, latestRunId: input.runId, retryReason: input.retryReason ?? null, + routingPolicy: "board_escalation_no_takeover_v1", }); if (input.cause === "execution_review_participant_recovery") { expect(action.nextAction).toContain("failed review participant path"); } else if (input.cause === "process_lost") { - expect(action.nextAction).toContain("Retry the original assignee from durable progress"); + expect(action.nextAction).toContain("explicitly retry the original owner"); } else { expect(action.nextAction).toContain( - input.kind === "missing_disposition" ? "valid issue disposition" : "Restore a live execution path", + input.kind === "missing_disposition" ? "valid issue disposition" : "Board operator", ); } @@ -1056,50 +1057,10 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { )); expect(recoveryIssues).toHaveLength(0); - const recoveryWakeup = await waitForValue(async () => { - const wakeups = await db - .select() - .from(agentWakeupRequests) - .where(eq(agentWakeupRequests.agentId, input.agentId)); - return wakeups.find((wakeup) => { - const payload = wakeup.payload as Record | null; - return payload?.issueId === input.issueId && - payload?.sourceIssueId === input.issueId && - payload?.recoveryActionId === action.id && - payload?.strandedRunId === input.runId; - }) ?? null; - }); - expect(recoveryWakeup).toMatchObject({ - companyId: input.companyId, - reason: "source_scoped_recovery_action", - source: "assignment", - payload: expect.objectContaining({ - modelProfile: "cheap", - allowDeliverableWork: false, - allowDocumentUpdates: false, - resumeRequiresNormalModel: true, - }), - }); - - const recoveryRun = recoveryWakeup?.runId - ? await db - .select() - .from(heartbeatRuns) - .where(eq(heartbeatRuns.id, recoveryWakeup.runId)) - .then((rows) => rows[0] ?? null) - : null; - expect(recoveryRun?.contextSnapshot).toMatchObject({ - issueId: input.issueId, - taskId: input.issueId, - source: "issue_recovery_action", - recoveryActionId: action.id, - sourceIssueId: input.issueId, - strandedRunId: input.runId, - modelProfile: "cheap", - allowDeliverableWork: false, - allowDocumentUpdates: false, - resumeRequiresNormalModel: true, - }); + const recoveryWakeups = await db.select().from(agentWakeupRequests).where( + sql`${agentWakeupRequests.payload} ->> 'recoveryActionId' = ${action.id}`, + ); + expect(recoveryWakeups).toHaveLength(0); await waitForHeartbeatIdle(db); const sourceIssue = await db .select() @@ -2539,7 +2500,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { expect(comments[0]?.presentation).toMatchObject({ kind: "system_notice", tone: "danger" }); expect(noticeMetadataReferencesRecoveryAction(comments[0]?.metadata, recoveryAction.id)).toBe(true); expect(commentMetadataRows(comments[0]).some((row) => - row.type === "agent_link" && row.label === "Recovery owner" && row.name === "CodexCoder", + row.type === "key_value" && row.label === "Recovery owner" && row.value === "Board decision required", )).toBe(true); }); @@ -3556,7 +3517,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { kind: "workspace_validation", cause: "workspace_validation_failed", status: "active", - ownerAgentId: agentId, + ownerType: "board", + ownerAgentId: null, recoveryIssueId: null, }); expect(recoveryAction?.evidence).toMatchObject({ @@ -3565,7 +3527,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { latestRunErrorCode: "workspace_validation_failed", recoveryCause: "workspace_validation_failed", }); - expect(recoveryAction?.nextAction).toContain("Repair the source issue workspace link"); + expect(recoveryAction?.nextAction).toContain("repair the source task workspace link"); const validationComment = await waitForValue(async () => { const rows = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId)); @@ -3649,10 +3611,11 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { kind: "configuration_validation", cause: "configuration_incomplete", status: "active", - ownerAgentId: agentId, + ownerType: "board", + ownerAgentId: null, recoveryIssueId: null, }); - expect(recoveryAction?.nextAction).toContain("Bind the missing secret"); + expect(recoveryAction?.nextAction).toContain("bind the missing secret"); const configurationComment = await waitForValue(async () => { const rows = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId)); @@ -4065,10 +4028,10 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { version: 1, sections: expect.arrayContaining([ expect.objectContaining({ - title: "Recovery owner", + title: "Recovery", rows: expect.arrayContaining([ expect.objectContaining({ type: "key_value", label: "Recovery action", value: recoveryAction.id }), - expect.objectContaining({ type: "agent_link", label: "Recovery owner", name: "CodexCoder" }), + expect.objectContaining({ type: "key_value", label: "Recovery owner", value: "Board decision required" }), ]), }), expect.objectContaining({ @@ -4665,7 +4628,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { }); }); - it("refuses source and manager attempt six without transferring the source", async () => { + it("escalates exhausted source-owner repair to the board without a substitute wake", async () => { const { companyId, agentId, runId, issueId } = await seedStrandedIssueFixture({ status: "in_progress", runStatus: "cancelled", @@ -4686,179 +4649,62 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { }); await db.update(agents).set({ reportsTo: managerId }).where(eq(agents.id, agentId)); - const sourceIssue = await db - .select() - .from(issues) - .where(eq(issues.id, issueId)) - .then((rows) => rows[0]!); + const sourceIssue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0]!); const state = await collectDispositionRepairSourceState(db, { issue: sourceIssue }); - await db - .update(heartbeatRuns) - .set({ - contextSnapshot: { - issueId, - taskId: issueId, - wakeReason: "issue_disposition_repair", - retryReason: "issue_disposition_repair", - dispositionRepairFingerprint: state.fingerprint, - dispositionRepairAttempt: 5, - dispositionRepairMaxAttempts: 5, - }, - }) - .where(eq(heartbeatRuns.id, runId)); + await db.update(heartbeatRuns).set({ + contextSnapshot: { + issueId, + taskId: issueId, + wakeReason: "issue_disposition_repair", + retryReason: "issue_disposition_repair", + dispositionRepairFingerprint: state.fingerprint, + dispositionRepairAttempt: 5, + dispositionRepairMaxAttempts: 5, + }, + }).where(eq(heartbeatRuns.id, runId)); const result = await heartbeatService(db).reconcileStrandedAssignedIssues(); - expect(result.dispositionRepairRequeued).toBe(0); - expect(result.escalated).toBe(1); + expect(result).toMatchObject({ dispositionRepairRequeued: 0, escalated: 1 }); - const sourceAfter = await db - .select() - .from(issues) - .where(eq(issues.id, issueId)) - .then((rows) => rows[0] ?? null); - expect(sourceAfter).toMatchObject({ - status: "blocked", - assigneeAgentId: agentId, - }); - - const action = await db - .select() - .from(issueRecoveryActions) - .where(and( + const [sourceAfter, action, substituteWakes, sourceAttemptSix] = await Promise.all([ + db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0] ?? null), + db.select().from(issueRecoveryActions).where(and( eq(issueRecoveryActions.companyId, companyId), eq(issueRecoveryActions.sourceIssueId, issueId), - )) - .then((rows) => rows[0] ?? null); - expect(action).toMatchObject({ - kind: "deliberate_wait_without_target", - status: "active", - ownerAgentId: managerId, - previousOwnerAgentId: agentId, - returnOwnerAgentId: agentId, - attemptCount: 1, - maxAttempts: 5, - resolutionNote: "unchanged_source_state_exhausted", - }); - expect(action?.evidence).toMatchObject({ - terminalReason: "unchanged_source_state_exhausted", - sourceAttemptCount: 5, - sourceMaxAttempts: 5, - }); - const sourceAttemptSix = await db - .select({ id: heartbeatRuns.id }) - .from(heartbeatRuns) - .where(and( + )).then((rows) => rows[0] ?? null), + db.select().from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.agentId, managerId), + )), + db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.agentId, agentId), sql`${heartbeatRuns.contextSnapshot} ->> 'dispositionRepairAttempt' = '6'`, - )); - expect(sourceAttemptSix).toHaveLength(0); - expect(action?.wakePolicy).toMatchObject({ - type: "bounded_recovery_owner", - attempt: 1, - maxAttempts: 5, - preservesSourceAssignee: true, - }); - - const firstManagerRun = await waitForValue(async () => db - .select() - .from(heartbeatRuns) - .where(and( - eq(heartbeatRuns.companyId, companyId), - eq(heartbeatRuns.agentId, managerId), - sql`${heartbeatRuns.contextSnapshot} ->> 'recoveryActionId' = ${action!.id}`, - )) - .then((rows) => rows[0] ?? null)); - await db - .update(heartbeatRuns) - .set({ status: "failed", errorCode: "process_lost", finishedAt: new Date() }) - .where(eq(heartbeatRuns.id, firstManagerRun.id)); - await db - .update(agentWakeupRequests) - .set({ status: "completed" }) - .where(eq(agentWakeupRequests.runId, firstManagerRun.id)); - - const restartedHeartbeat = heartbeatService(db); - for (let attempt = 2; attempt <= 5; attempt += 1) { - await restartedHeartbeat.reconcileStrandedAssignedIssues(); - const managerRun = await db - .select() - .from(heartbeatRuns) - .where(and( - eq(heartbeatRuns.companyId, companyId), - eq(heartbeatRuns.agentId, managerId), - sql`${heartbeatRuns.contextSnapshot} ->> 'recoveryActionId' = ${action!.id}`, - sql`${heartbeatRuns.contextSnapshot} ->> 'recoveryOwnerAttempt' = ${String(attempt)}`, - )) - .then((rows) => rows[0] ?? null); - expect(managerRun).toMatchObject({ - status: "scheduled_retry", - scheduledRetryAttempt: attempt, - scheduledRetryReason: "recovery_owner_retry", - }); - - await restartedHeartbeat.reconcileStrandedAssignedIssues(); - const duplicateRuns = await db - .select({ id: heartbeatRuns.id }) - .from(heartbeatRuns) - .where(and( - eq(heartbeatRuns.companyId, companyId), - eq(heartbeatRuns.agentId, managerId), - sql`${heartbeatRuns.contextSnapshot} ->> 'recoveryActionId' = ${action!.id}`, - sql`${heartbeatRuns.contextSnapshot} ->> 'recoveryOwnerAttempt' = ${String(attempt)}`, - )); - expect(duplicateRuns).toHaveLength(1); - - await db - .update(heartbeatRuns) - .set({ status: "failed", errorCode: "process_lost", finishedAt: new Date() }) - .where(eq(heartbeatRuns.id, managerRun!.id)); - await db - .update(agentWakeupRequests) - .set({ status: "completed" }) - .where(eq(agentWakeupRequests.runId, managerRun!.id)); - } - await restartedHeartbeat.reconcileStrandedAssignedIssues(); - - const exhaustedAction = await db - .select() - .from(issueRecoveryActions) - .where(eq(issueRecoveryActions.id, action!.id)) - .then((rows) => rows[0] ?? null); - expect(exhaustedAction).toMatchObject({ - status: "escalated", + )), + ]); + expect(sourceAfter).toMatchObject({ status: "blocked", assigneeAgentId: agentId }); + expect(action).toMatchObject({ + kind: "deliberate_wait_without_target", + status: "active", ownerType: "board", ownerAgentId: null, - attemptCount: 5, - maxAttempts: 5, - resolutionNote: "recovery_owner_retry_exhausted", - }); - expect(exhaustedAction?.wakePolicy).toMatchObject({ - type: "board_escalation", - reason: "recovery_owner_retry_exhausted", - attempt: 5, - maxAttempts: 5, - preservesSourceAssignee: true, - }); - const managerAttemptSix = await db - .select({ id: heartbeatRuns.id }) - .from(heartbeatRuns) - .where(and( - eq(heartbeatRuns.companyId, companyId), - eq(heartbeatRuns.agentId, managerId), - sql`${heartbeatRuns.contextSnapshot} ->> 'recoveryActionId' = ${action!.id}`, - sql`${heartbeatRuns.contextSnapshot} ->> 'recoveryOwnerAttempt' = '6'`, - )); - expect(managerAttemptSix).toHaveLength(0); - const sourceAfterManagerExhaustion = await db - .select() - .from(issues) - .where(eq(issues.id, issueId)) - .then((rows) => rows[0] ?? null); - expect(sourceAfterManagerExhaustion).toMatchObject({ - status: "blocked", - assigneeAgentId: agentId, + previousOwnerAgentId: agentId, + returnOwnerAgentId: agentId, + maxAttempts: null, + resolutionNote: "unchanged_source_state_exhausted", + wakePolicy: expect.objectContaining({ + type: "board_escalation", + reason: "unchanged_source_state_exhausted", + preservesSourceAssignee: true, + }), + evidence: expect.objectContaining({ + routingPolicy: "board_escalation_no_takeover_v1", + sourceAttemptCount: 5, + sourceMaxAttempts: 5, + }), }); + expect(substituteWakes).toHaveLength(0); + expect(sourceAttemptSix).toHaveLength(0); }); it("routes a non-invokable source owner to recovery without reassigning the source", async () => { @@ -4893,17 +4739,74 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { .then((rows) => rows[0] ?? null); expect(action).toMatchObject({ kind: "deliberate_wait_without_target", - status: "escalated", + status: "active", ownerType: "board", ownerAgentId: null, previousOwnerAgentId: agentId, returnOwnerAgentId: agentId, attemptCount: 0, - maxAttempts: 5, + maxAttempts: null, resolutionNote: "owner_not_invokable", }); }); + it("keeps a legacy agent-owned recovery action readable without scheduling another takeover wake", async () => { + const { companyId, agentId, issueId } = await seedStrandedIssueFixture({ + status: "in_progress", + runStatus: "failed", + }); + const legacyOwnerId = randomUUID(); + await db.insert(agents).values({ + id: legacyOwnerId, + companyId, + name: "Legacy recovery owner", + role: "cto", + status: "idle", + adapterType: "codex_local", + adapterConfig: {}, + runtimeConfig: {}, + permissions: {}, + }); + await db.update(issues).set({ status: "blocked" }).where(eq(issues.id, issueId)); + const [legacyAction] = await db.insert(issueRecoveryActions).values({ + companyId, + sourceIssueId: issueId, + kind: "stranded_assigned_issue", + status: "active", + ownerType: "agent", + ownerAgentId: legacyOwnerId, + previousOwnerAgentId: agentId, + returnOwnerAgentId: agentId, + cause: "process_lost", + fingerprint: `legacy:${issueId}`, + evidence: { latestRunId: null }, + nextAction: "Legacy recovery action", + wakePolicy: { + type: "bounded_recovery_owner", + ownerAgentId: legacyOwnerId, + attempt: 1, + maxAttempts: 5, + }, + attemptCount: 1, + maxAttempts: 5, + }).returning(); + + await heartbeatService(db).reconcileStrandedAssignedIssues(); + + const [persisted, takeoverWakes] = await Promise.all([ + db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, legacyAction!.id)).then((rows) => rows[0]), + db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.agentId, legacyOwnerId)), + ]); + expect(persisted).toMatchObject({ + status: "active", + ownerType: "agent", + ownerAgentId: legacyOwnerId, + attemptCount: 1, + maxAttempts: 5, + }); + expect(takeoverWakes).toHaveLength(0); + }); + it("does not consume a disposition-repair attempt when on-demand wakes are disabled", async () => { const { companyId, agentId, issueId } = await seedStrandedIssueFixture({ status: "in_progress", @@ -4940,10 +4843,11 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { ]); expect(action).toMatchObject({ kind: "deliberate_wait_without_target", - status: "escalated", + status: "active", ownerType: "board", ownerAgentId: null, attemptCount: 0, + maxAttempts: null, resolutionNote: "owner_not_invokable", }); expect(repairWakeups).toHaveLength(0); @@ -5184,7 +5088,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { expect(runs).toHaveLength(0); }); - it("skips budget-blocked assigned todo work with no prior run and continues the sweep", async () => { + it("creates a board recovery action for budget-blocked assigned work and continues the sweep", async () => { const blocked = await seedAssignedTodoNoRunFixture(); const unblocked = await seedAssignedTodoNoRunFixture(); await db.insert(budgetPolicies).values({ @@ -5214,9 +5118,9 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { expect(result.assignmentDispatched).toBe(1); expect(result.dispatchRequeued).toBe(0); expect(result.continuationRequeued).toBe(0); - expect(result.escalated).toBe(0); - expect(result.skipped).toBe(1); - expect(result.issueIds).toEqual([unblocked.issueId]); + expect(result.escalated).toBe(1); + expect(result.skipped).toBe(0); + expect(result.issueIds).toEqual([blocked.issueId, unblocked.issueId]); const blockedWakeups = await db .select() @@ -5231,7 +5135,21 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { .from(issues) .where(eq(issues.id, blocked.issueId)) .then((rows) => rows[0] ?? null); - expect(blockedIssue?.status).toBe("todo"); + expect(blockedIssue).toMatchObject({ + status: "blocked", + assigneeAgentId: blocked.agentId, + }); + const blockedAction = await db.select().from(issueRecoveryActions).where( + eq(issueRecoveryActions.sourceIssueId, blocked.issueId), + ).then((rows) => rows[0] ?? null); + expect(blockedAction).toMatchObject({ + ownerType: "board", + ownerAgentId: null, + returnOwnerAgentId: blocked.agentId, + evidence: expect.objectContaining({ + routingPolicy: "board_escalation_no_takeover_v1", + }), + }); const unblockedWakeups = await db .select() @@ -5256,22 +5174,47 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { } }); - it("does not dispatch assigned todo work with no prior run when the agent is paused", async () => { - const { agentId, issueId } = await seedAssignedTodoNoRunFixture({ agentStatus: "paused" }); + it("routes paused assigned work to the board without waking available executives", async () => { + const { companyId, agentId, issueId } = await seedAssignedTodoNoRunFixture({ agentStatus: "paused" }); + const executiveIds = [randomUUID(), randomUUID()]; + await db.insert(agents).values(executiveIds.map((id, index) => ({ + id, + companyId, + name: index === 0 ? "Available CTO" : "Available CEO", + role: index === 0 ? "cto" : "ceo", + status: "idle" as const, + adapterType: "codex_local" as const, + adapterConfig: {}, + runtimeConfig: {}, + permissions: {}, + }))); const heartbeat = heartbeatService(db); const result = await heartbeat.reconcileStrandedAssignedIssues(); expect(result.assignmentDispatched).toBe(0); expect(result.dispatchRequeued).toBe(0); expect(result.continuationRequeued).toBe(0); - expect(result.escalated).toBe(0); - expect(result.skipped).toBe(1); - expect(result.issueIds).toEqual([]); + expect(result.escalated).toBe(1); + expect(result.skipped).toBe(0); + expect(result.issueIds).toEqual([issueId]); const issue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0] ?? null); - expect(issue?.status).toBe("todo"); - const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.agentId, agentId)); + expect(issue).toMatchObject({ status: "blocked", assigneeAgentId: agentId }); + const action = await db.select().from(issueRecoveryActions).where( + eq(issueRecoveryActions.sourceIssueId, issueId), + ).then((rows) => rows[0] ?? null); + expect(action).toMatchObject({ + ownerType: "board", + ownerAgentId: null, + returnOwnerAgentId: agentId, + wakePolicy: expect.objectContaining({ type: "board_escalation" }), + }); + const runs = await db.select().from(heartbeatRuns).where(inArray(heartbeatRuns.agentId, [agentId, ...executiveIds])); expect(runs).toHaveLength(0); + const wakes = await db.select().from(agentWakeupRequests).where( + inArray(agentWakeupRequests.agentId, executiveIds), + ); + expect(wakes).toHaveLength(0); }); it("re-enqueues assigned todo work when the last issue run died and no wake remains", async () => { @@ -6092,11 +6035,13 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { .then((rows) => rows[0] ?? null); expect(action).toMatchObject({ kind: "deliberate_wait_without_target", - status: "escalated", + status: "active", + ownerType: "board", + ownerAgentId: null, previousOwnerAgentId: agentId, returnOwnerAgentId: agentId, - attemptCount: 0, - maxAttempts: 5, + attemptCount: 5, + maxAttempts: null, resolutionNote: "unchanged_source_state_exhausted", }); expect(action?.evidence).toMatchObject({ @@ -6545,7 +6490,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { expect(comments[0]?.presentation).toMatchObject({ kind: "system_notice", tone: "danger" }); expect(noticeMetadataReferencesRecoveryAction(comments[0]?.metadata, recoveryAction.id)).toBe(true); expect(commentMetadataRows(comments[0]).some((row) => - row.type === "agent_link" && row.label === "Recovery owner" && row.name === longRecoveryOwnerName.slice(0, 160), + row.type === "key_value" && row.label === "Recovery owner" && row.value === "Board decision required", )).toBe(true); }); @@ -6953,7 +6898,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { expect(comments[0]?.presentation).toMatchObject({ kind: "system_notice", tone: "danger" }); expect(noticeMetadataReferencesRecoveryAction(comments[0]?.metadata, recoveryAction.id)).toBe(true); expect(commentMetadataRows(comments[0]).some((row) => - row.type === "agent_link" && row.label === "Recovery owner" && row.name === "CodexCoder", + row.type === "key_value" && row.label === "Recovery owner" && row.value === "Board decision required", )).toBe(true); }); @@ -7610,7 +7555,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { }); const followupRuns = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.agentId, agentId)); - expect(followupRuns).toHaveLength(2); + expect(followupRuns).toHaveLength(1); }); it("preserves a persisted issue monitor as the durable external-wait path", async () => { @@ -7719,7 +7664,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { expect(comments[0]?.body).toContain("still has no live execution path"); expect(noticeMetadataReferencesRecoveryAction(comments[0]?.metadata, recoveryAction.id)).toBe(true); expect(commentMetadataRows(comments[0]).some((row) => - row.type === "agent_link" && row.label === "Recovery owner" && row.name === "CodexCoder", + row.type === "key_value" && row.label === "Recovery owner" && row.value === "Board decision required", )).toBe(true); }); diff --git a/server/src/__tests__/heartbeat-workspace-branch-containment.test.ts b/server/src/__tests__/heartbeat-workspace-branch-containment.test.ts index e8032b9847..10514830ae 100644 --- a/server/src/__tests__/heartbeat-workspace-branch-containment.test.ts +++ b/server/src/__tests__/heartbeat-workspace-branch-containment.test.ts @@ -646,6 +646,8 @@ async function expectContainedWorkspaceBranchFailure(input: { executionRunId: null, checkoutRunId: null, }); + const sourceAssigneeAgentId = issueById.get(input.sourceIssueId)?.assigneeAgentId; + expect(sourceAssigneeAgentId).toEqual(expect.any(String)); expect(issueById.get(input.sameWorkspaceSiblingId)).toMatchObject({ status: "in_progress", executionRunId: null, @@ -664,6 +666,11 @@ async function expectContainedWorkspaceBranchFailure(input: { kind: "workspace_validation", cause: "workspace_validation_failed", status: "active", + ownerType: "board", + ownerAgentId: null, + ownerUserId: null, + previousOwnerAgentId: sourceAssigneeAgentId, + returnOwnerAgentId: sourceAssigneeAgentId, fingerprint: expect.stringContaining(String(workspaceValidation.fingerprint)), attemptCount: 1, evidence: expect.objectContaining({ @@ -671,6 +678,7 @@ async function expectContainedWorkspaceBranchFailure(input: { latestRunId: input.runId, latestRunErrorCode: "workspace_validation_failed", recoveryCause: "workspace_validation_failed", + routingPolicy: "board_escalation_no_takeover_v1", workspaceValidation: expect.objectContaining({ fingerprint: workspaceValidation.fingerprint, expectedBranch: input.expectedBranch, @@ -686,9 +694,9 @@ async function expectContainedWorkspaceBranchFailure(input: { }), nextAction: expect.stringContaining("choose a new execution workspace"), wakePolicy: expect.objectContaining({ - type: "wake_owner", - reason: "source_scoped_recovery_action", - ownerAgentId: expect.any(String), + type: "board_escalation", + reason: "workspace_validation_failed", + preservesSourceAssignee: true, }), }); diff --git a/server/src/__tests__/issue-recovery-actions.test.ts b/server/src/__tests__/issue-recovery-actions.test.ts index 4fe95f14d4..cda5e7729b 100644 --- a/server/src/__tests__/issue-recovery-actions.test.ts +++ b/server/src/__tests__/issue-recovery-actions.test.ts @@ -272,8 +272,60 @@ describeEmbeddedPostgres("issue recovery actions", () => { expect(await svc.getActiveForIssue(randomUUID(), sourceIssueId)).toBeNull(); }); + it("preserves legacy recovery ownership when new evidence is folded into an active action", async () => { + const { companyId, managerId, coderId, sourceIssueId } = await seedCompany(); + const svc = issueRecoveryActionService(db); + const legacy = await svc.upsertSourceScoped({ + companyId, + sourceIssueId, + kind: "stranded_assigned_issue", + ownerType: "agent", + ownerAgentId: managerId, + previousOwnerAgentId: coderId, + returnOwnerAgentId: coderId, + cause: "process_lost", + fingerprint: "legacy-recovery", + evidence: { latestRunId: "run-1" }, + nextAction: "Repair the execution path.", + wakePolicy: { type: "bounded_recovery_owner", ownerAgentId: managerId, attempt: 1, maxAttempts: 5 }, + attemptCount: 1, + maxAttempts: 5, + }); + + const updated = await svc.upsertSourceScoped({ + companyId, + sourceIssueId, + kind: "stranded_assigned_issue", + ownerType: "board", + ownerAgentId: null, + previousOwnerAgentId: coderId, + returnOwnerAgentId: coderId, + cause: "process_lost", + fingerprint: "legacy-recovery", + evidence: { latestRunId: "run-2" }, + evidenceOnCreate: { routingPolicy: "board_escalation_no_takeover_v1" }, + nextAction: "Board decision required.", + wakePolicy: { type: "board_escalation" }, + preserveExistingOwner: true, + }); + + expect(updated).toMatchObject({ + id: legacy.id, + ownerType: "agent", + ownerAgentId: managerId, + previousOwnerAgentId: coderId, + returnOwnerAgentId: coderId, + attemptCount: 2, + maxAttempts: 5, + nextAction: "Repair the execution path.", + evidence: expect.objectContaining({ latestRunId: "run-2" }), + wakePolicy: expect.objectContaining({ type: "bounded_recovery_owner" }), + }); + expect(updated.evidence).not.toHaveProperty("routingPolicy"); + }); + it("escalates stranded assigned work into a source action instead of a recovery issue", async () => { - const { companyId, managerId, coderId, sourceIssue } = await seedCompany(); + const { companyId, coderId, sourceIssue } = await seedCompany(); const enqueueWakeup = vi.fn(async () => null); const recovery = recoveryService(db, { enqueueWakeup }); const latestRun = { @@ -286,18 +338,20 @@ describeEmbeddedPostgres("issue recovery actions", () => { livenessState: "needs_followup", } as const; - await recovery.escalateStrandedAssignedIssue({ - issue: sourceIssue, - previousStatus: "in_progress", - latestRun, - comment: "Automatic continuation recovery failed.", - }); - await recovery.escalateStrandedAssignedIssue({ - issue: sourceIssue, - previousStatus: "in_progress", - latestRun, - comment: "Automatic continuation recovery failed.", - }); + await Promise.all([ + recovery.escalateStrandedAssignedIssue({ + issue: sourceIssue, + previousStatus: "in_progress", + latestRun, + comment: "Automatic continuation recovery failed.", + }), + recovery.escalateStrandedAssignedIssue({ + issue: sourceIssue, + previousStatus: "in_progress", + latestRun, + comment: "Automatic continuation recovery failed.", + }), + ]); const actionRows = await db .select() @@ -308,10 +362,15 @@ describeEmbeddedPostgres("issue recovery actions", () => { companyId, kind: "stranded_assigned_issue", status: "active", + ownerType: "board", + ownerAgentId: null, previousOwnerAgentId: coderId, returnOwnerAgentId: coderId, cause: "stranded_assigned_issue", attemptCount: 2, + evidence: expect.objectContaining({ + routingPolicy: "board_escalation_no_takeover_v1", + }), }); const [updatedIssue] = await db.select().from(issues).where(eq(issues.id, sourceIssue.id)); @@ -323,12 +382,8 @@ describeEmbeddedPostgres("issue recovery actions", () => { .from(issues) .where(and(eq(issues.companyId, companyId), eq(issues.originKind, "stranded_issue_recovery"))); expect(recoveryIssues).toHaveLength(0); - expect(enqueueWakeup).toHaveBeenCalledTimes(2); - expect(enqueueWakeup.mock.calls[0]?.[1]?.payload).toMatchObject({ - issueId: sourceIssue.id, - sourceIssueId: sourceIssue.id, - recoveryCause: "stranded_assigned_issue", - }); + expect(updatedIssue?.assigneeAgentId).toBe(coderId); + expect(enqueueWakeup).not.toHaveBeenCalled(); }); // Model the production payload: `requestedRef` keeps the operator spelling, @@ -506,15 +561,15 @@ describeEmbeddedPostgres("issue recovery actions", () => { }); it.each([ - ["process_lost", undefined, "coder"], - ["adapter_failed", "successful_run_missing_state", "coder"], - ["codex_output_inactivity_monitor", undefined, "coder"], - ["workspace_validation_failed", "workspace_validation_failed", "manager"], - ["adapter_failed", undefined, "manager"], + ["process_lost", undefined], + ["adapter_failed", "successful_run_missing_state"], + ["codex_output_inactivity_monitor", undefined], + ["workspace_validation_failed", "workspace_validation_failed"], + ["adapter_failed", undefined], ] as const)( "routes %s recovery through the cause-keyed playbook", - async (errorCode, explicitCause, expectedOwner) => { - const { managerId, coderId, sourceIssue } = await seedCompany(); + async (errorCode, explicitCause) => { + const { coderId, sourceIssue } = await seedCompany(); const enqueueWakeup = vi.fn(async () => null); const recovery = recoveryService(db, { enqueueWakeup }); const latestRun = { @@ -539,21 +594,24 @@ describeEmbeddedPostgres("issue recovery actions", () => { ...(explicitCause ? { recoveryCause: explicitCause } : {}), }); - const expectedOwnerId = expectedOwner === "coder" ? coderId : managerId; const [action] = await db .select() .from(issueRecoveryActions) .where(eq(issueRecoveryActions.sourceIssueId, sourceIssue.id)); - expect(action?.ownerAgentId).toBe(expectedOwnerId); - expect(enqueueWakeup).toHaveBeenCalledWith( - expectedOwnerId, - expect.objectContaining({ - reason: "source_scoped_recovery_action", - payload: expect.objectContaining({ - recoveryCause: explicitCause ?? (errorCode === "adapter_failed" ? "stranded_assigned_issue" : errorCode), - }), + expect(action).toMatchObject({ + ownerType: "board", + ownerAgentId: null, + previousOwnerAgentId: coderId, + returnOwnerAgentId: coderId, + evidence: expect.objectContaining({ + routingPolicy: "board_escalation_no_takeover_v1", }), - ); + wakePolicy: expect.objectContaining({ + type: "board_escalation", + preservesSourceAssignee: true, + }), + }); + expect(enqueueWakeup).not.toHaveBeenCalled(); }, ); @@ -1063,7 +1121,8 @@ describeEmbeddedPostgres("issue recovery actions", () => { const [action] = await db.select().from(issueRecoveryActions); expect(action).toMatchObject({ sourceIssueId, - ownerAgentId: managerId, + ownerType: "board", + ownerAgentId: null, previousOwnerAgentId: coderId, cause: "configuration_incomplete", recoveryIssueId: null, @@ -1211,13 +1270,7 @@ describeEmbeddedPostgres("issue recovery actions", () => { attemptCount: 2, }); expect(actionRows[0]?.evidence).toMatchObject({ latestRunId: secondLatestRun.id }); - expect(enqueueWakeup).toHaveBeenCalledTimes(2); - expect(enqueueWakeup.mock.calls[1]?.[1]?.payload).toMatchObject({ - issueId: sourceIssue.id, - sourceIssueId: sourceIssue.id, - strandedRunId: secondLatestRun.id, - recoveryCause: "stranded_assigned_issue", - }); + expect(enqueueWakeup).not.toHaveBeenCalled(); }); it("deduplicates workspace-incoherence recovery actions by the typed workspace fingerprint", async () => { @@ -1311,9 +1364,9 @@ describeEmbeddedPostgres("issue recovery actions", () => { }), nextAction: expect.stringContaining("git worktree branch incoherence"), wakePolicy: expect.objectContaining({ - type: "wake_owner", - reason: "source_scoped_recovery_action", - ownerAgentId: expect.any(String), + type: "board_escalation", + reason: "workspace_validation_failed", + preservesSourceAssignee: true, }), }); @@ -1327,14 +1380,7 @@ describeEmbeddedPostgres("issue recovery actions", () => { tone: "danger", title: "Workspace validation failed", }); - expect(enqueueWakeup).toHaveBeenCalledTimes(2); - expect(enqueueWakeup).toHaveBeenCalledWith( - expect.any(String), - expect.objectContaining({ - reason: "source_scoped_recovery_action", - payload: expect.objectContaining({ recoveryCause: "workspace_validation_failed" }), - }), - ); + expect(enqueueWakeup).not.toHaveBeenCalled(); }); it("keeps the source issue blocked when source-scoped wakeup is claimed synchronously", async () => { diff --git a/server/src/__tests__/recovery-observability.test.ts b/server/src/__tests__/recovery-observability.test.ts index edb9cc5247..53653a83f0 100644 --- a/server/src/__tests__/recovery-observability.test.ts +++ b/server/src/__tests__/recovery-observability.test.ts @@ -395,4 +395,28 @@ describeEmbeddedPostgres("recovery observability report", () => { expect(report.window.since).not.toContain("T"); expect(report.weekly).toHaveLength(MAX_WINDOW_WEEKS); }); + + it("counts an active board recovery action without reporting an active takeover", async () => { + const { companyId, coderId } = await seedBaseline(); + await seedRecoveryAction({ + companyId, + n: 101, + createdAt: latestWeek, + cause: "process_lost", + errorCode: "process_lost", + status: "active", + outcome: null, + ownerAgentId: null, + returnOwnerAgentId: coderId, + finalAssigneeAgentId: coderId, + finalIssueStatus: "blocked", + }); + + const report = await recoveryObservabilityService(db).report(companyId, { now, weeks: 8 }); + + expect(report.handoff).toMatchObject({ boardOwned: 1, activeTakeovers: 0 }); + expect(report.perCauseRouting.find((entry) => entry.cause === "process_lost")).toMatchObject({ + active: 1, + }); + }); }); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index f5d9e1841e..611f310fbb 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -17475,7 +17475,6 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) previousStatus: issue.status, notice: buildExecutionReviewParticipantRecoveryNoticeSeed(), recoveryCause: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE, - recoveryOwnerAgentId: currentParticipant.agentId, }; } @@ -17731,7 +17730,6 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) : promotionResult.recoveryCause === EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE ? EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE : undefined, - recoveryOwnerAgentId: promotionResult.recoveryOwnerAgentId, }); return; } diff --git a/server/src/services/issue-recovery-actions.ts b/server/src/services/issue-recovery-actions.ts index dfb4f3978e..4bdc581dfc 100644 --- a/server/src/services/issue-recovery-actions.ts +++ b/server/src/services/issue-recovery-actions.ts @@ -16,6 +16,10 @@ type IssueRecoveryActionRow = typeof issueRecoveryActions.$inferSelect; type DbTransaction = Parameters[0]>[0]; type DbOrTransaction = Db | DbTransaction; +function asDatabaseDate(value: string | Date | null) { + return typeof value === "string" ? new Date(value) : value; +} + export type UpsertIssueRecoveryActionInput = { companyId: string; sourceIssueId: string; @@ -29,6 +33,8 @@ export type UpsertIssueRecoveryActionInput = { cause: string; fingerprint: string; evidence?: Record; + /** Evidence written only when this upsert creates a new action row. */ + evidenceOnCreate?: Record; nextAction: string; wakePolicy?: Record | null; monitorPolicy?: Record | null; @@ -41,6 +47,10 @@ export type UpsertIssueRecoveryActionInput = { // one. The new failure then gets a distinct recovery identity and a fresh // operator notice, and the prior identity stays as a resolved record. supersedeOnIdentityChange?: boolean; + // Rollout compatibility for active pre-policy actions. Refresh their + // evidence/attempt metadata without silently changing the recorded owner or + // the wake/monitor contract that made that owner authoritative. + preserveExistingOwner?: boolean; }; export type ResolveIssueRecoveryActionInput = { @@ -202,7 +212,10 @@ export function issueRecoveryActionService(db: Db) { returnOwnerAgentId: input.returnOwnerAgentId ?? null, cause: input.cause, fingerprint: input.fingerprint, - evidence: input.evidence ?? {}, + evidence: { + ...(input.evidence ?? {}), + ...(input.evidenceOnCreate ?? {}), + }, nextAction: input.nextAction, wakePolicy: input.wakePolicy ?? null, monitorPolicy: input.monitorPolicy ?? null, @@ -279,26 +292,51 @@ export function issueRecoveryActionService(db: Db) { const [updated] = await db .update(issueRecoveryActions) .set({ - recoveryIssueId: input.recoveryIssueId ?? null, - kind: input.kind, - status: "active", - ownerType, - ownerAgentId: input.ownerAgentId ?? null, - ownerUserId: input.ownerUserId ?? null, - previousOwnerAgentId: input.previousOwnerAgentId ?? existing.previousOwnerAgentId, - returnOwnerAgentId: input.returnOwnerAgentId ?? existing.returnOwnerAgentId, - cause: input.cause, - fingerprint: input.fingerprint, - evidence: input.evidence ?? existing.evidence, - nextAction: input.nextAction, - wakePolicy: input.wakePolicy ?? null, - monitorPolicy: input.monitorPolicy ?? null, + recoveryIssueId: input.preserveExistingOwner + ? existing.recoveryIssueId + : input.recoveryIssueId ?? null, + kind: input.preserveExistingOwner ? existing.kind : input.kind, + status: input.preserveExistingOwner ? existing.status : "active", + ownerType: input.preserveExistingOwner ? existing.ownerType : ownerType, + ownerAgentId: input.preserveExistingOwner + ? existing.ownerAgentId + : input.ownerAgentId ?? null, + ownerUserId: input.preserveExistingOwner + ? existing.ownerUserId + : input.ownerUserId ?? null, + previousOwnerAgentId: input.preserveExistingOwner + ? existing.previousOwnerAgentId + : input.previousOwnerAgentId ?? existing.previousOwnerAgentId, + returnOwnerAgentId: input.preserveExistingOwner + ? existing.returnOwnerAgentId + : input.returnOwnerAgentId ?? existing.returnOwnerAgentId, + cause: input.preserveExistingOwner ? existing.cause : input.cause, + fingerprint: input.preserveExistingOwner ? existing.fingerprint : input.fingerprint, + evidence: input.preserveExistingOwner + ? { + ...(existing.evidence ?? {}), + ...(input.evidence ?? {}), + } + : input.evidence ?? existing.evidence, + nextAction: input.preserveExistingOwner ? existing.nextAction : input.nextAction, + wakePolicy: input.preserveExistingOwner + ? existing.wakePolicy + : input.wakePolicy ?? null, + monitorPolicy: input.preserveExistingOwner + ? existing.monitorPolicy + : input.monitorPolicy ?? null, attemptCount: input.attemptCount ?? existing.attemptCount + 1, - maxAttempts: input.maxAttempts ?? null, - timeoutAt: input.timeoutAt ?? null, - lastAttemptAt: input.lastAttemptAt ?? now, - outcome: null, - resolutionNote: null, + maxAttempts: input.preserveExistingOwner + ? existing.maxAttempts + : input.maxAttempts ?? null, + timeoutAt: input.preserveExistingOwner + ? asDatabaseDate(existing.timeoutAt) + : input.timeoutAt ?? null, + lastAttemptAt: input.preserveExistingOwner + ? asDatabaseDate(existing.lastAttemptAt) + : input.lastAttemptAt ?? now, + outcome: input.preserveExistingOwner ? existing.outcome : null, + resolutionNote: input.preserveExistingOwner ? existing.resolutionNote : null, resolvedAt: null, updatedAt: now, }) diff --git a/server/src/services/recovery-observability.ts b/server/src/services/recovery-observability.ts index 3324207c40..7e866cdb8a 100644 --- a/server/src/services/recovery-observability.ts +++ b/server/src/services/recovery-observability.ts @@ -44,7 +44,7 @@ export type RecoveryCauseGroup = { }; export type RecoveryHandoffSummary = { - /** Genuine manager takeovers (recovery owner != original assignee) that resolved. */ + /** Historical agent takeovers (recovery owner != original assignee) that resolved. */ resolvedTakeovers: number; handedBack: number; ownerCompleted: number; @@ -309,7 +309,9 @@ export function recoveryObservabilityService(db: Db) { if (klass === "active") { routing.active += 1; if (row.status === "escalated") routing.escalated += 1; - if (row.ownerAgentId && row.ownerAgentId !== row.returnOwnerAgentId) { + if (!row.ownerAgentId) { + handoff.boardOwned += 1; + } else if (row.ownerAgentId !== row.returnOwnerAgentId) { handoff.activeTakeovers += 1; } continue; diff --git a/server/src/services/recovery/disposition-repair.ts b/server/src/services/recovery/disposition-repair.ts index 2c526a325d..ed315f19e1 100644 --- a/server/src/services/recovery/disposition-repair.ts +++ b/server/src/services/recovery/disposition-repair.ts @@ -17,8 +17,6 @@ const ACTIVE_RUN_STATUSES = ["queued", "running", "scheduled_retry"] as const; export const DISPOSITION_REPAIR_MAX_ATTEMPTS = 5; export const DISPOSITION_REPAIR_BASE_DELAYS_MS = [0, 60_000, 120_000, 240_000, 480_000] as const; -export const RECOVERY_OWNER_MAX_ATTEMPTS = 5; -export const RECOVERY_OWNER_BASE_DELAYS_MS = [0, 60_000, 120_000, 240_000, 480_000] as const; type DispositionRepairIssue = Pick< typeof issues.$inferSelect, @@ -77,15 +75,6 @@ export function dispositionRepairDelayMs(attempt: number, fingerprint: string) { ); } -export function recoveryOwnerDelayMs(attempt: number, fingerprint: string) { - return boundedRecoveryDelayMs( - attempt, - fingerprint, - RECOVERY_OWNER_BASE_DELAYS_MS, - "recovery owner", - ); -} - export async function collectDispositionRepairSourceState( db: Db, input: { diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 8b00c7d07b..f715479178 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -90,8 +90,6 @@ import { collectDispositionRepairSourceState, dispositionRepairDelayMs, DISPOSITION_REPAIR_MAX_ATTEMPTS, - recoveryOwnerDelayMs, - RECOVERY_OWNER_MAX_ATTEMPTS, } from "./disposition-repair.js"; const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_retry"] as const; @@ -105,7 +103,7 @@ const STRANDED_ISSUE_RECOVERY_ORIGIN_KIND = RECOVERY_ORIGIN_KINDS.strandedIssueR const STALE_ACTIVE_RUN_EVALUATION_ORIGIN_KIND = RECOVERY_ORIGIN_KINDS.staleActiveRunEvaluation; const DEFERRED_WAKE_CONTEXT_KEY = "_paperclipWakeContext"; const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_REASON = "execution_review_participant_recovery"; -const RECOVERY_OWNER_RETRY_REASON = "recovery_owner_retry"; +const STRANDED_BOARD_ESCALATION_POLICY = "board_escalation_no_takeover_v1"; const DISPOSITION_REPAIR_IDEMPOTENCY_INDEX = "agent_wakeup_requests_disposition_repair_idempotency_uq"; const RESOLVED_DEPENDENCY_WAKE_BACKSTOP_CANDIDATE_LIMIT = 500; const SESSIONED_LOCAL_ADAPTERS = new Set([ @@ -2016,17 +2014,6 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) ); } - function isUniqueStrandedIssueRecoveryConflict(error: unknown) { - const maybe = unwrapDatabaseConflictError(error); - if (!maybe) return false; - return maybe.code === "23505" && - ( - maybe.constraint === "issues_active_stranded_issue_recovery_uq" || - maybe.constraint_name === "issues_active_stranded_issue_recovery_uq" || - typeof maybe.message === "string" && maybe.message.includes("issues_active_stranded_issue_recovery_uq") - ); - } - async function ensureSourceIssueCommentedForStaleEvaluation(input: { sourceIssue: typeof issues.$inferSelect | null; evaluationIssue: { id: string; identifier: string | null }; @@ -2512,24 +2499,6 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) return row; } - async function findOpenStrandedIssueRecoveryIssue(companyId: string, sourceIssueId: string) { - return db - .select() - .from(issues) - .where( - and( - eq(issues.companyId, companyId), - eq(issues.originKind, STRANDED_ISSUE_RECOVERY_ORIGIN_KIND), - eq(issues.originId, sourceIssueId), - visibleIssueCondition(), - notInArray(issues.status, ["done", "cancelled"]), - ), - ) - .orderBy(desc(issues.createdAt)) - .limit(1) - .then((rows) => rows[0] ?? null); - } - function isStrandedIssueRecoveryIssue(issue: typeof issues.$inferSelect) { return issue.originKind === STRANDED_ISSUE_RECOVERY_ORIGIN_KIND; } @@ -2557,291 +2526,16 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) ].join("\n"); } - async function resolveStrandedIssueRecoveryOwnerAgentId( - issue: typeof issues.$inferSelect, - preferredOwnerAgentId?: string | null, - excludedAgentIds: ReadonlySet = new Set(), - ) { - const candidateIds: string[] = []; - if (preferredOwnerAgentId) candidateIds.push(preferredOwnerAgentId); - if (issue.assigneeAgentId) { - const assignee = await getAgent(issue.assigneeAgentId); - if (assignee?.reportsTo) candidateIds.push(assignee.reportsTo); - } - if (issue.createdByAgentId) { - const creator = await getAgent(issue.createdByAgentId); - if (creator?.reportsTo) candidateIds.push(creator.reportsTo); - candidateIds.push(issue.createdByAgentId); - } - - const roleCandidates = await db - .select() - .from(agents) - .where(and(eq(agents.companyId, issue.companyId), inArray(agents.role, ["cto", "ceo"]))) - .orderBy(sql`case when ${agents.role} = 'cto' then 0 else 1 end`, asc(agents.createdAt)); - candidateIds.push(...roleCandidates.map((agent) => agent.id)); - if (issue.assigneeAgentId) candidateIds.push(issue.assigneeAgentId); - - const seen = new Set(); - for (const agentId of candidateIds) { - if (seen.has(agentId)) continue; - seen.add(agentId); - if (excludedAgentIds.has(agentId)) continue; - const candidate = await getAgent(agentId); - if (!candidate || candidate.companyId !== issue.companyId) continue; - const budgetBlock = await budgets.getInvocationBlock(issue.companyId, candidate.id, { - issueId: issue.id, - projectId: issue.projectId, - }); - if ( - (await isAgentInvokable(candidate)) && - isHeartbeatWakeOnDemandEnabled(candidate) && - !budgetBlock - ) { - return candidate.id; - } - } - - return null; - } - - async function resolveInvokableRecoveryAgentId( - issue: typeof issues.$inferSelect, - agentId: string | null | undefined, - ) { - if (!agentId) return null; - const candidate = await getAgent(agentId); - if (!candidate || candidate.companyId !== issue.companyId) return null; - const budgetBlock = await budgets.getInvocationBlock(issue.companyId, candidate.id, { - issueId: issue.id, - projectId: issue.projectId, - }); - return (await isAgentInvokable(candidate)) && - isHeartbeatWakeOnDemandEnabled(candidate) && - !budgetBlock - ? candidate.id - : null; - } - - async function resolveStrandedRecoveryRouting(input: { + function resolveStrandedRecoveryRouting(input: { issue: typeof issues.$inferSelect; latestRun: LatestIssueRun; - recoveryCause: StrandedRecoveryCause; - preferredOwnerAgentId?: string | null; }) { - const originalAgentId = input.latestRun?.agentId ?? input.issue.assigneeAgentId; - const returnOwnerAgentId = input.issue.assigneeAgentId ?? originalAgentId; - const routeToOriginal = input.recoveryCause === "process_lost" || - input.recoveryCause === SUCCESSFUL_RUN_MISSING_STATE_REASON || - input.recoveryCause === "codex_output_inactivity_monitor"; - if (input.recoveryCause === "provider_quota") { - const retryAgentId = await resolveInvokableRecoveryAgentId(input.issue, originalAgentId); - if (!retryAgentId) { - return { - ownerAgentId: await resolveStrandedIssueRecoveryOwnerAgentId(input.issue), - returnOwnerAgentId: originalAgentId, - routingFallbackReason: "The original assignee is not invokable; quota recovery fell through to the manager ladder.", - }; - } - return { - ownerAgentId: null, - returnOwnerAgentId: retryAgentId, - routingFallbackReason: null, - }; - } - if (routeToOriginal) { - const ownerAgentId = await resolveInvokableRecoveryAgentId(input.issue, originalAgentId); - if (ownerAgentId) { - return { ownerAgentId, returnOwnerAgentId: originalAgentId, routingFallbackReason: null }; - } - return { - ownerAgentId: await resolveStrandedIssueRecoveryOwnerAgentId(input.issue), - returnOwnerAgentId: originalAgentId, - routingFallbackReason: "The original assignee is not invokable; recovery fell through to the manager ladder.", - }; - } + const originalAgentId = input.issue.assigneeAgentId ?? input.latestRun?.agentId ?? null; return { - ownerAgentId: await resolveStrandedIssueRecoveryOwnerAgentId( - input.issue, - input.preferredOwnerAgentId, - ), - returnOwnerAgentId, - routingFallbackReason: null, + returnOwnerAgentId: originalAgentId, }; } - function buildStrandedIssueRecoveryDescription(input: { - issue: typeof issues.$inferSelect; - latestRun: LatestIssueRun; - previousStatus: StrandedPreviousStatus; - prefix: string; - recoveryCause?: StrandedRecoveryCause; - successfulRunHandoffEvidence?: SuccessfulRunHandoffRecoveryEvidence | null; - sourceAssignee?: Pick | null; - }) { - const sourceIssue = issueUiLink({ identifier: input.issue.identifier, id: input.issue.id }, input.prefix); - const runLink = input.latestRun - ? `[\`${input.latestRun.id}\`](/${input.prefix}/agents/${input.latestRun.agentId}/runs/${input.latestRun.id})` - : "none"; - if (input.recoveryCause === SUCCESSFUL_RUN_MISSING_STATE_REASON) { - const sourceRunId = input.successfulRunHandoffEvidence?.sourceRunId; - const sourceRunLink = sourceRunId && input.latestRun - ? `[\`${sourceRunId}\`](/${input.prefix}/agents/${input.latestRun.agentId}/runs/${sourceRunId})` - : "unknown"; - const missingDisposition = input.successfulRunHandoffEvidence?.missingDisposition ?? "clear_next_step"; - return [ - "Paperclip exhausted the bounded corrective handoff for a successful run that still has no valid issue disposition.", - "", - "This is not a runtime/adapter crash report. The source run succeeded; the remaining problem is the missing `done`, `in_review`, `blocked`, delegated follow-up, or explicit continuation path.", - "", - "## Safe Evidence", - "", - `- Source issue: ${sourceIssue}`, - `- Source run: ${sourceRunLink}`, - `- Corrective handoff run: ${runLink}`, - `- Source assignee: ${agentUiLink(input.sourceAssignee ?? null, input.prefix)}`, - `- Latest issue status: \`${input.issue.status}\``, - `- Latest handoff run status: \`${input.latestRun?.status ?? "unknown"}\``, - `- Normalized cause: \`${SUCCESSFUL_RUN_MISSING_STATE_REASON}\``, - `- Missing disposition: \`${missingDisposition}\``, - "- Suggested manager action: choose and record a valid issue disposition without copying transcript content.", - "", - "## Required Action", - "", - "- Inspect the source issue and run metadata, not raw transcript excerpts.", - "- Choose a valid issue disposition: `done`/`cancelled`, `in_review` with an owner, `blocked` with first-class blockers, delegated follow-up work, or an explicit continuation path.", - "- When the source issue has a clear owner and disposition, mark this recovery issue done.", - ].join("\n"); - } - - const retryReason = readNonEmptyString(parseObject(input.latestRun?.contextSnapshot)?.retryReason) ?? "unknown"; - const failureSummary = summarizeRunFailureForIssueComment(input.latestRun); - const isReviewParticipantRecovery = input.recoveryCause === "execution_review_participant_recovery"; - const detectedInvariant = isReviewParticipantRecovery - ? "execution_review_participant_recovery" - : "stranded_assigned_issue"; - const requiredAction = isReviewParticipantRecovery - ? [ - "- Inspect the latest reviewer run and the pending execution-review stage.", - "- Fix the reviewer runtime, restore the source issue to `in_review` with a live participant, or record an intentional manual resolution.", - "- When the source issue has a live review path or has been intentionally resolved, mark this recovery issue done.", - ] - : [ - "- Inspect the latest run and source issue state.", - "- Fix the runtime/adapter problem, reassign the source issue, or convert the source issue into a clear manual-review state.", - "- When the source issue has a live execution path or has been intentionally resolved, mark this recovery issue done.", - ]; - - return [ - isReviewParticipantRecovery - ? "Paperclip exhausted automatic recovery for a pending execution-review participant and created this explicit recovery task." - : "Paperclip exhausted automatic recovery for an assigned issue and created this explicit recovery task.", - "", - "## Source", - "", - `- Source issue: ${sourceIssue}`, - `- Previous source status: \`${input.previousStatus}\``, - `- Latest retry run: ${runLink}`, - `- Latest retry status: \`${input.latestRun?.status ?? "unknown"}\``, - `- Detected invariant: \`${detectedInvariant}\``, - `- Retry reason: \`${retryReason}\``, - failureSummary ? `- Failure: ${failureSummary.trim()}` : "- Failure: none recorded", - "", - "## Ownership", - "", - "- Selected owner: the first invokable manager/creator/executive candidate with budget available.", - "", - "## Required Action", - "", - ...requiredAction, - ].join("\n"); - } - - async function ensureStrandedIssueRecoveryIssue(input: { - issue: typeof issues.$inferSelect; - latestRun: LatestIssueRun; - previousStatus: StrandedPreviousStatus; - recoveryCause?: StrandedRecoveryCause; - successfulRunHandoffEvidence?: SuccessfulRunHandoffRecoveryEvidence | null; - }) { - if (isStrandedIssueRecoveryIssue(input.issue)) return null; - - const existing = await findOpenStrandedIssueRecoveryIssue(input.issue.companyId, input.issue.id); - if (existing) return existing; - - const ownerAgentId = await resolveStrandedIssueRecoveryOwnerAgentId(input.issue); - if (!ownerAgentId) return null; - - const prefix = await getCompanyIssuePrefix(input.issue.companyId); - const sourceAssignee = input.issue.assigneeAgentId ? await getAgent(input.issue.assigneeAgentId) : null; - const recoveryCause = input.recoveryCause ?? "stranded_assigned_issue"; - let recovery: Awaited>; - try { - recovery = await issuesSvc.create(input.issue.companyId, { - title: recoveryCause === SUCCESSFUL_RUN_MISSING_STATE_REASON - ? `Recover missing next step ${input.issue.identifier ?? input.issue.title}` - : `Recover stalled issue ${input.issue.identifier ?? input.issue.title}`, - description: buildStrandedIssueRecoveryDescription({ - issue: input.issue, - latestRun: input.latestRun, - previousStatus: input.previousStatus, - prefix, - recoveryCause, - successfulRunHandoffEvidence: input.successfulRunHandoffEvidence, - sourceAssignee, - }), - status: "todo", - priority: input.issue.priority, - parentId: input.issue.id, - projectId: input.issue.projectId, - goalId: input.issue.goalId, - assigneeAgentId: ownerAgentId, - assigneeAdapterOverrides: recoveryAssigneeAdapterOverrides("status_only"), - originKind: STRANDED_ISSUE_RECOVERY_ORIGIN_KIND, - originId: input.issue.id, - originRunId: input.latestRun?.id ?? null, - originFingerprint: [ - STRANDED_ISSUE_RECOVERY_ORIGIN_KIND, - input.issue.companyId, - input.issue.id, - recoveryCause, - input.latestRun?.id ?? "no-run", - ].join(":"), - billingCode: input.issue.billingCode, - inheritExecutionWorkspaceFromIssueId: input.issue.id, - }); - } catch (error) { - if (!isUniqueStrandedIssueRecoveryConflict(error)) throw error; - const raced = await findOpenStrandedIssueRecoveryIssue(input.issue.companyId, input.issue.id); - if (!raced) throw error; - return raced; - } - - await deps.enqueueWakeup(ownerAgentId, { - source: "assignment", - triggerDetail: "system", - reason: "issue_assigned", - payload: withRecoveryModelProfileHint({ - issueId: recovery.id, - sourceIssueId: input.issue.id, - strandedRunId: input.latestRun?.id ?? null, - recoveryCause, - }, "status_only"), - requestedByActorType: "system", - requestedByActorId: null, - contextSnapshot: withRecoveryModelProfileHint({ - issueId: recovery.id, - taskId: recovery.id, - wakeReason: "issue_assigned", - source: STRANDED_ISSUE_RECOVERY_ORIGIN_KIND, - sourceIssueId: input.issue.id, - strandedRunId: input.latestRun?.id ?? null, - recoveryCause, - }, "status_only"), - }); - - return recovery; - } function strandedRecoveryActionKind(cause: StrandedRecoveryCause) { return cause === SUCCESSFUL_RUN_MISSING_STATE_REASON @@ -2932,17 +2626,14 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) latestRun: LatestIssueRun; previousStatus: StrandedPreviousStatus; recoveryCause?: StrandedRecoveryCause; - recoveryOwnerAgentId?: string | null; successfulRunHandoffEvidence?: SuccessfulRunHandoffRecoveryEvidence | null; }) { const recoveryCause = resolveStrandedRecoveryCause(input.latestRun, input.recoveryCause); - const routing = await resolveStrandedRecoveryRouting({ + const routing = resolveStrandedRecoveryRouting({ issue: input.issue, latestRun: input.latestRun, - recoveryCause, - preferredOwnerAgentId: input.recoveryOwnerAgentId, }); - const ownerAgentId = routing.ownerAgentId; + const isProviderQuotaWait = recoveryCause === "provider_quota"; const now = new Date(); const action = await recoveryActionsSvc.upsertSourceScoped({ companyId: input.issue.companyId, @@ -2952,9 +2643,11 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) // distinct blocker, so it must get a new recovery action and notify the // operator, not overwrite the active action of the prior ref. supersedeOnIdentityChange: recoveryCause === "configuration_incomplete", + preserveExistingOwner: true, kind: strandedRecoveryActionKind(recoveryCause), - ownerType: recoveryCause === "provider_quota" && !ownerAgentId ? "system" : ownerAgentId ? "agent" : "board", - ownerAgentId, + ownerType: isProviderQuotaWait ? "system" : "board", + ownerAgentId: null, + ownerUserId: null, previousOwnerAgentId: input.issue.assigneeAgentId, returnOwnerAgentId: routing.returnOwnerAgentId, cause: recoveryCause, @@ -2972,49 +2665,40 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) successfulRunHandoffEvidence: input.successfulRunHandoffEvidence, }), failureSummary: summarizeRunFailureForIssueComment(input.latestRun)?.trim() ?? null, - routingFallbackReason: routing.routingFallbackReason, }, + evidenceOnCreate: isProviderQuotaWait + ? {} + : { routingPolicy: STRANDED_BOARD_ESCALATION_POLICY }, nextAction: recoveryCause === SUCCESSFUL_RUN_MISSING_STATE_REASON - ? "Choose and record a valid issue disposition without copying transcript content." + ? "Board operator: inspect the run evidence, then explicitly choose a valid issue disposition, retry the original owner, reassign, or intentionally resolve the task." : recoveryCause === "process_lost" - ? "Retry the original assignee from durable progress without redoing completed steps." + ? "Board operator: inspect the retry history, then explicitly retry the original owner, reassign, or intentionally resolve the task." : recoveryCause === "provider_quota" ? "Wait for provider quota recovery, then retry the original assignee; do not wake a takeover owner." : recoveryCause === "codex_output_inactivity_monitor" - ? "Retry the same agent from durable progress after the output-inactivity termination." + ? "Board operator: inspect the inactivity evidence, then explicitly retry the original owner, reassign, or intentionally resolve the task." : recoveryCause === "workspace_validation_failed" ? readWorkspaceValidationPayload(input.latestRun)?.reason === "git_worktree_branch_incoherence" - ? "Repair the source issue git worktree branch incoherence, or choose a new execution workspace, before resuming adapter execution." + ? "Board operator: repair the source task git worktree branch incoherence or choose a new execution workspace, then explicitly retry or reassign." : readWorkspaceValidationPayload(input.latestRun)?.reason === "git_worktree_base_materialization_failed" - ? "Repair the project workspace repository URL or clone access, or configure a local checkout cwd, before resuming adapter execution." - : "Repair the source issue workspace link, project workspace cwd, or git checkout before resuming adapter execution." + ? "Board operator: repair the project workspace repository URL or clone access, or configure a local checkout cwd, then explicitly retry or reassign." + : "Board operator: repair the source task workspace link, project workspace cwd, or git checkout, then explicitly retry or reassign." : recoveryCause === "configuration_incomplete" - ? "Bind the missing secret(s) named in the run failure to the agent/project/routine env before resuming adapter execution." + ? "Board operator: bind the missing secret(s) named in the run failure, then explicitly retry the original owner or reassign." : recoveryCause === "execution_review_participant_recovery" - ? "Repair the failed review participant path, restore the source issue to in_review with a live reviewer, or record an intentional manual resolution." - : "Restore a live execution path, fix the runtime/adapter failure, or record an intentional manual resolution.", - wakePolicy: recoveryCause === "provider_quota" && !ownerAgentId + ? "Board operator: repair the failed review participant path, restore a live reviewer, explicitly reassign, or record an intentional resolution." + : "Board operator: inspect the evidence, repair the runtime if appropriate, then explicitly retry the original owner, reassign, or intentionally resolve the task.", + wakePolicy: isProviderQuotaWait ? { type: "monitor_only", reason: recoveryCause, } - : recoveryCause === "configuration_incomplete" - ? { - type: "manual_repair_required", - reason: recoveryCause, - ownerAgentId, - } - : ownerAgentId - ? { - type: "wake_owner", - reason: "source_scoped_recovery_action", - ownerAgentId, - } : { type: "board_escalation", - reason: "no_invokable_recovery_owner", + reason: recoveryCause, + preservesSourceAssignee: true, }, - monitorPolicy: recoveryCause === "provider_quota" && !ownerAgentId + monitorPolicy: isProviderQuotaWait ? { type: "wait_recovery", retryAgentId: routing.returnOwnerAgentId } : null, maxAttempts: null, @@ -3024,43 +2708,6 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) return action; } - async function enqueueSourceScopedStrandedRecoveryWake(input: { - action: Awaited>; - issue: typeof issues.$inferSelect; - latestRun: LatestIssueRun; - recoveryCause: StrandedRecoveryCause; - }) { - if (input.recoveryCause === "provider_quota" && !input.action.ownerAgentId) return; - if (input.recoveryCause === "configuration_incomplete") return; - if (!input.action.ownerAgentId) return; - await deps.enqueueWakeup(input.action.ownerAgentId, { - source: "assignment", - triggerDetail: "system", - reason: "source_scoped_recovery_action", - idempotencyKey: `source_scoped_recovery_action:${input.action.id}:${input.action.attemptCount}`, - payload: withRecoveryModelProfileHint({ - issueId: input.issue.id, - sourceIssueId: input.issue.id, - recoveryActionId: input.action.id, - strandedRunId: input.latestRun?.id ?? null, - recoveryCause: input.recoveryCause, - }, "status_only"), - requestedByActorType: "system", - requestedByActorId: null, - contextSnapshot: withRecoveryModelProfileHint({ - issueId: input.issue.id, - taskId: input.issue.id, - wakeReason: "source_scoped_recovery_action", - skipIssueComment: true, - source: "issue_recovery_action", - recoveryActionId: input.action.id, - sourceIssueId: input.issue.id, - strandedRunId: input.latestRun?.id ?? null, - recoveryCause: input.recoveryCause, - }, "status_only"), - }); - } - function readProviderQuotaRetryAt(latestRun: LatestIssueRun, now: Date) { const result = parseObject(latestRun?.resultJson); const context = parseObject(latestRun?.contextSnapshot); @@ -3687,36 +3334,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) return scheduledRun; } - async function hasLiveRecoveryOwnerPath(action: typeof issueRecoveryActions.$inferSelect) { - if (!action.ownerAgentId) return false; - const [run, wake] = await Promise.all([ - db - .select({ id: heartbeatRuns.id }) - .from(heartbeatRuns) - .where(and( - eq(heartbeatRuns.companyId, action.companyId), - eq(heartbeatRuns.agentId, action.ownerAgentId), - inArray(heartbeatRuns.status, [...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES]), - sql`${heartbeatRuns.contextSnapshot} ->> 'recoveryActionId' = ${action.id}`, - )) - .limit(1) - .then((rows) => rows[0] ?? null), - db - .select({ id: agentWakeupRequests.id }) - .from(agentWakeupRequests) - .where(and( - eq(agentWakeupRequests.companyId, action.companyId), - eq(agentWakeupRequests.agentId, action.ownerAgentId), - inArray(agentWakeupRequests.status, ["queued", "claimed", "deferred_issue_execution"]), - sql`${agentWakeupRequests.payload} ->> 'recoveryActionId' = ${action.id}`, - )) - .limit(1) - .then((rows) => rows[0] ?? null), - ]); - return Boolean(run || wake); - } - - async function latestRecoveryOwnerRun(action: typeof issueRecoveryActions.$inferSelect) { + async function latestRecoveryActionRun(action: typeof issueRecoveryActions.$inferSelect) { return db .select() .from(heartbeatRuns) @@ -3759,204 +3377,6 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) return Boolean(run || wake); } - async function scheduleRecoveryOwnerAttempt(input: { - action: typeof issueRecoveryActions.$inferSelect; - issue: typeof issues.$inferSelect; - attempt: number; - retryOfRunId?: string | null; - }) { - const ownerAgentId = input.action.ownerAgentId; - if (!ownerAgentId) return null; - const timing = recoveryOwnerDelayMs(input.attempt, input.action.fingerprint); - const now = new Date(); - const retryAt = new Date(now.getTime() + timing.delayMs); - const idempotencyKey = `recovery_owner_retry:${input.action.id}:${input.attempt}`; - const context = withRecoveryModelProfileHint({ - issueId: input.issue.id, - taskId: input.issue.id, - sourceIssueId: input.issue.id, - wakeReason: "source_scoped_recovery_action", - retryReason: RECOVERY_OWNER_RETRY_REASON, - source: "issue_recovery_action.owner_retry", - recoveryActionId: input.action.id, - recoveryCause: input.action.cause, - recoveryOwnerAttempt: input.attempt, - recoveryOwnerMaxAttempts: RECOVERY_OWNER_MAX_ATTEMPTS, - skipIssueComment: true, - ...(input.retryOfRunId ? { retryOfRunId: input.retryOfRunId } : {}), - }, "status_only"); - - let scheduledRun: typeof heartbeatRuns.$inferSelect | null = null; - if (timing.delayMs === 0) { - scheduledRun = await deps.enqueueWakeup(ownerAgentId, { - source: "assignment", - triggerDetail: "system", - reason: "source_scoped_recovery_action", - idempotencyKey, - payload: context, - requestedByActorType: "system", - requestedByActorId: null, - contextSnapshot: context, - }); - } else { - scheduledRun = await db.transaction(async (tx) => { - const existing = await tx - .select({ run: heartbeatRuns }) - .from(agentWakeupRequests) - .innerJoin(heartbeatRuns, eq(heartbeatRuns.wakeupRequestId, agentWakeupRequests.id)) - .where(and( - eq(agentWakeupRequests.companyId, input.action.companyId), - eq(agentWakeupRequests.idempotencyKey, idempotencyKey), - inArray(heartbeatRuns.status, [...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES]), - )) - .limit(1) - .then((rows) => rows[0]?.run ?? null); - if (existing) return existing; - - const wakeup = await tx - .insert(agentWakeupRequests) - .values({ - companyId: input.action.companyId, - agentId: ownerAgentId, - source: "automation", - triggerDetail: "system", - reason: "source_scoped_recovery_action", - payload: context, - status: "queued", - requestedByActorType: "system", - requestedByActorId: null, - idempotencyKey, - updatedAt: now, - }) - .returning() - .then((rows) => rows[0]!); - const run = await tx - .insert(heartbeatRuns) - .values({ - companyId: input.action.companyId, - agentId: ownerAgentId, - invocationSource: "automation", - triggerDetail: "system", - status: "scheduled_retry", - wakeupRequestId: wakeup.id, - retryOfRunId: input.retryOfRunId ?? null, - scheduledRetryAt: retryAt, - scheduledRetryAttempt: input.attempt, - scheduledRetryReason: RECOVERY_OWNER_RETRY_REASON, - contextSnapshot: context, - updatedAt: now, - }) - .returning() - .then((rows) => rows[0]!); - await tx - .update(agentWakeupRequests) - .set({ runId: run.id, updatedAt: now }) - .where(eq(agentWakeupRequests.id, wakeup.id)); - return run; - }); - } - - await db - .update(issueRecoveryActions) - .set({ - status: "active", - attemptCount: input.attempt, - maxAttempts: RECOVERY_OWNER_MAX_ATTEMPTS, - wakePolicy: { - type: "bounded_recovery_owner", - ownerAgentId, - attempt: input.attempt, - maxAttempts: RECOVERY_OWNER_MAX_ATTEMPTS, - baseBackoffMs: timing.baseDelayMs, - jitterMs: timing.jitterMs, - retryAt: retryAt.toISOString(), - scheduledRunId: scheduledRun?.id ?? null, - preservesSourceAssignee: true, - }, - timeoutAt: retryAt, - lastAttemptAt: now, - updatedAt: now, - }) - .where(and( - eq(issueRecoveryActions.id, input.action.id), - eq(issueRecoveryActions.companyId, input.action.companyId), - inArray(issueRecoveryActions.status, ["active", "escalated"]), - )); - - await logActivity(db, { - companyId: input.action.companyId, - actorType: "system", - actorId: "recovery", - agentId: null, - runId: input.retryOfRunId ?? null, - action: "issue.recovery_owner_retry_scheduled", - entityType: "issue_recovery_action", - entityId: input.action.id, - details: { - sourceIssueId: input.issue.id, - recoveryOwnerAgentId: ownerAgentId, - attempt: input.attempt, - maxAttempts: RECOVERY_OWNER_MAX_ATTEMPTS, - retryAt: retryAt.toISOString(), - scheduledRunId: scheduledRun?.id ?? null, - sourceAssigneeAgentId: input.issue.assigneeAgentId, - sourceAssigneePreserved: true, - }, - }); - return scheduledRun; - } - - async function escalateRecoveryOwnerToBoard(input: { - action: typeof issueRecoveryActions.$inferSelect; - issue: typeof issues.$inferSelect; - reason: string; - }) { - const now = new Date(); - const [updated] = await db - .update(issueRecoveryActions) - .set({ - status: "escalated", - ownerType: "board", - ownerAgentId: null, - wakePolicy: { - type: "board_escalation", - reason: input.reason, - attempt: input.action.attemptCount, - maxAttempts: RECOVERY_OWNER_MAX_ATTEMPTS, - preservesSourceAssignee: true, - }, - timeoutAt: null, - resolutionNote: input.reason, - updatedAt: now, - }) - .where(and( - eq(issueRecoveryActions.id, input.action.id), - eq(issueRecoveryActions.companyId, input.action.companyId), - inArray(issueRecoveryActions.status, ["active", "escalated"]), - )) - .returning(); - if (!updated) return false; - - await logActivity(db, { - companyId: input.action.companyId, - actorType: "system", - actorId: "recovery", - agentId: null, - runId: null, - action: "issue.recovery_owner_retry_exhausted", - entityType: "issue_recovery_action", - entityId: input.action.id, - details: { - sourceIssueId: input.issue.id, - attemptCount: input.action.attemptCount, - maxAttempts: RECOVERY_OWNER_MAX_ATTEMPTS, - reason: input.reason, - sourceAssigneeAgentId: input.issue.assigneeAgentId, - sourceAssigneePreserved: true, - }, - }); - return true; - } async function reconcileActiveRecoveryActions() { const rows = await db @@ -3978,7 +3398,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) if ( wakePolicyType !== "bounded_recovery_owner" && wakePolicyType !== "bounded_owner_disposition_repair" && - !(action.status === "escalated" && action.ownerType === "board") + action.ownerType !== "board" ) { continue; } @@ -3999,17 +3419,13 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) continue; } - if (action.ownerType === "board" && action.status === "escalated") { - result.skipped += 1; - continue; - } - const [sourceState, healthyChildren, hasNewSourcePath] = await Promise.all([ collectDispositionRepairSourceState(db, { issue }), healthyOpenChildIssues(issue), sourceHasNewPathOutsideRecoveryAction(action), ]); - if (sourceState.hasDurableWaitingPath || healthyChildren.length > 0 || hasNewSourcePath) { + const durablePathRestored = action.ownerType !== "board" && sourceState.hasDurableWaitingPath; + if (durablePathRestored || healthyChildren.length > 0 || hasNewSourcePath) { if (healthyChildren.length > 0 && !sourceState.hasDurableWaitingPath) { const blockerIds = await existingUnresolvedBlockerIssueIds(issue.companyId, issue.id); await issuesSvc.update(issue.id, { @@ -4018,7 +3434,6 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) ...blockerIds, ...healthyChildren.map((child) => child.id), ])], - assigneeAgentId: issue.assigneeAgentId, }); } const resolved = await recoveryActionsSvc.resolveActiveForIssue({ @@ -4027,7 +3442,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) actionId: action.id, status: "resolved", outcome: "restored", - resolutionNote: sourceState.hasDurableWaitingPath + resolutionNote: durablePathRestored ? `durable_path_restored:${sourceState.durablePathReason ?? "unknown"}` : healthyChildren.length > 0 ? "durable_path_restored:healthy_child" @@ -4051,7 +3466,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) continue; } - const latestRun = await latestRecoveryOwnerRun(action); + const latestRun = await latestRecoveryActionRun(action); const persistedAttempt = Math.max( action.attemptCount, Math.max(0, Math.floor(asNumber(wakePolicy.attempt, action.attemptCount))), @@ -4071,52 +3486,11 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) continue; } - if (!action.ownerAgentId) { - if (await escalateRecoveryOwnerToBoard({ action, issue, reason: "recovery_owner_missing" })) { - result.escalated += 1; - result.issueIds.push(issue.id); - } - continue; - } + if (action.ownerType === "board") continue; - const ownerAgentId = await resolveInvokableRecoveryAgentId(issue, action.ownerAgentId); - if (!ownerAgentId) { - if (await escalateRecoveryOwnerToBoard({ action, issue, reason: "recovery_owner_not_invokable" })) { - result.escalated += 1; - result.issueIds.push(issue.id); - } - continue; - } - - if (await hasLiveRecoveryOwnerPath(action)) { - result.skipped += 1; - continue; - } - - const latestRun = await latestRecoveryOwnerRun(action); - const persistedAttempt = wakePolicyType === "bounded_recovery_owner" - ? Math.max(0, Math.floor(asNumber(wakePolicy.attempt, action.attemptCount))) - : 0; - if (latestRun && !TERMINAL_HEARTBEAT_RUN_STATUSES.has(latestRun.status)) { - result.skipped += 1; - continue; - } - if (persistedAttempt >= RECOVERY_OWNER_MAX_ATTEMPTS) { - if (await escalateRecoveryOwnerToBoard({ action, issue, reason: "recovery_owner_retry_exhausted" })) { - result.escalated += 1; - result.issueIds.push(issue.id); - } - continue; - } - - await scheduleRecoveryOwnerAttempt({ - action, - issue, - attempt: persistedAttempt + 1, - retryOfRunId: latestRun?.id ?? null, - }); - result.requeued += 1; - result.issueIds.push(issue.id); + // Legacy takeover actions remain readable and resolvable, but recovery no + // longer schedules another agent-owned wake for them. + result.skipped += 1; } return result; } @@ -4134,23 +3508,15 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) fingerprint: input.fingerprint, attemptCount: input.attemptCount, }); - const excludedOwners = new Set( - input.issue.assigneeAgentId ? [input.issue.assigneeAgentId] : [], - ); - const recoveryOwnerAgentId = await resolveStrandedIssueRecoveryOwnerAgentId( - input.issue, - null, - excludedOwners, - ); const now = new Date(); await db .update(issueRecoveryActions) .set({ - status: recoveryOwnerAgentId ? "active" : "escalated", - ownerType: recoveryOwnerAgentId ? "agent" : "board", - ownerAgentId: recoveryOwnerAgentId, - attemptCount: 0, - maxAttempts: RECOVERY_OWNER_MAX_ATTEMPTS, + status: "active", + ownerType: "board", + ownerAgentId: null, + ownerUserId: null, + maxAttempts: null, evidence: { ...action.evidence, latestRunId: input.latestRun?.id ?? null, @@ -4159,22 +3525,15 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) terminalReason: input.terminalReason, sourceAttemptCount: input.attemptCount, sourceMaxAttempts: DISPOSITION_REPAIR_MAX_ATTEMPTS, + routingPolicy: STRANDED_BOARD_ESCALATION_POLICY, }, nextAction: - "Repair the source issue disposition or request an explicit reassignment decision without taking source ownership.", - wakePolicy: recoveryOwnerAgentId - ? { - type: "bounded_recovery_owner", - ownerAgentId: recoveryOwnerAgentId, - attempt: 0, - maxAttempts: RECOVERY_OWNER_MAX_ATTEMPTS, - preservesSourceAssignee: true, - } - : { - type: "board_escalation", - reason: "no_invokable_recovery_owner", - preservesSourceAssignee: true, - }, + "Inspect the evidence and choose whether to repair, retry the original owner, explicitly reassign, or resolve the source issue.", + wakePolicy: { + type: "board_escalation", + reason: input.terminalReason, + preservesSourceAssignee: true, + }, timeoutAt: null, resolutionNote: input.terminalReason, updatedAt: now, @@ -4186,11 +3545,12 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) const updated = await issuesSvc.update(input.issue.id, { status: "blocked", - assigneeAgentId: input.issue.assigneeAgentId, }); if (!updated) return null; + const sourceAssigneePreserved = + updated.assigneeAgentId === input.issue.assigneeAgentId && + updated.assigneeUserId === input.issue.assigneeUserId; - const recoveryOwner = recoveryOwnerAgentId ? await getAgent(recoveryOwnerAgentId) : null; await issuesSvc.addComment( input.issue.id, [ @@ -4198,7 +3558,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) "", `- Attempts: ${input.attemptCount}/${DISPOSITION_REPAIR_MAX_ATTEMPTS}`, `- Terminal reason: \`${input.terminalReason}\``, - `- Recovery owner: ${recoveryOwner?.name ?? "board"}`, + "- Recovery owner: board", "- Source ownership: unchanged; reassignment requires an explicit decision or a policy-defined serious failure.", "", "Next action: repair the liveness disposition or request an explicit source-owner decision.", @@ -4212,7 +3572,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) latestRun: input.latestRun, recoveryActionId: action.id, previousStatus: input.issue.status, - recoveryOwner, + recoveryOwner: null, }), }, ); @@ -4235,29 +3595,28 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) maxAttempts: DISPOSITION_REPAIR_MAX_ATTEMPTS, terminalReason: input.terminalReason, recoveryActionId: action.id, - recoveryOwnerAgentId, - sourceAssigneeAgentId: input.issue.assigneeAgentId, - sourceAssigneePreserved: true, + recoveryOwnerAgentId: null, + recoveryOwnerType: "board", + routingPolicy: STRANDED_BOARD_ESCALATION_POLICY, + sourceAssigneeBefore: { + agentId: input.issue.assigneeAgentId, + userId: input.issue.assigneeUserId, + }, + sourceAssigneeAfter: { + agentId: updated.assigneeAgentId, + userId: updated.assigneeUserId, + }, + sourceAssigneePreserved, }, }); - - const escalatedAction = await db - .select() - .from(issueRecoveryActions) - .where(and( - eq(issueRecoveryActions.companyId, input.issue.companyId), - eq(issueRecoveryActions.sourceIssueId, input.issue.id), - inArray(issueRecoveryActions.status, ["active", "escalated"]), - )) - .limit(1) - .then((rows) => rows[0] ?? null); - if (escalatedAction && recoveryOwnerAgentId) { - await scheduleRecoveryOwnerAttempt({ - action: escalatedAction, - issue: input.issue, - attempt: 1, - retryOfRunId: input.latestRun?.id ?? null, - }); + if (!sourceAssigneePreserved) { + logger.error({ + issueId: input.issue.id, + beforeAssigneeAgentId: input.issue.assigneeAgentId, + afterAssigneeAgentId: updated.assigneeAgentId, + beforeAssigneeUserId: input.issue.assigneeUserId, + afterAssigneeUserId: updated.assigneeUserId, + }, "automatic disposition recovery observed a concurrent source-owner change"); } return updated; } @@ -4361,7 +3720,6 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) comment?: string; notice?: StrandedRecoveryNoticeSeed | null; recoveryCause?: StrandedRecoveryCause; - recoveryOwnerAgentId?: string | null; successfulRunHandoffEvidence?: SuccessfulRunHandoffRecoveryEvidence | null; }) { if (isStrandedIssueRecoveryIssue(input.issue)) { @@ -4378,7 +3736,6 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) previousStatus: input.previousStatus, latestRun: input.latestRun, recoveryCause, - recoveryOwnerAgentId: input.recoveryOwnerAgentId, successfulRunHandoffEvidence: input.successfulRunHandoffEvidence, }); const isProviderQuotaWait = recoveryCause === "provider_quota" && @@ -4396,12 +3753,12 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) const updated = await issuesSvc.update(input.issue.id, { status: "blocked", blockedByIssueIds: blockerIds, - // Recovery-action ownership is intentionally separate from deliverable - // ownership. Automatic escalation must never transfer the source task. - assigneeAgentId: input.issue.assigneeAgentId, }); if (!updated) return null; if (isProviderQuotaWait) return updated; + const sourceAssigneePreserved = + updated.assigneeAgentId === input.issue.assigneeAgentId && + updated.assigneeUserId === input.issue.assigneeUserId; const recoveryOwner = recoveryAction.ownerAgentId ? await getAgent(recoveryAction.ownerAgentId) : null; const sourceAssignee = input.issue.assigneeAgentId ? await getAgent(input.issue.assigneeAgentId) : null; @@ -4524,41 +3881,32 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) latestRunStatus: input.latestRun?.status ?? null, latestRunErrorCode: input.latestRun?.errorCode ?? null, recoveryActionId: recoveryAction.id, + recoveryOwnerType: recoveryAction.ownerType, recoveryOwnerAgentId: recoveryAction.ownerAgentId, previousOwnerAgentId: recoveryAction.previousOwnerAgentId, returnOwnerAgentId: recoveryAction.returnOwnerAgentId, + routingPolicy: parseObject(recoveryAction.evidence).routingPolicy ?? null, + sourceAssigneeBefore: { + agentId: input.issue.assigneeAgentId, + userId: input.issue.assigneeUserId, + }, + sourceAssigneeAfter: { + agentId: updated.assigneeAgentId, + userId: updated.assigneeUserId, + }, + sourceAssigneePreserved, blockerIssueIds: blockerIds, }, }); - await enqueueSourceScopedStrandedRecoveryWake({ - action: recoveryAction, - issue: input.issue, - latestRun: input.latestRun, - recoveryCause, - }); - - if (input.issue.assigneeAgentId) { - const [currentIssue] = await db - .select({ - status: issues.status, - assigneeAgentId: issues.assigneeAgentId, - }) - .from(issues) - .where(eq(issues.id, input.issue.id)) - .limit(1); - if ( - currentIssue && - (currentIssue.status !== "blocked" || - currentIssue.assigneeAgentId !== input.issue.assigneeAgentId) - ) { - const reblocked = await issuesSvc.update(input.issue.id, { - status: "blocked", - blockedByIssueIds: blockerIds, - assigneeAgentId: input.issue.assigneeAgentId, - }); - if (reblocked) return reblocked; - } + if (!sourceAssigneePreserved) { + logger.error({ + issueId: input.issue.id, + beforeAssigneeAgentId: input.issue.assigneeAgentId, + afterAssigneeAgentId: updated.assigneeAgentId, + beforeAssigneeUserId: input.issue.assigneeUserId, + afterAssigneeUserId: updated.assigneeUserId, + }, "automatic stranded recovery observed a concurrent source-owner change"); } return updated; @@ -4769,7 +4117,20 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) result.skipped += 1; } } else { - result.skipped += 1; + const updated = await escalateStrandedAssignedIssue({ + issue, + previousStatus: issue.status as StrandedPreviousStatus, + latestRun, + comment: + "Paperclip cannot safely continue automatic recovery because the original assignee is not invokable. " + + "The source assignment is unchanged and the board must choose the next action.", + }); + if (updated) { + result.escalated += 1; + result.issueIds.push(issue.id); + } else { + result.skipped += 1; + } } continue; } @@ -4797,6 +4158,40 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) result.operatorCancelExempted += 1; continue; } + if (await isInvocationBudgetBlocked(issue, agentId)) { + const classification = classifyContinuationFailure(latestRun); + if ( + classification.kind === "deliberate_wait_without_target" || + readDispositionRepairAttempt(latestRun) + ) { + const outcome = await reconcileDispositionRepair(issue, latestRun); + if (outcome === "escalated") { + result.escalated += 1; + result.issueIds.push(issue.id); + } else { + result.skipped += 1; + } + } else { + const updated = await escalateStrandedAssignedIssue({ + issue, + previousStatus: issue.status as StrandedPreviousStatus, + latestRun, + recoveryCause: issue.status === "in_review" + ? EXECUTION_REVIEW_PARTICIPANT_RECOVERY_REASON + : undefined, + comment: + "Paperclip cannot safely continue automatic recovery because the original recovery target is over budget. " + + "The source assignment is unchanged and the board must choose the next action.", + }); + if (updated) { + result.escalated += 1; + result.issueIds.push(issue.id); + } else { + result.skipped += 1; + } + } + continue; + } if (latestRun?.status === "succeeded" && await hasPersistedDurableWaitPath(issue)) { result.skipped += 1; continue; @@ -5007,7 +4402,6 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) latestRun: participantLatestRun, notice: buildExecutionReviewParticipantUnavailableNoticeSeed(), recoveryCause: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_REASON, - recoveryOwnerAgentId: participantAgentId, }); if (updated) { result.escalated += 1; @@ -5048,7 +4442,6 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) previousStatus: "in_review", latestRun: participantLatestRun, recoveryCause: "configuration_incomplete", - recoveryOwnerAgentId: participantAgentId, comment: "Paperclip classified the active review participant's latest adapter failure as " + "`configuration_incomplete`. Moving the issue to `blocked` with the configuration fix " + @@ -5074,7 +4467,6 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) latestRun: participantLatestRun, notice: buildExecutionReviewParticipantUnavailableNoticeSeed(), recoveryCause: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_REASON, - recoveryOwnerAgentId: participantAgentId, }); if (updated) { result.escalated += 1; @@ -5092,7 +4484,6 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) latestRun: participantLatestRun, notice: buildExecutionReviewParticipantRecoveryNoticeSeed(), recoveryCause: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_REASON, - recoveryOwnerAgentId: participantAgentId, }); if (updated) { result.escalated += 1; @@ -5438,6 +4829,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) result.escalated += activeRecovery.escalated; result.skipped += activeRecovery.skipped; result.issueIds.push(...activeRecovery.issueIds); + result.issueIds = [...new Set(result.issueIds)]; return result; } diff --git a/server/src/services/recovery/stranded-notice.test.ts b/server/src/services/recovery/stranded-notice.test.ts index 3fff23b5fc..4a2878ac32 100644 --- a/server/src/services/recovery/stranded-notice.test.ts +++ b/server/src/services/recovery/stranded-notice.test.ts @@ -101,7 +101,7 @@ describe("buildStrandedRecoveryEscalationNotice", () => { const ownerRow = allRows(notice.metadata).find((row) => row.label === "Recovery owner"); expect(ownerRow?.type).toBe("key_value"); - expect(String(ownerRow?.value)).toContain("Board escalation"); + expect(ownerRow?.value).toBe("Board decision required"); }); it("derives title from the recovery cause and body from the plain-comment fallback", () => { diff --git a/server/src/services/recovery/stranded-notice.ts b/server/src/services/recovery/stranded-notice.ts index 7705ba328f..79b78f83dc 100644 --- a/server/src/services/recovery/stranded-notice.ts +++ b/server/src/services/recovery/stranded-notice.ts @@ -74,7 +74,7 @@ export function buildExecutionReviewParticipantRecoveryNoticeSeed(): StrandedRec return { body: "Paperclip retried the pending execution-review participant once, but the review stage still has no " + - "completed decision or live reviewer run. Moving it to `blocked` so the recovery owner can repair the " + + "completed decision or live reviewer run. Moving it to `blocked` so the board can inspect the evidence, repair the " + "reviewer runtime, restore the review stage, or record an intentional manual resolution.", title: "Review recovery stalled", tone: "danger", @@ -86,7 +86,7 @@ export function buildExecutionReviewParticipantUnavailableNoticeSeed(): Stranded body: "Paperclip cannot continue the pending execution-review participant because the participant is not " + "invokable and the review stage has no completed decision or live reviewer run. Moving it to `blocked` " + - "so the recovery owner can repair the reviewer runtime, restore the review stage, or record an " + + "so the board can inspect the evidence, repair the reviewer runtime, restore the review stage, or record an " + "intentional manual resolution.", title: "Review recovery stalled", tone: "danger", @@ -122,13 +122,13 @@ export function buildStrandedRecoveryEscalationNotice(input: { ? agentLinkRow("Recovery owner", input.recoveryOwner) : keyValueRow( "Recovery owner", - "Board escalation - Paperclip could not find an invokable manager, creator, or executive owner with budget available", + "Board decision required", ), keyValueRow( "Next action", input.recoveryOwner ? "The recovery owner should either restore a live execution path or record the manual resolution on the source issue" - : "A board operator should assign an invokable recovery owner, fix the agent/runtime state, or record an intentional manual resolution", + : "Inspect the evidence, then retry the original owner, explicitly reassign, repair the execution path, or record an intentional resolution", ), ]; diff --git a/server/src/services/recovery/successful-run-handoff.test.ts b/server/src/services/recovery/successful-run-handoff.test.ts index 936aca4564..6b0f0ca9b2 100644 --- a/server/src/services/recovery/successful-run-handoff.test.ts +++ b/server/src/services/recovery/successful-run-handoff.test.ts @@ -531,7 +531,7 @@ describe("successful run handoff decision", () => { expect(notice.metadata.sourceRunId).toBe("22222222-2222-4222-8222-222222222222"); expect(notice.metadata.sections).toEqual(expect.arrayContaining([ expect.objectContaining({ - title: "Recovery owner", + title: "Recovery", rows: expect.arrayContaining([ expect.objectContaining({ type: "key_value", label: "Recovery action", value: "77777777-7777-4777-8777-777777777777" }), expect.objectContaining({ type: "agent_link", label: "Recovery owner", name: "CTO" }), diff --git a/server/src/services/recovery/successful-run-handoff.ts b/server/src/services/recovery/successful-run-handoff.ts index 0c5943606a..b0ad4826f7 100644 --- a/server/src/services/recovery/successful-run-handoff.ts +++ b/server/src/services/recovery/successful-run-handoff.ts @@ -17,7 +17,7 @@ export const DEFAULT_MAX_SUCCESSFUL_RUN_HANDOFF_ATTEMPTS = 1; export const SUCCESSFUL_RUN_HANDOFF_REQUIRED_NOTICE_BODY = "Paperclip needs a disposition before this issue can continue."; export const SUCCESSFUL_RUN_HANDOFF_EXHAUSTED_NOTICE_BODY = - "Paperclip could not resolve this issue's missing disposition automatically. The issue is blocked on a recovery owner."; + "Paperclip could not resolve this issue's missing disposition automatically. The source assignment is unchanged and a board decision is required."; export const LEGACY_SUCCESSFUL_RUN_HANDOFF_NOTICE_PREFIXES = [ "## This issue still needs a next step", "## Successful run missing issue disposition", @@ -212,15 +212,17 @@ export function buildSuccessfulRunHandoffExhaustedNotice(input: { sourceRunId: input.sourceRun?.id ?? null, sections: [ { - title: "Recovery owner", + title: "Recovery", rows: [ issueLinkRow("Source issue", input.issue), input.recoveryActionId ? keyValueRow("Recovery action", input.recoveryActionId) : issueLinkRow("Recovery issue", input.recoveryIssue), - agentLinkRow("Recovery owner", input.recoveryOwner), + input.recoveryOwner + ? agentLinkRow("Recovery owner", input.recoveryOwner) + : keyValueRow("Recovery owner", "Board decision required"), agentLinkRow("Source assignee", input.sourceAssignee), - keyValueRow("Suggested action", "choose and record a valid issue disposition without copying transcript content"), + keyValueRow("Suggested action", "inspect the evidence, then retry the original owner, explicitly reassign, or record a valid issue disposition"), ], }, { diff --git a/ui/src/components/IssueRecoveryActionCard.test.tsx b/ui/src/components/IssueRecoveryActionCard.test.tsx index 7b09c766a2..94eca857bb 100644 --- a/ui/src/components/IssueRecoveryActionCard.test.tsx +++ b/ui/src/components/IssueRecoveryActionCard.test.tsx @@ -999,32 +999,35 @@ describe("IssueRecoveryActionCard owner-sticky retry lineage", () => { expect(node.textContent).not.toContain("Times out"); }); - it("shows the board escalation without implying the board owns the task", () => { + it("shows a current-policy board recovery action without implying the board owns the task", () => { const node = render( , ); const section = node.querySelector("section[aria-label]"); - expect(section?.getAttribute("data-recovery-state")).toBe("escalated"); + expect(section?.getAttribute("data-recovery-state")).toBe("needed"); expect(section?.getAttribute("data-recovery-lane")).toBe("board"); expect(node.textContent).toContain("Automatic recovery is exhausted"); + expect(node.textContent).toContain("Board decision required"); const recoveryOwner = node.querySelector("[data-testid='recovery-recovery-owner']"); expect(recoveryOwner?.textContent).toContain("Board"); expect(recoveryOwner?.textContent).toContain("decides the next step only"); diff --git a/ui/src/components/IssueRecoveryActionCard.tsx b/ui/src/components/IssueRecoveryActionCard.tsx index fda6fa6da6..9e17faaa10 100644 --- a/ui/src/components/IssueRecoveryActionCard.tsx +++ b/ui/src/components/IssueRecoveryActionCard.tsx @@ -763,7 +763,7 @@ function readWakePolicySummary(action: IssueRecoveryAction): string | null { return "Paperclip is retrying the original owner"; } if (type === "bounded_recovery_owner") return "A recovery owner is repairing the next step"; - if (type === "board_escalation") return "Board will decide"; + if (type === "board_escalation") return "Board decision required"; if (type === "manual") return "Manual follow-up needed"; if (type === "manual_repair_required") return "Repair needed before retry"; if (type === "monitor") { diff --git a/ui/src/fixtures/systemNoticeFixtures.ts b/ui/src/fixtures/systemNoticeFixtures.ts index a7cb6f75c5..c1fece4f76 100644 --- a/ui/src/fixtures/systemNoticeFixtures.ts +++ b/ui/src/fixtures/systemNoticeFixtures.ts @@ -155,7 +155,7 @@ export const systemNoticeFixtures: readonly SystemNoticeFixture[] = [ label: "System alert", source: { label: "Paperclip", href: "/PAP/agents" }, timestamp: "2026-05-04T16:48:00.000Z", - body: "Paperclip could not resolve this issue's missing disposition automatically. The issue is blocked on a recovery owner.", + body: "Paperclip could not resolve this issue's missing disposition automatically. The source assignment is unchanged and a board decision is required.", metadata: HANDOFF_METADATA, detailsDefaultOpen: false, }, @@ -166,7 +166,7 @@ export const systemNoticeFixtures: readonly SystemNoticeFixture[] = [ label: "System alert", source: { label: "Paperclip", href: "/PAP/agents" }, timestamp: "2026-05-04T16:48:00.000Z", - body: "Paperclip could not resolve this issue's missing disposition automatically. The issue is blocked on a recovery owner.", + body: "Paperclip could not resolve this issue's missing disposition automatically. The source assignment is unchanged and a board decision is required.", metadata: HANDOFF_METADATA, detailsDefaultOpen: true, }, diff --git a/ui/src/lib/successful-run-handoff.ts b/ui/src/lib/successful-run-handoff.ts index 1f54db8b81..2ffac2d77e 100644 --- a/ui/src/lib/successful-run-handoff.ts +++ b/ui/src/lib/successful-run-handoff.ts @@ -6,7 +6,7 @@ export const SUCCESSFUL_RUN_HANDOFF_ESCALATED_ACTION = "issue.successful_run_han export const SUCCESSFUL_RUN_HANDOFF_REQUIRED_NOTICE_BODY = "Paperclip needs a disposition before this issue can continue."; export const SUCCESSFUL_RUN_HANDOFF_EXHAUSTED_NOTICE_BODY = - "Paperclip could not resolve this issue's missing disposition automatically. The issue is blocked on a recovery owner."; + "Paperclip could not resolve this issue's missing disposition automatically. The source assignment is unchanged and a board decision is required."; export function isSuccessfulRunHandoffActivity(action: string) { return action === SUCCESSFUL_RUN_HANDOFF_REQUIRED_ACTION diff --git a/ui/src/pages/SystemNoticeUxLab.tsx b/ui/src/pages/SystemNoticeUxLab.tsx index 00297a780f..e32dd4ab6d 100644 --- a/ui/src/pages/SystemNoticeUxLab.tsx +++ b/ui/src/pages/SystemNoticeUxLab.tsx @@ -243,7 +243,7 @@ export function SystemNoticeUxLab() { label="System alert" source={{ label: "Paperclip", href: "/PAP/agents" }} timestamp="2026-05-04T16:48:00.000Z" - body="Paperclip could not resolve this issue's missing disposition automatically. The issue is blocked on a recovery owner." + body="Paperclip could not resolve this issue's missing disposition automatically. The source assignment is unchanged and a board decision is required." metadata={[ { title: "Recovery owner",