From b98badb2468df7dee10c6d330158440fbec414de Mon Sep 17 00:00:00 2001 From: Magnus <21985329+im0xMagnus@users.noreply.github.com> Date: Fri, 4 Sep 2026 13:05:20 +1000 Subject: [PATCH] fix(recovery): exclude hidden issues from stranded recovery and continuation wakes (#5648) ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The recovery subsystem watches assigned issues and re-wakes an agent whose run ended without finishing the work > - Intake can hide a duplicate issue by setting `hiddenAt` while leaving its status and assignee in place > - The stranded-issue query and the terminal-run cleanup both ignore `hiddenAt`, so a hidden issue is re-woken on every cycle > - Nothing on the board shows the hidden issue, so the repeated wakes have no visible cause > - This pull request adds a hidden-issue guard to both predicates and a test for each > - The benefit is that hiding an issue stops recovery work on it, with no other change in behavior for visible issues ## Linked Issues or Issue Description **What happened?** When intake marks an issue as a duplicate it sets `hiddenAt` but leaves the status at `todo` or `in_progress` with the agent still assigned. The stranded-issue recovery timer selects that issue on every tick and queues an `issue_continuation_needed` wake for it. The agent's run on the hidden issue fails or is cancelled, the terminal-run cleanup queues immediate recovery for the same issue, and the cycle repeats indefinitely. Hidden issues are invisible on the board, so nothing a person can see explains the wakes. **Expected behavior** A hidden issue is never a recovery candidate. Stranded-issue reconciliation skips it, and a failed, timed-out or cancelled run on it releases the issue without queuing a continuation. **Steps to reproduce** 1. Assign an issue to an agent and leave it `in_progress`. 2. Hide the issue (set `hiddenAt`, for example by marking it a duplicate through intake) without changing its status or assignee. 3. Let a run on that issue fail, or wait for the stranded-issue recovery timer. 4. Observe a new `issue_continuation_needed` heartbeat run queued for the hidden issue on every cycle. **Paperclip version or commit** Reproduced on `master` when this PR was opened (May 2026). The two predicates are unchanged on current `master`; this branch is rebased onto it. **Deployment mode** Not deployment-specific: both guards are in the server's recovery and heartbeat services and apply in every mode. ## What Changed - `server/src/services/recovery/service.ts`: `isNull(issues.hiddenAt)` added to the `reconcileStrandedAssignedIssues` candidate query, so hidden issues never enter the stranded set. - `server/src/services/heartbeat.ts`: `!issue.hiddenAt` added to `issueNeedsImmediateRecovery`, so terminal-run cleanup releases a hidden issue instead of queuing a continuation. - `server/src/__tests__/heartbeat-process-recovery.test.ts`: one test per guard. A failed run on a hidden issue queues no recovery run, and a hidden stranded issue is left out of reconciliation. ## Verification - `heartbeat-process-recovery.test.ts` covers both guards; CI runs it against embedded Postgres. ## Risks Low. Both changes narrow an existing predicate to exclude rows that already carry `hiddenAt`; visible issues take exactly the path they take today. A hidden issue that genuinely needs recovery would have to be unhidden first, which matches how hidden issues behave everywhere else in the board. ## Model Used The original two-line fix was authored by @im0xMagnus. The rebase onto current `master`, the two regression tests, and this description were produced with Claude (claude-fable-5-1, extended thinking, tool use) driven by a Paperclip maintainer through Prospector's triage flow. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [ ] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Andrew Aymeloglu --- .../heartbeat-process-recovery.test.ts | 51 ++++++++++++++++++- server/src/services/heartbeat.ts | 1 + server/src/services/recovery/service.ts | 1 + 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index b2f1c703d0..c958b27641 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -1325,6 +1325,53 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { expect(agent).toEqual({ status: "running", errorReason: null }); }); + it("does not queue immediate recovery when the failed run's issue is hidden", async () => { + mockAdapterExecute.mockResolvedValueOnce({ + exitCode: 1, + signal: null, + timedOut: false, + errorMessage: null, + provider: "test", + model: "test-model", + }); + + const { runId, issueId } = await seedQueuedIssueRunFixture(); + await db + .update(issues) + .set({ hiddenAt: new Date("2026-03-19T00:05:00.000Z") }) + .where(eq(issues.id, issueId)); + const heartbeat = heartbeatService(db); + + await heartbeat.resumeQueuedRuns(); + await waitForRunToSettle(heartbeat, runId); + await heartbeat.waitForRunExecutionDrain(runId); + + const run = await heartbeat.getRun(runId); + const recoveryRuns = await db + .select({ id: heartbeatRuns.id }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.retryOfRunId, runId)); + + expect(run).toMatchObject({ status: "failed" }); + expect(recoveryRuns).toHaveLength(0); + }); + + it("leaves hidden issues out of stranded-issue reconciliation", async () => { + const { issueId } = await seedStrandedIssueFixture({ + status: "in_progress", + runStatus: "failed", + }); + await db + .update(issues) + .set({ hiddenAt: new Date("2026-03-19T00:05:00.000Z") }) + .where(eq(issues.id, issueId)); + + const result = await heartbeatService(db).reconcileStrandedAssignedIssues(); + + expect(result.issueIds).not.toContain(issueId); + expect(result.continuationRequeued).toBe(0); + }); + it("keeps a local run active when the recorded pid is still alive", async () => { const child = spawnAliveProcess(); childProcesses.add(child); @@ -6028,7 +6075,9 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { expect(result.continuationRequeued).toBe(0); expect(result.escalated).toBe(1); expect(result.skipped).toBe(0); - expect(result.issueIds).toEqual([blocked.issueId, unblocked.issueId]); + expect([...result.issueIds].sort()).toEqual( + [blocked.issueId, unblocked.issueId].sort(), + ); const blockedWakeups = await db .select() diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index faa8b46f68..af753d2cf5 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -22814,6 +22814,7 @@ export function heartbeatService( const issueNeedsImmediateRecovery = (issue.status === "todo" || issue.status === "in_progress") && !issue.assigneeUserId && + !issue.hiddenAt && issue.assigneeAgentId === run.agentId && (run.status === "failed" || run.status === "timed_out" || diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 2d8cbb5d86..8d7655d6d6 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -3451,6 +3451,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) eq(issues.status, "in_review"), ), opts?.issueCreatedAtGte ? gte(issues.createdAt, opts.issueCreatedAtGte) : undefined, + isNull(issues.hiddenAt), ), );