From a0dbe2104567a06b90a7f34f276b1366eff161cf Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Sat, 1 Aug 2026 16:51:13 -0700 Subject: [PATCH] fix(server): stand down recovery while an operator-cancelled run is the latest activity (#10656) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Operators can cancel a running agent from the board when a run is unwanted — most acutely while cleaning up a runaway loop > - The recovery machinery treats a cancelled run like any other unsuccessful terminal run: the stranded-issue sweep classifies the issue as stranded, creates a recovery action, and wakes the agent again > - So cancelling runs to stop a loop *fed* the loop: each operator cancel spawned a recovery action that re-woke the agent the operator had just stopped > - This pull request stamps board-initiated cancellations with operator attribution and makes the sweep stand down while such a run is the issue's latest activity > - The benefit is that an operator's cancel is final until something new happens, instead of being fought by automation ## Linked Issues or Issue Description Fixes #10646 ## What Changed - `POST /heartbeat-runs/:runId/cancel` (board-only) now cancels with an explicit reason ("Cancelled by a board operator") and stamps `resultJson.cancelledByActorType: "user"` / `cancelledByUserId`. - `reconcileStrandedAssignedIssues` gains an early stand-down: when the issue's latest run is operator-cancelled (the new stamp, or the existing `operator_interrupted` error code from interrupt-by-comment), the issue is skipped entirely — no recovery action, no wake — and counted in a new `operatorCancelExempted` result field. The exemption is inherently self-limiting: any newer run or wake supersedes it because the gate only looks at the *latest* run. - System cancellations without operator attribution (lease expiry, assignee changes, terminal-status cancels, pause holds) keep today's recovery behavior unchanged. ## Verification - `pnpm vitest run server/src/__tests__/issue-recovery-actions.test.ts` (embedded Postgres) — 3 new cases: a stamped operator cancel produces zero recovery actions and zero wakes; an `operator_interrupted` cancel likewise; an unattributed system cancel still flows into pre-existing recovery (wake observed), proving the stand-down is scoped to operator attribution. - `pnpm vitest run server/src/__tests__/heartbeat-process-recovery.test.ts server/src/__tests__/issue-scheduled-retry-routes.test.ts` — unchanged (109 tests). - `cd server && pnpm run typecheck`. ## Risks - Low. The only suppressed behavior is recovery of runs a human explicitly cancelled from the board; everything else is byte-identical. If an operator cancels and walks away, the issue stays quiet until any new activity — which is the intent (the operator owns the next step). ## Model Used Claude Fable 5 (`claude-fable-5`, Anthropic) via Claude Code — extended thinking, agentic tool use. No other models involved. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --- .../__tests__/issue-recovery-actions.test.ts | 74 +++++++++++++++++++ server/src/routes/agents.ts | 10 ++- server/src/services/recovery/service.ts | 20 +++++ 3 files changed, 103 insertions(+), 1 deletion(-) diff --git a/server/src/__tests__/issue-recovery-actions.test.ts b/server/src/__tests__/issue-recovery-actions.test.ts index 302065e96e..911137dc79 100644 --- a/server/src/__tests__/issue-recovery-actions.test.ts +++ b/server/src/__tests__/issue-recovery-actions.test.ts @@ -380,6 +380,80 @@ describeEmbeddedPostgres("issue recovery actions", () => { }, ); + it("stands down while the latest run was cancelled by a board operator", async () => { + const { companyId, coderId, sourceIssueId } = await seedCompany(); + await db.insert(heartbeatRuns).values({ + id: randomUUID(), + companyId, + agentId: coderId, + invocationSource: "manual", + status: "cancelled", + error: "Cancelled by a board operator", + errorCode: "cancelled", + resultJson: { cancelledByActorType: "user", cancelledByUserId: "board-user" }, + startedAt: new Date("2026-07-15T20:00:00.000Z"), + finishedAt: new Date("2026-07-15T20:01:00.000Z"), + contextSnapshot: { issueId: sourceIssueId }, + }); + const enqueueWakeup = vi.fn(async () => null); + const recovery = recoveryService(db, { enqueueWakeup }); + + const result = await recovery.reconcileStrandedAssignedIssues(); + + expect(result.operatorCancelExempted).toBe(1); + expect(await db.select().from(issueRecoveryActions)).toHaveLength(0); + expect(enqueueWakeup).not.toHaveBeenCalled(); + }); + + it("stands down after an operator interrupt cancellation", async () => { + const { companyId, coderId, sourceIssueId } = await seedCompany(); + await db.insert(heartbeatRuns).values({ + id: randomUUID(), + companyId, + agentId: coderId, + invocationSource: "manual", + status: "cancelled", + error: "Interrupted by board comment", + errorCode: "operator_interrupted", + startedAt: new Date("2026-07-15T20:00:00.000Z"), + finishedAt: new Date("2026-07-15T20:01:00.000Z"), + contextSnapshot: { issueId: sourceIssueId }, + }); + const enqueueWakeup = vi.fn(async () => null); + const recovery = recoveryService(db, { enqueueWakeup }); + + const result = await recovery.reconcileStrandedAssignedIssues(); + + expect(result.operatorCancelExempted).toBe(1); + expect(enqueueWakeup).not.toHaveBeenCalled(); + }); + + it("still recovers system-cancelled runs with no operator attribution", async () => { + const { companyId, coderId, sourceIssueId } = await seedCompany(); + await db.insert(heartbeatRuns).values({ + id: randomUUID(), + companyId, + agentId: coderId, + invocationSource: "manual", + status: "cancelled", + error: "Cancelled because the workspace lease expired", + errorCode: "cancelled", + startedAt: new Date("2026-07-15T20:00:00.000Z"), + finishedAt: new Date("2026-07-15T20:01:00.000Z"), + contextSnapshot: { issueId: sourceIssueId }, + }); + const enqueueWakeup = vi.fn(async () => null); + const recovery = recoveryService(db, { enqueueWakeup }); + + const result = await recovery.reconcileStrandedAssignedIssues(); + + expect(result.operatorCancelExempted).toBe(0); + // The system-cancelled run still flows into the pre-existing recovery + // behavior (a continuation requeue or escalation — either produces a + // wake), proving the stand-down is scoped to operator attribution. + expect(enqueueWakeup).toHaveBeenCalled(); + }); + it("schedules a provider-quota monitor for the original assignee without creating recovery work", async () => { const { companyId, coderId, sourceIssueId } = await seedCompany(); const runId = randomUUID(); diff --git a/server/src/routes/agents.ts b/server/src/routes/agents.ts index b154931e01..d3659591b2 100644 --- a/server/src/routes/agents.ts +++ b/server/src/routes/agents.ts @@ -3756,7 +3756,15 @@ export function agentRoutes( const runId = req.params.runId as string; const existing = await getAccessibleResource(req, res, heartbeat.getRun(runId), "Heartbeat run not found"); if (!existing) return; - const run = await heartbeat.cancelRun(runId); + // Stamp the cancellation as operator-initiated (this route is board-only). + // Recovery reads this to stand down instead of classifying the cancelled + // run as agent stranding and re-waking the agent the operator just stopped. + const run = await heartbeat.cancelRun(runId, "Cancelled by a board operator", { + resultJson: { + cancelledByActorType: "user", + cancelledByUserId: req.actor.userId ?? null, + }, + }); if (run) { await logActivity(db, { diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 0610362e3a..fbeb4aacb9 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -645,6 +645,21 @@ function isStrandedIssueRecoveryIssue(issue: Pick