From 5acb4274cb4cfa5a70dce48cd5e706cfebba1527 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 10 Sep 2026 02:50:57 -0500 Subject: [PATCH] Honor operator cancellation before immediate task recovery Reuse the periodic recovery policy so an operator-cancelled run does not immediately dispatch a replacement and resume its sandbox. Keep explicit deferred wake promotion intact. Co-Authored-By: Paperclip --- doc/sandbox-work-folders.md | 3 +++ .../heartbeat-process-recovery.test.ts | 19 +++++++++++++++++++ server/src/services/heartbeat.ts | 6 ++++++ server/src/services/recovery/service.ts | 2 +- 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/doc/sandbox-work-folders.md b/doc/sandbox-work-folders.md index fc4a57e3f0..3ff8e8cb97 100644 --- a/doc/sandbox-work-folders.md +++ b/doc/sandbox-work-folders.md @@ -431,6 +431,9 @@ before acknowledgement, preventing cancelled startup work from dispatching. The supervisor preserves externally delivered child termination signals. Failed stop requests remain visible and can be retried explicitly. Local and native runner cancellation retain their existing authorities. +Immediate recovery honors the same operator-cancellation attribution as periodic +recovery, so cancelling a run does not synthesize a continuation that restarts its +sandbox. Explicitly queued work can still run through normal promotion. Automated tests do not qualify a deployed runner image. Before merging, use a new pinned staging stack with the branch's Cloud image and matching migrator. diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index b207227a63..c75625d096 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -6009,6 +6009,25 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { expect(runningProcesses.has(runId)).toBe(false); }); + it.each(["user", "board", "interrupt"])("does not immediately recover an operator-cancelled run (%s)", async (actor) => { + const { companyId, agentId, runId, issueId, wakeupRequestId } = await seedRunFixture({ + agentStatus: "running", includeIssue: true, + }); + const heartbeat = heartbeatService(db); + try { + const cancelled = await heartbeat.cancelRun(runId, "Operator stopped this run", actor === "interrupt" + ? { errorCode: "operator_interrupted" } + : { resultJson: { cancelledByActorType: actor } }); + expect(cancelled?.status).toBe("cancelled"); + const wakeups = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId)); + expect(wakeups.map((wake) => wake.id)).toEqual([wakeupRequestId]); + const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.agentId, agentId)); + expect(runs.map((run) => run.id)).toEqual([runId]); + const [issue] = await db.select().from(issues).where(eq(issues.id, issueId)); + expect(issue).toMatchObject({ status: "in_progress", executionRunId: null, checkoutRunId: null }); + } finally { await heartbeat.drainActiveRunExecutions(); } + }); + it("records manual cancellation stop metadata", async () => { const { runId } = await seedRunFixture({ agentStatus: "running", diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index f89d5d4223..a83ecc2eb4 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -371,6 +371,7 @@ import { import { withRecoveryContext } from "./recovery/status-only-context.js"; import { ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS as RECOVERY_ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS, + isOperatorCancelledRun, recoveryService, } from "./recovery/service.js"; import { collectDispositionRepairSourceState } from "./recovery/disposition-repair.js"; @@ -23689,6 +23690,11 @@ export function heartbeatService( }; } + // Match the periodic recovery policy: an operator's stop must not create + // a fresh continuation that immediately resumes the cancelled sandbox. + // Explicit deferred wakes above remain eligible for normal promotion. + if (isOperatorCancelledRun(run)) return { kind: "released" as const }; + const findExistingExecutionPath = (agentId?: string | null) => tx .select({ id: heartbeatRuns.id }) diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index f5f31334c3..12357e55a4 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -608,7 +608,7 @@ function isStrandedIssueRecoveryIssue(issue: Pick