From 7d3b1be21f6fde2d469ccb395b29028897de83e0 Mon Sep 17 00:00:00 2001 From: Harold Kim Date: Fri, 11 Sep 2026 22:44:24 +0000 Subject: [PATCH] fix: distinguish controller lease loss from an explicit stop Co-Authored-By: Paperclip --- .../automatic-sandbox-continuation.test.ts | 39 ++++++++++++++++++- server/src/services/heartbeat.ts | 14 ++++--- .../src/services/legacy-controller-lease.ts | 9 ++++- 3 files changed, 55 insertions(+), 7 deletions(-) diff --git a/server/src/services/automatic-sandbox-continuation.test.ts b/server/src/services/automatic-sandbox-continuation.test.ts index 945cd1aad8..18bcf750c3 100644 --- a/server/src/services/automatic-sandbox-continuation.test.ts +++ b/server/src/services/automatic-sandbox-continuation.test.ts @@ -1,6 +1,6 @@ import { randomUUID } from "node:crypto"; import { and, eq } from "drizzle-orm"; -import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { agents, authUsers, companies, createDb, environmentLeases, environments, heartbeatRuns, issueComments, issueRecoveryActions, issues, issueThreadInteractions } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; @@ -11,6 +11,10 @@ import { getExecutionBlocker } from "./execution-blocker.js"; import { CONVERSATION_CONTINUATION_POLICY } from "./conversation-continuation.js"; import { buildExecutionContinuation } from "./execution-continuation.js"; +import { registerServerAdapter, unregisterServerAdapter } from "../adapters/index.js"; +import { adapterExecutionControls } from "./adapter-execution-control.js"; +import { LegacyControllerLeaseLostError } from "./legacy-controller-lease.js"; + const support = await getEmbeddedPostgresTestSupport(); (support.supported ? describe : describe.skip)("automatic sandbox conversation recovery", () => { let database: Awaited>; @@ -53,6 +57,39 @@ const support = await getEmbeddedPostgresTestSupport(); async function successors(runId: string) { return db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, runId)); } + it.each([ + { leaseLost: true, throws: false }, { leaseLost: true, throws: true }, + { leaseLost: false, throws: false }, { leaseLost: false, throws: true }, + ])("distinguishes lease loss from user cancellation ($leaseLost, throws: $throws)", async ({ leaseLost, throws }) => { + const f = await seed(); + const adapterType = "lease_loss_test"; + registerServerAdapter({ type: adapterType, + execute: async ({ onCancellationReady }) => { + await onCancellationReady?.(); + adapterExecutionControls.get(f.run.id)!.controller.abort(leaseLost ? new LegacyControllerLeaseLostError() : new Error("Operator stopped run")); + if (throws) throw new Error("Adapter interrupted"); + return { exitCode: 1, signal: null, timedOut: false }; + }, + testEnvironment: async () => ({ adapterType, status: "pass", checks: [], testedAt: new Date().toISOString() }), + }); + const heartbeat = heartbeatService(db); + try { + await db.delete(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId)); + await db.update(agents).set({ adapterType }).where(eq(agents.id, f.agentId)); + await db.update(heartbeatRuns).set({ status: "queued", errorCode: null, error: null, finishedAt: null }) + .where(eq(heartbeatRuns.id, f.run.id)); + await heartbeat.resumeQueuedRuns(); + await vi.waitFor(async () => { + const [saved] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)); + expect(saved.status).toBe(leaseLost ? "failed" : "cancelled"); + if (leaseLost) expect(saved.errorCode).toBe("process_lost"); + else expect(saved.errorCode).not.toBe("process_lost"); + }, { timeout: 10000 }); + } finally { + await heartbeat.drainActiveRunExecutions(); + unregisterServerAdapter(adapterType); + } + }); it("automatically resumes a historical startup failure after exact provider termination", async () => { const f = await seed(); await heartbeatService(db).resumeInterruptedSandboxRuns(); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 5e21584f10..5150f9b79b 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -1,6 +1,6 @@ import { PROCESS_IDENTITY_RECORDED, recordNativeLocalProcessStop } from "./native-local-process-stop.js"; import { prepareAutomaticSandboxContinuation, runHasUnconfirmedRemoteExecution, SANDBOX_INFRASTRUCTURE_ERRORS } from "./automatic-sandbox-continuation.js"; -import { legacyControllerBootId, legacyControllerClaim, hasLiveLegacyController, revokeExpiredLegacyController, watchLegacyControllerLease } from "./legacy-controller-lease.js"; +import { LegacyControllerLeaseLostError, legacyControllerBootId, legacyControllerClaim, hasLiveLegacyController, revokeExpiredLegacyController, watchLegacyControllerLease } from "./legacy-controller-lease.js"; import { completeTerminatedRemoteNativeSessionCleanup } from "../vendor/paperclip-runner/index.js"; import { remoteExecutionHasStopped, remoteTerminationReceipt, stoppedRemoteCleanupScopes } from "./remote-execution-termination.js"; import { applyConnectorSkills, prepareConnectorSkillDelivery, resolveConnectorAssignments } from "./connector-runtime.js"; @@ -23642,12 +23642,13 @@ export function heartbeatService( failedProcessRunCancellations.get(run.id)) : undefined; await processCancellation?.settled; + const controllerLost = executionControl.controller.signal.reason instanceof LegacyControllerLeaseLostError; let outcome: RunSessionOutcome; const latestRun = await getRun(run.id); if (isHeartbeatRunTerminalStatus(latestRun?.status)) { outcome = latestRun.status; } else if (executionControl.controller.signal.aborted) { - outcome = "cancelled"; + outcome = controllerLost ? "failed" : "cancelled"; } else if (adapterResult.nativeFinalization) { const nativeTerminal = adapterResult.nativeFinalization.terminal.runTerminalState; @@ -23689,7 +23690,7 @@ export function heartbeatService( usageBasis: adapterResult.usageBasis ?? null, }); const normalizedUsage = sessionUsageResolution.normalizedUsage; - const runErrorMessage = + const runErrorMessage = controllerLost ? "Legacy controller lease lost" : outcome === "cancelled" ? (latestRun?.error ?? adapterResult.errorMessage ?? "Cancelled") : outcome === "succeeded" @@ -23701,7 +23702,7 @@ export function heartbeatService( ); const recordedResponsibleUserDenialCode = normalizeResponsibleUserDenialCode(latestRun?.errorCode); - const runErrorCode = + const runErrorCode = controllerLost ? "process_lost" : outcome === "timed_out" ? "timeout" : outcome === "cancelled" @@ -24394,7 +24395,9 @@ export function heartbeatService( : null; }) .catch(() => null); + const controllerLost = executionControl.controller.signal.reason instanceof LegacyControllerLeaseLostError; const failureErrorCode = + (controllerLost ? "process_lost" : null) ?? workspaceValidationFailure?.code ?? configurationIncompleteFailure?.code ?? nonRetryablePreflightFailureCode(err) ?? @@ -24429,7 +24432,7 @@ export function heartbeatService( ); }); - const stoppedDuringFailure = executionControl.controller.signal.aborted; + const stoppedDuringFailure = executionControl.controller.signal.aborted && !controllerLost; const stopSnapshot = stoppedDuringFailure ? await getRun(run.id) : null; const failureOutcome = stoppedDuringFailure ? "cancelled" : "failed"; const failedRunWrite = await setRunStatusIfRunning(run.id, failureOutcome, { @@ -24640,6 +24643,7 @@ export function heartbeatService( const nonRetryablePreflightCode = nonRetryablePreflightFailureCode(outerErr); const setupFailureErrorCode = + (executionControl.controller.signal.reason instanceof LegacyControllerLeaseLostError ? "process_lost" : null) ?? workspaceValidationSetupFailure?.code ?? configurationIncompleteSetupFailure?.code ?? (unresolvedBaseRefSetupFailure || diff --git a/server/src/services/legacy-controller-lease.ts b/server/src/services/legacy-controller-lease.ts index 4fc746fe48..454068b4fa 100644 --- a/server/src/services/legacy-controller-lease.ts +++ b/server/src/services/legacy-controller-lease.ts @@ -7,6 +7,10 @@ export const legacyControllerBootId = randomUUID(); export const LEGACY_CONTROLLER_LEASE_MS = 60_000; export const LEGACY_CONTROLLER_RENEW_MS = 10_000; +export class LegacyControllerLeaseLostError extends Error { + constructor() { super("Legacy controller lease lost"); this.name = "LegacyControllerLeaseLostError"; } +} + type Run = typeof heartbeatRuns.$inferSelect; /** Commit these fields in the same UPDATE that claims a queued run. */ @@ -70,7 +74,7 @@ export function watchLegacyControllerLease(db: Db, run: Run, controller: AbortCo } let stopped = false; let pending = false; - const lost = () => { if (!stopped) controller.abort(new Error("Legacy controller lease lost")); }; + const lost = () => { if (!stopped) controller.abort(new LegacyControllerLeaseLostError()); }; let deadline = setTimeout(lost, Math.max(0, (run.controllerLeaseExpiresAt?.getTime() ?? 0) - Date.now())); deadline.unref(); @@ -86,6 +90,9 @@ export function watchLegacyControllerLease(db: Db, run: Run, controller: AbortCo let renewed: boolean; try { renewed = await Promise.race([renewLegacyControllerLease(db, run, stage), aborted]); + } catch { + lost(); + throw controller.signal.reason; } finally { controller.signal.removeEventListener("abort", onAbort); }