fix: distinguish controller lease loss from an explicit stop

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/services/automatic-sandbox-continuation.test.ts

@@ -1,6 +1,6 @@
 import { randomUUID } from "node:crypto";
 import { and, eq } from "drizzle-orm";
-import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
 import { agents, authUsers, companies, createDb, environmentLeases, environments, heartbeatRuns,
   issueComments, issueRecoveryActions, issues, issueThreadInteractions } from "@paperclipai/db";
 import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js";
@@ -11,6 +11,10 @@ import { getExecutionBlocker } from "./execution-blocker.js";
 import { CONVERSATION_CONTINUATION_POLICY } from "./conversation-continuation.js";
 import { buildExecutionContinuation } from "./execution-continuation.js";
 
+import { registerServerAdapter, unregisterServerAdapter } from "../adapters/index.js";
+import { adapterExecutionControls } from "./adapter-execution-control.js";
+import { LegacyControllerLeaseLostError } from "./legacy-controller-lease.js";
+
 const support = await getEmbeddedPostgresTestSupport();
 (support.supported ? describe : describe.skip)("automatic sandbox conversation recovery", () => {
   let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
@@ -53,6 +57,39 @@ const support = await getEmbeddedPostgresTestSupport();
   async function successors(runId: string) {
     return db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, runId));
   }
+  it.each([
+    { leaseLost: true, throws: false }, { leaseLost: true, throws: true },
+    { leaseLost: false, throws: false }, { leaseLost: false, throws: true },
+  ])("distinguishes lease loss from user cancellation ($leaseLost, throws: $throws)", async ({ leaseLost, throws }) => {
+    const f = await seed();
+    const adapterType = "lease_loss_test";
+    registerServerAdapter({ type: adapterType,
+      execute: async ({ onCancellationReady }) => {
+        await onCancellationReady?.();
+        adapterExecutionControls.get(f.run.id)!.controller.abort(leaseLost ? new LegacyControllerLeaseLostError() : new Error("Operator stopped run"));
+        if (throws) throw new Error("Adapter interrupted");
+        return { exitCode: 1, signal: null, timedOut: false };
+      },
+      testEnvironment: async () => ({ adapterType, status: "pass", checks: [], testedAt: new Date().toISOString() }),
+    });
+    const heartbeat = heartbeatService(db);
+    try {
+      await db.delete(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId));
+      await db.update(agents).set({ adapterType }).where(eq(agents.id, f.agentId));
+      await db.update(heartbeatRuns).set({ status: "queued", errorCode: null, error: null, finishedAt: null })
+        .where(eq(heartbeatRuns.id, f.run.id));
+      await heartbeat.resumeQueuedRuns();
+      await vi.waitFor(async () => {
+        const [saved] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id));
+        expect(saved.status).toBe(leaseLost ? "failed" : "cancelled");
+        if (leaseLost) expect(saved.errorCode).toBe("process_lost");
+        else expect(saved.errorCode).not.toBe("process_lost");
+      }, { timeout: 10000 });
+    } finally {
+      await heartbeat.drainActiveRunExecutions();
+      unregisterServerAdapter(adapterType);
+    }
+  });
   it("automatically resumes a historical startup failure after exact provider termination", async () => {
     const f = await seed();
     await heartbeatService(db).resumeInterruptedSandboxRuns();

server/src/services/heartbeat.ts

@@ -1,6 +1,6 @@
 import { PROCESS_IDENTITY_RECORDED, recordNativeLocalProcessStop } from "./native-local-process-stop.js";
 import { prepareAutomaticSandboxContinuation, runHasUnconfirmedRemoteExecution, SANDBOX_INFRASTRUCTURE_ERRORS } from "./automatic-sandbox-continuation.js";
-import { legacyControllerBootId, legacyControllerClaim, hasLiveLegacyController, revokeExpiredLegacyController, watchLegacyControllerLease } from "./legacy-controller-lease.js";
+import { LegacyControllerLeaseLostError, legacyControllerBootId, legacyControllerClaim, hasLiveLegacyController, revokeExpiredLegacyController, watchLegacyControllerLease } from "./legacy-controller-lease.js";
 import { completeTerminatedRemoteNativeSessionCleanup } from "../vendor/paperclip-runner/index.js";
 import { remoteExecutionHasStopped, remoteTerminationReceipt, stoppedRemoteCleanupScopes } from "./remote-execution-termination.js";
 import { applyConnectorSkills, prepareConnectorSkillDelivery, resolveConnectorAssignments } from "./connector-runtime.js";
@@ -23642,12 +23642,13 @@ export function heartbeatService(
               failedProcessRunCancellations.get(run.id))
             : undefined;
         await processCancellation?.settled;
+        const controllerLost = executionControl.controller.signal.reason instanceof LegacyControllerLeaseLostError;
         let outcome: RunSessionOutcome;
         const latestRun = await getRun(run.id);
         if (isHeartbeatRunTerminalStatus(latestRun?.status)) {
           outcome = latestRun.status;
         } else if (executionControl.controller.signal.aborted) {
-          outcome = "cancelled";
+          outcome = controllerLost ? "failed" : "cancelled";
         } else if (adapterResult.nativeFinalization) {
           const nativeTerminal =
             adapterResult.nativeFinalization.terminal.runTerminalState;
@@ -23689,7 +23690,7 @@ export function heartbeatService(
           usageBasis: adapterResult.usageBasis ?? null,
         });
         const normalizedUsage = sessionUsageResolution.normalizedUsage;
-        const runErrorMessage =
+        const runErrorMessage = controllerLost ? "Legacy controller lease lost" :
           outcome === "cancelled"
             ? (latestRun?.error ?? adapterResult.errorMessage ?? "Cancelled")
             : outcome === "succeeded"
@@ -23701,7 +23702,7 @@ export function heartbeatService(
                 );
         const recordedResponsibleUserDenialCode =
           normalizeResponsibleUserDenialCode(latestRun?.errorCode);
-        const runErrorCode =
+        const runErrorCode = controllerLost ? "process_lost" :
           outcome === "timed_out"
             ? "timeout"
             : outcome === "cancelled"
@@ -24394,7 +24395,9 @@ export function heartbeatService(
               : null;
           })
           .catch(() => null);
+        const controllerLost = executionControl.controller.signal.reason instanceof LegacyControllerLeaseLostError;
         const failureErrorCode =
+          (controllerLost ? "process_lost" : null) ??
           workspaceValidationFailure?.code ??
           configurationIncompleteFailure?.code ??
           nonRetryablePreflightFailureCode(err) ??
@@ -24429,7 +24432,7 @@ export function heartbeatService(
           );
         });
 
-        const stoppedDuringFailure = executionControl.controller.signal.aborted;
+        const stoppedDuringFailure = executionControl.controller.signal.aborted && !controllerLost;
         const stopSnapshot = stoppedDuringFailure ? await getRun(run.id) : null;
         const failureOutcome = stoppedDuringFailure ? "cancelled" : "failed";
         const failedRunWrite = await setRunStatusIfRunning(run.id, failureOutcome, {
@@ -24640,6 +24643,7 @@ export function heartbeatService(
         const nonRetryablePreflightCode =
           nonRetryablePreflightFailureCode(outerErr);
         const setupFailureErrorCode =
+          (executionControl.controller.signal.reason instanceof LegacyControllerLeaseLostError ? "process_lost" : null) ??
           workspaceValidationSetupFailure?.code ??
           configurationIncompleteSetupFailure?.code ??
           (unresolvedBaseRefSetupFailure ||

server/src/services/legacy-controller-lease.ts

@@ -7,6 +7,10 @@ export const legacyControllerBootId = randomUUID();
 export const LEGACY_CONTROLLER_LEASE_MS = 60_000;
 export const LEGACY_CONTROLLER_RENEW_MS = 10_000;
 
+export class LegacyControllerLeaseLostError extends Error {
+  constructor() { super("Legacy controller lease lost"); this.name = "LegacyControllerLeaseLostError"; }
+}
+
 type Run = typeof heartbeatRuns.$inferSelect;
 
 /** Commit these fields in the same UPDATE that claims a queued run. */
@@ -70,7 +74,7 @@ export function watchLegacyControllerLease(db: Db, run: Run, controller: AbortCo
   }
   let stopped = false;
   let pending = false;
-  const lost = () => { if (!stopped) controller.abort(new Error("Legacy controller lease lost")); };
+  const lost = () => { if (!stopped) controller.abort(new LegacyControllerLeaseLostError()); };
   let deadline = setTimeout(lost, Math.max(0,
     (run.controllerLeaseExpiresAt?.getTime() ?? 0) - Date.now()));
   deadline.unref();
@@ -86,6 +90,9 @@ export function watchLegacyControllerLease(db: Db, run: Run, controller: AbortCo
     let renewed: boolean;
     try {
       renewed = await Promise.race([renewLegacyControllerLease(db, run, stage), aborted]);
+    } catch {
+      lost();
+      throw controller.signal.reason;
     } finally {
       controller.signal.removeEventListener("abort", onAbort);
     }