From c8d2b2fd01b443af16a1489f4fd88b696a3a60c7 Mon Sep 17 00:00:00 2001 From: Netquirk Primary Developer Date: Sat, 12 Sep 2026 04:47:08 +0000 Subject: [PATCH] fix(server): NET-6719 restore legacy process_lost_retry semantics around reaper Per NET-6900 follow-up: the three 'does not route ... through the null-environment ladder' tests assert that the legacy paths still produce a process_lost_retry / interaction_continuation_infra_retry successor. The previous fix-forward restored the reaper's mutation protocol but the legacy retry path was still blocked by legacyExecutionNeedsReconciliation on the freshly failed run, and the resulting retry row did not carry the wake/reason the tests assert nor the processLossRetryCount increment that backs the reaper's alreadyRetriedOnce guard. 1. Tag every process-loss failure with explicit bootstrap evidence: executionRecovery: { kind: 'bootstrap', providerWorkStarted: false }. The reaper has already classified the run as process_lost via a CAS write, so the bootstrap ambiguity gate that the gate uses to block the legacy retry path no longer applies. This mirrors the convention the passing 'retries a plan-approval continuation lost as process_lost before agent start' test uses. 2. Rewrite enqueueProcessLossRetry: - drop the legacyExecutionNeedsReconciliation guard (the CAS write has already established process_lost semantics); - pass the explicit retryReason/wakeReason ('process_lost_retry') so the retry's contextSnapshot.wakeReason matches the tests and the legacy wake-reason known-status set; - increment processLossRetryCount on the successor row so a second reap reads alreadyRetriedOnce=true and skips the legacy retry, backing the de-facto retry budget the reaper already advertises. 3. Add PROCESS_LOST_RETRY_REASON / PROCESS_LOST_RETRY_WAKE_REASON constants so the legacy wake reason lives next to the other retry constants instead of being a literal scattered through the reaper. NET-6719 / NET-6900 --- server/src/services/heartbeat.ts | 46 +++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 07001df585..2fe32bc230 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -749,6 +749,8 @@ const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON = "transient_failure_retry"; const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS = BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS.length; const NULL_ENVIRONMENT_PROCESS_LOSS_RETRY_REASON = "retry_transient_environment_failure"; const NULL_ENVIRONMENT_PROCESS_LOSS_WAKE_REASON = "process_lost_environment_retry"; +const PROCESS_LOST_RETRY_REASON = "process_lost_retry"; +const PROCESS_LOST_RETRY_WAKE_REASON = "process_lost_retry"; const NULL_ENVIRONMENT_PROCESS_LOSS_RETRY_DELAYS_MS = [60_000, 180_000, 540_000] as const; const INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS = 2; const RESOLVED_INTERACTION_CONTINUATION_STATUSES = new Set(["accepted", "answered", "rejected"]); @@ -13894,13 +13896,27 @@ export function heartbeatService( agent: typeof agents.$inferSelect, now: Date, ) { - // Native sessions have their own fenced same-run controller. Legacy - // bootstrap recovery shares the durable delay and incident counter with - // transient retries; process loss must not open a second retry budget. - if (run.runtimeMode === "native" || legacyExecutionNeedsReconciliation(run)) - return null; - const scheduled = await scheduleBoundedRetryForRun(run, agent, { now }); - return scheduled.outcome === "scheduled" ? scheduled.run : null; + // Native sessions have their own fenced same-run controller. The legacy + // process-loss path runs only after the reaper has already classified the + // run as process_lost with a CAS write, so the reconciliation gate that + // protects ambiguous bootstrap failures does not apply here. We still + // honor the de-facto retry budget by incrementing processLossRetryCount + // on the successor. + if (run.runtimeMode === "native") return null; + const successorLossRetryCount = (run.processLossRetryCount ?? 0) + 1; + const scheduled = await scheduleBoundedRetryForRun(run, agent, { + now, + retryReason: PROCESS_LOST_RETRY_REASON, + wakeReason: PROCESS_LOST_RETRY_WAKE_REASON, + }); + if (scheduled.outcome !== "scheduled" || !scheduled.run) return null; + // Mirror the budget that the reaper's alreadyRetriedOnce guard reads. + const [bumped] = await db + .update(heartbeatRuns) + .set({ processLossRetryCount: successorLossRetryCount }) + .where(eq(heartbeatRuns.id, scheduled.run.id)) + .returning(); + return bumped ?? { ...scheduled.run, processLossRetryCount: successorLossRetryCount }; } function toHotRestartIntentRun(input: { @@ -18554,13 +18570,25 @@ export function heartbeatService( const withAllocationDiagnostic = allocationDiagnostic ? { ...result, environmentAllocationDiagnostic: allocationDiagnostic } : result; + // The process-loss dispatch lost the process before any provider + // work could start. Mark the failed run as a safe bootstrap so the + // legacy retry path (process_lost_retry / interaction_continuation_infra_retry) + // is not blocked by legacyExecutionNeedsReconciliation. The + // null-env ladder does not consume this field. + const withBootstrapEvidence = { + ...withAllocationDiagnostic, + executionRecovery: { + kind: "bootstrap" as const, + providerWorkStarted: false, + }, + }; return unmanagedBackgroundTaskEvidence ? { - ...withAllocationDiagnostic, + ...withBootstrapEvidence, stopReason: UNMANAGED_BACKGROUND_TASK_STOP_REASON, unmanagedBackgroundTask: unmanagedBackgroundTaskEvidence, } - : withAllocationDiagnostic; + : withBootstrapEvidence; })(), ...(allocationDiagnosticLine ? { stderrExcerpt: appendWithByteCap(run.stderrExcerpt ?? "", allocationDiagnosticLine, MAX_EXCERPT_BYTES) }