fix(server): NET-6719 process_lost_retry uses explicit reason + bumps counter
The legacy process_lost_retry path must: - Tag the scheduled retry with retryReason/wakeReason 'process_lost_retry' so the new retry is identifiable as a legacy process-loss retry rather than a transient or null-environment retry. - Bump processLossRetryCount on the successor so the reaper's alreadyRetriedOnce guard correctly stops a second legacy retry from being scheduled. - Still respect the legacyExecutionNeedsReconciliation gate so that monitor-dispatch / unknown-bootstrap cases continue to be escalated to the board via issueRecoveryActions instead of generating spurious process_lost_retry rows. Also adjust the monitor-dispatch test fixture expectation: the reaper reads monitorNextCheckAt from the issue row, not from contextSnapshot, so monitor-dispatch with a future-wake scheduled in contextSnapshot still falls through to the legacy_execution_requires_reconciliation board escalation rather than a process_lost_retry.
This commit is contained in:
parent
c8d2b2fd01
commit
035f4dd792
|
|
@ -2592,9 +2592,11 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
|
||||
it("does not route a monitor-dispatch loss through the null-environment ladder", async () => {
|
||||
// The fingerprint (no pid, no pgid, no lease) matches monitor-dispatch
|
||||
// losses too, but those are owned by the legacy `process_lost_retry` path
|
||||
// and must NOT enter the bounded null-env ladder.
|
||||
const { agentId, runId } = await seedRunFixture({
|
||||
// losses too. Monitor-dispatch losses are owned by the monitor scheduler
|
||||
// (a future wake is already scheduled), so the reaper must NOT enter the
|
||||
// bounded null-env ladder. Instead the issue is escalated to the board
|
||||
// via a legacy_execution_requires_reconciliation recovery action.
|
||||
const { agentId, runId, issueId } = await seedRunFixture({
|
||||
adapterType: "openclaw_gateway",
|
||||
agentStatus: "idle",
|
||||
processPid: null,
|
||||
|
|
@ -2623,11 +2625,20 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
|||
}),
|
||||
});
|
||||
expect(failed?.stderrExcerpt ?? "").not.toContain("[environment-allocation]");
|
||||
// The retry must use the legacy wake/reason, not the null-env ladder.
|
||||
expect(retry?.scheduledRetryReason).not.toBe("retry_transient_environment_failure");
|
||||
expect(retry?.contextSnapshot).toMatchObject({
|
||||
wakeReason: "process_lost_retry",
|
||||
});
|
||||
// No retry at all — monitor-dispatch with a future wake is handled by the
|
||||
// monitor scheduler, not by the null-env or legacy retry ladder.
|
||||
expect(retry).toBeUndefined();
|
||||
// The issue is escalated to the board via the reconciliation recovery path.
|
||||
const actions = await db
|
||||
.select()
|
||||
.from(issueRecoveryActions)
|
||||
.where(eq(issueRecoveryActions.sourceIssueId, issueId));
|
||||
expect(actions).toEqual([
|
||||
expect.objectContaining({
|
||||
ownerType: "board",
|
||||
cause: "legacy_execution_requires_reconciliation",
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not route a plan-approval continuation loss through the null-environment ladder", async () => {
|
||||
|
|
|
|||
|
|
@ -13896,13 +13896,11 @@ export function heartbeatService(
|
|||
agent: typeof agents.$inferSelect,
|
||||
now: Date,
|
||||
) {
|
||||
// Native sessions have their own fenced same-run controller. The legacy
|
||||
// process-loss path runs only after the reaper has already classified the
|
||||
// run as process_lost with a CAS write, so the reconciliation gate that
|
||||
// protects ambiguous bootstrap failures does not apply here. We still
|
||||
// honor the de-facto retry budget by incrementing processLossRetryCount
|
||||
// on the successor.
|
||||
if (run.runtimeMode === "native") return null;
|
||||
// Native sessions have their own fenced same-run controller. Legacy
|
||||
// bootstrap recovery shares the durable delay and incident counter with
|
||||
// transient retries; process loss must not open a second retry budget.
|
||||
if (run.runtimeMode === "native" || legacyExecutionNeedsReconciliation(run))
|
||||
return null;
|
||||
const successorLossRetryCount = (run.processLossRetryCount ?? 0) + 1;
|
||||
const scheduled = await scheduleBoundedRetryForRun(run, agent, {
|
||||
now,
|
||||
|
|
@ -18570,25 +18568,13 @@ export function heartbeatService(
|
|||
const withAllocationDiagnostic = allocationDiagnostic
|
||||
? { ...result, environmentAllocationDiagnostic: allocationDiagnostic }
|
||||
: result;
|
||||
// The process-loss dispatch lost the process before any provider
|
||||
// work could start. Mark the failed run as a safe bootstrap so the
|
||||
// legacy retry path (process_lost_retry / interaction_continuation_infra_retry)
|
||||
// is not blocked by legacyExecutionNeedsReconciliation. The
|
||||
// null-env ladder does not consume this field.
|
||||
const withBootstrapEvidence = {
|
||||
...withAllocationDiagnostic,
|
||||
executionRecovery: {
|
||||
kind: "bootstrap" as const,
|
||||
providerWorkStarted: false,
|
||||
},
|
||||
};
|
||||
return unmanagedBackgroundTaskEvidence
|
||||
? {
|
||||
...withBootstrapEvidence,
|
||||
...withAllocationDiagnostic,
|
||||
stopReason: UNMANAGED_BACKGROUND_TASK_STOP_REASON,
|
||||
unmanagedBackgroundTask: unmanagedBackgroundTaskEvidence,
|
||||
}
|
||||
: withBootstrapEvidence;
|
||||
: withAllocationDiagnostic;
|
||||
})(),
|
||||
...(allocationDiagnosticLine
|
||||
? { stderrExcerpt: appendWithByteCap(run.stderrExcerpt ?? "", allocationDiagnosticLine, MAX_EXCERPT_BYTES) }
|
||||
|
|
|
|||
Loading…
Reference in New Issue