fix(server): NET-6719 process_lost_retry uses explicit reason + bumps counter
The legacy process_lost_retry path must: - Tag the scheduled retry with retryReason/wakeReason 'process_lost_retry' so the new retry is identifiable as a legacy process-loss retry rather than a transient or null-environment retry. - Bump processLossRetryCount on the successor so the reaper's alreadyRetriedOnce guard correctly stops a second legacy retry from being scheduled. - Still respect the legacyExecutionNeedsReconciliation gate so that monitor-dispatch / unknown-bootstrap cases continue to be escalated to the board via issueRecoveryActions instead of generating spurious process_lost_retry rows. Also adjust the monitor-dispatch test fixture expectation: the reaper reads monitorNextCheckAt from the issue row, not from contextSnapshot, so monitor-dispatch with a future-wake scheduled in contextSnapshot still falls through to the legacy_execution_requires_reconciliation board escalation rather than a process_lost_retry.
This commit is contained in:
parent
c8d2b2fd01
commit
035f4dd792
|
|
@ -2592,9 +2592,11 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
||||||
|
|
||||||
it("does not route a monitor-dispatch loss through the null-environment ladder", async () => {
|
it("does not route a monitor-dispatch loss through the null-environment ladder", async () => {
|
||||||
// The fingerprint (no pid, no pgid, no lease) matches monitor-dispatch
|
// The fingerprint (no pid, no pgid, no lease) matches monitor-dispatch
|
||||||
// losses too, but those are owned by the legacy `process_lost_retry` path
|
// losses too. Monitor-dispatch losses are owned by the monitor scheduler
|
||||||
// and must NOT enter the bounded null-env ladder.
|
// (a future wake is already scheduled), so the reaper must NOT enter the
|
||||||
const { agentId, runId } = await seedRunFixture({
|
// bounded null-env ladder. Instead the issue is escalated to the board
|
||||||
|
// via a legacy_execution_requires_reconciliation recovery action.
|
||||||
|
const { agentId, runId, issueId } = await seedRunFixture({
|
||||||
adapterType: "openclaw_gateway",
|
adapterType: "openclaw_gateway",
|
||||||
agentStatus: "idle",
|
agentStatus: "idle",
|
||||||
processPid: null,
|
processPid: null,
|
||||||
|
|
@ -2623,11 +2625,20 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
expect(failed?.stderrExcerpt ?? "").not.toContain("[environment-allocation]");
|
expect(failed?.stderrExcerpt ?? "").not.toContain("[environment-allocation]");
|
||||||
// The retry must use the legacy wake/reason, not the null-env ladder.
|
// No retry at all — monitor-dispatch with a future wake is handled by the
|
||||||
expect(retry?.scheduledRetryReason).not.toBe("retry_transient_environment_failure");
|
// monitor scheduler, not by the null-env or legacy retry ladder.
|
||||||
expect(retry?.contextSnapshot).toMatchObject({
|
expect(retry).toBeUndefined();
|
||||||
wakeReason: "process_lost_retry",
|
// The issue is escalated to the board via the reconciliation recovery path.
|
||||||
});
|
const actions = await db
|
||||||
|
.select()
|
||||||
|
.from(issueRecoveryActions)
|
||||||
|
.where(eq(issueRecoveryActions.sourceIssueId, issueId));
|
||||||
|
expect(actions).toEqual([
|
||||||
|
expect.objectContaining({
|
||||||
|
ownerType: "board",
|
||||||
|
cause: "legacy_execution_requires_reconciliation",
|
||||||
|
}),
|
||||||
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("does not route a plan-approval continuation loss through the null-environment ladder", async () => {
|
it("does not route a plan-approval continuation loss through the null-environment ladder", async () => {
|
||||||
|
|
|
||||||
|
|
@ -13896,13 +13896,11 @@ export function heartbeatService(
|
||||||
agent: typeof agents.$inferSelect,
|
agent: typeof agents.$inferSelect,
|
||||||
now: Date,
|
now: Date,
|
||||||
) {
|
) {
|
||||||
// Native sessions have their own fenced same-run controller. The legacy
|
// Native sessions have their own fenced same-run controller. Legacy
|
||||||
// process-loss path runs only after the reaper has already classified the
|
// bootstrap recovery shares the durable delay and incident counter with
|
||||||
// run as process_lost with a CAS write, so the reconciliation gate that
|
// transient retries; process loss must not open a second retry budget.
|
||||||
// protects ambiguous bootstrap failures does not apply here. We still
|
if (run.runtimeMode === "native" || legacyExecutionNeedsReconciliation(run))
|
||||||
// honor the de-facto retry budget by incrementing processLossRetryCount
|
return null;
|
||||||
// on the successor.
|
|
||||||
if (run.runtimeMode === "native") return null;
|
|
||||||
const successorLossRetryCount = (run.processLossRetryCount ?? 0) + 1;
|
const successorLossRetryCount = (run.processLossRetryCount ?? 0) + 1;
|
||||||
const scheduled = await scheduleBoundedRetryForRun(run, agent, {
|
const scheduled = await scheduleBoundedRetryForRun(run, agent, {
|
||||||
now,
|
now,
|
||||||
|
|
@ -18570,25 +18568,13 @@ export function heartbeatService(
|
||||||
const withAllocationDiagnostic = allocationDiagnostic
|
const withAllocationDiagnostic = allocationDiagnostic
|
||||||
? { ...result, environmentAllocationDiagnostic: allocationDiagnostic }
|
? { ...result, environmentAllocationDiagnostic: allocationDiagnostic }
|
||||||
: result;
|
: result;
|
||||||
// The process-loss dispatch lost the process before any provider
|
|
||||||
// work could start. Mark the failed run as a safe bootstrap so the
|
|
||||||
// legacy retry path (process_lost_retry / interaction_continuation_infra_retry)
|
|
||||||
// is not blocked by legacyExecutionNeedsReconciliation. The
|
|
||||||
// null-env ladder does not consume this field.
|
|
||||||
const withBootstrapEvidence = {
|
|
||||||
...withAllocationDiagnostic,
|
|
||||||
executionRecovery: {
|
|
||||||
kind: "bootstrap" as const,
|
|
||||||
providerWorkStarted: false,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
return unmanagedBackgroundTaskEvidence
|
return unmanagedBackgroundTaskEvidence
|
||||||
? {
|
? {
|
||||||
...withBootstrapEvidence,
|
...withAllocationDiagnostic,
|
||||||
stopReason: UNMANAGED_BACKGROUND_TASK_STOP_REASON,
|
stopReason: UNMANAGED_BACKGROUND_TASK_STOP_REASON,
|
||||||
unmanagedBackgroundTask: unmanagedBackgroundTaskEvidence,
|
unmanagedBackgroundTask: unmanagedBackgroundTaskEvidence,
|
||||||
}
|
}
|
||||||
: withBootstrapEvidence;
|
: withAllocationDiagnostic;
|
||||||
})(),
|
})(),
|
||||||
...(allocationDiagnosticLine
|
...(allocationDiagnosticLine
|
||||||
? { stderrExcerpt: appendWithByteCap(run.stderrExcerpt ?? "", allocationDiagnosticLine, MAX_EXCERPT_BYTES) }
|
? { stderrExcerpt: appendWithByteCap(run.stderrExcerpt ?? "", allocationDiagnosticLine, MAX_EXCERPT_BYTES) }
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue