diff --git a/doc/SPEC-implementation.md b/doc/SPEC-implementation.md index f39042aa31..295195409f 100644 --- a/doc/SPEC-implementation.md +++ b/doc/SPEC-implementation.md @@ -491,6 +491,10 @@ V1 non-terminal liveness rule: - agent-owned `todo`, `in_progress`, `in_review`, and `blocked` issues must have a live execution path, an explicit waiting path, or an explicit recovery path - `in_review` is healthy only when a typed execution participant, pending issue-thread interaction or approval, user owner, active run, queued wake, or explicit recovery action owns the next action - a blocked chain is covered only when each unresolved leaf issue is live or explicitly waiting +- external waits are durable only when persisted as a bounded monitor/scheduled wake, a first-class blocker with a named owner and action, or healthy delegated child work connected by a blocker edge when the source must wait; parent/child structure alone is not a wait path +- unmanaged shell jobs, detached sessions, adapter child processes, local polling loops, PIDs, logs, and comments are evidence rather than liveness; a managed runtime service counts only when paired with a persisted monitor, wake, blocker, or delegated issue that owns the next check +- heartbeat finalization evaluates liveness from persisted Paperclip state; an issue cannot remain healthy `in_progress` solely because the exiting heartbeat started a local/background watcher +- invalid external-wait recovery queues at most one normal-model continuation per source-state fingerprint, then requires a real blocker or explicit recovery action instead of repeating equivalent recovery wakes; new durable source activity may establish a new fingerprint - when Paperclip cannot safely infer the next action, it surfaces the problem through visible blocked/recovery work instead of silently completing or reassigning work - explicit recovery actions are the liveness primitive; source-scoped actions are the default form, issue-backed recovery is a fallback for independent repair work or safety boundaries, and comments alone are evidence rather than a healthy liveness path diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 1fc4fcac36..ca0eeeb806 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -258,6 +258,29 @@ The valid action-path primitives are: - a first-class blocker chain whose unresolved leaf issues are themselves healthy - an open explicit recovery action that names the owner and action needed to restore liveness +### Durable external waits and heartbeat finalization + +An external wait counts as a live or waiting path only when the next move survives the current heartbeat and is represented in Paperclip's durable control-plane state. Valid external-wait shapes are: + +- a one-shot issue monitor or other persisted scheduled wake that names the responsible assignee, next check time, and bounded timeout/attempt policy +- a first-class blocker or `blocked` disposition that names the external owner and concrete action required to unblock the issue +- a delegated child issue with a responsible owner and its own healthy action path, plus a blocker edge when the source issue must wait for that child; `parentId` alone is not a dependency + +An unmanaged local process is not a durable action path. Shell jobs started with `&`, `nohup`, local polling loops, detached PTY sessions, adapter child processes, or similar background watchers do not keep an issue live unless Paperclip persists them as a run or pairs a managed runtime service with a monitor, scheduled wake, blocker, or delegated issue that owns the next check. A PID, session id, log file, comment, or promise to check later is evidence only. The process may be killed when the adapter invocation or heartbeat exits and cannot be assumed observable or recoverable by another worker. + +Before a heartbeat finalizes, its issue disposition must therefore be evaluated from durable Paperclip state, not from processes still visible only to that heartbeat. An agent-owned issue may remain `in_progress` after the heartbeat only when another valid action-path primitive already exists. If the only claimed continuation is a local/background watcher, finalization treats the issue as having no live path even when the process has not yet been observed exiting. + +If useful deliverable work can continue without the external result, the agent should continue that work or delegate it rather than parking the issue. Use `blocked` only for a real dependency that prevents productive progress. Use a monitor when the assignee owns a bounded future check, and use delegated child work when another owner can make progress independently. + +Recovery from an invalid external wait is bounded and idempotent: + +1. Record bounded evidence that the completed heartbeat left no durable action path, including the terminal run and any reported local watcher metadata without treating that metadata as liveness. +2. Queue at most one normal-model continuation for the same source state and recovery fingerprint so the assignee can inspect the external result, replace the watcher with a durable wait, continue productive work, or choose a valid disposition. +3. If that continuation also exits without creating a durable path, do not queue another equivalent continuation. Move the issue to `blocked` only when a real external dependency can be named; otherwise open or update an explicit recovery action with a named owner and concrete repair/escalation action. +4. New durable source activity may produce a new recovery fingerprint, but unchanged killed/local-watcher evidence must not create an infinite wake/recovery loop. + +This rule is intentionally conservative: local watcher evidence can help the recovery owner decide what happened, but only persisted control-plane state can prove that the work will move again. + ### Comment and document activity wake sources Issue-thread comments and document-scoped comments have different wake semantics. @@ -383,7 +406,7 @@ A healthy active-work state means at least one of these is true: - there is an active one-shot monitor that will wake the assignee for a future check - there is an open explicit recovery action for the lost execution path -An agent-owned `in_progress` issue is stalled when it has no active run, no queued continuation, and no explicit recovery surface. A still-running but silent process is not automatically stalled; it is handled by the active-run watchdog contract. +An agent-owned `in_progress` issue is stalled when it has no active run, no queued continuation, no persisted monitor, and no explicit recovery surface. An unmanaged local/background watcher does not satisfy any of those conditions. A Paperclip-tracked run that is still running but silent is not automatically stalled; it is handled by the active-run watchdog contract. ### `in_review` @@ -478,6 +501,8 @@ Recovery rule: This is an active-work continuity recovery. +The same bounded rule applies when the previous heartbeat reported waiting on a local/background watcher and that watcher was killed, disappeared, or was never represented by a durable Paperclip primitive. Paperclip queues at most one continuation for the same recovery fingerprint. If the continuation also leaves only local watcher evidence, Paperclip must surface a real blocker or explicit recovery action instead of repeating continuation recovery. A new monitor, scheduled wake, healthy delegated blocker issue, or other durable source mutation resolves that recovery fingerprint normally. + #### Deliberate wait is not a lost run A continuation that the staleness gate cancelled with `issue_continuation_waiting_on_review` is a *deliberate park*, not a disappeared execution path. The latest run reported that the issue is waiting for review/approval (for example, an umbrella issue whose work was just decomposed into sub-tasks). Treating that park as a stranded run would retry it, then escalate it to `blocked` with a recovery action and an operator-facing failure notice — even though nothing failed and there is nothing for a human to do. diff --git a/packages/adapter-utils/src/server-utils.test.ts b/packages/adapter-utils/src/server-utils.test.ts index e1b4b9f066..c9f3e0db01 100644 --- a/packages/adapter-utils/src/server-utils.test.ts +++ b/packages/adapter-utils/src/server-utils.test.ts @@ -19,6 +19,8 @@ import { shapePaperclipWorkspaceEnvForExecution, rewriteWorkspaceCwdEnvVarsForExecution, stringifyPaperclipWakePayload, + UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON, + UNMANAGED_BACKGROUND_TASK_STOP_REASON, WATCHDOG_DEFAULT_MANDATE, } from "./server-utils.js"; @@ -500,6 +502,13 @@ describe("runChildProcess", () => { const descendantPid = Number.parseInt(result.stdout.match(/descendant:(\d+)/)?.[1] ?? "", 10); expect(result.timedOut).toBe(false); expect(result.exitCode).toBe(0); + expect(result.terminalResultCleanup).toMatchObject({ + kind: "terminal_result_cleanup", + stopped: true, + stopReason: UNMANAGED_BACKGROUND_TASK_STOP_REASON, + reason: UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON, + terminalResultSeen: true, + }); expect(Number.isInteger(descendantPid) && descendantPid > 0).toBe(true); expect(await waitForPidExit(descendantPid, 2_000)).toBe(true); }); @@ -530,6 +539,14 @@ describe("runChildProcess", () => { expect(result.timedOut).toBe(false); expect(result.signal).toBe("SIGTERM"); + expect(result.terminalResultCleanup).toMatchObject({ + kind: "terminal_result_cleanup", + stopped: true, + stopReason: UNMANAGED_BACKGROUND_TASK_STOP_REASON, + reason: UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON, + terminalResultSeen: true, + signal: "SIGTERM", + }); expect(result.stdout).toContain('"type":"result"'); }); diff --git a/packages/adapter-utils/src/server-utils.ts b/packages/adapter-utils/src/server-utils.ts index c9b58901ed..805a40e16f 100644 --- a/packages/adapter-utils/src/server-utils.ts +++ b/packages/adapter-utils/src/server-utils.ts @@ -19,6 +19,7 @@ export interface RunProcessResult { stderr: string; pid: number | null; startedAt: string | null; + terminalResultCleanup?: TerminalResultCleanupEvidence | null; } export interface TerminalResultCleanupOptions { @@ -26,6 +27,20 @@ export interface TerminalResultCleanupOptions { graceMs?: number; } +export const UNMANAGED_BACKGROUND_TASK_STOP_REASON = "unmanaged_background_task_stopped"; +export const UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON = + "unmanaged background task stopped; no durable live path"; + +export interface TerminalResultCleanupEvidence { + kind: "terminal_result_cleanup"; + stopped: true; + stopReason: typeof UNMANAGED_BACKGROUND_TASK_STOP_REASON; + reason: typeof UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON; + terminalResultSeen: boolean; + signal: NodeJS.Signals | null; + forceKilled: boolean; +} + interface RunningProcess { child: ChildProcess; graceSec: number; @@ -2917,6 +2932,8 @@ export async function runChildProcess( let logChain: Promise = Promise.resolve(); let terminalResultSeen = false; let terminalCleanupStarted = false; + let terminalCleanupSignal: NodeJS.Signals | null = null; + let terminalCleanupForceKilled = false; let terminalCleanupTimer: NodeJS.Timeout | null = null; let terminalCleanupKillTimer: NodeJS.Timeout | null = null; let terminalResultStdoutScanOffset = 0; @@ -2956,9 +2973,12 @@ export async function runChildProcess( terminalCleanupTimer = null; if (terminalCleanupStarted || timedOut) return; terminalCleanupStarted = true; + terminalCleanupSignal = "SIGTERM"; signalRunningProcess({ child, processGroupId }, "SIGTERM"); terminalCleanupKillTimer = setTimeout(() => { terminalCleanupKillTimer = null; + terminalCleanupSignal = "SIGKILL"; + terminalCleanupForceKilled = true; signalRunningProcess({ child, processGroupId }, "SIGKILL"); }, Math.max(1, opts.graceSec) * 1000); }, graceMs); @@ -3051,6 +3071,17 @@ export async function runChildProcess( stderr, pid: child.pid ?? null, startedAt, + terminalResultCleanup: terminalCleanupStarted + ? { + kind: "terminal_result_cleanup", + stopped: true, + stopReason: UNMANAGED_BACKGROUND_TASK_STOP_REASON, + reason: UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON, + terminalResultSeen, + signal: terminalCleanupSignal, + forceKilled: terminalCleanupForceKilled, + } + : null, }); }); }); diff --git a/packages/adapters/claude-local/src/server/execute.ts b/packages/adapters/claude-local/src/server/execute.ts index 38d6d218e8..97ca1994ca 100644 --- a/packages/adapters/claude-local/src/server/execute.ts +++ b/packages/adapters/claude-local/src/server/execute.ts @@ -921,6 +921,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise { livenessState?: "completed" | "advanced" | "plan_only" | "empty_response" | "blocked" | "failed" | "needs_followup" | null; runErrorCode?: string | null; runError?: string | null; + resultJson?: Record | null; + monitorNextCheckAt?: Date | null; }) { const companyId = randomUUID(); const agentId = randomUUID(); @@ -729,6 +735,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { ? null : ("runError" in input ? input.runError : "run failed before issue advanced"), livenessState: input.livenessState ?? null, + resultJson: input.resultJson ?? null, }); await db.insert(issues).values([ @@ -755,6 +762,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { assigneeUserId: input.assignToUser ? "user-1" : null, checkoutRunId: input.status === "in_progress" ? runId : null, executionRunId: null, + monitorNextCheckAt: input.monitorNextCheckAt ?? null, responsibleUserId: "responsible-user", issueNumber: input.activePauseHold ? 2 : 1, identifier: `${issuePrefix}-${input.activePauseHold ? 2 : 1}`, @@ -1554,6 +1562,17 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { expect(failedRun?.status).toBe("failed"); expect(failedRun?.errorCode).toBe("process_lost"); expect(failedRun?.error).toContain("descendant process group"); + expect(failedRun?.resultJson).toMatchObject({ + stopReason: UNMANAGED_BACKGROUND_TASK_STOP_REASON, + unmanagedBackgroundTask: { + kind: "orphaned_process_group_cleanup", + stopped: true, + stopReason: UNMANAGED_BACKGROUND_TASK_STOP_REASON, + reason: UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON, + processPid: orphan.processPid, + processGroupId: orphan.processGroupId, + }, + }); const retryRun = runs.find((row) => row.id !== runId); expect(["queued", "running"]).toContain(retryRun?.status); @@ -5287,6 +5306,155 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { expect(wakeups).toHaveLength(2); }); + + it("does not accept unmanaged local-background wait evidence as a live continuation path", async () => { + const localWaitEvidence = { + summary: "Started a local polling watcher and will check the log later.", + externalWait: { + kind: "local_background", + pid: 12345, + logPath: "run/watch.log", + durable: false, + }, + }; + const { agentId, issueId, runId } = await seedStrandedIssueFixture({ + status: "in_progress", + runStatus: "succeeded", + livenessState: "advanced", + resultJson: localWaitEvidence, + }); + const heartbeat = heartbeatService(db); + + const result = await heartbeat.reconcileStrandedAssignedIssues(); + expect(result.continuationRequeued).toBe(1); + expect(result.escalated).toBe(0); + expect(result.issueIds).toEqual([issueId]); + + const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.agentId, agentId)); + const retryRun = runs.find((row) => row.id !== runId); + expect(retryRun?.contextSnapshot as Record | undefined).toMatchObject({ + issueId, + retryReason: "issue_continuation_needed", + retryOfRunId: runId, + source: "issue.productive_terminal_continuation_recovery", + }); + expect(retryRun?.contextSnapshot as Record).not.toHaveProperty("modelProfile"); + }); + + it("escalates repeated unmanaged local-background waits instead of retrying forever", async () => { + const localWaitEvidence = { + summary: "Still waiting on the local background watcher.", + externalWait: { + kind: "local_background", + pid: 12345, + logPath: "run/watch.log", + durable: false, + }, + }; + const { companyId, agentId, issueId, runId } = await seedStrandedIssueFixture({ + status: "in_progress", + runStatus: "succeeded", + retryReason: "issue_continuation_needed", + runSource: "issue.productive_terminal_continuation_recovery", + livenessState: "advanced", + resultJson: localWaitEvidence, + }); + const heartbeat = heartbeatService(db); + + const result = await heartbeat.reconcileStrandedAssignedIssues(); + expect(result.continuationRequeued).toBe(0); + expect(result.escalated).toBe(1); + expect(result.issueIds).toEqual([issueId]); + + const issue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0] ?? null); + expect(issue?.status).toBe("blocked"); + + await expectSourceScopedStrandedRecoveryAction({ + companyId, + agentId, + issueId, + runId, + previousStatus: "in_progress", + retryReason: "issue_continuation_needed", + }); + + const followupRuns = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.agentId, agentId)); + expect(followupRuns).toHaveLength(2); + }); + + it("preserves a persisted issue monitor as the durable external-wait path", async () => { + const { companyId, agentId, issueId } = await seedStrandedIssueFixture({ + status: "in_progress", + runStatus: "succeeded", + livenessState: "advanced", + monitorNextCheckAt: new Date("2026-03-19T01:00:00.000Z"), + resultJson: { + summary: "Waiting for the deploy to settle; monitor is scheduled.", + externalWait: { kind: "issue_monitor", durable: true }, + }, + }); + const heartbeat = heartbeatService(db); + + const result = await heartbeat.reconcileStrandedAssignedIssues(); + expect(result.continuationRequeued).toBe(0); + expect(result.escalated).toBe(0); + expect(result.skipped).toBe(1); + + const issue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0] ?? null); + expect(issue?.status).toBe("in_progress"); + expect(issue?.monitorNextCheckAt?.toISOString()).toBe("2026-03-19T01:00:00.000Z"); + + const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.agentId, agentId)); + expect(runs).toHaveLength(1); + const recoveryIssues = await db + .select() + .from(issues) + .where(and(eq(issues.companyId, companyId), eq(issues.originKind, "stranded_issue_recovery"))); + expect(recoveryIssues).toHaveLength(0); + }); + + it("preserves a delegated blocker edge as the durable external-wait path", async () => { + const { companyId, agentId, issueId } = await seedStrandedIssueFixture({ + status: "in_progress", + runStatus: "succeeded", + livenessState: "advanced", + resultJson: { + summary: "Delegated the external account check to a child task.", + externalWait: { kind: "delegated_child", durable: true }, + }, + }); + const blockerIssueId = randomUUID(); + await db.insert(issues).values({ + id: blockerIssueId, + companyId, + parentId: issueId, + title: "Check external account approval", + status: "todo", + priority: "medium", + assigneeUserId: "external-owner", + responsibleUserId: "responsible-user", + issueNumber: 2, + identifier: "PAP-2", + }); + await db.insert(issueRelations).values({ + companyId, + issueId: blockerIssueId, + relatedIssueId: issueId, + type: "blocks", + }); + const heartbeat = heartbeatService(db); + + const result = await heartbeat.reconcileStrandedAssignedIssues(); + expect(result.continuationRequeued).toBe(0); + expect(result.escalated).toBe(0); + expect(result.skipped).toBe(1); + + const source = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0] ?? null); + expect(source?.status).toBe("in_progress"); + const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.agentId, agentId)); + expect(runs).toHaveLength(1); + }); + it("blocks stranded in-progress work after a productive continuation retry was already used", async () => { const { companyId, agentId, issueId, runId } = await seedStrandedIssueFixture({ status: "in_progress", diff --git a/server/src/__tests__/run-liveness.test.ts b/server/src/__tests__/run-liveness.test.ts index 2feeb07668..92c5560d95 100644 --- a/server/src/__tests__/run-liveness.test.ts +++ b/server/src/__tests__/run-liveness.test.ts @@ -195,6 +195,27 @@ describe("run liveness classifier", () => { expect(classification.nextAction).toBe("deploy to production and verify live traffic."); }); + + it("uses killed background-task evidence instead of a generic failed-run reason", () => { + const classification = classifyRunLiveness({ + ...baseInput, + runStatus: "failed", + errorCode: "process_lost", + resultJson: { + stopReason: "unmanaged_background_task_stopped", + unmanagedBackgroundTask: { + kind: "orphaned_process_group_cleanup", + stopped: true, + stopReason: "unmanaged_background_task_stopped", + reason: "unmanaged background task stopped; no durable live path", + }, + }, + }); + + expect(classification.livenessState).toBe("failed"); + expect(classification.livenessReason).toBe("unmanaged background task stopped; no durable live path"); + }); + it("marks unclear useful output as unknown actionability", () => { const classification = classifyRunLiveness({ ...baseInput, diff --git a/server/src/services/heartbeat-stop-metadata.ts b/server/src/services/heartbeat-stop-metadata.ts index 268a3722fc..ef89175c29 100644 --- a/server/src/services/heartbeat-stop-metadata.ts +++ b/server/src/services/heartbeat-stop-metadata.ts @@ -9,6 +9,7 @@ export type HeartbeatRunStopReason = | "paused" | "max_turns_exhausted" | "process_lost" + | "unmanaged_background_task_stopped" | "adapter_failed"; export interface HeartbeatRunTimeoutPolicy { @@ -88,6 +89,7 @@ export function inferHeartbeatRunStopReason(input: { const maxTurnStopReason = normalizeMaxTurnStopReason(input.errorCode); if (maxTurnStopReason) return maxTurnStopReason; if (input.outcome === "timed_out") return "timeout"; + if (input.outcome === "failed" && input.errorCode === "unmanaged_background_task_stopped") return "unmanaged_background_task_stopped"; if (input.outcome === "failed" && input.errorCode === "process_lost") return "process_lost"; if (input.outcome === "cancelled") { const message = (input.errorMessage ?? "").toLowerCase(); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index ad8948503a..67bad29384 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -215,6 +215,8 @@ import { } from "@paperclipai/adapter-utils"; import { readPaperclipSkillSyncPreference, + UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON, + UNMANAGED_BACKGROUND_TASK_STOP_REASON, writePaperclipSkillSyncPreference, } from "@paperclipai/adapter-utils/server-utils"; import { extractSkillMentionIds, isUuidLike } from "@paperclipai/shared"; @@ -7392,9 +7394,24 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) return `[${label}](/${prefix}/issues/${label})`; } + function hasUnmanagedBackgroundTaskEvidence(resultJson: Record | null | undefined) { + const evidence = parseObject(resultJson?.unmanagedBackgroundTask); + return evidence.stopped === true && + (evidence.stopReason === UNMANAGED_BACKGROUND_TASK_STOP_REASON || + evidence.reason === UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON); + } + + function withUnmanagedBackgroundTaskStopReason(resultJson: Record | null | undefined) { + return { + ...(resultJson ?? {}), + stopReason: UNMANAGED_BACKGROUND_TASK_STOP_REASON, + }; + } + async function buildDetectedSuccessfulRunProgressSummary(run: typeof heartbeatRuns.$inferSelect) { const resultJson = parseObject(run.resultJson); const candidates = [ + hasUnmanagedBackgroundTaskEvidence(resultJson) ? UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON : null, readNonEmptyString(run.nextAction) ? `Next action noted: ${readNonEmptyString(run.nextAction)}` : null, readNonEmptyString(run.livenessReason), readNonEmptyString(resultJson.summary), @@ -7458,6 +7475,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId, executionState: issues.executionState, + monitorNextCheckAt: issues.monitorNextCheckAt, projectId: issues.projectId, }) .from(issues) @@ -7634,6 +7652,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) hasActiveExecutionPath: Boolean(activeExecutionPath), hasQueuedWake: Boolean(queuedWake), hasPendingInteractionOrApproval: Boolean(pendingInteraction || pendingApproval), + hasPersistedMonitor: Boolean(issue?.monitorNextCheckAt), hasExplicitBlockerPath: Boolean(explicitBlocker), hasOpenRecoveryIssue: Boolean(openRecoveryIssue), hasPauseHold: Boolean(pauseHold), @@ -7644,6 +7663,17 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) if (decision.kind !== "enqueue" || !issue) return; + if (hasUnmanagedBackgroundTaskEvidence(parseObject(run.resultJson))) { + await db + .update(heartbeatRuns) + .set({ + livenessReason: UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON, + resultJson: withUnmanagedBackgroundTaskStopReason(parseObject(run.resultJson)), + updatedAt: new Date(), + }) + .where(eq(heartbeatRuns.id, run.id)); + } + const handoffRun = await enqueueWakeup(run.agentId, { source: "automation", triggerDetail: "system", @@ -10622,20 +10652,39 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const shouldRetry = tracksLocalChild && (!!run.processPid || !!run.processGroupId) && (run.processLossRetryCount ?? 0) < 1; const baseMessage = buildProcessLossMessage(run, descendantOnlyCleanup ? { descendantOnly: true } : undefined); + const unmanagedBackgroundTaskEvidence = descendantOnlyCleanup + ? { + kind: "orphaned_process_group_cleanup", + stopped: true, + stopReason: UNMANAGED_BACKGROUND_TASK_STOP_REASON, + reason: UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON, + processPid: run.processPid ?? null, + processGroupId: run.processGroupId ?? null, + } + : null; let finalizedRun = await setRunStatus(run.id, "failed", { error: shouldRetry ? `${baseMessage}; retrying once` : baseMessage, errorCode: "process_lost", finishedAt: now, - resultJson: mergeRunStopMetadataForAgent( - { adapterType, adapterConfig }, - "failed", - { - resultJson: parseObject(run.resultJson), - errorCode: "process_lost", - errorMessage: shouldRetry ? `${baseMessage}; retrying once` : baseMessage, - }, - ), + resultJson: (() => { + const result = mergeRunStopMetadataForAgent( + { adapterType, adapterConfig }, + "failed", + { + resultJson: parseObject(run.resultJson), + errorCode: "process_lost", + errorMessage: shouldRetry ? `${baseMessage}; retrying once` : baseMessage, + }, + ); + return unmanagedBackgroundTaskEvidence + ? { + ...result, + stopReason: UNMANAGED_BACKGROUND_TASK_STOP_REASON, + unmanagedBackgroundTask: unmanagedBackgroundTaskEvidence, + } + : result; + })(), }); await setWakeupStatus(run.wakeupRequestId, "failed", { finishedAt: now, @@ -13848,9 +13897,24 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .limit(1) .then((rows) => rows[0] ?? null); - const issueHasScheduledMonitor = - issue.monitorNextCheckAt instanceof Date && - issue.monitorNextCheckAt.getTime() > Date.now(); + const issueHasPersistedMonitor = Boolean(issue.monitorNextCheckAt); + const findExplicitBlockerPath = () => + tx + .select({ id: issueRelations.issueId }) + .from(issueRelations) + .innerJoin(issues, eq(issueRelations.issueId, issues.id)) + .where( + and( + eq(issueRelations.companyId, issue.companyId), + eq(issueRelations.relatedIssueId, issue.id), + eq(issueRelations.type, "blocks"), + eq(issues.companyId, issue.companyId), + notInArray(issues.status, ["done", "cancelled"]), + isNull(issues.hiddenAt), + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null); const executionState = parseIssueExecutionState(issue.executionState); const currentParticipant = executionState?.status === "pending" ? executionState.currentParticipant @@ -13870,7 +13934,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) if ( options.suppressImmediateRecovery || existingReviewParticipantExecutionPath || - issueHasScheduledMonitor || + issueHasPersistedMonitor || await isAutomaticRecoverySuppressedByPauseHold(db, issue.companyId, issue.id, treeControlSvc) ) { return { kind: "released" as const }; @@ -13989,7 +14053,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } const existingExecutionPath = await findExistingExecutionPath(); - if (existingExecutionPath) { + if (existingExecutionPath || issueHasPersistedMonitor || await findExplicitBlockerPath()) { return { kind: "released" as const }; } diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 2ef1c1d051..53a58e96e2 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -699,6 +699,27 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) .then((rows) => Boolean(rows[0])); } + async function hasPersistedDurableWaitPath(issue: typeof issues.$inferSelect) { + if (issue.monitorNextCheckAt) return true; + + return db + .select({ id: issueRelations.issueId }) + .from(issueRelations) + .innerJoin(issues, eq(issueRelations.issueId, issues.id)) + .where( + and( + eq(issueRelations.companyId, issue.companyId), + eq(issueRelations.relatedIssueId, issue.id), + eq(issueRelations.type, "blocks"), + eq(issues.companyId, issue.companyId), + notInArray(issues.status, ["done", "cancelled"]), + isNull(issues.hiddenAt), + ), + ) + .limit(1) + .then((rows) => Boolean(rows[0])); + } + async function hasQueuedIssueWake(companyId: string, issueId: string, agentId?: string | null) { return db .select({ id: agentWakeupRequests.id }) @@ -3011,6 +3032,10 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) } const latestRun = await getLatestIssueRun(issue.companyId, issue.id); + if (latestRun?.status === "succeeded" && await hasPersistedDurableWaitPath(issue)) { + result.skipped += 1; + continue; + } if (isStrandedIssueRecoveryIssue(issue) && isUnsuccessfulTerminalIssueRun(latestRun)) { const updated = await escalateStrandedRecoveryIssueInPlace({ issue, diff --git a/server/src/services/recovery/successful-run-handoff.test.ts b/server/src/services/recovery/successful-run-handoff.test.ts index 184930b1f7..25a99cbb21 100644 --- a/server/src/services/recovery/successful-run-handoff.test.ts +++ b/server/src/services/recovery/successful-run-handoff.test.ts @@ -12,6 +12,7 @@ import { isSuccessfulRunHandoffRequiredNoticeBody, noticeMetadataReferencesRecoveryAction, } from "./successful-run-handoff.js"; +import { UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON } from "@paperclipai/adapter-utils/server-utils"; const run = { id: "run-1", @@ -49,6 +50,7 @@ function decide(overrides: Partial hasActiveExecutionPath: false, hasQueuedWake: false, hasPendingInteractionOrApproval: false, + hasPersistedMonitor: false, hasExplicitBlockerPath: false, hasOpenRecoveryIssue: false, hasPauseHold: false, @@ -114,17 +116,36 @@ describe("successful run handoff decision", () => { kind: "skip", reason: "pending interaction or approval owns the next action", }); + expect(decide({ hasPersistedMonitor: true })).toEqual({ + kind: "skip", + reason: "persisted issue monitor owns the next action", + }); expect(decide({ hasActiveExecutionPath: true })).toEqual({ kind: "skip", reason: "issue already has an active execution path", }); }); + it("does not treat killed background-task evidence as a missing live path when a durable monitor owns the wait", () => { + expect(decide({ + detectedProgressSummary: UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON, + livenessState: "needs_followup", + hasPersistedMonitor: true, + })).toEqual({ + kind: "skip", + reason: "persisted issue monitor owns the next action", + }); + }); + it("does not queue when another wake or dependency path already owns the next action", () => { expect(decide({ hasQueuedWake: true })).toEqual({ kind: "skip", reason: "issue already has a queued or deferred wake", }); + expect(decide({ hasPersistedMonitor: true })).toEqual({ + kind: "skip", + reason: "persisted issue monitor owns the next action", + }); expect(decide({ hasExplicitBlockerPath: true })).toEqual({ kind: "skip", reason: "explicit blocker path owns the next action", diff --git a/server/src/services/recovery/successful-run-handoff.ts b/server/src/services/recovery/successful-run-handoff.ts index 5ac73473f5..b0f0b94dbd 100644 --- a/server/src/services/recovery/successful-run-handoff.ts +++ b/server/src/services/recovery/successful-run-handoff.ts @@ -347,6 +347,7 @@ export function decideSuccessfulRunHandoff(input: { hasActiveExecutionPath: boolean; hasQueuedWake: boolean; hasPendingInteractionOrApproval: boolean; + hasPersistedMonitor: boolean; hasExplicitBlockerPath: boolean; hasOpenRecoveryIssue: boolean; hasPauseHold: boolean; @@ -388,6 +389,7 @@ export function decideSuccessfulRunHandoff(input: { if (input.hasPendingInteractionOrApproval) { return { kind: "skip", reason: "pending interaction or approval owns the next action" }; } + if (input.hasPersistedMonitor) return { kind: "skip", reason: "persisted issue monitor owns the next action" }; if (input.hasExplicitBlockerPath) return { kind: "skip", reason: "explicit blocker path owns the next action" }; if (input.hasOpenRecoveryIssue) return { kind: "skip", reason: "open recovery issue owns the ambiguity" }; if (input.hasPauseHold) return { kind: "skip", reason: "issue is under an active pause hold" }; diff --git a/server/src/services/run-liveness.ts b/server/src/services/run-liveness.ts index 87b53285c7..1d7a05bba5 100644 --- a/server/src/services/run-liveness.ts +++ b/server/src/services/run-liveness.ts @@ -75,6 +75,8 @@ const RUNNABLE_RE = const PLAN_TASK_TITLE_RE = /\b(?:plan|planning|analysis|investigation|research|report|proposal|design doc|write-?up)\b/i; const PLAN_TASK_DESCRIPTION_RE = /\b(?:create|write|produce|draft|update|revise|prepare)\s+(?:a\s+|the\s+)?(?:plan|analysis|investigation|research report|report|proposal|design doc|write-?up)\b/i; +const UNMANAGED_BACKGROUND_TASK_STOP_REASON = "unmanaged_background_task_stopped"; +const UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON = "unmanaged background task stopped; no durable live path"; function compactReason(reason: string) { return reason.length <= 500 ? reason : `${reason.slice(0, 497)}...`; @@ -94,6 +96,17 @@ function readText(value: unknown): string | null { return trimmed.length > 0 ? trimmed : null; } +function hasUnmanagedBackgroundTaskEvidence(resultJson: Record | null | undefined) { + if (!resultJson) return false; + if (resultJson.stopReason === UNMANAGED_BACKGROUND_TASK_STOP_REASON) return true; + const evidence = resultJson.unmanagedBackgroundTask; + if (!evidence || typeof evidence !== "object" || Array.isArray(evidence)) return false; + const record = evidence as Record; + return record.stopped === true && + (record.stopReason === UNMANAGED_BACKGROUND_TASK_STOP_REASON || + record.reason === UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON); +} + function resultFinalText(resultJson: Record | null | undefined) { if (!resultJson) return ""; return [ @@ -314,6 +327,9 @@ export function classifyRunLiveness(input: RunLivenessClassificationInput): RunL } if (input.runStatus !== "succeeded") { + if (hasUnmanagedBackgroundTaskEvidence(input.resultJson)) { + return output("failed", UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON); + } return output("failed", input.errorCode ? `Run ended with ${input.runStatus} (${input.errorCode})` : `Run ended with ${input.runStatus}`); } diff --git a/ui/src/components/IssueRecoveryActionCard.test.tsx b/ui/src/components/IssueRecoveryActionCard.test.tsx index 24efe9c8c2..5613da8638 100644 --- a/ui/src/components/IssueRecoveryActionCard.test.tsx +++ b/ui/src/components/IssueRecoveryActionCard.test.tsx @@ -196,6 +196,48 @@ describe("IssueRecoveryActionCard", () => { expect(node.textContent).toContain("Manual repair required"); }); + it("renders a human evidence summary as prose, not a mono log line", () => { + const node = render( + , + ); + const summary = Array.from(node.querySelectorAll("span")).find((el) => + el.textContent === "Unmanaged background task stopped; no durable live path.", + ); + expect(summary).toBeTruthy(); + expect(summary?.className).toContain("text-xs"); + expect(summary?.className).not.toContain("font-mono"); + }); + + it("keeps code-shaped evidence (error code, no summary) in the mono treatment", () => { + const node = render( + , + ); + const code = Array.from(node.querySelectorAll("span")).find((el) => + el.textContent === "workspace_validation_failed", + ); + expect(code).toBeTruthy(); + expect(code?.className).toContain("font-mono"); + }); + it("renders the resolved label and outcome when resolved", () => { const node = render( , diff --git a/ui/src/components/IssueRecoveryActionCard.tsx b/ui/src/components/IssueRecoveryActionCard.tsx index 45d3320087..da9197221c 100644 --- a/ui/src/components/IssueRecoveryActionCard.tsx +++ b/ui/src/components/IssueRecoveryActionCard.tsx @@ -214,20 +214,20 @@ function readEvidenceString(value: unknown): string | null { return trimmed.length > 240 ? `${trimmed.slice(0, 237)}…` : trimmed; } -function pickEvidenceSummary(action: IssueRecoveryAction): string | null { +// Human-sentence evidence sources render as prose; code-shaped sources +// (error codes, statuses) stay in the mono treatment used for run ids. +const PROSE_EVIDENCE_KEYS = ["summary", "detectedProgressSummary", "missingDisposition", "retryReason"] as const; +const CODE_EVIDENCE_KEYS = ["latestRunErrorCode", "latestRunStatus", "latestIssueStatus"] as const; + +function pickEvidenceSummary(action: IssueRecoveryAction): { text: string; isCode: boolean } | null { const evidence = action.evidence ?? {}; - const candidates = [ - "summary", - "detectedProgressSummary", - "missingDisposition", - "retryReason", - "latestRunErrorCode", - "latestRunStatus", - "latestIssueStatus", - ] as const; - for (const key of candidates) { + for (const key of PROSE_EVIDENCE_KEYS) { const next = readEvidenceString(evidence[key]); - if (next) return next; + if (next) return { text: next, isCode: false }; + } + for (const key of CODE_EVIDENCE_KEYS) { + const next = readEvidenceString(evidence[key]); + if (next) return { text: next, isCode: true }; } return null; } @@ -1075,7 +1075,13 @@ export function IssueRecoveryActionCard({ ) : null} {evidenceSummary ? ( - {evidenceSummary} + evidenceSummary.isCode ? ( + + {evidenceSummary.text} + + ) : ( + {evidenceSummary.text} + ) ) : ( )} diff --git a/ui/src/components/IssueRunLedger.test.tsx b/ui/src/components/IssueRunLedger.test.tsx index 751b4fe789..eca7cfa804 100644 --- a/ui/src/components/IssueRunLedger.test.tsx +++ b/ui/src/components/IssueRunLedger.test.tsx @@ -375,6 +375,11 @@ describe("IssueRunLedger", () => { resultJson: { stopReason: "budget_paused" }, createdAt: "2026-04-18T19:56:00.000Z", }), + createRun({ + runId: "run-background-task", + resultJson: { stopReason: "unmanaged_background_task_stopped" }, + createdAt: "2026-04-18T19:55:30.000Z", + }), createRun({ runId: "run-paused", resultJson: { stopReason: "paused" }, @@ -386,6 +391,7 @@ describe("IssueRunLedger", () => { expect(container.textContent).toContain("timeout (30s timeout)"); expect(container.textContent).toContain("cancelled"); expect(container.textContent).toContain("budget paused"); + expect(container.textContent).toContain("unmanaged background task stopped"); expect(container.textContent).toContain("paused by board"); }); diff --git a/ui/src/components/IssueRunLedger.tsx b/ui/src/components/IssueRunLedger.tsx index 40e51043e5..a69693b4b7 100644 --- a/ui/src/components/IssueRunLedger.tsx +++ b/ui/src/components/IssueRunLedger.tsx @@ -322,6 +322,7 @@ function stopReasonLabel(run: RunForIssue) { if (stopReason === "cancelled") return "cancelled"; if (stopReason === "paused") return "paused by board"; if (stopReason === "process_lost") return "process lost"; + if (stopReason === "unmanaged_background_task_stopped") return "unmanaged background task stopped"; if (stopReason === "adapter_failed") return "adapter failed"; if (stopReason === "completed") return timeoutText ? `completed (${timeoutText})` : "completed"; return timeoutText; diff --git a/ui/storybook/stories/source-issue-recovery.stories.tsx b/ui/storybook/stories/source-issue-recovery.stories.tsx index f252b15919..de29f1f11e 100644 --- a/ui/storybook/stories/source-issue-recovery.stories.tsx +++ b/ui/storybook/stories/source-issue-recovery.stories.tsx @@ -144,6 +144,48 @@ function AllStatesPanel() { })} canFalsePositive /> + + ); }