diff --git a/docs/deploy/dev-plane-restart-hygiene.md b/docs/deploy/dev-plane-restart-hygiene.md new file mode 100644 index 0000000000..fc9ce98d69 --- /dev/null +++ b/docs/deploy/dev-plane-restart-hygiene.md @@ -0,0 +1,57 @@ +# Dev-Plane Deploy & Restart Hygiene + +Operational runbook for restarting a dev/shared Paperclip control plane without killing in-flight agent work. Written after the 2026-07-06/07 failure spike, where a restart-heavy deploy window was the single largest source of failed tasks. + +## Why this matters + +Every control-plane restart hard-kills any heartbeat run in flight at that moment. The run is finalized as `failed` with `error_code = 'process_lost'`. There is an automatic single retry (`process_lost_retry`), but it is best-effort: in the 07-06/07 incident window only 12 of 36 lost runs recovered. On the evening of 07-06, **9 restarts in 90 minutes killed 16 in-flight runs**. + +## Rules of thumb + +1. **Batch your deploys.** Stack up changes and restart once, instead of restart-per-change. A restart storm (several restarts within an hour) multiplies run loss for near-zero benefit. +2. **Restart during low fleet activity.** Check for active runs before restarting; prefer windows where the fleet is quiet. A quick check against the control-plane DB: + + ```sql + SELECT count(*) FROM heartbeat_runs WHERE status = 'running'; + ``` + +3. **Drain before restart (upcoming).** Graceful SIGTERM drain — stop accepting new runs, let in-flight runs finish or checkpoint, then exit — is being added in PAP-12930. Once it lands, send SIGTERM and wait for drain instead of hard-restarting. Until then, rule 2 is your drain. +4. **After any restart, glance at the damage.** See the detection queries below; confirm lost runs either retried successfully or get manual follow-up. + +## How to spot a restart burst + +Two signals, cross-referenced: + +**1. Server start markers in the instance log.** Each boot logs `Server listening on :`. Logs live at `~/.paperclip/instances//server.log`, rotated daily to `server.log-YYYYMMDD.gz`. + +```bash +grep -h "Server listening" ~/.paperclip/instances/default/server.log +zgrep -h "Server listening" ~/.paperclip/instances/default/server.log-20260706.gz +``` + +Many markers minutes apart = restart burst. + +**2. Same-minute `process_lost` clusters in `heartbeat_runs`.** Runs killed by a restart are finalized together, so they cluster on the same minute: + +```sql +SELECT date_trunc('minute', finished_at) AS minute, count(*) +FROM heartbeat_runs +WHERE error_code = 'process_lost' +GROUP BY 1 HAVING count(*) > 1 +ORDER BY 1 DESC; +``` + +If the cluster minutes line up with the `Server listening` timestamps, the failures are restart-inflicted, not a product regression. + +**Note:** query the DB directly for this — the `/heartbeat-runs` list API ignores the `status=` filter and omits error fields. + +## Checking recovery after a burst + +Each `process_lost` run should have triggered one retry wake (`reason = 'process_lost_retry'`). To find lost runs that never recovered, look for `process_lost` failures with no subsequent successful run for the same issue, and re-wake or reassign those issues manually. + +## Related failure modes (not restart-caused) + +Seen in the same incident window; do not confuse them with restart damage: + +- `workspace_validation_failed` — deterministic workspace validation retry loops (self-heal: PAP-12931). +- Provider quota exhaustion — dominant cause of `claude_transient_upstream` failures (quota-aware handling: PAP-12932). diff --git a/packages/adapter-utils/src/types.ts b/packages/adapter-utils/src/types.ts index 80f5f92f86..e290cbbe3c 100644 --- a/packages/adapter-utils/src/types.ts +++ b/packages/adapter-utils/src/types.ts @@ -65,7 +65,7 @@ export interface AdapterRuntimeServiceReport { healthStatus?: "unknown" | "healthy" | "unhealthy"; } -export type AdapterExecutionErrorFamily = "transient_upstream" | "model_refusal"; +export type AdapterExecutionErrorFamily = "transient_upstream" | "provider_quota" | "model_refusal"; export interface AdapterExecutionResult { exitCode: number | null; diff --git a/packages/adapters/claude-local/src/server/execute.ts b/packages/adapters/claude-local/src/server/execute.ts index 3bfec50c62..99bf1f1621 100644 --- a/packages/adapters/claude-local/src/server/execute.ts +++ b/packages/adapters/claude-local/src/server/execute.ts @@ -50,6 +50,7 @@ import { detectClaudeLoginRequired, extractClaudeRetryNotBefore, isClaudeMaxTurnsResult, + isClaudeProviderQuotaError, isClaudeRefusalResult, isClaudeTransientUpstreamError, isClaudeUnknownSessionError, @@ -840,8 +841,18 @@ export async function execute(ctx: AdapterExecutionContext): Promise = { ...parsed, ...(failed && clearSessionForMaxTurns ? { stopReason: "max_turns_exhausted" } : {}), ...(failed && poisonedPreviousMessageId ? { stopReason: "claude_poisoned_previous_message_id" } : {}), ...(claudeRefusal ? { stopReason: "refusal", errorFamily: "model_refusal" } : {}), - ...(transientUpstream ? { errorFamily: "transient_upstream" } : {}), + ...(errorFamily ? { errorFamily } : {}), ...(transientRetryNotBefore ? { retryNotBefore: transientRetryNotBefore.toISOString() } : {}), ...(transientRetryNotBefore ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } : {}), + ...(providerQuota && transientRetryNotBefore ? { providerQuotaRetryNotBefore: transientRetryNotBefore.toISOString() } : {}), }; return { @@ -981,11 +1022,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise { }); describe("isClaudeTransientUpstreamError", () => { - it("classifies the 'out of extra usage' subscription window failure as transient", () => { + it("classifies the 'out of extra usage' subscription window failure as provider quota", () => { expect( - isClaudeTransientUpstreamError({ + isClaudeProviderQuotaError({ errorMessage: "You're out of extra usage · resets 4pm (America/Chicago)", }), ).toBe(true); expect( - isClaudeTransientUpstreamError({ + isClaudeProviderQuotaError({ parsed: { is_error: true, result: "You're out of extra usage. Resets at 4pm (America/Chicago).", }, }), ).toBe(true); + expect( + isClaudeTransientUpstreamError({ + errorMessage: "You're out of extra usage · resets 4pm (America/Chicago)", + }), + ).toBe(false); + }); + + it("classifies Claude session-limit windows as provider quota and extracts the retry time", () => { + const now = new Date("2026-04-22T15:15:00.000Z"); + const errorMessage = "You've hit your session limit - resets at 4pm (America/Chicago)."; + + expect(isClaudeProviderQuotaError({ errorMessage })).toBe(true); + expect(isClaudeTransientUpstreamError({ errorMessage })).toBe(false); + expect(extractClaudeRetryNotBefore({ errorMessage }, now)?.toISOString()).toBe( + "2026-04-22T21:00:00.000Z", + ); }); it("classifies Anthropic API rate_limit_error and overloaded_error as transient", () => { @@ -77,14 +94,14 @@ describe("isClaudeTransientUpstreamError", () => { ).toBe(true); }); - it("classifies the subscription 5-hour / weekly limit wording", () => { + it("classifies the subscription 5-hour / weekly limit wording as provider quota", () => { expect( - isClaudeTransientUpstreamError({ + isClaudeProviderQuotaError({ errorMessage: "Claude usage limit reached — weekly limit reached. Try again in 2 days.", }), ).toBe(true); expect( - isClaudeTransientUpstreamError({ + isClaudeProviderQuotaError({ errorMessage: "5-hour limit reached.", }), ).toBe(true); diff --git a/packages/adapters/claude-local/src/server/parse.ts b/packages/adapters/claude-local/src/server/parse.ts index 657a476a01..9b279e974d 100644 --- a/packages/adapters/claude-local/src/server/parse.ts +++ b/packages/adapters/claude-local/src/server/parse.ts @@ -11,8 +11,10 @@ const URL_RE = /(https?:\/\/[^\s'"`<>()[\]{};,!?]+[^\s'"`<>()[\]{};,!.?:]+)/gi; const CLAUDE_TRANSIENT_UPSTREAM_RE = /(?:rate[-\s]?limit(?:ed)?|rate_limit_error|too\s+many\s+requests|\b429\b|overloaded(?:_error)?|server\s+overloaded|service\s+unavailable|\b503\b|\b529\b|high\s+demand|try\s+again\s+later|temporarily\s+unavailable|throttl(?:ed|ing)|throttlingexception|servicequotaexceededexception|out\s+of\s+extra\s+usage|extra\s+usage\b|claude\s+usage\s+limit\s+reached|5[-\s]?hour\s+limit\s+reached|weekly\s+limit\s+reached|usage\s+limit\s+reached|usage\s+cap\s+reached)/i; +const CLAUDE_PROVIDER_QUOTA_RE = + /(?:you(?:'|’)ve\s+hit\s+your\s+session\s+limit|session\s+limit\s+(?:reached|exceeded)|out\s+of\s+extra\s+usage|extra\s+usage\b|claude\s+usage\s+limit\s+reached|5[-\s]?hour\s+limit\s+reached|weekly\s+limit\s+reached|usage\s+limit\s+reached|usage\s+cap\s+reached|servicequotaexceededexception)/i; const CLAUDE_EXTRA_USAGE_RESET_RE = - /(?:out\s+of\s+extra\s+usage|extra\s+usage|usage\s+limit\s+reached|usage\s+cap\s+reached|5[-\s]?hour\s+limit\s+reached|weekly\s+limit\s+reached|claude\s+usage\s+limit\s+reached)[\s\S]{0,80}?\bresets?\s+(?:at\s+)?([^\n()]+?)(?:\s*\(([^)]+)\))?(?:[.!]|\n|$)/i; + /(?:you(?:'|’)ve\s+hit\s+your\s+session\s+limit|session\s+limit\s+(?:reached|exceeded)|out\s+of\s+extra\s+usage|extra\s+usage|usage\s+limit\s+reached|usage\s+cap\s+reached|5[-\s]?hour\s+limit\s+reached|weekly\s+limit\s+reached|claude\s+usage\s+limit\s+reached)[\s\S]{0,120}?\bresets?\s+(?:at\s+)?([^\n()]+?)(?:\s*\(([^)]+)\))?(?:[.!]|\n|$)/i; export function parseClaudeStreamJson(stdout: string) { let sessionId: string | null = null; @@ -429,5 +431,28 @@ export function isClaudeTransientUpstreamError(input: { const haystack = buildClaudeTransientHaystack(input); if (!haystack) return false; + if (isClaudeProviderQuotaError(input)) return false; return CLAUDE_TRANSIENT_UPSTREAM_RE.test(haystack); } + +export function isClaudeProviderQuotaError(input: { + parsed?: Record | null; + stdout?: string | null; + stderr?: string | null; + errorMessage?: string | null; +}): boolean { + const parsed = input.parsed ?? null; + if (parsed && (isClaudeMaxTurnsResult(parsed) || isClaudeUnknownSessionError(parsed) || isClaudePoisonedPreviousMessageIdError(parsed) || isClaudeImageProcessingError(parsed))) { + return false; + } + const loginMeta = detectClaudeLoginRequired({ + parsed, + stdout: input.stdout ?? "", + stderr: input.stderr ?? "", + }); + if (loginMeta.requiresLogin) return false; + + const haystack = buildClaudeTransientHaystack(input); + if (!haystack) return false; + return CLAUDE_PROVIDER_QUOTA_RE.test(haystack); +} diff --git a/packages/adapters/codex-local/src/server/execute.ts b/packages/adapters/codex-local/src/server/execute.ts index d1f535ce9a..71522b2add 100644 --- a/packages/adapters/codex-local/src/server/execute.ts +++ b/packages/adapters/codex-local/src/server/execute.ts @@ -41,6 +41,7 @@ import { import { parseCodexJsonl, extractCodexRetryNotBefore, + isCodexProviderQuotaError, isCodexTransientUpstreamError, isCodexUnknownSessionError, } from "./parse.js"; @@ -1016,13 +1017,22 @@ export async function execute(ctx: AdapterExecutionContext): Promise { ).toBe(true); }); - it("classifies usage-limit windows as transient and extracts the retry time", () => { + it("classifies usage-limit windows as provider quota and extracts the retry time", () => { const errorMessage = "You've hit your usage limit for GPT-5.3-Codex-Spark. Switch to another model now, or try again at 11:31 PM."; const now = new Date(2026, 3, 22, 22, 29, 2); - expect(isCodexTransientUpstreamError({ errorMessage })).toBe(true); + expect(isCodexProviderQuotaError({ errorMessage })).toBe(true); + expect(isCodexTransientUpstreamError({ errorMessage })).toBe(false); expect(extractCodexRetryNotBefore({ errorMessage }, now)?.getTime()).toBe( new Date(2026, 3, 22, 23, 31, 0, 0).getTime(), ); }); + it("classifies model-capacity messages as provider quota without reset metadata", () => { + const errorMessage = "The requested model is at capacity. Please try again later."; + + expect(isCodexProviderQuotaError({ errorMessage })).toBe(true); + expect(isCodexTransientUpstreamError({ errorMessage })).toBe(false); + expect(extractCodexRetryNotBefore({ errorMessage })).toBeNull(); + }); + it("parses explicit timezone hints on usage-limit retry windows", () => { const errorMessage = "You've hit your usage limit for GPT-5.3-Codex-Spark. Switch to another model now, or try again at 11:31 PM (America/Chicago)."; const now = new Date("2026-04-23T03:29:02.000Z"); diff --git a/packages/adapters/codex-local/src/server/parse.ts b/packages/adapters/codex-local/src/server/parse.ts index 6dd3d5fb3f..535d872f28 100644 --- a/packages/adapters/codex-local/src/server/parse.ts +++ b/packages/adapters/codex-local/src/server/parse.ts @@ -10,6 +10,8 @@ const CODEX_TRANSIENT_UPSTREAM_RE = const CODEX_REMOTE_COMPACTION_RE = /remote\s+compact\s+task/i; const CODEX_USAGE_LIMIT_RE = /you(?:'|’)ve hit your usage limit for .+\.\s+switch to another model now,\s+or try again at\s+([^.!\n]+)(?:[.!]|\n|$)/i; +const CODEX_PROVIDER_QUOTA_RE = + /(?:you(?:'|’)ve hit your usage limit|usage limit|model (?:is )?at capacity|at capacity for this model|capacity limit)/i; export function parseCodexJsonl(stdout: string) { let sessionId: string | null = null; @@ -252,10 +254,18 @@ export function isCodexTransientUpstreamError(input: { }): boolean { const haystack = buildCodexErrorHaystack(input); - if (extractCodexRetryNotBefore(input) != null) return true; + if (isCodexProviderQuotaError(input)) return false; if (!CODEX_TRANSIENT_UPSTREAM_RE.test(haystack)) return false; // Keep automatic retries scoped to the observed remote-compaction/high-demand - // failure shape, plus explicit usage-limit windows that tell us when retrying - // becomes safe again. + // failure shape. return CODEX_REMOTE_COMPACTION_RE.test(haystack) || /high\s+demand|temporary\s+errors/i.test(haystack); } + +export function isCodexProviderQuotaError(input: { + stdout?: string | null; + stderr?: string | null; + errorMessage?: string | null; +}): boolean { + const haystack = buildCodexErrorHaystack(input); + return CODEX_PROVIDER_QUOTA_RE.test(haystack) || extractCodexRetryNotBefore(input) != null; +} diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.ts index 978aa4a5ed..cf100e91c9 100644 --- a/packages/shared/src/constants.ts +++ b/packages/shared/src/constants.ts @@ -768,6 +768,7 @@ export const HEARTBEAT_RUN_STATUSES = [ "scheduled_retry", "running", "succeeded", + "interrupted", "failed", "cancelled", "timed_out", diff --git a/packages/shared/src/types/dashboard.ts b/packages/shared/src/types/dashboard.ts index e4225f4394..977b124ce1 100644 --- a/packages/shared/src/types/dashboard.ts +++ b/packages/shared/src/types/dashboard.ts @@ -1,9 +1,27 @@ export interface DashboardRunActivityDay { date: string; succeeded: number; + /** + * True failures for the day, excluding process-loss/restart kills that were + * later recovered by a successful retry (those are surfaced in `recovered`). + */ failed: number; + /** + * Runs that terminated in a failure state (failed/timed_out) but whose retry + * chain eventually succeeded — e.g. restart-killed runs that recovered. Kept + * out of `failed` so the headline failure count reflects true, unrecovered + * failures. + */ + recovered: number; other: number; total: number; + /** + * Per-error-code breakdown of the (true) `failed` count for the day, so a + * spike can be attributed to an error class (e.g. `process_lost`, + * `provider_quota`, `workspace_validation_failed`). Recovered runs are not + * included here. Runs with no error code are bucketed under `unknown`. + */ + failedByErrorCode: Record; } export interface DashboardSummary { diff --git a/server/src/__tests__/claude-local-execute.test.ts b/server/src/__tests__/claude-local-execute.test.ts index 0316ca7c1f..bb212f7858 100644 --- a/server/src/__tests__/claude-local-execute.test.ts +++ b/server/src/__tests__/claude-local-execute.test.ts @@ -1272,7 +1272,7 @@ describe("claude execute", () => { } }, 15_000); - it("classifies Claude 'out of extra usage' failures as transient upstream errors", async () => { + it("classifies Claude 'out of extra usage' failures as provider quota errors", async () => { const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-claude-execute-transient-")); const workspace = path.join(root, "workspace"); const commandPath = path.join(root, "claude"); @@ -1320,11 +1320,12 @@ describe("claude execute", () => { }); expect(result.exitCode).toBe(1); - expect(result.errorCode).toBe("claude_transient_upstream"); - expect(result.errorFamily).toBe("transient_upstream"); + expect(result.errorCode).toBe("provider_quota"); + expect(result.errorFamily).toBe("provider_quota"); const expectedRetryNotBefore = "2026-04-22T21:00:00.000Z"; expect(result.retryNotBefore).toBe(expectedRetryNotBefore); expect(result.resultJson?.retryNotBefore).toBe(expectedRetryNotBefore); + expect(result.resultJson?.providerQuotaRetryNotBefore).toBe(expectedRetryNotBefore); expect(result.errorMessage ?? "").toContain("extra usage"); expect(new Date(String(result.resultJson?.transientRetryNotBefore)).getTime()).toBe( new Date("2026-04-22T21:00:00.000Z").getTime(), @@ -1337,6 +1338,63 @@ describe("claude execute", () => { } }); + it("treats subtype=success results as successful even when the process exits nonzero", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-claude-execute-success-subtype-")); + const workspace = path.join(root, "workspace"); + const commandPath = path.join(root, "claude"); + await fs.mkdir(workspace, { recursive: true }); + await writeFailingClaudeCommand(commandPath, { + exitCode: 1, + resultEvent: { + type: "result", + subtype: "success", + session_id: "ffffffff-ffff-4fff-8fff-ffffffffffff", + is_error: false, + result: "Implemented the requested change.", + usage: { input_tokens: 1, cache_read_input_tokens: 0, output_tokens: 1 }, + }, + }); + + const previousHome = process.env.HOME; + process.env.HOME = root; + + try { + const result = await execute({ + runId: "run-claude-success-subtype", + agent: { + id: "agent-1", + companyId: "company-1", + name: "Claude Coder", + adapterType: "claude_local", + adapterConfig: {}, + }, + runtime: { + sessionId: null, + sessionParams: null, + sessionDisplayId: null, + taskKey: null, + }, + config: { + command: commandPath, + cwd: workspace, + promptTemplate: "Follow the paperclip heartbeat.", + }, + context: {}, + authToken: "run-jwt-token", + onLog: async () => {}, + }); + + expect(result.exitCode).toBe(1); + expect(result.errorMessage).toBeNull(); + expect(result.errorCode).toBeNull(); + expect(result.summary).toBe("Implemented the requested change."); + } finally { + if (previousHome === undefined) delete process.env.HOME; + else process.env.HOME = previousHome; + await fs.rm(root, { recursive: true, force: true }); + } + }); + it("classifies rate-limit / overloaded failures without reset metadata as transient", async () => { const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-claude-execute-rate-limit-")); const workspace = path.join(root, "workspace"); diff --git a/server/src/__tests__/codex-local-execute.test.ts b/server/src/__tests__/codex-local-execute.test.ts index a4aff6c87b..d8b1c5b104 100644 --- a/server/src/__tests__/codex-local-execute.test.ts +++ b/server/src/__tests__/codex-local-execute.test.ts @@ -554,7 +554,7 @@ describe("codex execute", () => { } }); - it("persists retry-not-before metadata for codex usage-limit failures", async () => { + it("persists retry-not-before metadata for codex provider quota failures", async () => { const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-codex-execute-usage-limit-")); const workspace = path.join(root, "workspace"); const commandPath = path.join(root, "codex"); @@ -601,11 +601,12 @@ describe("codex execute", () => { }); expect(result.exitCode).toBe(1); - expect(result.errorCode).toBe("codex_transient_upstream"); - expect(result.errorFamily).toBe("transient_upstream"); + expect(result.errorCode).toBe("provider_quota"); + expect(result.errorFamily).toBe("provider_quota"); const expectedRetryNotBefore = new Date(2026, 3, 22, 23, 31, 0, 0).toISOString(); expect(result.retryNotBefore).toBe(expectedRetryNotBefore); expect(result.resultJson?.retryNotBefore).toBe(expectedRetryNotBefore); + expect(result.resultJson?.providerQuotaRetryNotBefore).toBe(expectedRetryNotBefore); expect(new Date(String(result.resultJson?.transientRetryNotBefore)).getTime()).toBe( new Date(2026, 3, 22, 23, 31, 0, 0).getTime(), ); diff --git a/server/src/__tests__/dashboard-service.test.ts b/server/src/__tests__/dashboard-service.test.ts index 119f88964b..05a1a738c9 100644 --- a/server/src/__tests__/dashboard-service.test.ts +++ b/server/src/__tests__/dashboard-service.test.ts @@ -156,14 +156,85 @@ describeEmbeddedPostgres("dashboard service", () => { expect(todayBucket).toMatchObject({ succeeded: 105, failed: 0, + recovered: 0, other: 0, total: 105, + failedByErrorCode: {}, }); expect(weekAgoBucket).toMatchObject({ succeeded: 0, failed: 2, + recovered: 0, other: 1, total: 3, + // failed + timed_out with no error code both bucket under "unknown" + failedByErrorCode: { unknown: 2 }, }); }); + + it("separates recovered restart kills from true failures and breaks failures down by error code", async () => { + const companyId = randomUUID(); + const agentId = randomUUID(); + const day = utcDay(-2); + + await db.insert(companies).values({ + id: companyId, + name: "Paperclip", + issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`, + requireBoardApprovalForNewAgents: false, + }); + + await db.insert(agents).values({ + id: agentId, + companyId, + name: "CodexCoder", + role: "engineer", + status: "running", + adapterType: "codex_local", + adapterConfig: {}, + runtimeConfig: {}, + permissions: {}, + }); + + const base = { + companyId, + agentId, + invocationSource: "assignment", + createdAt: day, + }; + + // Direct recovery: a process-loss kill whose retry succeeded. + const original = randomUUID(); + const retry = randomUUID(); + // Chained recovery: kill -> failed retry -> succeeded retry (both kills recovered). + const chainedOriginal = randomUUID(); + const chainedRetry = randomUUID(); + const chainedRetrySuccess = randomUUID(); + // A genuine, unrecovered failure that should remain in the failed count. + const trueFailure = randomUUID(); + + await db.insert(heartbeatRuns).values([ + { ...base, id: original, status: "failed", errorCode: "process_lost" }, + { ...base, id: retry, status: "succeeded", retryOfRunId: original }, + { ...base, id: chainedOriginal, status: "failed", errorCode: "process_lost" }, + { ...base, id: chainedRetry, status: "failed", errorCode: "process_lost", retryOfRunId: chainedOriginal }, + { ...base, id: chainedRetrySuccess, status: "succeeded", retryOfRunId: chainedRetry }, + { ...base, id: trueFailure, status: "failed", errorCode: "provider_quota" }, + ]); + + const summary = await dashboardService(db).summary(companyId); + const bucket = summary.runActivity.find((b) => b.date === utcDateKey(day)); + + expect(bucket).toMatchObject({ + succeeded: 2, + // original + chainedOriginal + chainedRetry all recovered via a later success + recovered: 3, + failed: 1, + other: 0, + total: 6, + failedByErrorCode: { provider_quota: 1 }, + }); + // process_lost kills that recovered must not leak into the failed breakdown. + expect(bucket?.failedByErrorCode.process_lost).toBeUndefined(); + }); }); diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index 22154cb9b3..d0af367378 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -1293,6 +1293,213 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { expect(checkoutReleasedIssue?.checkoutRunId).toBeNull(); }); + it("interrupts running runs on graceful shutdown and queues restart recovery without recording a failure", async () => { + const { agentId, runId, issueId, wakeupRequestId } = await seedRunFixture({ + agentStatus: "running", + contextSnapshot: { + modelProfile: "cheap", + allowDeliverableWork: false, + allowDocumentUpdates: false, + resumeRequiresNormalModel: true, + }, + }); + const heartbeat = heartbeatService(db); + + const result = await heartbeat.drainRunningRunsForShutdown( + "SIGTERM", + new Date("2026-03-19T00:06:00.000Z"), + ); + expect(result.interrupted).toBe(1); + expect(result.interruptedRunIds).toEqual([runId]); + + const runs = await db + .select() + .from(heartbeatRuns) + .where(eq(heartbeatRuns.agentId, agentId)); + expect(runs).toHaveLength(2); + + const interruptedRun = runs.find((row) => row.id === runId); + const retryRun = runs.find((row) => row.retryOfRunId === runId); + expect(interruptedRun).toMatchObject({ + status: "interrupted", + errorCode: "server_shutdown_interrupted", + signal: "SIGTERM", + livenessState: "needs_followup", + }); + expect(interruptedRun?.resultJson).toMatchObject({ + stopReason: "interrupted", + timeoutConfigured: false, + timeoutFired: false, + }); + expect(retryRun).toMatchObject({ + status: "queued", + retryOfRunId: runId, + processLossRetryCount: 1, + }); + expect(retryRun?.contextSnapshot as Record).toMatchObject({ + retryReason: "process_lost", + retryOfRunId: runId, + }); + expect(retryRun?.contextSnapshot as Record).not.toHaveProperty("modelProfile"); + + const wakeup = await db + .select() + .from(agentWakeupRequests) + .where(eq(agentWakeupRequests.id, wakeupRequestId)) + .then((rows) => rows[0] ?? null); + expect(wakeup?.status).toBe("cancelled"); + + const issue = await db + .select() + .from(issues) + .where(eq(issues.id, issueId)) + .then((rows) => rows[0] ?? null); + expect(issue?.checkoutRunId).toBeNull(); + expect(issue?.executionRunId).toBe(retryRun?.id); + }); + + it("does not overwrite a run that is no longer running during graceful shutdown drain", async () => { + const { runId, wakeupRequestId } = await seedRunFixture({ + agentStatus: "running", + }); + const heartbeat = heartbeatService(db); + + await db + .update(heartbeatRuns) + .set({ + status: "succeeded", + finishedAt: new Date("2026-03-19T00:05:30.000Z"), + updatedAt: new Date("2026-03-19T00:05:30.000Z"), + }) + .where(eq(heartbeatRuns.id, runId)); + + const result = await heartbeat.drainRunningRunsForShutdown( + "SIGTERM", + new Date("2026-03-19T00:06:00.000Z"), + ); + + expect(result).toMatchObject({ + interrupted: 0, + interruptedRunIds: [], + retryRunIds: [], + }); + const run = await db + .select() + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, runId)) + .then((rows) => rows[0] ?? null); + expect(run).toMatchObject({ + status: "succeeded", + errorCode: null, + signal: null, + }); + const wakeup = await db + .select() + .from(agentWakeupRequests) + .where(eq(agentWakeupRequests.id, wakeupRequestId)) + .then((rows) => rows[0] ?? null); + expect(wakeup?.status).toBe("claimed"); + }); + + it("does not enqueue duplicate restart recovery for the same interrupted run", async () => { + const { agentId, runId, issueId, wakeupRequestId } = await seedRunFixture({ + agentStatus: "running", + }); + const heartbeat = heartbeatService(db); + + await heartbeat.drainRunningRunsForShutdown("SIGTERM", new Date("2026-03-19T00:06:00.000Z")); + const firstRetry = await db + .select() + .from(heartbeatRuns) + .where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.retryOfRunId, runId))) + .then((rows) => rows[0] ?? null); + expect(firstRetry?.id).toBeTruthy(); + + await db + .update(heartbeatRuns) + .set({ status: "running", finishedAt: null, updatedAt: new Date("2026-03-19T00:07:00.000Z") }) + .where(eq(heartbeatRuns.id, runId)); + await db + .update(agentWakeupRequests) + .set({ status: "claimed", finishedAt: null, updatedAt: new Date("2026-03-19T00:07:00.000Z") }) + .where(eq(agentWakeupRequests.id, wakeupRequestId)); + await db + .update(issues) + .set({ checkoutRunId: runId, executionRunId: runId, updatedAt: new Date("2026-03-19T00:07:00.000Z") }) + .where(eq(issues.id, issueId)); + + const secondDrain = await heartbeat.drainRunningRunsForShutdown( + "SIGTERM", + new Date("2026-03-19T00:08:00.000Z"), + ); + expect(secondDrain.retryRunIds).toEqual([firstRetry?.id]); + + const retryRuns = await db + .select() + .from(heartbeatRuns) + .where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.retryOfRunId, runId))); + expect(retryRuns).toHaveLength(1); + expect(retryRuns[0]?.id).toBe(firstRetry?.id); + }); + + it("chains a single retry when restart recovery is interrupted by a second graceful shutdown", async () => { + const { agentId, runId, issueId } = await seedRunFixture({ + agentStatus: "running", + }); + const heartbeat = heartbeatService(db); + + await heartbeat.drainRunningRunsForShutdown("SIGTERM", new Date("2026-03-19T00:06:00.000Z")); + const firstRetry = await db + .select() + .from(heartbeatRuns) + .where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.retryOfRunId, runId))) + .then((rows) => rows[0] ?? null); + expect(firstRetry?.id).toBeTruthy(); + + await db + .update(heartbeatRuns) + .set({ status: "running", startedAt: new Date("2026-03-19T00:07:00.000Z"), updatedAt: new Date("2026-03-19T00:07:00.000Z") }) + .where(eq(heartbeatRuns.id, firstRetry!.id)); + await db + .update(agentWakeupRequests) + .set({ status: "claimed", claimedAt: new Date("2026-03-19T00:07:00.000Z"), updatedAt: new Date("2026-03-19T00:07:00.000Z") }) + .where(eq(agentWakeupRequests.id, firstRetry!.wakeupRequestId)); + await db + .update(issues) + .set({ checkoutRunId: firstRetry!.id, executionRunId: firstRetry!.id, updatedAt: new Date("2026-03-19T00:07:00.000Z") }) + .where(eq(issues.id, issueId)); + + const secondDrain = await heartbeat.drainRunningRunsForShutdown( + "SIGTERM", + new Date("2026-03-19T00:08:00.000Z"), + ); + expect(secondDrain.interruptedRunIds).toEqual([firstRetry!.id]); + + const runs = await db + .select() + .from(heartbeatRuns) + .where(eq(heartbeatRuns.agentId, agentId)); + expect(runs).toHaveLength(3); + expect(runs.find((row) => row.id === runId)?.status).toBe("interrupted"); + expect(runs.find((row) => row.id === firstRetry!.id)?.status).toBe("interrupted"); + + const originalRetries = runs.filter((row) => row.retryOfRunId === runId); + expect(originalRetries).toHaveLength(1); + const secondRetry = runs.find((row) => row.retryOfRunId === firstRetry!.id); + expect(secondRetry).toMatchObject({ + status: "queued", + processLossRetryCount: 2, + }); + + const issue = await db + .select() + .from(issues) + .where(eq(issues.id, issueId)) + .then((rows) => rows[0] ?? null); + expect(issue?.checkoutRunId).toBeNull(); + expect(issue?.executionRunId).toBe(secondRetry?.id); + }); + it("releases active environment leases when an orphaned run is reaped", async () => { const { runId, issueId, companyId } = await seedRunFixture({ processPid: 999_999_999, diff --git a/server/src/__tests__/heartbeat-retry-scheduling.test.ts b/server/src/__tests__/heartbeat-retry-scheduling.test.ts index 4e59091057..80b05e5ef4 100644 --- a/server/src/__tests__/heartbeat-retry-scheduling.test.ts +++ b/server/src/__tests__/heartbeat-retry-scheduling.test.ts @@ -5,8 +5,10 @@ import { agents, agentRuntimeState, agentWakeupRequests, + activityLog, budgetPolicies, companies, + companySkills, createDb, environmentLeases, heartbeatRunEvents, @@ -18,6 +20,7 @@ import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; +import { registerServerAdapter, unregisterServerAdapter } from "../adapters/index.ts"; import { BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS, MAX_TURN_CONTINUATION_RETRY_REASON, @@ -27,6 +30,7 @@ import { const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; +const PROVIDER_QUOTA_TEST_ADAPTER = "provider_quota_test"; if (!embeddedPostgresSupport.supported) { console.warn( @@ -34,6 +38,20 @@ if (!embeddedPostgresSupport.supported) { ); } +async function waitForRunToFinish( + heartbeat: ReturnType, + runId: string, + timeoutMs = 5_000, +) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const run = await heartbeat.getRun(runId); + if (run && !["queued", "running"].includes(run.status)) return run; + await new Promise((resolve) => setTimeout(resolve, 50)); + } + return await heartbeat.getRun(runId); +} + describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => { let db!: ReturnType; let heartbeat!: ReturnType; @@ -43,22 +61,49 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => { tempDb = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-retry-scheduling-"); db = createDb(tempDb.connectionString); heartbeat = heartbeatService(db); + registerServerAdapter({ + type: PROVIDER_QUOTA_TEST_ADAPTER, + execute: async () => ({ + exitCode: 1, + signal: null, + timedOut: false, + errorMessage: "You've hit your session limit - resets at 4pm (America/Chicago).", + errorCode: "provider_quota", + errorFamily: "provider_quota", + retryNotBefore: "2030-04-22T21:00:00.000Z", + resultJson: { + errorFamily: "provider_quota", + retryNotBefore: "2030-04-22T21:00:00.000Z", + providerQuotaRetryNotBefore: "2030-04-22T21:00:00.000Z", + }, + }), + testEnvironment: async () => ({ + adapterType: PROVIDER_QUOTA_TEST_ADAPTER, + status: "pass", + checks: [], + testedAt: new Date().toISOString(), + }), + }); }, 20_000); afterEach(async () => { + await db.delete(activityLog); await db.delete(heartbeatRunEvents); await db.delete(environmentLeases); await db.delete(issueRelations); await db.delete(issues); + await db.delete(activityLog); await db.delete(heartbeatRuns); await db.delete(agentWakeupRequests); await db.delete(agentRuntimeState); await db.delete(budgetPolicies); await db.delete(agents); + await db.delete(companySkills); await db.delete(companies); }); afterAll(async () => { + unregisterServerAdapter(PROVIDER_QUOTA_TEST_ADAPTER); await tempDb?.cleanup(); }); @@ -68,11 +113,11 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => { agentId: string; now: Date; errorCode: string; - errorFamily?: "transient_upstream" | null; + errorFamily?: "transient_upstream" | "provider_quota" | null; retryNotBefore?: string | null; scheduledRetryAttempt?: number; resultJson?: Record | null; - adapterType?: "codex_local" | "claude_local"; + adapterType?: string; agentName?: string; }) { const adapterType = input.adapterType ?? "codex_local"; @@ -131,6 +176,88 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => { }); } + it("records provider quota failures, schedules the reset-time retry, and leaves the agent idle", async () => { + const companyId = randomUUID(); + const agentId = randomUUID(); + + await db.insert(companies).values({ + id: companyId, + name: "Paperclip", + issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`, + requireBoardApprovalForNewAgents: false, + defaultResponsibleUserId: "responsible-user", + }); + + await db.insert(agents).values({ + id: agentId, + companyId, + name: "Quota Test", + role: "engineer", + status: "idle", + adapterType: PROVIDER_QUOTA_TEST_ADAPTER, + adapterConfig: {}, + runtimeConfig: { + heartbeat: { + wakeOnDemand: true, + maxConcurrentRuns: 1, + }, + }, + permissions: {}, + }); + + const run = await heartbeat.invoke(agentId, "on_demand", {}, "manual"); + expect(run).not.toBeNull(); + + const failedRun = await waitForRunToFinish(heartbeat, run!.id); + expect(failedRun?.status).toBe("failed"); + expect(failedRun?.errorCode).toBe("provider_quota"); + expect((failedRun?.resultJson as Record | null)?.errorFamily).toBe("provider_quota"); + + await expect + .poll( + () => + db + .select({ id: heartbeatRuns.id }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.retryOfRunId, run!.id)) + .then((rows) => rows.length), + { timeout: 5_000, interval: 50 }, + ) + .toBe(1); + + const retryRun = await db + .select({ + id: heartbeatRuns.id, + status: heartbeatRuns.status, + scheduledRetryAt: heartbeatRuns.scheduledRetryAt, + scheduledRetryReason: heartbeatRuns.scheduledRetryReason, + contextSnapshot: heartbeatRuns.contextSnapshot, + }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.retryOfRunId, run!.id)) + .then((rows) => rows[0] ?? null); + expect(retryRun?.status).toBe("scheduled_retry"); + expect(retryRun?.scheduledRetryReason).toBe("transient_failure"); + expect(retryRun?.scheduledRetryAt?.toISOString()).toBe("2030-04-22T21:00:00.000Z"); + expect((retryRun?.contextSnapshot as Record | null)?.errorFamily).toBe("provider_quota"); + expect((retryRun?.contextSnapshot as Record | null)?.providerQuotaRetryNotBefore).toBe( + "2030-04-22T21:00:00.000Z", + ); + expect((retryRun?.contextSnapshot as Record | null)?.codexTransientFallbackMode ?? null).toBeNull(); + + await expect + .poll( + () => + db + .select({ status: agents.status, errorReason: agents.errorReason }) + .from(agents) + .where(eq(agents.id, agentId)) + .then((rows) => rows[0] ?? null), + { timeout: 5_000, interval: 50 }, + ) + .toEqual({ status: "idle", errorReason: null }); + }); + async function seedMaxTurnFixture(input?: { companyId?: string; agentId?: string; @@ -684,6 +811,7 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => { await db.delete(agentWakeupRequests); await db.delete(agentRuntimeState); await db.delete(agents); + await db.delete(companySkills); await db.delete(companies); const dependencyBlocked = await seedMaxTurnFixture({ now: new Date("2026-04-20T17:00:00.000Z") }); @@ -1350,6 +1478,7 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => { await db.delete(heartbeatRuns); await db.delete(agentWakeupRequests); await db.delete(agents); + await db.delete(companySkills); await db.delete(companies); } }); diff --git a/server/src/__tests__/heartbeat-workspace-branch-containment.test.ts b/server/src/__tests__/heartbeat-workspace-branch-containment.test.ts index c49152d254..1e23f826da 100644 --- a/server/src/__tests__/heartbeat-workspace-branch-containment.test.ts +++ b/server/src/__tests__/heartbeat-workspace-branch-containment.test.ts @@ -139,10 +139,17 @@ async function createForwardBranchMismatch(input: { worktreePath: string; expectedBranch: string; actualBranch: string; + divergeRecordedBranch?: boolean; }) { await mkdir(path.dirname(input.worktreePath), { recursive: true }); await runGit(input.repoRoot, ["branch", input.expectedBranch]); await runGit(input.repoRoot, ["worktree", "add", "-b", input.actualBranch, input.worktreePath, input.expectedBranch]); + if (input.divergeRecordedBranch) { + await runGit(input.repoRoot, ["checkout", input.expectedBranch]); + await writeFile(path.join(input.repoRoot, "recorded-branch.txt"), "recorded branch work\n", "utf8"); + await runGit(input.repoRoot, ["add", "recorded-branch.txt"]); + await runGit(input.repoRoot, ["commit", "-m", "Add recorded branch work"]); + } await writeFile(path.join(input.worktreePath, "actual-branch.txt"), "actual branch work\n", "utf8"); await runGit(input.worktreePath, ["add", "actual-branch.txt"]); await runGit(input.worktreePath, ["commit", "-m", "Add actual branch work"]); @@ -362,6 +369,7 @@ async function seedBranchContainmentRun( worktreePath, expectedBranch, actualBranch, + divergeRecordedBranch: opts.enableWorkspaceBranchReconcileForward !== true, }); } @@ -594,12 +602,12 @@ async function expectContainedWorkspaceBranchFailure(input: { expectedBranchExists: true, actualBranchExists: true, sameHead: false, - ancestryVerdict: "ancestor", + ancestryVerdict: "diverged", }); expect(provenance.expectedHeadSha).toEqual(expect.stringMatching(/^[a-f0-9]{40}$/)); expect(provenance.actualHeadSha).toEqual(expect.stringMatching(/^[a-f0-9]{40}$/)); expect(provenance.expectedHeadSha).not.toBe(provenance.actualHeadSha); - expect(provenance.plainLanguageReason).toEqual(expect.stringContaining("forward of the recorded branch")); + expect(provenance.plainLanguageReason).toEqual(expect.stringContaining("cannot prove a forward-only reconciliation")); const { issueRows, actionRows, comments } = await waitForContainmentSideEffects({ db: input.db, @@ -647,7 +655,7 @@ async function expectContainedWorkspaceBranchFailure(input: { provenance: expect.objectContaining({ expectedHeadSha: provenance.expectedHeadSha, actualHeadSha: provenance.actualHeadSha, - ancestryVerdict: "ancestor", + ancestryVerdict: "diverged", plainLanguageReason: provenance.plainLanguageReason, }), }), @@ -710,9 +718,10 @@ async function expectForwardBranchReconciled(input: { }) .from(executionWorkspaces) .where(eq(executionWorkspaces.id, activeWorkspaceId)); + const expectedDurableBranch = input.expectsExistingRecordUpdate ? input.actualBranch : input.expectedBranch; expect(activeWorkspace).toMatchObject({ - name: input.actualBranch, - branchName: input.actualBranch, + name: expectedDurableBranch, + branchName: expectedDurableBranch, providerRef: input.worktreePath, }); @@ -738,19 +747,21 @@ async function expectForwardBranchReconciled(input: { .select() .from(workspaceOperations) .where(eq(workspaceOperations.heartbeatRunId, input.runId)); - expect(operations).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - status: "succeeded", - metadata: expect.objectContaining({ - branchIncoherenceReconcileForward: true, - expectedBranchName: input.expectedBranch, - actualBranchName: input.actualBranch, - fingerprint: expect.stringMatching(/^workspace_incoherence:v1:sha256:/), + if (input.expectsExistingRecordUpdate) { + expect(operations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + status: "succeeded", + metadata: expect.objectContaining({ + branchIncoherenceReconcileForward: true, + expectedBranchName: input.expectedBranch, + actualBranchName: input.actualBranch, + fingerprint: expect.stringMatching(/^workspace_incoherence:v1:sha256:/), + }), }), - }), - ]), - ); + ]), + ); + } if (input.expectsExistingRecordUpdate) { const [updatedWorkspace] = await input.db @@ -881,7 +892,6 @@ describeEmbeddedPostgres("heartbeat workspace branch containment", () => { it.each([ ["workspace-runtime fresh worktree reuse", "fresh_realize" as const, null], ["workspace-runtime persisted restore", "persisted_restore" as const, "source-workspace"], - ["heartbeat finalization", "finalize" as const, "runtime-workspace"], ])("contains mid-change branch divergence at %s", async (_name, callSite, expectedWorkspaceId) => { const repoRoot = await createGitRepo(); tempRoots.push(repoRoot); @@ -939,9 +949,9 @@ describeEmbeddedPostgres("heartbeat workspace branch containment", () => { }, 30_000); it.each([ - ["workspace-runtime fresh worktree reuse", "fresh_realize" as const, true], - ["workspace-runtime persisted restore", "persisted_restore" as const, true], - ["heartbeat finalization", "finalize" as const, true], + ["workspace-runtime fresh worktree reuse", "fresh_realize" as const, false], + ["workspace-runtime persisted restore", "persisted_restore" as const, false], + ["heartbeat finalization", "finalize" as const, false], ])("auto-reconciles forward branch divergence at %s when the flag is enabled", async (_name, callSite, expectsExistingRecordUpdate) => { const repoRoot = await createGitRepo(); tempRoots.push(repoRoot); @@ -954,7 +964,7 @@ describeEmbeddedPostgres("heartbeat workspace branch containment", () => { status: callSite === "finalize" ? "" : await readGit(seeded.worktreePath, ["status", "--porcelain", "--untracked-files=all"]), }; let expectedResolvedRecoveryActionFingerprint: string | null = null; - if (callSite === "fresh_realize") { + if (callSite === "fresh_realize" && expectsExistingRecordUpdate) { const expectedHeadSha = await readGit(seeded.worktreePath, ["rev-parse", seeded.expectedBranch]); const actualHeadSha = await readGit(seeded.worktreePath, ["rev-parse", seeded.actualBranch]); expectedResolvedRecoveryActionFingerprint = fingerprintWorkspaceBranchIncoherenceForTest({ diff --git a/server/src/__tests__/heartbeat-workspace-finalize-branch.test.ts b/server/src/__tests__/heartbeat-workspace-finalize-branch.test.ts index fb30b06cfd..b4428786a5 100644 --- a/server/src/__tests__/heartbeat-workspace-finalize-branch.test.ts +++ b/server/src/__tests__/heartbeat-workspace-finalize-branch.test.ts @@ -387,7 +387,7 @@ describeEmbeddedPostgres("heartbeat workspace finalization branch guard", () => }); }, 20_000); - it("fails unrecorded branch drift when the checked-out branch has different commits", async () => { + it("adopts unrecorded forward branch drift for finalization without persisting it", async () => { const repoRoot = await createGitRepo(); tempRoots.push(repoRoot); const { agentId, issueId } = await seedRunTarget(db, repoRoot); @@ -420,51 +420,40 @@ describeEmbeddedPostgres("heartbeat workspace finalization branch guard", () => const finishedRun = await waitForRunToFinish(heartbeat, run!.id); expect(finishedRun).toMatchObject({ - status: "failed", - errorCode: "workspace_validation_failed", - }); - const workspaceValidation = (finishedRun?.resultJson as Record | null)?.workspaceValidation; - expect(workspaceValidation).toMatchObject({ - reason: "git_worktree_branch_incoherence", - sourceIssueId: issueId, - executionWorkspaceId, - expectedBranch: recordedBranch, - actualBranch: publishBranch, - cleanliness: "clean", - provenance: expect.objectContaining({ - expectedBranchExists: true, - actualBranchExists: true, - sameHead: false, - }), - safeRepair: expect.objectContaining({ - eligible: false, - attempted: false, - succeeded: false, - reason: "expected branch and current HEAD differ", - }), + status: "succeeded", + errorCode: null, + error: null, }); await waitForRuntimeStateLastRun(db, agentId, run!.id); expect(adapterExecute).toHaveBeenCalledTimes(1); + const finalizedWorkspace = await db + .select({ branchName: executionWorkspaces.branchName }) + .from(executionWorkspaces) + .where(eq(executionWorkspaces.id, executionWorkspaceId!)) + .then((rows) => rows[0] ?? null); + expect(finalizedWorkspace?.branchName).toBe(recordedBranch); + const finalizeOps = await listFinalizeOperations(db, run!.id); expect(finalizeOps).toHaveLength(1); expect(finalizeOps[0]).toMatchObject({ - status: "failed", + status: "succeeded", executionWorkspaceId, - stderrExcerpt: expect.stringContaining("Managed git worktree branch check failed"), }); expect(finalizeOps[0]?.metadata).toMatchObject({ managedGitWorktreeBranch: expect.objectContaining({ executionWorkspaceId, - valid: false, - reasonCode: "branch_mismatch", - expectedBranchName: recordedBranch, + valid: true, + reasonCode: null, + expectedBranchName: publishBranch, actualBranchName: publishBranch, }), - workspaceValidation: expect.objectContaining({ - reason: "git_worktree_branch_incoherence", + managedGitWorktreeBranchRepair: expect.objectContaining({ + attempted: true, + succeeded: true, }), }); + expect(recordedBranch).not.toBe(publishBranch); }, 20_000); it("allows a successful adapter run when the branch transition is recorded before finalization", async () => { diff --git a/server/src/__tests__/heartbeat-workspace-session.test.ts b/server/src/__tests__/heartbeat-workspace-session.test.ts index 3b505cb12d..cde79bf787 100644 --- a/server/src/__tests__/heartbeat-workspace-session.test.ts +++ b/server/src/__tests__/heartbeat-workspace-session.test.ts @@ -1334,7 +1334,7 @@ describe("effective run execution workspace config freshness", () => { }); }); - it("fails explicit reuse restore errors without realizing a fallback workspace", async () => { + it("fails loudly when explicit reuse restore errors", async () => { const base = buildWorkspaceConfigMetadata(); const next = buildWorkspaceConfigMetadata({ repoRef: "origin/release", @@ -1350,7 +1350,7 @@ describe("effective run execution workspace config freshness", () => { existingWorkspaceMetadata: persistedWorkspaceConfigFingerprint(base), nextMetadata: next, }); - const realizeWorkspace = vi.fn(async () => ({ id: "fallback-workspace" })); + const realizeWorkspace = vi.fn(async () => ({ id: "fallback-workspace", warnings: [] })); await expect(provisionExecutionWorkspaceForFreshnessDecision({ requestedShouldReuseExisting: true, @@ -1362,28 +1362,14 @@ describe("effective run execution workspace config freshness", () => { throw new Error("restore command failed"); }, realizeWorkspace, - })).rejects.toMatchObject({ - code: "workspace_validation_failed", - resultJson: { - workspaceValidation: expect.objectContaining({ - reason: "inherited_workspace_reuse_failed", - issueId: "issue-1", - issueIdentifier: "PAP-42", - executionWorkspaceId: "workspace-old", - workspaceConfigFreshnessAction: "replace", - requestedReuseExisting: true, - replacementWorkspaceRealized: false, - remediation: expect.stringContaining("restore/provision logs"), - }), - }, - }); + })).rejects.toThrow(/restore command failed/); expect(realizeWorkspace).not.toHaveBeenCalled(); }); it.each([ { name: "missing", status: null }, { name: "archived", status: "archived" }, - ])("fails explicit reuse when the inherited workspace row is $name", async ({ status }) => { + ])("fails loudly when the inherited workspace row is $name", async ({ status }) => { const reuseRequest = resolveExecutionWorkspaceReuseRequestForIssue({ issueExecutionWorkspaceId: "workspace-old", issueExecutionWorkspacePreference: "reuse_existing", @@ -1403,7 +1389,7 @@ describe("effective run execution workspace config freshness", () => { existingWorkspaceMetadata: null, nextMetadata: metadata, }); - const realizeWorkspace = vi.fn(async () => ({ id: "fallback-workspace" })); + const realizeWorkspace = vi.fn(async () => ({ id: "fallback-workspace", warnings: [] })); await expect(provisionExecutionWorkspaceForFreshnessDecision({ requestedShouldReuseExisting: reuseRequest.requestedShouldReuseExisting, @@ -1412,35 +1398,21 @@ describe("effective run execution workspace config freshness", () => { runId: "run-1", workspaceConfigFreshness: decision, restoreExistingWorkspace: reuseRequest.existingExecutionWorkspaceAvailable - ? async () => ({ id: "workspace-old" }) + ? async () => ({ id: "workspace-old", warnings: [] }) : null, realizeWorkspace, - })).rejects.toMatchObject({ - code: "workspace_validation_failed", - resultJson: { - workspaceValidation: expect.objectContaining({ - reason: "inherited_workspace_reuse_unavailable", - issueId: "issue-1", - issueIdentifier: "PAP-42", - executionWorkspaceId: "workspace-old", - workspaceConfigFreshnessAction: "create", - requestedReuseExisting: true, - replacementWorkspaceRealized: false, - remediation: expect.stringContaining("clear the issue's reuse_existing workspace binding"), - }), - }, - }); + })).rejects.toThrow(/could not be restored/); expect(realizeWorkspace).not.toHaveBeenCalled(); }); - it("fails explicit reuse restore misses without realizing a fallback workspace", async () => { + it("fails loudly when explicit reuse restore returns no workspace", async () => { const metadata = buildWorkspaceConfigMetadata(); const decision = resolveExecutionWorkspaceConfigFreshness({ hasExistingWorkspace: true, existingWorkspaceMetadata: persistedWorkspaceConfigFingerprint(metadata), nextMetadata: metadata, }); - const realizeWorkspace = vi.fn(async () => ({ id: "fallback-workspace" })); + const realizeWorkspace = vi.fn(async () => ({ id: "fallback-workspace", warnings: [] })); await expect(provisionExecutionWorkspaceForFreshnessDecision({ requestedShouldReuseExisting: true, @@ -1450,18 +1422,7 @@ describe("effective run execution workspace config freshness", () => { workspaceConfigFreshness: decision, restoreExistingWorkspace: async () => null, realizeWorkspace, - })).rejects.toMatchObject({ - code: "workspace_validation_failed", - resultJson: { - workspaceValidation: expect.objectContaining({ - reason: "inherited_workspace_reuse_unavailable", - workspaceConfigFreshnessAction: "reuse", - requestedReuseExisting: true, - replacementWorkspaceRealized: false, - remediation: expect.stringContaining("clear the issue's reuse_existing workspace binding"), - }), - }, - }); + })).rejects.toThrow(/could not be restored/); expect(realizeWorkspace).not.toHaveBeenCalled(); }); diff --git a/server/src/__tests__/workspace-runtime.test.ts b/server/src/__tests__/workspace-runtime.test.ts index bccd6c1e36..61c612bfdc 100644 --- a/server/src/__tests__/workspace-runtime.test.ts +++ b/server/src/__tests__/workspace-runtime.test.ts @@ -2337,6 +2337,60 @@ describe("realizeExecutionWorkspace", () => { ); }, 15_000); + it("reattaches a clean forward detached HEAD to the recorded persisted git worktree branch", async () => { + const repoRoot = await createTempRepo(); + const branchName = "PAP-454-reattach-detached-head"; + const worktreePath = path.join(repoRoot, ".paperclip", "worktrees", branchName); + await fs.mkdir(path.dirname(worktreePath), { recursive: true }); + await runGit(repoRoot, ["branch", branchName]); + await runGit(repoRoot, ["worktree", "add", worktreePath, branchName]); + await runGit(worktreePath, ["checkout", "--detach"]); + await fs.writeFile(path.join(worktreePath, "detached.txt"), "detached work\n", "utf8"); + await runGit(worktreePath, ["add", "detached.txt"]); + await runGit(worktreePath, ["commit", "-m", "Add detached work"]); + const detachedHead = await readGit(worktreePath, ["rev-parse", "HEAD"]); + + const restored = await ensurePersistedExecutionWorkspaceAvailable({ + base: { + baseCwd: repoRoot, + source: "project_primary", + projectId: "project-1", + workspaceId: "workspace-1", + repoUrl: null, + repoRef: "HEAD", + }, + workspace: { + id: "execution-workspace-detached", + mode: "isolated_workspace", + strategyType: "git_worktree", + cwd: worktreePath, + providerRef: worktreePath, + projectId: "project-1", + projectWorkspaceId: "workspace-1", + repoUrl: null, + baseRef: "HEAD", + branchName, + }, + issue: { + id: "issue-detached", + identifier: "PAP-454", + title: "Repair detached branch mismatch", + }, + agent: { + id: "agent-1", + name: "Codex Coder", + companyId: "company-1", + }, + }); + + expect(restored?.branchName).toBe(branchName); + expect(restored?.warnings).toEqual(expect.arrayContaining([ + expect.stringContaining("moved the recorded branch to that HEAD"), + ])); + await expect(readGit(worktreePath, ["branch", "--show-current"])).resolves.toBe(branchName); + await expect(readGit(worktreePath, ["rev-parse", "HEAD"])).resolves.toBe(detachedHead); + }, 15_000); + it("rejects dirty persisted git worktree branch incoherence with bounded recovery evidence", async () => { const repoRoot = await createTempRepo(); const expectedBranch = "PAP-455-reject-dirty-branch-mismatch"; @@ -2460,7 +2514,7 @@ describe("realizeExecutionWorkspace", () => { }); }, 15_000); - it("rejects an existing persisted git worktree when the checked-out branch changed to a different commit", async () => { + it("adopts an existing persisted git worktree when the checked-out branch is forward of the recorded branch", async () => { const repoRoot = await createTempRepo(); const initial = await realizeExecutionWorkspace({ @@ -2497,85 +2551,44 @@ describe("realizeExecutionWorkspace", () => { await runGit(initial.cwd, ["commit", "-m", "Add publish branch work"]); if (!initial.branchName) throw new Error("expected realized worktree branch name"); - const expectedHeadSha = await readGit(repoRoot, ["rev-parse", `refs/heads/${initial.branchName}^{commit}`]); - const actualHeadSha = await readGit(initial.cwd, ["rev-parse", "HEAD"]); - const expectedFingerprint = workspaceBranchIncoherenceFingerprintForTest({ - sourceIssueId: "issue-3", - executionWorkspaceId: "execution-workspace-3", - worktreePath: initial.cwd, - expectedBranch: initial.branchName, - actualBranch, - cleanliness: "clean", - expectedHeadSha, - actualHeadSha, - }); - - let error: unknown = null; - try { - await ensurePersistedExecutionWorkspaceAvailable({ - base: { - baseCwd: repoRoot, - source: "project_primary", - projectId: "project-1", - workspaceId: "workspace-1", - repoUrl: null, - repoRef: "HEAD", - }, - workspace: { - id: "execution-workspace-3", - mode: "isolated_workspace", - strategyType: "git_worktree", - cwd: initial.cwd, - providerRef: initial.worktreePath, - projectId: "project-1", - projectWorkspaceId: "workspace-1", - repoUrl: null, - baseRef: "HEAD", - branchName: initial.branchName, - }, - issue: { - id: "issue-3", - identifier: "PAP-456", - title: "Keep persisted branch coherent", - }, - agent: { - id: "agent-1", - name: "Codex Coder", - companyId: "company-1", - }, - }); - } catch (err) { - error = err; - } - - expect(error).toMatchObject({ - code: "workspace_validation_failed", - resultJson: { - workspaceValidation: expect.objectContaining({ - reason: "git_worktree_branch_incoherence", - fingerprint: expectedFingerprint, - sourceIssueId: "issue-3", - sourceIdentifier: "PAP-456", - executionWorkspaceId: "execution-workspace-3", - expectedBranch: initial.branchName, - actualBranch, - cleanliness: "clean", - provenance: expect.objectContaining({ - expectedBranchExists: true, - actualBranchExists: true, - sameHead: false, - ancestryVerdict: "ancestor", - plainLanguageReason: expect.stringContaining("forward of the recorded branch"), - }), - safeRepair: expect.objectContaining({ - eligible: false, - attempted: false, - succeeded: false, - reason: "expected branch and current HEAD differ", - }), - }), + const restored = await ensurePersistedExecutionWorkspaceAvailable({ + base: { + baseCwd: repoRoot, + source: "project_primary", + projectId: "project-1", + workspaceId: "workspace-1", + repoUrl: null, + repoRef: "HEAD", + }, + workspace: { + id: "execution-workspace-3", + mode: "isolated_workspace", + strategyType: "git_worktree", + cwd: initial.cwd, + providerRef: initial.worktreePath, + projectId: "project-1", + projectWorkspaceId: "workspace-1", + repoUrl: null, + baseRef: "HEAD", + branchName: initial.branchName, + }, + issue: { + id: "issue-3", + identifier: "PAP-456", + title: "Keep persisted branch coherent", + }, + agent: { + id: "agent-1", + name: "Codex Coder", + companyId: "company-1", }, }); + + expect(restored?.branchName).toBe(actualBranch); + expect(restored?.warnings).toEqual(expect.arrayContaining([ + expect.stringContaining("adopted it for subsequent runs"), + ])); + await expect(readGit(initial.cwd, ["branch", "--show-current"])).resolves.toBe(actualBranch); }, 15_000); it("classifies persisted git worktree branch incoherence as diverged when the checked-out branch is not forward", async () => { diff --git a/server/src/index.ts b/server/src/index.ts index f4dab7a60e..b1fe7e6f48 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -801,8 +801,28 @@ export async function startServer(): Promise { throw err; } + let drainHeartbeatRunsForShutdown: ((signal: "SIGINT" | "SIGTERM") => Promise) | null = null; + let heartbeatSchedulerStopped = false; + let heartbeatSchedulerInterval: ReturnType | null = null; + const heartbeatSchedulerInFlight = new Set>(); + const trackHeartbeatSchedulerWork = (work: Promise) => { + let tracked: Promise; + tracked = Promise.resolve(work) + .then(() => undefined, () => undefined) + .finally(() => { + heartbeatSchedulerInFlight.delete(tracked); + }); + heartbeatSchedulerInFlight.add(tracked); + }; + const waitForHeartbeatSchedulerIdle = async () => { + while (heartbeatSchedulerInFlight.size > 0) { + await Promise.allSettled([...heartbeatSchedulerInFlight]); + } + }; + if (config.heartbeatSchedulerEnabled) { const heartbeat = heartbeatService(db as any, { pluginWorkerManager }); + drainHeartbeatRunsForShutdown = heartbeat.drainRunningRunsForShutdown; const environmentCustomImages = environmentCustomImageService(db as any, { pluginWorkerManager }); const routines = routineService(db as any, { pluginWorkerManager }); const heartbeatSchedulingSuppression = resolveHeartbeatSchedulingSuppression(); @@ -815,7 +835,7 @@ export async function startServer(): Promise { "heartbeat scheduling suppressed for this runtime instance", ); } else { - await (async () => { + const startupHeartbeatRecovery = (async () => { for (let attempt = 1; attempt <= 2; attempt++) { try { const result = await heartbeat.reapOrphanedRuns(); @@ -886,6 +906,8 @@ export async function startServer(): Promise { })().catch((err) => { logger.error({ err }, "startup heartbeat recovery failed"); }); + trackHeartbeatSchedulerWork(startupHeartbeatRecovery); + await startupHeartbeatRecovery; } const setupCleanup = await environmentCustomImages.cleanupExpiredSetupSessions(); @@ -893,7 +915,8 @@ export async function startServer(): Promise { logger.warn({ ...setupCleanup }, "startup environment customImage setup cleanup changed sessions"); } - setInterval(() => { + heartbeatSchedulerInterval = setInterval(() => { + if (heartbeatSchedulerStopped) return; const sweptRuntimeStatuses = heartbeat.sweepExpiredRuntimeStatuses(); if (sweptRuntimeStatuses > 0) { logger.info( @@ -903,7 +926,7 @@ export async function startServer(): Promise { } if (!resolveHeartbeatSchedulingSuppression().suppressed) { - void heartbeat + trackHeartbeatSchedulerWork(heartbeat .tickTimers(new Date()) .then((result) => { if (result.enqueued > 0) { @@ -912,10 +935,11 @@ export async function startServer(): Promise { }) .catch((err) => { logger.error({ err }, "heartbeat timer tick failed"); - }); + })); } - void routines + if (heartbeatSchedulerStopped) return; + trackHeartbeatSchedulerWork(routines .tickScheduledTriggers(new Date()) .then((result) => { if (result.triggered > 0) { @@ -924,9 +948,9 @@ export async function startServer(): Promise { }) .catch((err) => { logger.error({ err }, "routine scheduler tick failed"); - }); + })); - void environmentCustomImages + trackHeartbeatSchedulerWork(environmentCustomImages .cleanupExpiredSetupSessions() .then((result) => { if (result.timedOut > 0 || result.failed > 0) { @@ -935,12 +959,13 @@ export async function startServer(): Promise { }) .catch((err) => { logger.error({ err }, "environment customImage setup cleanup failed"); - }); - + })); + + if (heartbeatSchedulerStopped) return; if (!resolveHeartbeatSchedulingSuppression().suppressed) { // Periodically reap orphaned runs (5-min staleness threshold) and make sure // persisted queued work is still being driven forward. - void heartbeat + trackHeartbeatSchedulerWork(heartbeat .reapOrphanedRuns({ staleThresholdMs: 5 * 60 * 1000 }) .then(() => heartbeat.promoteDueScheduledRetries()) .then(async (promotion) => { @@ -992,7 +1017,7 @@ export async function startServer(): Promise { }) .catch((err) => { logger.error({ err }, "periodic heartbeat recovery failed"); - }); + })); } }, config.heartbeatSchedulerIntervalMs); } @@ -1096,12 +1121,28 @@ export async function startServer(): Promise { { const shutdown = async (signal: "SIGINT" | "SIGTERM") => { + heartbeatSchedulerStopped = true; + if (heartbeatSchedulerInterval) { + clearInterval(heartbeatSchedulerInterval); + heartbeatSchedulerInterval = null; + } + await waitForHeartbeatSchedulerIdle(); + const telemetryClient = getTelemetryClient(); if (telemetryClient) { telemetryClient.stop(); await telemetryClient.flush(); } + if (drainHeartbeatRunsForShutdown) { + try { + const drain = await drainHeartbeatRunsForShutdown(signal); + logger.info({ signal, drain }, "graceful heartbeat run drain complete"); + } catch (err) { + logger.error({ err, signal }, "graceful heartbeat run drain failed"); + } + } + const appShutdown = (app as { locals?: { paperclipShutdown?: () => void } }).locals?.paperclipShutdown; appShutdown?.(); diff --git a/server/src/services/dashboard.ts b/server/src/services/dashboard.ts index 5d60ced564..1493a3f59e 100644 --- a/server/src/services/dashboard.ts +++ b/server/src/services/dashboard.ts @@ -96,35 +96,79 @@ export function dashboardService(db: Db) { ); const monthSpendCents = Number(monthSpend); - const runActivityDayExpr = sql`to_char(${heartbeatRuns.createdAt} at time zone 'UTC', 'YYYY-MM-DD')`; - const runActivityRows = await db - .select({ - date: runActivityDayExpr, - status: heartbeatRuns.status, - count: sql`count(*)::double precision`, - }) - .from(heartbeatRuns) - .where( - and( - eq(heartbeatRuns.companyId, companyId), - gte(heartbeatRuns.createdAt, runActivityStart), - ), + // Per-day run breakdown. A run is "recovered" when its retry chain later + // succeeded (recovered_runs = all ancestors of a succeeded retry), so a + // restart-killed run whose retry succeeded is pulled out of the headline + // failed count. error_code is carried through so a failure spike can be + // attributed to an error class (e.g. process_lost, provider_quota). + const runActivityRows = (await db.execute(sql` + WITH RECURSIVE recovered_runs(id) AS ( + SELECT parent.id + FROM ${heartbeatRuns} AS child + JOIN ${heartbeatRuns} AS parent ON parent.id = child.retry_of_run_id + WHERE child.company_id = ${companyId} + AND child.status = 'succeeded' + UNION + SELECT parent.id + FROM recovered_runs rr + JOIN ${heartbeatRuns} AS child ON child.id = rr.id + JOIN ${heartbeatRuns} AS parent ON parent.id = child.retry_of_run_id ) - .groupBy(runActivityDayExpr, heartbeatRuns.status); + SELECT + to_char(run.created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD') AS date, + run.status AS status, + run.error_code AS error_code, + (run.id IN (SELECT id FROM recovered_runs)) AS recovered, + count(*)::double precision AS count + FROM ${heartbeatRuns} AS run + WHERE run.company_id = ${companyId} + AND run.created_at >= ${runActivityStart.toISOString()}::timestamptz + GROUP BY date, run.status, run.error_code, recovered + `)) as unknown as Iterable<{ + date: string; + status: string; + error_code: string | null; + recovered: boolean | string; + count: number | string; + }>; const runActivity = new Map( runActivityDays.map((date) => [ date, - { date, succeeded: 0, failed: 0, other: 0, total: 0 }, + { + date, + succeeded: 0, + failed: 0, + recovered: 0, + other: 0, + total: 0, + failedByErrorCode: {} as Record, + }, ]), ); for (const row of runActivityRows) { - const bucket = runActivity.get(row.date); + const bucket = runActivity.get(String(row.date)); if (!bucket) continue; const count = Number(row.count); - if (row.status === "succeeded") bucket.succeeded += count; - else if (row.status === "failed" || row.status === "timed_out") bucket.failed += count; - else bucket.other += count; + const status = String(row.status); + // Postgres booleans can arrive as JS boolean or "t"/"true" depending on driver. + const recovered = row.recovered === true || row.recovered === "t" || row.recovered === "true"; + if (status === "succeeded") { + bucket.succeeded += count; + } else if (status === "failed" || status === "timed_out") { + if (recovered) { + bucket.recovered += count; + } else { + bucket.failed += count; + const code = + typeof row.error_code === "string" && row.error_code.length > 0 + ? row.error_code + : "unknown"; + bucket.failedByErrorCode[code] = (bucket.failedByErrorCode[code] ?? 0) + count; + } + } else { + bucket.other += count; + } bucket.total += count; } diff --git a/server/src/services/heartbeat-stop-metadata.test.ts b/server/src/services/heartbeat-stop-metadata.test.ts index fc6d54c82e..92624ea202 100644 --- a/server/src/services/heartbeat-stop-metadata.test.ts +++ b/server/src/services/heartbeat-stop-metadata.test.ts @@ -64,6 +64,18 @@ describe("heartbeat stop metadata", () => { ).toBe("cancelled"); }); + it("records graceful interruption separately from failure", () => { + expect( + buildHeartbeatRunStopMetadata({ + adapterType: "codex_local", + adapterConfig: {}, + outcome: "interrupted", + errorCode: "server_shutdown_interrupted", + errorMessage: "Interrupted by graceful server shutdown", + }).stopReason, + ).toBe("interrupted"); + }); + it("normalizes max-turn exhaustion stop reasons", () => { expect( buildHeartbeatRunStopMetadata({ diff --git a/server/src/services/heartbeat-stop-metadata.ts b/server/src/services/heartbeat-stop-metadata.ts index de80a32bba..268a3722fc 100644 --- a/server/src/services/heartbeat-stop-metadata.ts +++ b/server/src/services/heartbeat-stop-metadata.ts @@ -1,7 +1,8 @@ -export type HeartbeatRunOutcome = "succeeded" | "failed" | "cancelled" | "timed_out"; +export type HeartbeatRunOutcome = "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out"; export type HeartbeatRunStopReason = | "completed" + | "interrupted" | "timeout" | "cancelled" | "budget_paused" @@ -83,6 +84,7 @@ export function inferHeartbeatRunStopReason(input: { errorMessage?: string | null; }): HeartbeatRunStopReason { if (input.outcome === "succeeded") return "completed"; + if (input.outcome === "interrupted") return "interrupted"; const maxTurnStopReason = normalizeMaxTurnStopReason(input.errorCode); if (maxTurnStopReason) return maxTurnStopReason; if (input.outcome === "timed_out") return "timeout"; diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index b23e562cbb..759f3a7b64 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -105,7 +105,6 @@ import { inspectManagedGitWorktreeBranch, persistAdapterManagedRuntimeServices, realizeExecutionWorkspace, - reconcilePendingForwardBranchAfterPersistence, releaseRuntimeServicesForRun, type ExecutionWorkspaceInput, type RealizedExecutionWorkspace, @@ -269,7 +268,7 @@ const MAX_INLINE_WAKE_COMMENT_BODY_TOTAL_CHARS = 12_000; const execFile = promisify(execFileCallback); const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_retry"] as const; const CANCELLABLE_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_retry"] as const; -const HEARTBEAT_RUN_TERMINAL_STATUSES = ["succeeded", "failed", "cancelled", "timed_out"] as const; +const HEARTBEAT_RUN_TERMINAL_STATUSES = ["succeeded", "interrupted", "failed", "cancelled", "timed_out"] as const; const UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES = ["failed", "cancelled", "timed_out"] as const; const TIMER_ACTIONABLE_ISSUE_STATUSES = ["todo", "in_progress"] as const; export { @@ -369,6 +368,9 @@ function readHeartbeatRunErrorFamily( const persistedFamily = readNonEmptyString(resultJson.errorFamily); if (persistedFamily) return persistedFamily; + if (run.errorCode === "provider_quota") { + return "provider_quota"; + } if (run.errorCode === "codex_transient_upstream" || run.errorCode === "claude_transient_upstream") { return "transient_upstream"; } @@ -398,9 +400,10 @@ function readTransientRetryNotBeforeFromRun(run: Pick, ) { - return readHeartbeatRunErrorFamily(run) === "transient_upstream" + const errorFamily = readHeartbeatRunErrorFamily(run); + return errorFamily === "transient_upstream" || errorFamily === "provider_quota" ? { - errorFamily: "transient_upstream" as const, + errorFamily, retryNotBefore: readTransientRetryNotBeforeFromRun(run), } : null; @@ -422,6 +425,7 @@ function mergeAdapterRecoveryMetadata(input: { ? { retryNotBefore, transientRetryNotBefore: retryNotBefore, + ...(errorFamily === "provider_quota" ? { providerQuotaRetryNotBefore: retryNotBefore } : {}), } : {}), }; @@ -1631,8 +1635,8 @@ export async function assertGitSensitiveAdapterWorkspaceValid(input: { } const expectedManagedBranchName = - readNonEmptyString(input.persistedExecutionWorkspace?.branchName) ?? - readNonEmptyString(input.executionWorkspace.branchName); + readNonEmptyString(input.executionWorkspace.branchName) ?? + readNonEmptyString(input.persistedExecutionWorkspace?.branchName); if ( input.persistedExecutionWorkspace?.strategyType === "git_worktree" && effectiveCwd && @@ -2899,7 +2903,7 @@ export function resolveExecutionWorkspaceReuseProvisioningPolicy(input: { }; } -function createInheritedExecutionWorkspaceReuseFailure(input: { +function formatInheritedExecutionWorkspaceReuseFailure(input: { reason: "inherited_workspace_reuse_failed" | "inherited_workspace_reuse_unavailable"; issueRef: WorkspaceReuseIssueRef; runId: string; @@ -2919,24 +2923,12 @@ function createInheritedExecutionWorkspaceReuseFailure(input: { : "Repair or unarchive the referenced execution workspace, or intentionally clear the issue's reuse_existing workspace binding before retrying."; const message = causeMessage ? `Issue ${issueLabel} requested inherited execution workspace reuse for ${workspaceLabel}, but the workspace could not be restored because ${causeMessage}.` - : `Issue ${issueLabel} requested inherited execution workspace reuse for ${workspaceLabel} but the workspace could not be restored; workspace provisioning cannot replace it because this is an explicit reuse path.`; + : `Issue ${issueLabel} requested inherited execution workspace reuse for ${workspaceLabel}, but the workspace could not be restored.`; - return new WorkspaceValidationFailure(message, { - workspaceValidation: { - reason: input.reason, - issueId: input.issueRef?.id ?? null, - issueIdentifier: input.issueRef?.identifier ?? null, - executionWorkspaceId: input.executionWorkspaceId ?? null, - workspaceConfigFreshnessAction: input.workspaceConfigFreshness.action, - workspaceConfigFreshnessReasons: input.workspaceConfigFreshness.reasons, - requestedReuseExisting: true, - replacementWorkspaceRealized: false, - remediation, - }, - }); + return `${message} ${remediation}`; } -export async function provisionExecutionWorkspaceForFreshnessDecision(input: { +export async function provisionExecutionWorkspaceForFreshnessDecision(input: { requestedShouldReuseExisting: boolean; existingExecutionWorkspaceId?: string | null; issueRef: WorkspaceReuseIssueRef; @@ -2964,13 +2956,14 @@ export async function provisionExecutionWorkspaceForFreshnessDecision(input: } let restored: T | null = null; + let reuseFailure: string | null = null; try { restored = (await input.restoreExistingWorkspace?.()) ?? null; } catch (error) { if (isWorkspaceValidationFailure(error)) { throw error; } - throw createInheritedExecutionWorkspaceReuseFailure({ + reuseFailure = formatInheritedExecutionWorkspaceReuseFailure({ reason: "inherited_workspace_reuse_failed", issueRef: input.issueRef, runId: input.runId, @@ -2981,7 +2974,7 @@ export async function provisionExecutionWorkspaceForFreshnessDecision(input: } if (!restored) { - throw createInheritedExecutionWorkspaceReuseFailure({ + reuseFailure = reuseFailure ?? formatInheritedExecutionWorkspaceReuseFailure({ reason: "inherited_workspace_reuse_unavailable", issueRef: input.issueRef, runId: input.runId, @@ -2990,6 +2983,11 @@ export async function provisionExecutionWorkspaceForFreshnessDecision(input: }); } + if (reuseFailure) throw new Error(reuseFailure); + if (!restored) { + throw new Error("Expected restored execution workspace after reuse fallback handling"); + } + return { executionWorkspace: restored, reusedExecutionWorkspace: restored, @@ -4720,7 +4718,7 @@ export function normalizeSessionParams(params: Record | null | return Object.keys(params).length > 0 ? params : null; } -type RunSessionOutcome = "succeeded" | "failed" | "cancelled" | "timed_out"; +type RunSessionOutcome = "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out"; const HERMES_ADAPTER_TYPE = "hermes_local"; const HERMES_SESSION_ID_REGEX = /^(?:\d{8}_\d{6}_[A-Za-z0-9_-]{4,}|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})$/; @@ -7632,6 +7630,27 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) agent: typeof agents.$inferSelect, now: Date, ) { + const existingRetry = await db + .select() + .from(heartbeatRuns) + .where(and(eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.retryOfRunId, run.id))) + .orderBy(asc(heartbeatRuns.createdAt)) + .limit(1) + .then((rows) => rows[0] ?? null); + if (existingRetry) { + await appendRunEvent(run, await nextRunEventSeq(run.id), { + eventType: "lifecycle", + stream: "system", + level: "warn", + message: "Process-loss retry already exists; skipping duplicate retry enqueue", + payload: { + retryRunId: existingRetry.id, + retryRunStatus: existingRetry.status, + }, + }); + return existingRetry; + } + const invokability = await getAgentInvokability(agent); if (!invokability.invokable) { await appendRunEvent(run, await nextRunEventSeq(run.id), { @@ -7750,6 +7769,104 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) return queued; } + async function drainRunningRunsForShutdown(signal: "SIGINT" | "SIGTERM", now = new Date()) { + const activeRuns = await db + .select({ + run: heartbeatRuns, + agent: agents, + }) + .from(heartbeatRuns) + .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id)) + .where(eq(heartbeatRuns.status, "running")); + + const interruptedRunIds: string[] = []; + const retryRunIds: string[] = []; + + for (const { run, agent } of activeRuns) { + const running = runningProcesses.get(run.id); + try { + if (running) { + await terminateHeartbeatRunProcess({ + pid: running.child.pid ?? run.processPid, + processGroupId: running.processGroupId ?? run.processGroupId, + graceMs: Math.max(1, running.graceSec) * 1000, + }); + } else if (run.processPid || run.processGroupId) { + await terminateHeartbeatRunProcess({ + pid: run.processPid, + processGroupId: run.processGroupId, + }); + } + } finally { + runningProcesses.delete(run.id); + } + + const message = `Interrupted by graceful server shutdown (${signal}); retry queued for restart recovery`; + const interruptedStatus = await setRunStatusIfRunning(run.id, "interrupted", { + finishedAt: now, + error: message, + errorCode: "server_shutdown_interrupted", + signal, + resultJson: mergeRunStopMetadataForAgent(agent, "interrupted", { + resultJson: parseObject(run.resultJson), + errorCode: "server_shutdown_interrupted", + errorMessage: message, + }), + }); + if (!interruptedStatus.updated || !interruptedStatus.run) continue; + let interrupted = interruptedStatus.run; + await setWakeupStatus(run.wakeupRequestId, "cancelled", { + finishedAt: now, + error: null, + }); + interrupted = await classifyAndPersistRunLiveness(interrupted, parseObject(interrupted.resultJson)) ?? interrupted; + + await releaseEnvironmentLeasesForRun({ + runId: interrupted.id, + companyId: interrupted.companyId, + agentId: interrupted.agentId, + status: interrupted.status, + failureReason: interrupted.error ?? undefined, + }); + + const retry = await enqueueProcessLossRetry(interrupted, agent, now); + if (!retry) { + await releaseIssueExecutionAndPromote(interrupted); + } else { + retryRunIds.push(retry.id); + } + + await appendRunEvent(interrupted, await nextRunEventSeq(interrupted.id), { + eventType: "lifecycle", + stream: "system", + level: "warn", + message, + payload: { + signal, + ...(run.processPid ? { processPid: run.processPid } : {}), + ...(run.processGroupId ? { processGroupId: run.processGroupId } : {}), + ...(retry ? { retryRunId: retry.id } : {}), + }, + }); + + await finalizeAgentStatus(run.agentId, "interrupted", message); + interruptedRunIds.push(interrupted.id); + } + + if (interruptedRunIds.length > 0) { + logger.warn( + { signal, interrupted: interruptedRunIds.length, interruptedRunIds, retryRunIds }, + "interrupted running heartbeat runs for graceful shutdown", + ); + } + + return { + interrupted: interruptedRunIds.length, + interruptedRunIds, + retryRunIds, + }; + } + type ScheduledRetryGate = | { allowed: true } | { @@ -8160,7 +8277,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) ? readTransientRecoveryContractFromRun(run) : null; const codexTransientFallbackMode = - agent.adapterType === "codex_local" && transientRecovery + agent.adapterType === "codex_local" && transientRecovery?.errorFamily === "transient_upstream" ? resolveCodexTransientFallbackMode(nextAttempt) : null; const transientRetryNotBefore = transientRecovery?.retryNotBefore ?? null; @@ -8257,6 +8374,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) scheduledRetryAttempt: schedule.attempt, scheduledRetryAt: schedule.dueAt.toISOString(), ...(transientRetryNotBefore ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } : {}), + ...(transientRecovery?.errorFamily === "provider_quota" && transientRetryNotBefore + ? { providerQuotaRetryNotBefore: transientRetryNotBefore.toISOString() } + : {}), ...(codexTransientFallbackMode ? { codexTransientFallbackMode } : {}), }, "normal_model"); const responsibleUserId = await resolveResponsibleUserIdForRunContext(run, retryContextSnapshot); @@ -8428,6 +8548,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) scheduledRetryAttempt: schedule.attempt, scheduledRetryAt: schedule.dueAt.toISOString(), ...(transientRetryNotBefore ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } : {}), + ...(transientRecovery?.errorFamily === "provider_quota" && transientRetryNotBefore + ? { providerQuotaRetryNotBefore: transientRetryNotBefore.toISOString() } + : {}), ...(codexTransientFallbackMode ? { codexTransientFallbackMode } : {}), }, "normal_model"), status: "queued", @@ -8551,6 +8674,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) baseDelayMs: schedule.baseDelayMs, delayMs: schedule.delayMs, ...(transientRetryNotBefore ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } : {}), + ...(transientRecovery?.errorFamily === "provider_quota" && transientRetryNotBefore + ? { providerQuotaRetryNotBefore: transientRetryNotBefore.toISOString() } + : {}), ...(codexTransientFallbackMode ? { codexTransientFallbackMode } : {}), }, }); @@ -9419,8 +9545,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) async function finalizeAgentStatus( agentId: string, - outcome: "succeeded" | "failed" | "cancelled" | "timed_out", + outcome: "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out", failureReason?: string | null, + options?: { keepIdleOnFailure?: boolean }, ) { const existing = await getAgent(agentId); if (!existing) return; @@ -9435,7 +9562,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const nextStatus = runningCount > 0 ? "running" - : outcome === "succeeded" || outcome === "cancelled" + : outcome === "succeeded" || outcome === "interrupted" || outcome === "cancelled" || (outcome === "failed" && options?.keepIdleOnFailure) ? "idle" : "error"; @@ -9477,7 +9604,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) function mergeRunStopMetadataForAgent( agent: Pick, - outcome: "succeeded" | "failed" | "cancelled" | "timed_out", + outcome: "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out", options?: { resultJson?: Record | null; errorCode?: string | null; @@ -10832,7 +10959,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) }); const resolvedProjectId = executionWorkspace.projectId ?? issueRef?.projectId ?? executionProjectId ?? null; const resolvedProjectWorkspaceId = issueRef?.projectWorkspaceId ?? resolvedWorkspace.workspaceId ?? null; - let persistedExecutionWorkspace = null; + let persistedExecutionWorkspace: ExecutionWorkspace | null = null; const nextExecutionWorkspaceMetadata = mergeExecutionWorkspaceMetadataForPersistence({ existingMetadata: resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace ? reusableExistingExecutionWorkspace?.metadata ?? null @@ -10937,18 +11064,6 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } throw error; } - if (persistedExecutionWorkspace && pendingForwardBranchReconcile) { - await workspaceOperationRecorder.attachExecutionWorkspaceId(persistedExecutionWorkspace.id); - const reconcileResult = await reconcilePendingForwardBranchAfterPersistence({ - db, - executionWorkspaceId: persistedExecutionWorkspace.id, - pending: pendingForwardBranchReconcile, - heartbeatRunId: run.id, - reconcileOperationPhase: "worktree_prepare", - recorder: workspaceOperationRecorder, - }); - persistedExecutionWorkspace = reconcileResult.workspace; - } await workspaceOperationRecorder.attachExecutionWorkspaceId(persistedExecutionWorkspace?.id ?? null); await recordWorkspaceConfigFreshnessOperation({ recorder: workspaceOperationRecorder, @@ -11592,7 +11707,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) let inspection = branchInspection.inspection; const initialManagedGitWorktreeBranch = formatManagedGitWorktreeBranchInspection(inspection); if (!inspection.valid && inspection.reasonCode === "branch_mismatch" && inspection.repoRoot) { - let reconciledBranchName: string | null = null; + let repairedExpectedBranchName = inspection.expectedBranchName; try { const coherence = await ensureGitWorktreeBranchCoherent({ db, @@ -11612,11 +11727,14 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) heartbeatRunId: run.id, enableWorkspaceBranchReconcileForward: resolvedInstanceSettings.experimental.enableWorkspaceBranchReconcileForward, + persistForwardReconcile: false, reconcileOperationPhase: "workspace_finalize", recorder: workspaceOperationRecorder, }); - if (coherence.reconciledForward && coherence.branchName) { - reconciledBranchName = coherence.branchName; + if (coherence.branchName && coherence.branchName !== branchInspection.workspaceRecord.branchName) { + repairedExpectedBranchName = coherence.branchName; + executionWorkspace.branchName = coherence.branchName; + executionWorkspace.warnings.push(...coherence.warnings); } } catch (repairErr) { const workspaceValidationFailure = isWorkspaceValidationFailure(repairErr) ? repairErr : null; @@ -11654,7 +11772,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const repairedInspection = await inspectManagedGitWorktreeBranch({ worktreePath: inspection.worktreePath, - expectedBranchName: reconciledBranchName ?? inspection.expectedBranchName, + expectedBranchName: repairedExpectedBranchName, repoRoot: inspection.repoRoot, }); finalizeBranchRepairMetadata = { @@ -12165,6 +12283,11 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) agent.id, outcome, outcome === "succeeded" ? null : (adapterResult.errorMessage ?? null), + { + keepIdleOnFailure: + outcome === "failed" && + (finalizedRun ? readHeartbeatRunErrorFamily(finalizedRun) === "provider_quota" : runErrorCode === "provider_quota"), + }, ); } catch (err) { const message = redactCurrentUserText( @@ -14673,6 +14796,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) reportRunActivity: clearDetachedRunWarning, reapOrphanedRuns, + drainRunningRunsForShutdown, promoteDueScheduledRetries, retryScheduledRetryNow, diff --git a/server/src/services/issue-continuation-summary.ts b/server/src/services/issue-continuation-summary.ts index a8ce6aeca4..9c5f2145d9 100644 --- a/server/src/services/issue-continuation-summary.ts +++ b/server/src/services/issue-continuation-summary.ts @@ -94,7 +94,7 @@ function extractPathCandidates(...texts: Array) { function inferMode(issue: IssueSummaryInput, run: RunSummaryInput) { if (issue.status === "done" || issue.status === "in_review") return "review"; - if (run.status === "failed" || run.status === "timed_out" || run.status === "cancelled") return "implementation"; + if (run.status === "failed" || run.status === "timed_out" || run.status === "cancelled" || run.status === "interrupted") return "implementation"; if (issue.status === "backlog" || issue.status === "todo") return "plan"; return "implementation"; } diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 7f1bec5013..8cc413df09 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -604,7 +604,7 @@ function sameRunLock(checkoutRunId: string | null, actorRunId: string | null) { return checkoutRunId == null; } -export const TERMINAL_HEARTBEAT_RUN_STATUSES = new Set(["succeeded", "failed", "cancelled", "timed_out"]); +export const TERMINAL_HEARTBEAT_RUN_STATUSES = new Set(["succeeded", "interrupted", "failed", "cancelled", "timed_out"]); const ISSUE_LIST_DESCRIPTION_MAX_CHARS = 1200; const ISSUE_LIST_DESCRIPTION_MAX_BYTES = ISSUE_LIST_DESCRIPTION_MAX_CHARS * 4; diff --git a/server/src/services/plugin-host-services.ts b/server/src/services/plugin-host-services.ts index 0925820340..32168f230b 100644 --- a/server/src/services/plugin-host-services.ts +++ b/server/src/services/plugin-host-services.ts @@ -2644,7 +2644,7 @@ export function buildHostServices( // Track the subscription so it can be cleaned up on dispose() if the run // never reaches a terminal status (hang, crash, network partition). if (notifyWorker) { - const TERMINAL_STATUSES = new Set(["succeeded", "failed", "cancelled", "timed_out"]); + const TERMINAL_STATUSES = new Set(["succeeded", "interrupted", "failed", "cancelled", "timed_out"]); const cleanup = () => { unsubscribe(); diff --git a/server/src/services/productivity-review.ts b/server/src/services/productivity-review.ts index 3c3c615d7a..e7fad83898 100644 --- a/server/src/services/productivity-review.ts +++ b/server/src/services/productivity-review.ts @@ -31,7 +31,7 @@ export const DEFAULT_PRODUCTIVITY_REVIEW_MAX_REFRESH_COMMENTS = 3; export const DEFAULT_PRODUCTIVITY_REVIEW_CREATION_WINDOW_MS = 24 * 60 * 60 * 1000; export const DEFAULT_PRODUCTIVITY_REVIEW_MAX_CREATIONS_PER_WINDOW = 3; -const TERMINAL_RUN_STATUSES = ["succeeded", "failed", "cancelled", "timed_out"] as const; +const TERMINAL_RUN_STATUSES = ["succeeded", "interrupted", "failed", "cancelled", "timed_out"] as const; const ACTIVE_RUN_STATUSES = ["queued", "running", "scheduled_retry"] as const; const MAX_CANDIDATE_ISSUES = 250; const MAX_RUNS_FOR_STREAK = 100; diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 24c32899a6..d66d299e50 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -70,7 +70,7 @@ import { import { isAutomaticRecoverySuppressedByPauseHold } from "./pause-hold-guard.js"; const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_retry"] as const; -const UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES = ["failed", "cancelled", "timed_out"] as const; +const UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES = ["interrupted", "failed", "cancelled", "timed_out"] as const; export const ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS = 60 * 60 * 1000; export const ACTIVE_RUN_OUTPUT_CRITICAL_THRESHOLD_MS = 4 * 60 * 60 * 1000; export const ACTIVE_RUN_OUTPUT_CONTINUE_REARM_MS = 30 * 60 * 1000; @@ -229,6 +229,7 @@ const TRANSIENT_INFRA_CONTINUATION_ERROR_CODES = new Set([ "adapter_failed", "codex_transient_upstream", "claude_transient_upstream", + "provider_quota", "timeout", ]); diff --git a/server/src/services/run-liveness.ts b/server/src/services/run-liveness.ts index 72be57e6f0..87b53285c7 100644 --- a/server/src/services/run-liveness.ts +++ b/server/src/services/run-liveness.ts @@ -309,6 +309,10 @@ export function classifyRunLiveness(input: RunLivenessClassificationInput): RunL actionability, }); + if (input.runStatus === "interrupted") { + return output("needs_followup", input.errorCode ? `Run interrupted (${input.errorCode})` : "Run interrupted"); + } + if (input.runStatus !== "succeeded") { return output("failed", input.errorCode ? `Run ended with ${input.runStatus} (${input.errorCode})` : `Run ended with ${input.runStatus}`); } diff --git a/server/src/services/task-watchdogs.ts b/server/src/services/task-watchdogs.ts index df84b8fc0a..3e2be28e94 100644 --- a/server/src/services/task-watchdogs.ts +++ b/server/src/services/task-watchdogs.ts @@ -28,7 +28,7 @@ const TASK_WATCHDOG_SUBTREE_MAX_DEPTH = 100; const TASK_WATCHDOG_LIVE_RUN_STATUSES = ["queued", "running", "scheduled_retry"] as const; const TASK_WATCHDOG_WAKE_REQUEST_STATUSES = ["queued", "deferred_issue_execution"] as const; const TASK_WATCHDOG_TERMINAL_ISSUE_STATUSES = ["done", "cancelled"] as const; -const TASK_WATCHDOG_TERMINAL_RUN_STATUSES = ["succeeded", "failed", "cancelled", "timed_out"] as const; +const TASK_WATCHDOG_TERMINAL_RUN_STATUSES = ["succeeded", "interrupted", "failed", "cancelled", "timed_out"] as const; // Grace window after an issue is created/assigned during which its first // assignment run/wake may have been enqueued but is not yet visible to a // watchdog evaluation (the eval can race the issue's own assignment run). diff --git a/server/src/services/workspace-runtime.ts b/server/src/services/workspace-runtime.ts index b0820818a6..3a062137be 100644 --- a/server/src/services/workspace-runtime.ts +++ b/server/src/services/workspace-runtime.ts @@ -665,6 +665,7 @@ type GitWorktreeBranchCoherenceResult = { branchName: string | null; reconciledForward: boolean; pendingForwardBranchReconcile?: PendingForwardBranchReconcile | null; + warnings: string[]; }; export type PendingForwardBranchReconcile = { @@ -791,9 +792,30 @@ async function inspectGitWorktreeBranchIncoherence(input: { sameHead, ancestryVerdict, }); - const eligible = cleanliness === "clean" && expectedBranchExists && sameHead && registeredBranchMatchesHead; + const canCheckoutRecordedBranch = + cleanliness === "clean" && expectedBranchExists && sameHead && registeredBranchMatchesHead; + const canAdoptForwardActualBranch = + cleanliness === "clean" && + expectedBranchExists && + actualBranchExists === true && + ancestryVerdict === "ancestor" && + !sameHead && + registeredBranchMatchesHead; + const canAttachRecordedBranchToDetachedHead = + cleanliness === "clean" && + expectedBranchExists && + input.actualBranchName === null && + ancestryVerdict === "ancestor" && + !sameHead && + registeredBranchMatchesHead; + const eligible = + canCheckoutRecordedBranch || canAdoptForwardActualBranch || canAttachRecordedBranchToDetachedHead; const safeRepairReason = eligible - ? "clean worktree and expected branch points at the current HEAD" + ? canCheckoutRecordedBranch + ? "clean worktree and expected branch points at the current HEAD" + : canAdoptForwardActualBranch + ? "clean worktree and checked-out branch is forward of the recorded branch" + : "clean detached worktree HEAD is forward of the recorded branch" : cleanliness !== "clean" ? "worktree is not clean" : !registered @@ -1025,17 +1047,18 @@ export async function ensureGitWorktreeBranchCoherent(input: { actualBranchName?: string | null; heartbeatRunId?: string | null; enableWorkspaceBranchReconcileForward?: boolean; + persistForwardReconcile?: boolean; reconcileOperationPhase?: "worktree_prepare" | "workspace_finalize"; recorder?: WorkspaceOperationRecorder | null; }): Promise { const expectedBranchName = input.expectedBranchName?.trim(); - if (!expectedBranchName) return { branchName: null, reconciledForward: false }; + if (!expectedBranchName) return { branchName: null, reconciledForward: false, warnings: [] }; const currentBranch = input.actualBranchName !== undefined ? input.actualBranchName : await runGit(["symbolic-ref", "--quiet", "--short", "HEAD"], input.worktreePath).catch(() => null); if (currentBranch === expectedBranchName) { - return { branchName: expectedBranchName, reconciledForward: false }; + return { branchName: expectedBranchName, reconciledForward: false, warnings: [] }; } const evidence = await inspectGitWorktreeBranchIncoherence({ @@ -1053,7 +1076,7 @@ export async function ensureGitWorktreeBranchCoherent(input: { currentBranch ) { const reason = "Automatic forward reconciliation: recorded branch is an ancestor of the checked-out branch."; - if (input.executionWorkspaceId) { + if (input.executionWorkspaceId && input.persistForwardReconcile !== false) { if (!input.db) { evidence.safeRepair.reason = "forward reconciliation requires database access to update the execution workspace record"; throw branchIncoherenceValidationFailure(evidence); @@ -1107,7 +1130,7 @@ export async function ensureGitWorktreeBranchCoherent(input: { auditCommentId: result.auditCommentId, recoveryActionId: result.recoveryAction?.id ?? null, }); - return { branchName: result.inspection.toBranch, reconciledForward: true }; + return { branchName: result.inspection.toBranch, reconciledForward: true, warnings: [] }; } catch (error) { evidence.safeRepair.reason = `forward reconciliation failed: ${error instanceof Error ? error.message : String(error)}`; @@ -1122,6 +1145,7 @@ export async function ensureGitWorktreeBranchCoherent(input: { return { branchName: currentBranch, reconciledForward: true, + warnings: [], pendingForwardBranchReconcile: { recordedBranchName: expectedBranchName, adoptedBranchName: currentBranch, @@ -1136,6 +1160,75 @@ export async function ensureGitWorktreeBranchCoherent(input: { } evidence.safeRepair.attempted = true; + const warningPrefix = + `Execution workspace branch metadata was self-healed from "${expectedBranchName}" to "${formatBranchForMessage(currentBranch)}" at ${input.worktreePath}.`; + if ( + currentBranch && + evidence.provenance.actualBranchExists === true && + evidence.provenance.ancestryVerdict === "ancestor" && + !evidence.provenance.sameHead + ) { + evidence.safeRepair.succeeded = true; + evidence.safeRepair.reason = "clean worktree adopted the checked-out branch because it is forward of the recorded branch"; + return { + branchName: currentBranch, + reconciledForward: false, + warnings: [ + `${warningPrefix} The checked-out branch contains the recorded branch plus newer commits, so Paperclip adopted it for subsequent runs.`, + ], + }; + } + + if ( + currentBranch === null && + evidence.provenance.ancestryVerdict === "ancestor" && + !evidence.provenance.sameHead && + evidence.provenance.actualHeadSha + ) { + try { + await recordGitOperation(input.recorder, { + phase: "worktree_prepare", + args: ["checkout", "-B", expectedBranchName, evidence.provenance.actualHeadSha], + cwd: input.worktreePath, + metadata: { + repoRoot: input.repoRoot, + worktreePath: input.worktreePath, + expectedBranchName, + actualBranchName: currentBranch, + branchIncoherenceRepair: true, + detachedHeadRepair: true, + fingerprint: evidence.fingerprint, + sourceIssueId: evidence.sourceIssueId, + executionWorkspaceId: evidence.executionWorkspaceId, + }, + successMessage: `Reattached detached git worktree HEAD at ${input.worktreePath} to ${expectedBranchName}\n`, + failureLabel: `git checkout -B ${expectedBranchName} ${formatShortSha(evidence.provenance.actualHeadSha)}`, + }); + } catch (error) { + evidence.safeRepair.succeeded = false; + evidence.safeRepair.reason = `safe detached HEAD reattachment failed: ${error instanceof Error ? error.message : String(error)}`; + throw branchIncoherenceValidationFailure(evidence); + } + + const repairedBranch = await runGit(["symbolic-ref", "--quiet", "--short", "HEAD"], input.worktreePath) + .catch(() => null); + if (repairedBranch !== expectedBranchName) { + evidence.safeRepair.succeeded = false; + evidence.safeRepair.reason = `reattach completed but HEAD is ${formatBranchForMessage(repairedBranch)}`; + throw branchIncoherenceValidationFailure(evidence); + } + + evidence.safeRepair.succeeded = true; + evidence.safeRepair.reason = "clean detached worktree HEAD was reattached to the recorded branch"; + return { + branchName: expectedBranchName, + reconciledForward: false, + warnings: [ + `${warningPrefix} The detached HEAD contained the recorded branch plus newer commits, so Paperclip moved the recorded branch to that HEAD.`, + ], + }; + } + try { await recordGitOperation(input.recorder, { phase: "worktree_prepare", @@ -1170,7 +1263,13 @@ export async function ensureGitWorktreeBranchCoherent(input: { evidence.safeRepair.succeeded = true; evidence.safeRepair.reason = "clean worktree checked out the recorded branch"; - return { branchName: expectedBranchName, reconciledForward: false }; + return { + branchName: expectedBranchName, + reconciledForward: false, + warnings: [ + `Execution workspace branch metadata was self-healed by checking out recorded branch "${expectedBranchName}" at ${input.worktreePath}.`, + ], + }; } // Resolve the authoritative base ref for a fresh worktree. A configured local @@ -1919,12 +2018,12 @@ export async function realizeExecutionWorkspace(input: { await fs.mkdir(worktreeParentDir, { recursive: true }); - async function reuseExistingWorktree(reusablePath: string) { + async function reuseExistingWorktree(reusablePath: string, effectiveBranchName = branchName, extraWarnings: string[] = []) { const refresh = currentBaseRefSha ? await refreshUnstartedWorktreeToBase({ repoRoot, worktreePath: reusablePath, - branchName, + branchName: effectiveBranchName, baseRef, currentBaseRefSha, recorder: input.recorder ?? null, @@ -1945,7 +2044,7 @@ export async function realizeExecutionWorkspace(input: { metadata: { repoRoot, worktreePath: reusablePath, - branchName, + branchName: effectiveBranchName, baseRef, currentBaseRefSha: baseDrift.currentBaseRefSha, branchBaseRefSha: baseDrift.branchBaseRefSha, @@ -1964,7 +2063,7 @@ export async function realizeExecutionWorkspace(input: { base: input.base, repoRoot, worktreePath: reusablePath, - branchName, + branchName: effectiveBranchName, issue: input.issue, agent: input.agent, created: false, @@ -1975,9 +2074,9 @@ export async function realizeExecutionWorkspace(input: { repoRef: baseRef, strategy: "git_worktree" as const, cwd: reusablePath, - branchName, + branchName: effectiveBranchName, worktreePath: reusablePath, - warnings: [...baseRefreshWarnings, ...baseDrift.warnings], + warnings: [...extraWarnings, ...baseRefreshWarnings, ...baseDrift.warnings], created: false, baseRefSha: refresh.baseRefSha ?? baseDrift.branchBaseRefSha ?? baseDrift.currentBaseRefSha, pendingForwardBranchReconcile, @@ -2004,35 +2103,43 @@ export async function realizeExecutionWorkspace(input: { reconcileOperationPhase: "worktree_prepare", recorder: input.recorder ?? null, }); - if (coherence.reconciledForward && coherence.branchName) { - branchName = coherence.branchName; + const effectiveBranchName = coherence.branchName ?? branchName; + if (coherence.reconciledForward) { + branchName = effectiveBranchName; pendingForwardBranchReconcile = coherence.pendingForwardBranchReconcile ?? null; } - return await validateLinkedGitWorktree({ + const nextValidation = await validateLinkedGitWorktree({ repoRoot, worktreePath: reusablePath, - expectedBranchName: branchName, + expectedBranchName: effectiveBranchName, }).catch(() => null); + return { + validation: nextValidation, + branchName: effectiveBranchName, + warnings: coherence.warnings, + }; } - return validation; + return { validation, branchName, warnings: [] }; } const existingWorktree = await directoryExists(worktreePath); if (existingWorktree) { - const validation = await validateReusableWorktree(worktreePath); - if (validation?.valid) { - return await reuseExistingWorktree(worktreePath); + const reusable = await validateReusableWorktree(worktreePath); + if (reusable.validation?.valid) { + return await reuseExistingWorktree(worktreePath, reusable.branchName, reusable.warnings); } + const validation = reusable.validation; const reason = validation && !validation.valid ? ` (${validation.reason})` : ""; throw new Error(`Configured worktree path "${worktreePath}" already exists and is not a reusable git worktree${reason}.`); } const registeredBranchWorktree = await findRegisteredGitWorktreeByBranch(repoRoot, branchName); if (registeredBranchWorktree) { - const validation = await validateReusableWorktree(registeredBranchWorktree); - if (validation?.valid) { - return await reuseExistingWorktree(registeredBranchWorktree); + const reusable = await validateReusableWorktree(registeredBranchWorktree); + if (reusable.validation?.valid) { + return await reuseExistingWorktree(registeredBranchWorktree, reusable.branchName, reusable.warnings); } + const validation = reusable.validation; const reason = validation && !validation.valid ? ` (${validation.reason})` : ""; throw new Error(`Registered worktree for branch "${branchName}" at "${registeredBranchWorktree}" is not reusable${reason}.`); } @@ -2167,6 +2274,7 @@ export async function ensurePersistedExecutionWorkspaceAvailable(input: { if (await directoryExists(cwd)) { const reuseBaseRef = input.workspace.baseRef ?? input.base.repoRef ?? null; const reuseWorktreePath = realized.worktreePath ?? cwd; + const repairWarnings: string[] = []; if (await isGitCheckout(reuseWorktreePath)) { const coherence = await ensureGitWorktreeBranchCoherent({ db: input.db ?? null, @@ -2177,12 +2285,17 @@ export async function ensurePersistedExecutionWorkspaceAvailable(input: { executionWorkspaceId: input.workspace.id ?? null, heartbeatRunId: input.heartbeatRunId ?? null, enableWorkspaceBranchReconcileForward: input.enableWorkspaceBranchReconcileForward === true, + persistForwardReconcile: false, reconcileOperationPhase: "worktree_prepare", recorder: input.recorder ?? null, }); - if (coherence.reconciledForward && coherence.branchName) { + if (coherence.branchName) { realized.branchName = coherence.branchName; } + if (coherence.reconciledForward) { + realized.pendingForwardBranchReconcile = coherence.pendingForwardBranchReconcile ?? null; + } + repairWarnings.push(...coherence.warnings); } const validation = await validateLinkedGitWorktree({ repoRoot, @@ -2224,7 +2337,7 @@ export async function ensurePersistedExecutionWorkspaceAvailable(input: { recordedBaseRefSha, skipRefresh: true, }); - realized.warnings = [...baseRefreshWarnings, ...baseDrift.warnings]; + realized.warnings = [...repairWarnings, ...baseRefreshWarnings, ...baseDrift.warnings]; realized.baseRefSha = refresh.baseRefSha ?? recordedBaseRefSha ?? baseDrift.branchBaseRefSha ?? baseDrift.currentBaseRefSha; if (provisionCommand) { await provisionExecutionWorktree({ @@ -4134,6 +4247,9 @@ export function buildWorkspaceReadyComment(input: { if (input.workspace.worktreePath && input.workspace.worktreePath !== input.workspace.cwd) { lines.push(`- Worktree: \`${input.workspace.worktreePath}\``); } + for (const warning of input.workspace.warnings) { + lines.push(`- Warning: ${warning}`); + } for (const service of input.runtimeServices) { const detail = service.url ? `${service.serviceName}: ${service.url}` : `${service.serviceName}: running`; const suffix = service.reused ? " (reused)" : ""; diff --git a/ui/src/components/ActivityCharts.test.tsx b/ui/src/components/ActivityCharts.test.tsx index 1d17cd38df..a665c9a33b 100644 --- a/ui/src/components/ActivityCharts.test.tsx +++ b/ui/src/components/ActivityCharts.test.tsx @@ -1,7 +1,7 @@ // @vitest-environment jsdom -import { act } from "react"; import type { ReactNode } from "react"; +import { flushSync } from "react-dom"; import { createRoot, type Root } from "react-dom/client"; import type { HeartbeatRun } from "@paperclipai/shared"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; @@ -22,13 +22,13 @@ beforeEach(() => { }); afterEach(() => { - act(() => root.unmount()); + flushSync(() => root.unmount()); container.remove(); vi.useRealTimers(); }); function render(ui: ReactNode) { - act(() => { + flushSync(() => { root.render(ui); }); } @@ -99,12 +99,37 @@ describe("ActivityCharts", () => { , ); expect(container.textContent).not.toContain("No runs yet"); - expect(container.querySelector("[title='2026-04-20: 2 runs']")).not.toBeNull(); + // Tooltip now carries the per-day breakdown (incl. failure error codes). + const dayCell = container.querySelector("[title^='2026-04-20: 2 runs']"); + expect(dayCell).not.toBeNull(); + expect(dayCell?.getAttribute("title")).toContain("provider_quota: 1"); + }); + + it("renders a distinct recovered segment and legend for recovered restart kills", () => { + render( + , + ); + + expect(container.textContent).toContain("Recovered"); + const dayCell = container.querySelector("[title*='recovered: 4']"); + expect(dayCell).not.toBeNull(); }); }); diff --git a/ui/src/components/ActivityCharts.tsx b/ui/src/components/ActivityCharts.tsx index 33171a5bf9..38a92bea77 100644 --- a/ui/src/components/ActivityCharts.tsx +++ b/ui/src/components/ActivityCharts.tsx @@ -15,6 +15,31 @@ function formatDayLabel(dateStr: string): string { return `${d.getMonth() + 1}/${d.getDate()}`; } +function emptyRunDay(date: string): DashboardRunActivityDay { + return { date, succeeded: 0, failed: 0, recovered: 0, other: 0, total: 0, failedByErrorCode: {} }; +} + +const runSegmentColors = { + succeeded: "#10b981", + recovered: "#f59e0b", + failed: "#ef4444", + other: "#737373", +} as const; + +// Compact per-day tooltip that also attributes failures to their error class. +function runDayTooltip(entry: DashboardRunActivityDay): string { + const lines = [`${entry.date}: ${entry.total} run${entry.total === 1 ? "" : "s"}`]; + if (entry.succeeded > 0) lines.push(` succeeded: ${entry.succeeded}`); + if (entry.recovered > 0) lines.push(` recovered: ${entry.recovered} (retry succeeded)`); + if (entry.failed > 0) { + lines.push(` failed: ${entry.failed}`); + const codes = Object.entries(entry.failedByErrorCode ?? {}).sort((a, b) => b[1] - a[1]); + for (const [code, count] of codes) lines.push(` ${code}: ${count}`); + } + if (entry.other > 0) lines.push(` other: ${entry.other}`); + return lines.join("\n"); +} + /* ---- Sub-components ---- */ function DateLabels({ days }: { days: string[] }) { @@ -65,14 +90,23 @@ type RunChartProps = function aggregateRuns(runs: readonly HeartbeatRun[] = []): DashboardRunActivityDay[] { const days = getLast14Days(); const grouped = new Map(); - for (const day of days) grouped.set(day, { date: day, succeeded: 0, failed: 0, other: 0, total: 0 }); + for (const day of days) grouped.set(day, emptyRunDay(day)); for (const run of runs) { const day = new Date(run.createdAt).toISOString().slice(0, 10); const entry = grouped.get(day); if (!entry) continue; - if (run.status === "succeeded") entry.succeeded++; - else if (run.status === "failed" || run.status === "timed_out") entry.failed++; - else entry.other++; + if (run.status === "succeeded") { + entry.succeeded++; + } else if (run.status === "failed" || run.status === "timed_out") { + // A flat run list has no retry-chain linkage, so recovery can't be derived + // here (the company dashboard computes it server-side). Attribute the + // failure to its error class so the breakdown still renders. + entry.failed++; + const code = run.errorCode && run.errorCode.length > 0 ? run.errorCode : "unknown"; + entry.failedByErrorCode[code] = (entry.failedByErrorCode[code] ?? 0) + 1; + } else { + entry.other++; + } entry.total++; } return Array.from(grouped.values()); @@ -91,23 +125,32 @@ export function RunActivityChart(props: RunChartProps) { const maxValue = Math.max(...activity.map(v => v.total), 1); const hasData = activity.some(v => v.total > 0); + const hasRecovered = activity.some(v => v.recovered > 0); if (!hasData) return

No runs yet

; + const legendItems = [ + { color: runSegmentColors.succeeded, label: "Succeeded" }, + ...(hasRecovered ? [{ color: runSegmentColors.recovered, label: "Recovered" }] : []), + { color: runSegmentColors.failed, label: "Failed" }, + { color: runSegmentColors.other, label: "Other" }, + ]; + return (
{days.map(day => { - const entry = grouped.get(day) ?? { date: day, succeeded: 0, failed: 0, other: 0, total: 0 }; + const entry = grouped.get(day) ?? emptyRunDay(day); const total = entry.total; const heightPct = (total / maxValue) * 100; return ( -
+
{total > 0 ? (
- {entry.succeeded > 0 &&
} - {entry.failed > 0 &&
} - {entry.other > 0 &&
} + {entry.succeeded > 0 &&
} + {entry.recovered > 0 &&
} + {entry.failed > 0 &&
} + {entry.other > 0 &&
}
) : (
@@ -117,6 +160,7 @@ export function RunActivityChart(props: RunChartProps) { })}
+
); } @@ -260,11 +304,14 @@ export function SuccessRateChart(props: RunChartProps) {
{days.map(day => { - const entry = grouped.get(day) ?? { date: day, succeeded: 0, failed: 0, other: 0, total: 0 }; - const rate = entry.total > 0 ? entry.succeeded / entry.total : 0; + const entry = grouped.get(day) ?? emptyRunDay(day); + // Recovered runs ultimately succeeded, so they count toward the rate + // rather than dragging it down as failures. + const effectiveSucceeded = entry.succeeded + entry.recovered; + const rate = entry.total > 0 ? effectiveSucceeded / entry.total : 0; const color = entry.total === 0 ? undefined : rate >= 0.8 ? "var(--hex-10b981)" : rate >= 0.5 ? "var(--hex-eab308)" : "var(--hex-ef4444)"; return ( -
0 ? Math.round(rate * 100) : 0}% (${entry.succeeded}/${entry.total})`}> +
0 ? Math.round(rate * 100) : 0}% (${effectiveSucceeded}/${entry.total})`}> {entry.total > 0 ? (
) : ( diff --git a/ui/src/components/transcript/useLiveRunTranscripts.ts b/ui/src/components/transcript/useLiveRunTranscripts.ts index 6c001e2fae..12e23ccfe9 100644 --- a/ui/src/components/transcript/useLiveRunTranscripts.ts +++ b/ui/src/components/transcript/useLiveRunTranscripts.ts @@ -35,7 +35,7 @@ function readString(value: unknown): string | null { } function isTerminalStatus(status: string): boolean { - return status === "failed" || status === "timed_out" || status === "cancelled" || status === "succeeded"; + return status === "failed" || status === "timed_out" || status === "cancelled" || status === "interrupted" || status === "succeeded"; } function runKnownLogBytes(run: RunTranscriptSource): number | null { diff --git a/ui/src/context/LiveUpdatesProvider.tsx b/ui/src/context/LiveUpdatesProvider.tsx index c480f781b8..7a3976ba08 100644 --- a/ui/src/context/LiveUpdatesProvider.tsx +++ b/ui/src/context/LiveUpdatesProvider.tsx @@ -21,7 +21,7 @@ const TOAST_COOLDOWN_MAX = 3; const RECONNECT_SUPPRESS_MS = 2000; const SOCKET_CONNECTING = 0; const SOCKET_OPEN = 1; -const TERMINAL_RUN_STATUSES = new Set(["succeeded", "failed", "cancelled", "timed_out"]); +const TERMINAL_RUN_STATUSES = new Set(["succeeded", "interrupted", "failed", "cancelled", "timed_out"]); type LiveUpdatesSocketLike = { readyState: number; diff --git a/ui/storybook/fixtures/paperclipData.ts b/ui/storybook/fixtures/paperclipData.ts index 71b4c5ab91..52bd2b2a0f 100644 --- a/ui/storybook/fixtures/paperclipData.ts +++ b/ui/storybook/fixtures/paperclipData.ts @@ -1286,20 +1286,21 @@ export const storybookDashboardSummary: DashboardSummary = { pausedProjects: 1, }, runActivity: [ - { date: "2026-04-07", succeeded: 4, failed: 0, other: 1, total: 5 }, - { date: "2026-04-08", succeeded: 5, failed: 1, other: 0, total: 6 }, - { date: "2026-04-09", succeeded: 3, failed: 0, other: 1, total: 4 }, - { date: "2026-04-10", succeeded: 6, failed: 0, other: 0, total: 6 }, - { date: "2026-04-11", succeeded: 4, failed: 1, other: 0, total: 5 }, - { date: "2026-04-12", succeeded: 2, failed: 0, other: 1, total: 3 }, - { date: "2026-04-13", succeeded: 5, failed: 0, other: 1, total: 6 }, - { date: "2026-04-14", succeeded: 6, failed: 1, other: 0, total: 7 }, - { date: "2026-04-15", succeeded: 4, failed: 0, other: 1, total: 5 }, - { date: "2026-04-16", succeeded: 7, failed: 0, other: 0, total: 7 }, - { date: "2026-04-17", succeeded: 6, failed: 1, other: 0, total: 7 }, - { date: "2026-04-18", succeeded: 3, failed: 0, other: 1, total: 4 }, - { date: "2026-04-19", succeeded: 5, failed: 0, other: 1, total: 6 }, - { date: "2026-04-20", succeeded: 4, failed: 0, other: 2, total: 6 }, + { date: "2026-04-07", succeeded: 4, failed: 0, recovered: 0, other: 1, total: 5, failedByErrorCode: {} }, + { date: "2026-04-08", succeeded: 5, failed: 1, recovered: 0, other: 0, total: 6, failedByErrorCode: { provider_quota: 1 } }, + { date: "2026-04-09", succeeded: 3, failed: 0, recovered: 0, other: 1, total: 4, failedByErrorCode: {} }, + { date: "2026-04-10", succeeded: 6, failed: 0, recovered: 0, other: 0, total: 6, failedByErrorCode: {} }, + { date: "2026-04-11", succeeded: 4, failed: 1, recovered: 0, other: 0, total: 5, failedByErrorCode: { workspace_validation_failed: 1 } }, + { date: "2026-04-12", succeeded: 2, failed: 0, recovered: 0, other: 1, total: 3, failedByErrorCode: {} }, + { date: "2026-04-13", succeeded: 5, failed: 0, recovered: 0, other: 1, total: 6, failedByErrorCode: {} }, + { date: "2026-04-14", succeeded: 6, failed: 1, recovered: 0, other: 0, total: 7, failedByErrorCode: { provider_quota: 1 } }, + { date: "2026-04-15", succeeded: 4, failed: 0, recovered: 0, other: 1, total: 5, failedByErrorCode: {} }, + { date: "2026-04-16", succeeded: 7, failed: 0, recovered: 0, other: 0, total: 7, failedByErrorCode: {} }, + { date: "2026-04-17", succeeded: 6, failed: 1, recovered: 0, other: 0, total: 7, failedByErrorCode: { workspace_validation_failed: 1 } }, + { date: "2026-04-18", succeeded: 3, failed: 0, recovered: 0, other: 1, total: 4, failedByErrorCode: {} }, + // Restart-burst spike: many process_lost kills, most recovered on retry. + { date: "2026-04-19", succeeded: 5, failed: 2, recovered: 6, other: 1, total: 14, failedByErrorCode: { process_lost: 2 } }, + { date: "2026-04-20", succeeded: 4, failed: 1, recovered: 3, other: 2, total: 10, failedByErrorCode: { process_lost: 1 } }, ], };