From 6d2eab742f69727b272e82edd793448972903223 Mon Sep 17 00:00:00 2001 From: Jannes Stubbemann Date: Thu, 13 Aug 2026 01:43:48 +0200 Subject: [PATCH] fix(server): retry runs that hit a sandbox provider worker restart window instead of failing setup (#10212) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Agent runs execute in sandbox environments acquired through provider plugins (e.g. the Kubernetes sandbox provider) > - Lease acquisition happens during run setup, before the adapter executes > - When a provider plugin's worker is momentarily unavailable (a server or plugin restart window), lease acquisition throws "Sandbox provider ... is installed via plugin ..., but its worker is not running." > - The heartbeat setup path records that as a terminal `setup_failed`: no retry classifier matches the message, so the run dies instantly even though the worker returns seconds later > - This PR classifies that transient condition as retryable infrastructure so the run is retried instead of being lost to a restart blip > - The benefit is that routine restarts no longer produce spurious instant run failures ## Linked Issues or Issue Description No public GitHub issue exists; describing inline following the bug report template. **What happened** During a brief sandbox-provider-worker restart window, several runs failed instantly with `setup_failed` ("... but its worker is not running."), while runs on the same agent moments earlier and later succeeded. **Expected behavior** A transient, self-healing worker-unavailable condition should schedule a bounded retry, not terminally fail the run. **Steps to reproduce** Trigger a run while the sandbox provider plugin worker is momentarily unavailable (a server or plugin restart). Lease acquisition throws the worker-not-running error and the run is finalized as `setup_failed` with no retry. The recovery test added here reproduces the classification path. **Deployment mode** Cloud multi-tenant execution (Kubernetes sandbox provider plugin). ## What Changed - Added a dedicated, readable predicate that recognizes the transient sandbox-provider-worker-unavailable lease failure and treats it as retryable infrastructure, so the heartbeat schedules a bounded continuation retry instead of finalizing terminally - The predicate is anchored to the full lease-failure phrasing (`is installed via plugin ... but its worker is not running`) so it cannot match the permanent "provider not installed" message emitted by config validation - Added tests proving the readiness poll already waits the full deadline while the worker handle is absent or `starting` (registered-late coverage); no poll behavior change was needed ## Verification - `cd server && npx vitest run src/__tests__/environment-runtime.test.ts` — poll exhaustion + registered-late cases - `npx vitest run src/__tests__/heartbeat-process-recovery.test.ts` — worker-unavailable message schedules a retry; a non-matching permanent provider failure still escalates terminally (negative case) ## Risks Low risk. The retry is bounded by the existing infrastructure-continuation attempt cap (max 3), the message match is narrow enough to exclude the permanent provider-not-installed failure (covered by a negative test), and no readiness-poll or lease-acquisition behavior changed. ## Model Used Claude (Anthropic) via Claude Code. Implementation and tests authored by a Claude Sonnet-class model (`claude-sonnet-5`) dispatched as isolated per-task implementer agents under a multi-agent orchestration workflow; root-cause investigation, planning, and two-stage adversarial code review performed by additional Claude agents. Extended thinking and tool use enabled throughout. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [ ] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --- .../src/__tests__/environment-runtime.test.ts | 174 ++++++++++ .../heartbeat-process-recovery.test.ts | 324 ++++++++++++++++++ server/src/services/heartbeat.ts | 26 +- 3 files changed, 523 insertions(+), 1 deletion(-) diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index 01210c1bda..b22dd69a99 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -1575,6 +1575,180 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", expect.anything(), 31234); }); + it("throws a worker-not-running error once the readiness deadline is exhausted", async () => { + const pluginId = randomUUID(); + const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); + const providerConfig = { + provider: "fake-plugin", + image: "fake:test", + timeoutMs: 1234, + reuseLease: false, + }; + const environment = { + ...baseEnvironment, + name: "Never Running Plugin Sandbox", + driver: "sandbox", + config: providerConfig, + }; + await environmentService(db).update(environment.id, { + driver: "sandbox", + name: environment.name, + config: providerConfig, + }); + await db.insert(plugins).values({ + id: pluginId, + pluginKey: "acme.never-running-sandbox-provider", + packageName: "@acme/never-running-sandbox-provider", + version: "1.0.0", + apiVersion: 1, + categories: ["automation"], + manifestJson: { + id: "acme.never-running-sandbox-provider", + apiVersion: 1, + version: "1.0.0", + displayName: "Never Running Sandbox Provider", + description: "Test plugin worker that never comes online before the readiness deadline", + author: "Acme", + categories: ["automation"], + capabilities: ["environment.drivers.register"], + entrypoints: { worker: "dist/worker.js" }, + environmentDrivers: [ + { + driverKey: "fake-plugin", + kind: "sandbox_provider", + displayName: "Fake Plugin", + configSchema: { type: "object" }, + }, + ], + }, + status: "ready", + installOrder: 1, + updatedAt: new Date(), + } as any); + + const workerManager = { + isRunning: vi.fn(() => false), + call: vi.fn(async (_pluginId: string, method: string) => { + throw new Error(`Unexpected plugin method: ${method}`); + }), + } as unknown as PluginWorkerManager; + const runtimeWithPlugin = environmentRuntimeService(db, { + pluginWorkerManager: workerManager, + pluginWorkerReadyTimeoutMs: 25, + pluginWorkerReadyPollMs: 1, + }); + + await expect( + runtimeWithPlugin.acquireRunLease({ + companyId, + environment, + issueId: null, + heartbeatRunId: runId, + persistedExecutionWorkspace: null, + }), + ).rejects.toThrow(/worker is not running/); + // Confirms the loop actually polled repeatedly across the deadline window + // instead of giving up after a single check. + expect(workerManager.isRunning.mock.calls.length).toBeGreaterThanOrEqual(2); + }); + + it("keeps polling across a longer worker restart window and succeeds once the handle registers late", async () => { + const pluginId = randomUUID(); + const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); + const providerConfig = { + provider: "fake-plugin", + image: "fake:test", + timeoutMs: 1234, + reuseLease: false, + }; + const environment = { + ...baseEnvironment, + name: "Registered Late Plugin Sandbox", + driver: "sandbox", + config: providerConfig, + }; + await environmentService(db).update(environment.id, { + driver: "sandbox", + name: environment.name, + config: providerConfig, + }); + await db.insert(plugins).values({ + id: pluginId, + pluginKey: "acme.registered-late-sandbox-provider", + packageName: "@acme/registered-late-sandbox-provider", + version: "1.0.0", + apiVersion: 1, + categories: ["automation"], + manifestJson: { + id: "acme.registered-late-sandbox-provider", + apiVersion: 1, + version: "1.0.0", + displayName: "Registered Late Sandbox Provider", + description: "Test plugin worker handle that stays unregistered/starting for most of the readiness window", + author: "Acme", + categories: ["automation"], + capabilities: ["environment.drivers.register"], + entrypoints: { worker: "dist/worker.js" }, + environmentDrivers: [ + { + driverKey: "fake-plugin", + kind: "sandbox_provider", + displayName: "Fake Plugin", + configSchema: { type: "object" }, + }, + ], + }, + status: "ready", + installOrder: 1, + updatedAt: new Date(), + } as any); + + // Simulates a worker process restart: the handle is absent (or "starting") + // for most of the readiness window and only reports running with several + // checks left before the deadline. A larger attempt count than the + // existing "waits briefly" sanity test, to prove the loop is bound by the + // deadline rather than by a small fixed number of attempts. + const readyAfterChecks = 8; + let checks = 0; + const workerManager = { + isRunning: vi.fn((id: string) => { + if (id !== pluginId) return false; + checks += 1; + return checks >= readyAfterChecks; + }), + call: vi.fn(async (_pluginId: string, method: string) => { + if (method === "environmentAcquireLease") { + return { + providerLeaseId: "sandbox-registered-late", + metadata: { + provider: "fake-plugin", + image: "fake:test", + timeoutMs: 1234, + reuseLease: false, + }, + }; + } + throw new Error(`Unexpected plugin method: ${method}`); + }), + } as unknown as PluginWorkerManager; + const runtimeWithPlugin = environmentRuntimeService(db, { + pluginWorkerManager: workerManager, + pluginWorkerReadyTimeoutMs: 200, + pluginWorkerReadyPollMs: 1, + }); + + const acquired = await runtimeWithPlugin.acquireRunLease({ + companyId, + environment, + issueId: null, + heartbeatRunId: runId, + persistedExecutionWorkspace: null, + }); + + expect(acquired.lease.providerLeaseId).toBe("sandbox-registered-late"); + expect(checks).toBe(readyAfterChecks); + }); + it("extends plugin-backed sandbox lease RPC timeouts from provider config", async () => { const pluginId = randomUUID(); const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index 4826ad0021..7ada8c774b 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -38,6 +38,7 @@ import { issueTreeHolds, issueWorkProducts, issues, + plugins, projects, projectWorkspaces, workspaceOperations, @@ -384,6 +385,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { await db.delete(workspaceOperations); await db.delete(environmentLeases); await db.delete(environments); + await db.delete(plugins); await db.delete(issuePlanDecompositions); await db.delete(issueThreadInteractions); await db.delete(documentAnnotationComments); @@ -2901,6 +2903,328 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { mockAdapterExecute.mockClear(); }); + it("schedules an infra retry for a setup failure caused by a transient sandbox provider worker restart", async () => { + // Reproduces the production incident: the "Kubernetes Sandbox" plugin + // worker was mid-restart when a run tried to acquire a lease. The lease + // acquisition fails BEFORE the adapter is ever dispatched (no call to + // mockAdapterExecute), so this hits the setup-failure catch (errorCode + // "setup_failed") rather than the adapter-failure catch. The condition is + // transient and self-healing, so it must be classified retryable + // infrastructure, not a terminal setup failure. + const { companyId, agentId, runId, wakeupRequestId, issueId } = await seedQueuedIssueRunFixture(); + const interactionId = randomUUID(); + const pluginId = randomUUID(); + const environmentId = randomUUID(); + + await db.insert(plugins).values({ + id: pluginId, + pluginKey: "paperclip.kubernetes-sandbox-provider", + packageName: "@paperclipai/kubernetes-sandbox-provider", + version: "1.0.0", + apiVersion: 1, + categories: ["automation"], + manifestJson: { + id: "paperclip.kubernetes-sandbox-provider", + apiVersion: 1, + version: "1.0.0", + displayName: "Kubernetes Sandbox Provider", + description: "Test Kubernetes sandbox provider whose worker is mid-restart", + author: "Paperclip", + categories: ["automation"], + capabilities: ["environment.drivers.register"], + entrypoints: { worker: "dist/worker.js" }, + environmentDrivers: [ + { + driverKey: "kubernetes", + kind: "sandbox_provider", + displayName: "Kubernetes Sandbox", + configSchema: { type: "object" }, + }, + ], + }, + status: "ready", + installOrder: 1, + updatedAt: new Date(), + } as any); + await db.insert(environments).values({ + id: environmentId, + companyId, + name: "Kubernetes Sandbox", + driver: "sandbox", + status: "active", + config: { + provider: "kubernetes", + image: "fake:test", + timeoutMs: 1234, + reuseLease: false, + }, + createdAt: new Date(), + updatedAt: new Date(), + }); + await db + .update(agents) + .set({ defaultEnvironmentId: environmentId }) + .where(eq(agents.id, agentId)); + + await db.insert(issueThreadInteractions).values({ + id: interactionId, + companyId, + issueId, + kind: "request_confirmation", + status: "accepted", + continuationPolicy: "wake_assignee_on_accept", + createdByAgentId: agentId, + resolvedByUserId: "responsible-user", + resolvedAt: new Date("2026-03-19T00:00:00.000Z"), + payload: { + version: 1, + prompt: "Approve the plan?", + target: { + type: "issue_document", + issueId, + key: "plan", + revisionId: randomUUID(), + }, + }, + result: { version: 1, outcome: "accepted" }, + }); + + await db + .update(agentWakeupRequests) + .set({ + source: "automation", + reason: "issue_commented", + payload: { + issueId, + interactionId, + interactionKind: "request_confirmation", + interactionStatus: "accepted", + mutation: "interaction", + }, + }) + .where(eq(agentWakeupRequests.id, wakeupRequestId)); + await db + .update(heartbeatRuns) + .set({ + invocationSource: "automation", + contextSnapshot: { + issueId, + taskId: issueId, + wakeReason: "issue_commented", + mutation: "interaction", + interactionId, + interactionKind: "request_confirmation", + interactionStatus: "accepted", + }, + }) + .where(eq(heartbeatRuns.id, runId)); + await db + .update(issues) + .set({ status: "in_review" }) + .where(eq(issues.id, issueId)); + + const heartbeat = heartbeatService(db); + await heartbeat.resumeQueuedRuns(); + + const runs = await waitForValue(async () => { + const rows = await db + .select() + .from(heartbeatRuns) + .where(eq(heartbeatRuns.agentId, agentId)); + return rows.length >= 2 ? rows : null; + }); + expect(runs).toHaveLength(2); + + const failedRun = runs?.find((row) => row.id === runId); + const retryRun = runs?.find((row) => row.id !== runId); + expect(failedRun?.status).toBe("failed"); + expect(failedRun?.errorCode).toBe("setup_failed"); + expect(failedRun?.error).toContain("worker is not running"); + expect(retryRun).toMatchObject({ + status: "scheduled_retry", + retryOfRunId: runId, + scheduledRetryAttempt: 1, + scheduledRetryReason: INTERACTION_CONTINUATION_INFRA_RETRY_REASON, + }); + expect(retryRun?.contextSnapshot).toMatchObject({ + issueId, + interactionId, + interactionStatus: "accepted", + retryReason: INTERACTION_CONTINUATION_INFRA_RETRY_REASON, + wakeReason: INTERACTION_CONTINUATION_INFRA_WAKE_REASON, + scheduledRetryAttempt: 1, + }); + + // The lease never succeeded, so the adapter was never dispatched. + expect(mockAdapterExecute).not.toHaveBeenCalled(); + + const issue = await db + .select({ status: issues.status, executionRunId: issues.executionRunId }) + .from(issues) + .where(eq(issues.id, issueId)) + .then((rows) => rows[0] ?? null); + expect(issue).toEqual({ + status: "in_review", + executionRunId: retryRun?.id ?? null, + }); + + mockAdapterExecute.mockClear(); + }); + + it("escalates (does not retry) an accepted-interaction-continuation setup failure whose message matches neither retryable pattern", async () => { + // Negative-case counterpart to "schedules an infra retry for a setup + // failure caused by a transient sandbox provider worker restart" above. + // The injected failure message is the real *permanent* "provider not + // installed" message plugin-environment-driver.ts throws (see :135 and + // :233): "... is not installed or its plugin worker is not running." + // That phrase is NOT the transient lease-failure phrasing this heartbeat + // classifier is meant to catch (environment-runtime.ts:808's "is + // installed via plugin ... but its worker is not running"), so a + // correctly narrow classifier must not treat it as retryable + // infrastructure: it must escalate straight to needs-attention at + // attempt 1, not schedule a retry. If the classifier's sandbox-worker + // regex over-matches on the coincidental "worker is not running" + // substring, this test fails by finding a scheduled_retry row instead. + const { companyId, agentId, runId, wakeupRequestId, issueId } = await seedQueuedIssueRunFixture(); + const interactionId = randomUUID(); + + await db.insert(issueThreadInteractions).values({ + id: interactionId, + companyId, + issueId, + kind: "request_confirmation", + status: "accepted", + continuationPolicy: "wake_assignee_on_accept", + createdByAgentId: agentId, + resolvedByUserId: "responsible-user", + resolvedAt: new Date("2026-03-19T00:00:00.000Z"), + payload: { + version: 1, + prompt: "Approve the plan?", + target: { + type: "issue_document", + issueId, + key: "plan", + revisionId: randomUUID(), + }, + }, + result: { version: 1, outcome: "accepted" }, + }); + + await db + .update(agentWakeupRequests) + .set({ + source: "automation", + reason: "issue_commented", + payload: { + issueId, + interactionId, + interactionKind: "request_confirmation", + interactionStatus: "accepted", + mutation: "interaction", + }, + }) + .where(eq(agentWakeupRequests.id, wakeupRequestId)); + await db + .update(heartbeatRuns) + .set({ + invocationSource: "automation", + contextSnapshot: { + issueId, + taskId: issueId, + wakeReason: "issue_commented", + mutation: "interaction", + interactionId, + interactionKind: "request_confirmation", + interactionStatus: "accepted", + }, + }) + .where(eq(heartbeatRuns.id, runId)); + await db + .update(issues) + .set({ status: "in_review" }) + .where(eq(issues.id, issueId)); + + mockAdapterExecute.mockRejectedValueOnce( + new Error('Sandbox provider "kubernetes" is not installed or its plugin worker is not running.'), + ); + + const heartbeat = heartbeatService(db); + await heartbeat.resumeQueuedRuns(); + + const failedRun = await waitForValue(async () => { + const row = await db + .select() + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, runId)) + .then((rows) => rows[0] ?? null); + return row?.status === "failed" ? row : null; + }); + expect(failedRun?.errorCode).toBe("adapter_failed"); + expect(failedRun?.error).toContain("is not installed or its plugin worker is not running"); + + const interaction = await waitForValue(async () => { + const row = await db + .select({ result: issueThreadInteractions.result }) + .from(issueThreadInteractions) + .where(eq(issueThreadInteractions.id, interactionId)) + .then((rows) => rows[0] ?? null); + const result = row?.result ?? null; + const resumeFailure = result && "resumeFailure" in result ? result.resumeFailure : null; + return resumeFailure?.status === "needs_attention" ? row : null; + }); + expect(interaction?.result).toMatchObject({ + version: 1, + outcome: "accepted", + resumeFailure: { + status: "needs_attention", + errorCode: "adapter_failed", + runId, + }, + }); + + // No scheduled retry: the classifier's FALSE branch must not schedule + // one for this run. (A downstream, unrelated recovery reassignment run + // may still exist for the issue once it's escalated and rerouted; the + // test only cares that *this* run's failure was not classified as + // retryable infrastructure.) + const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.agentId, agentId)); + expect(runs.some((row) => row.retryOfRunId === runId)).toBe(false); + expect( + runs.some( + (row) => row.status === "scheduled_retry" && row.scheduledRetryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON, + ), + ).toBe(false); + + // Instead it escalates straight to needs-attention, same as the + // retry-exhausted path. + const issue = await db + .select({ status: issues.status }) + .from(issues) + .where(eq(issues.id, issueId)) + .then((rows) => rows[0] ?? null); + expect(issue?.status).toBe("blocked"); + + const recoveryAction = await db + .select({ status: issueRecoveryActions.status, sourceIssueId: issueRecoveryActions.sourceIssueId }) + .from(issueRecoveryActions) + .where(eq(issueRecoveryActions.sourceIssueId, issueId)) + .then((rows) => rows[0] ?? null); + expect(recoveryAction).toMatchObject({ + status: "active", + sourceIssueId: issueId, + }); + + const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId)); + expect(comments).toHaveLength(1); + expect(comments[0]).toMatchObject({ + authorType: "system", + body: expect.stringContaining("Agent failed to resume after approval: `adapter_failed` — needs attention"), + }); + + mockAdapterExecute.mockClear(); + }); + it("escalates exhausted plan approval resume failures with a system comment and recovery action", async () => { const { companyId, agentId, runId, issueId } = await seedQueuedIssueRunFixture(); const interactionId = randomUUID(); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index a1f53697e6..2500ac42b6 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -619,6 +619,27 @@ function isSpawnLikeFailureMessage(value: unknown) { return /failed to start command|spawn\b|\bENOENT\b/i.test(value); } +// A sandbox provider plugin's worker can be briefly down during its own +// restart window (e.g. a rolling deploy of the plugin worker process). Lease +// acquisition fails immediately in that window, but the condition is +// transient and self-healing, so it must be treated as retryable +// infrastructure rather than a terminal setup failure. See +// resolveSandboxProviderPlugin's "worker_unavailable" message in +// environment-runtime.ts (":808"), e.g. 'Sandbox provider "kubernetes" is +// installed via plugin "acme.kubernetes-sandbox-provider", but its worker is +// not running.' +// +// This is anchored on both "is installed via plugin" and "but its worker is +// not running" so it does not also match plugin-environment-driver.ts's +// unrelated, permanent "provider not installed" message ('Sandbox provider +// "X" is not installed or its plugin worker is not running.'), which +// coincidentally contains the same "worker is not running" substring but +// describes a terminal condition that must not be retried. +function isSandboxProviderWorkerUnavailableFailureMessage(value: unknown) { + if (typeof value !== "string") return false; + return /sandbox provider .* is installed via plugin .* but its worker is not running/i.test(value); +} + function isRetryableInteractionContinuationInfrastructureFailure( run: Pick, ) { @@ -632,7 +653,10 @@ function isRetryableInteractionContinuationInfrastructureFailure( return ( isSpawnLikeFailureMessage(run.error) || isSpawnLikeFailureMessage(resultJson.errorMessage) || - isSpawnLikeFailureMessage(resultJson.message) + isSpawnLikeFailureMessage(resultJson.message) || + isSandboxProviderWorkerUnavailableFailureMessage(run.error) || + isSandboxProviderWorkerUnavailableFailureMessage(resultJson.errorMessage) || + isSandboxProviderWorkerUnavailableFailureMessage(resultJson.message) ); }