diff --git a/doc/plugins/PLUGIN_SPEC.md b/doc/plugins/PLUGIN_SPEC.md index 45fba6be86..08b859c135 100644 --- a/doc/plugins/PLUGIN_SPEC.md +++ b/doc/plugins/PLUGIN_SPEC.md @@ -495,6 +495,7 @@ If a worker fails: - keep the rest of the instance running - retry start with bounded backoff - do not drop other plugins or core services +- a bundled plugin (shipped with the release image) that is still `error` at the next server boot is moved back to `ready` once per boot, so the startup loader gets a fresh activation attempt; an operator-`disabled` plugin is never touched ## 12.5 Graceful Shutdown Policy diff --git a/server/src/__tests__/bundled-plugins.test.ts b/server/src/__tests__/bundled-plugins.test.ts index 1d52059aaa..6dbc45f522 100644 --- a/server/src/__tests__/bundled-plugins.test.ts +++ b/server/src/__tests__/bundled-plugins.test.ts @@ -202,6 +202,7 @@ type LooseRow = { status: string; version?: string; manifestJson?: Record; + lastError?: string | null; }; // Build a minimal manifest for a persisted row or a shipped bundle. The reconcile @@ -240,6 +241,7 @@ function makeDeps(overrides?: { return { manifest: { id: pluginKey } }; }); const update = vi.fn(async () => undefined); + const updateStatus = vi.fn(async () => undefined); const loadManifest = vi.fn(async (localPath: string) => { const entry = BUNDLED_PLUGIN_CATALOG.find((candidate) => localPath.endsWith(candidate.relativePath), @@ -255,13 +257,14 @@ function makeDeps(overrides?: { registry: { getByKey: vi.fn(async (pluginKey: string) => installedRows.get(pluginKey) ?? null), update, + updateStatus, } as unknown as BundledPluginProvisionerDeps["registry"], loader: { installPlugin, loadManifest } as unknown as BundledPluginProvisionerDeps["loader"], lifecycle: { load: vi.fn(async () => undefined) }, - logger: { info: vi.fn(), error: vi.fn() }, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, bundleManifestExists: overrides?.bundleManifestExists ?? (() => true), }; - return { deps, installPlugin, update, loadManifest }; + return { deps, installPlugin, update, updateStatus, loadManifest }; } const K8S: ResolvedBundledPlugin = { @@ -286,7 +289,7 @@ describe("ensureBundledPlugins", () => { }); it("skips a plugin present in any non-uninstalled state (disabled is not re-enabled)", async () => { - for (const status of ["installed", "ready", "disabled", "error"]) { + for (const status of ["installed", "ready", "disabled", "upgrade_pending"]) { const { deps, installPlugin } = makeDeps({ rows: { [K8S.pluginKey]: { id: "row-1", pluginKey: K8S.pluginKey, status }, @@ -295,9 +298,59 @@ describe("ensureBundledPlugins", () => { await ensureBundledPlugins([K8S], deps, { reinstallUninstalled: true }); expect(installPlugin).not.toHaveBeenCalled(); expect(deps.lifecycle.load).not.toHaveBeenCalled(); + expect(deps.registry.updateStatus).not.toHaveBeenCalled(); } }); + it("resets a bundled plugin that a previous activation left in error back to ready, without reinstalling it", async () => { + const { deps, installPlugin, updateStatus } = makeDeps({ + rows: { + [K8S.pluginKey]: { + id: "row-1", + pluginKey: K8S.pluginKey, + status: "error", + lastError: 'RPC call "initialize" timed out after 15000ms', + }, + }, + }); + await ensureBundledPlugins([K8S], deps, { reinstallUninstalled: false }); + expect(installPlugin).not.toHaveBeenCalled(); + // No lifecycle call: the row goes straight back to `ready` with its error + // cleared, and the startup loadAll() does the activation (and emits the + // lifecycle events only once the worker actually started). + expect(deps.lifecycle.load).not.toHaveBeenCalled(); + expect(updateStatus).toHaveBeenCalledTimes(1); + expect(updateStatus).toHaveBeenCalledWith("row-1", { status: "ready", lastError: null }); + // The prior failure is surfaced at warn level with its recorded cause, so + // an operator reading boot logs sees why the plugin needed a retry. + expect(deps.logger.warn).toHaveBeenCalledWith( + expect.objectContaining({ + pluginKey: K8S.pluginKey, + lastError: 'RPC call "initialize" timed out after 15000ms', + }), + expect.stringContaining("re-enabling"), + ); + expect(deps.logger.error).not.toHaveBeenCalled(); + }); + + it("continues boot when resetting an errored bundled plugin fails", async () => { + const { deps, installPlugin, updateStatus } = makeDeps({ + rows: { + [K8S.pluginKey]: { id: "row-1", pluginKey: K8S.pluginKey, status: "error" }, + }, + }); + updateStatus.mockRejectedValueOnce(new Error("db down")); + await expect( + ensureBundledPlugins([K8S, DAYTONA], deps, { reinstallUninstalled: true }), + ).resolves.toBeUndefined(); + expect(deps.logger.error).toHaveBeenCalledWith( + expect.objectContaining({ pluginKey: K8S.pluginKey }), + expect.stringContaining("continuing boot"), + ); + // The later entry is still provisioned. + expect(installPlugin).toHaveBeenCalledWith({ localPath: DAYTONA.localPath }); + }); + it("reconciles the persisted manifest of a present plugin when the bundle version changed", async () => { const { deps, installPlugin, update } = makeDeps({ rows: { diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index b207227a63..9eacb08e90 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -114,6 +114,7 @@ import { INTERACTION_CONTINUATION_INFRA_RETRY_REASON, INTERACTION_CONTINUATION_INFRA_WAKE_REASON, heartbeatService, + parseSandboxProviderPluginNotReadyFailureMessage, redactDetectedSuccessfulRunProgressSummaryForBoard, redactSuccessfulRunHandoffEvidence, } from "../services/heartbeat.ts"; @@ -3621,6 +3622,189 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { mockAdapterExecute.mockClear(); }); + it("classifies only the installed-but-not-ready sandbox provider plugin message as a configuration gap", () => { + expect( + parseSandboxProviderPluginNotReadyFailureMessage( + 'Sandbox provider "kubernetes" is installed via plugin "paperclip.kubernetes-sandbox-provider", but that plugin is currently error.', + ), + ).toEqual({ + provider: "kubernetes", + pluginKey: "paperclip.kubernetes-sandbox-provider", + pluginStatus: "error", + }); + expect( + parseSandboxProviderPluginNotReadyFailureMessage( + 'Failed to acquire lease: Sandbox provider "daytona" is installed via plugin "paperclip.daytona-sandbox-provider", but that plugin is currently upgrade_pending.', + ), + ).toMatchObject({ pluginStatus: "upgrade_pending" }); + expect( + parseSandboxProviderPluginNotReadyFailureMessage( + 'Sandbox provider "kubernetes" is installed via plugin "x", but that plugin is currently disabled.', + ), + ).toMatchObject({ pluginStatus: "disabled" }); + // The transient worker-restart message keeps its retryable classification. + expect( + parseSandboxProviderPluginNotReadyFailureMessage( + 'Sandbox provider "kubernetes" is installed via plugin "paperclip.kubernetes-sandbox-provider", but its worker is not running.', + ), + ).toBeNull(); + // The permanent "not installed" message is a different condition. + expect( + parseSandboxProviderPluginNotReadyFailureMessage( + 'Sandbox provider "kubernetes" is not installed or its plugin worker is not running.', + ), + ).toBeNull(); + expect(parseSandboxProviderPluginNotReadyFailureMessage(null)).toBeNull(); + }); + + it("blocks the issue instead of re-dispatching when the sandbox provider plugin is stuck in error", async () => { + // Reproduces a production incident: the bundled Kubernetes sandbox + // provider plugin was marked `error` after one failed activation and + // nothing ever cleared it. Every run for every agent on that provider + // failed lease acquisition before dispatch with "that plugin is currently + // error", and because that message matched neither retry classifier the + // scheduler re-dispatched the same failing run every tick for days. The + // condition needs an operator, so the setup catch must record it as a + // `configuration_incomplete` gap that routes the issue to `blocked` with + // one recovery action, not as a retryable `setup_failed`. + const { companyId, agentId, runId, issueId } = + await seedQueuedIssueRunFixture(); + const pluginId = randomUUID(); + const environmentId = randomUUID(); + + await db.insert(plugins).values({ + id: pluginId, + pluginKey: "paperclip.kubernetes-sandbox-provider", + packageName: "@paperclipai/kubernetes-sandbox-provider", + version: "1.0.0", + apiVersion: 1, + categories: ["automation"], + manifestJson: { + id: "paperclip.kubernetes-sandbox-provider", + apiVersion: 1, + version: "1.0.0", + displayName: "Kubernetes Sandbox Provider", + description: "Test Kubernetes sandbox provider stuck in error", + author: "Paperclip", + categories: ["automation"], + capabilities: ["environment.drivers.register"], + entrypoints: { worker: "dist/worker.js" }, + environmentDrivers: [ + { + driverKey: "kubernetes", + kind: "sandbox_provider", + displayName: "Kubernetes Sandbox", + configSchema: { type: "object" }, + }, + ], + }, + status: "error", + lastError: 'RPC call "initialize" timed out after 15000ms', + installOrder: 1, + updatedAt: new Date(), + } as any); + await db.insert(environments).values({ + id: environmentId, + companyId, + name: "Kubernetes Sandbox", + driver: "sandbox", + status: "active", + config: { + provider: "kubernetes", + image: "fake:test", + timeoutMs: 1234, + reuseLease: false, + }, + createdAt: new Date(), + updatedAt: new Date(), + }); + await db + .update(agents) + .set({ defaultEnvironmentId: environmentId }) + .where(eq(agents.id, agentId)); + + const heartbeat = heartbeatService(db); + await heartbeat.resumeQueuedRuns(); + await waitForRunToSettle(heartbeat, runId, 5_000); + + // The lease never succeeded, so the adapter was never dispatched. + expect(mockAdapterExecute).not.toHaveBeenCalled(); + + const failedRun = await db + .select() + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, runId)) + .then((rows) => rows[0] ?? null); + expect(failedRun).toMatchObject({ + status: "failed", + errorCode: "configuration_incomplete", + }); + expect(failedRun?.error).toContain("that plugin is currently error"); + expect(failedRun?.resultJson).toMatchObject({ + configurationIncomplete: { + reason: "sandbox_provider_plugin_not_ready", + sandboxProvider: "kubernetes", + pluginKey: "paperclip.kubernetes-sandbox-provider", + pluginStatus: "error", + fingerprint: + "sandbox_provider_plugin:paperclip.kubernetes-sandbox-provider:error", + }, + }); + + const issue = await waitForValue(async () => + db + .select() + .from(issues) + .where(eq(issues.id, issueId)) + .then((rows) => { + const row = rows[0] ?? null; + return row?.status === "blocked" ? row : null; + }), + ); + expect(issue?.executionRunId).toBeNull(); + + // No scheduled retry and no fresh dispatch: the failed run is the only + // run this agent has. + const agentRuns = await db + .select({ id: heartbeatRuns.id, status: heartbeatRuns.status }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.agentId, agentId)); + expect(agentRuns).toEqual([{ id: runId, status: "failed" }]); + + const recoveryAction = await db + .select() + .from(issueRecoveryActions) + .where( + and( + eq(issueRecoveryActions.companyId, companyId), + eq(issueRecoveryActions.sourceIssueId, issueId), + ), + ) + .then((rows) => rows[0] ?? null); + expect(recoveryAction).toMatchObject({ + kind: "configuration_validation", + cause: "configuration_incomplete", + status: "active", + ownerType: "board", + }); + expect(recoveryAction?.nextAction).toContain("sandbox provider plugin"); + expect(recoveryAction?.nextAction).toContain("enable the plugin"); + + const notice = await waitForValue(async () => { + const rows = await db + .select() + .from(issueComments) + .where(eq(issueComments.issueId, issueId)); + return ( + rows.find((comment) => + comment.body.includes("paperclip.kubernetes-sandbox-provider"), + ) ?? null + ); + }); + expect(notice?.body).toContain("is in status `error`"); + expect(notice?.body).not.toContain("secret/env bindings"); + }); + it("escalates (does not retry) an accepted-interaction-continuation setup failure whose message matches neither retryable pattern", async () => { // Negative-case counterpart to "schedules an infra retry for a setup // failure caused by a transient sandbox provider worker restart" above. diff --git a/server/src/services/bundled-plugins.ts b/server/src/services/bundled-plugins.ts index 6e79c27b7d..3990934ffb 100644 --- a/server/src/services/bundled-plugins.ts +++ b/server/src/services/bundled-plugins.ts @@ -193,6 +193,7 @@ interface RegistryPluginRow { status: string; version: string; manifestJson: PaperclipPluginManifestV1; + lastError?: string | null; } export interface BundledPluginProvisionerDeps { @@ -202,6 +203,7 @@ export interface BundledPluginProvisionerDeps { id: string, data: { version?: string; manifest?: PaperclipPluginManifestV1 }, ): Promise; + updateStatus(id: string, input: { status: "ready"; lastError: string | null }): Promise; }; loader: { installPlugin(options: { localPath: string }): Promise<{ @@ -214,6 +216,7 @@ export interface BundledPluginProvisionerDeps { }; logger: { info(obj: unknown, msg?: string): void; + warn(obj: unknown, msg?: string): void; error(obj: unknown, msg?: string): void; }; /** Overridable for tests; defaults to checking `dist/manifest.js`. */ @@ -267,6 +270,58 @@ async function reconcileBundledPluginManifest( } } +/** + * Re-enable a bundled plugin that a previous boot left in `error`. + * + * `error` is not an operator choice: the loader records it when activation + * fails (for example the worker's `initialize` RPC timed out once) and it + * also switches off the worker's auto-restart. Every automatic path + * afterwards (`loadAll()`, the lazy worker recovery, the run lease) only + * considers `ready` plugins, so a bundled plugin in `error` stays unusable + * across restarts until an operator enables it by hand, and every run that + * needs its provider fails with "that plugin is currently error". The bundle + * ships with the release image and is expected to work, so one fresh attempt + * per boot is the right default: the row goes back to `ready` (with its + * `lastError` cleared), and the startup `loadAll()` activates it. If + * activation fails again the loader marks `error` again and nothing retries + * until the next boot, so this cannot loop within one process. + * + * This is a plain registry status reset, not `lifecycle.enable()`: the + * lifecycle call would emit `plugin.enabled` before `loadAll()` has started + * the worker, and a consumer of that event (the dev watcher, activity + * listeners) would act on a plugin that may still fail to activate. + * Activation, and its own events, stay with `loadAll()`. + * + * Fail-safe like the rest of the provisioner: a failed status reset is + * logged and boot continues with the plugin unavailable. + */ +async function reenableErroredBundledPlugin( + existing: RegistryPluginRow, + install: ResolvedBundledPlugin, + deps: BundledPluginProvisionerDeps, +): Promise { + deps.logger.warn( + { + pluginId: existing.id, + pluginKey: install.pluginKey, + lastError: existing.lastError ?? null, + }, + "bundled plugin is in error status from a previous activation; re-enabling it for this boot", + ); + try { + await deps.registry.updateStatus(existing.id, { status: "ready", lastError: null }); + deps.logger.info( + { pluginId: existing.id, pluginKey: install.pluginKey }, + "bundled plugin reset to ready; the startup loader will activate it", + ); + } catch (err) { + deps.logger.error( + { err, pluginId: existing.id, pluginKey: install.pluginKey }, + "Failed to re-enable errored bundled plugin; continuing boot (degraded: plugin unavailable)", + ); + } +} + /** * Ensure each resolved bundled plugin is installed and loaded. * @@ -280,6 +335,10 @@ async function reconcileBundledPluginManifest( * operator-disabled plugin is not silently re-enabled on reboot. Before the * skip, the persisted manifest is reconciled to the shipped bundle version * (see `reconcileBundledPluginManifest`). + * - The one exception is `error`, which the loader sets when an activation + * fails and which no automatic path ever clears. A bundled plugin in + * `error` is moved back to `ready` once per boot so `loadAll()` gets a + * fresh attempt (see `reenableErroredBundledPlugin`). * - A soft-uninstalled plugin is reinstalled only when * `reinstallUninstalled` is set (managed mode, where the control plane * owns provisioning). Self-hosted keeps the pre-refactor behavior of @@ -303,6 +362,10 @@ export async function ensureBundledPlugins( // plugin. The reconcile updates only the stored manifest row; the // running worker already runs the shipped code. await reconcileBundledPluginManifest(existing, install, deps, bundleManifestExists); + if (existing.status === "error") { + await reenableErroredBundledPlugin(existing, install, deps); + continue; + } deps.logger.info( { pluginKey: install.pluginKey, status: existing.status }, "bundled plugin already present; skipping auto-install", diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 350877a00e..3d80e7af99 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -356,6 +356,7 @@ import { } from "./recovery/index.js"; import { isAutomaticRecoverySuppressedByPauseHold } from "./recovery/pause-hold-guard.js"; import { + SANDBOX_PROVIDER_PLUGIN_NOT_READY_REASON, buildConfigurationIncompleteRecoveryNoticeSeed, buildExecutionReviewParticipantRecoveryNoticeSeed, buildImmediateExecutionPathRecoveryNoticeSeed, @@ -744,6 +745,39 @@ export class ConfigurationIncompleteFailure extends Error { // branch (`fix/foo` and `origin/fix/foo`) share one fingerprint, so a repeated // failure reuses one active recovery action and does not reset the attempt // count or post a duplicate notice. A different branch makes a new action. +// Build the configuration-incomplete result payload for a sandbox provider +// plugin that is installed but not `ready`. The `fingerprint` is the plugin +// key plus its status, so every run that hits the same stuck plugin reuses +// one active recovery action instead of posting a fresh notice per attempt, +// while a status change (say `error` -> `disabled`) makes a new one. +function buildSandboxProviderPluginNotReadyResultJson( + run: typeof heartbeatRuns.$inferSelect, + failure: { provider: string; pluginKey: string; pluginStatus: string }, +): Record { + const context = parseObject(run.contextSnapshot); + return { + configurationIncomplete: { + reason: SANDBOX_PROVIDER_PLUGIN_NOT_READY_REASON, + companyId: run.companyId, + agentId: run.agentId, + issueId: readNonEmptyString(context.issueId) ?? null, + projectId: readNonEmptyString(context.projectId) ?? null, + sandboxProvider: failure.provider, + pluginKey: failure.pluginKey, + pluginStatus: failure.pluginStatus, + fingerprint: `sandbox_provider_plugin:${failure.pluginKey}:${failure.pluginStatus}`, + missingBindings: [], + }, + }; +} + +function readConfigurationIncompletePayload( + run: Pick | null | undefined, +): Record | null { + const payload = parseObject(parseObject(run?.resultJson).configurationIncomplete); + return Object.keys(payload).length > 0 ? payload : null; +} + function buildUnresolvedWorkspaceBaseRefResultJson( run: typeof heartbeatRuns.$inferSelect, error: UnresolvedWorkspaceBaseRefError, @@ -950,6 +984,33 @@ function isSandboxProviderWorkerUnavailableFailureMessage(value: unknown) { ); } +// environment-runtime.ts's resolveSandboxProviderPlugin "not_ready" message, +// e.g. 'Sandbox provider "kubernetes" is installed via plugin +// "acme.kubernetes-sandbox-provider", but that plugin is currently error.' +// The plugin row exists but its status is `error` (a failed activation), +// `disabled` (an operator switched it off) or `upgrade_pending`. Unlike the +// worker restart window above, nothing on the run path ever changes that +// status: only an operator enabling the plugin, or a server boot that +// re-activates a bundled plugin, does. Re-running the agent produces the +// identical failure every time, so the setup catch classifies it as +// `configuration_incomplete` (routed to a human owner) instead of a retryable +// `setup_failed` that the scheduler would keep re-dispatching. +const SANDBOX_PROVIDER_PLUGIN_NOT_READY_RE = + /sandbox provider "([^"]*)" is installed via plugin "([^"]*)", but that plugin is currently (error|disabled|upgrade_pending)\b/i; + +export function parseSandboxProviderPluginNotReadyFailureMessage( + value: unknown, +): { provider: string; pluginKey: string; pluginStatus: string } | null { + if (typeof value !== "string") return null; + const match = SANDBOX_PROVIDER_PLUGIN_NOT_READY_RE.exec(value); + if (!match) return null; + return { + provider: match[1] ?? "", + pluginKey: match[2] ?? "", + pluginStatus: (match[3] ?? "").toLowerCase(), + }; +} + function isRetryableInteractionContinuationInfrastructureFailure( run: Pick< typeof heartbeatRuns.$inferSelect, @@ -22690,6 +22751,13 @@ export function heartbeatService( ) ? outerErr : null; + // A sandbox provider plugin stuck in error/disabled/upgrade_pending + // fails every lease the same way until an operator acts, so it is a + // configuration gap, not a transient setup failure. + const sandboxProviderPluginNotReadySetupFailure = + parseSandboxProviderPluginNotReadyFailureMessage( + outerErr instanceof Error ? outerErr.message : null, + ); const recordedResponsibleUserDenialCode = normalizeResponsibleUserDenialCode( (await getRun(runId).catch(() => null))?.errorCode, @@ -22697,7 +22765,7 @@ export function heartbeatService( const setupFailureErrorCode = workspaceValidationSetupFailure?.code ?? configurationIncompleteSetupFailure?.code ?? - (unresolvedBaseRefSetupFailure + (unresolvedBaseRefSetupFailure || sandboxProviderPluginNotReadySetupFailure ? CONFIGURATION_INCOMPLETE_FAILURE_CODE : null) ?? recordedResponsibleUserDenialCode ?? @@ -22707,6 +22775,24 @@ export function heartbeatService( "heartbeat execution setup failed", ); const setupFailureAgent = await getAgent(run.agentId).catch(() => null); + // The structured failure payload drives the recovery notice and next + // action, so it is persisted even when the agent lookup failed and the + // agent-scoped stop metadata cannot be merged in. + const setupFailureResultJson = + workspaceValidationSetupFailure?.resultJson ?? + configurationIncompleteSetupFailure?.resultJson ?? + (unresolvedBaseRefSetupFailure + ? buildUnresolvedWorkspaceBaseRefResultJson( + run, + unresolvedBaseRefSetupFailure, + ) + : null) ?? + (sandboxProviderPluginNotReadySetupFailure + ? buildSandboxProviderPluginNotReadyResultJson( + run, + sandboxProviderPluginNotReadySetupFailure, + ) + : null); const setupFailureWrite = await setRunStatusIfRunning(runId, "failed", { error: message, errorCode: setupFailureErrorCode, @@ -22719,19 +22805,13 @@ export function heartbeatService( { errorCode: setupFailureErrorCode, errorMessage: message, - resultJson: - workspaceValidationSetupFailure?.resultJson ?? - configurationIncompleteSetupFailure?.resultJson ?? - (unresolvedBaseRefSetupFailure - ? buildUnresolvedWorkspaceBaseRefResultJson( - run, - unresolvedBaseRefSetupFailure, - ) - : null), + resultJson: setupFailureResultJson, }, ), } - : {}), + : setupFailureResultJson + ? { resultJson: setupFailureResultJson } + : {}), }).catch(() => ({ run: null, updated: false as const })); if (!setupFailureWrite.updated) { logger.info( @@ -23116,7 +23196,7 @@ export function heartbeatService( issue, previousStatus: issue.status, notice: configurationIncomplete - ? buildConfigurationIncompleteRecoveryNoticeSeed() + ? buildConfigurationIncompleteRecoveryNoticeSeed(readConfigurationIncompletePayload(run)) : buildWorkspaceValidationRecoveryNoticeSeed(), recoveryCause: configurationIncomplete ? CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE @@ -23764,7 +23844,7 @@ export function heartbeatService( const notice = workspaceValidationFailure ? buildWorkspaceValidationRecoveryNoticeSeed() : configurationIncompleteFailure - ? buildConfigurationIncompleteRecoveryNoticeSeed() + ? buildConfigurationIncompleteRecoveryNoticeSeed(readConfigurationIncompletePayload(run)) : buildImmediateExecutionPathRecoveryNoticeSeed({ status: issue.status as "todo" | "in_progress", }); diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index f5f31334c3..906d0e8672 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -60,6 +60,8 @@ import { type SuccessfulRunHandoffNotice, } from "./successful-run-handoff.js"; import { + SANDBOX_PROVIDER_PLUGIN_NOT_READY_REASON, + sandboxProviderPluginRemedy, buildExecutionReviewParticipantRecoveryNoticeSeed, buildExecutionReviewParticipantUnavailableNoticeSeed, buildStrandedRecoveryEscalationNotice, @@ -297,9 +299,13 @@ function readWorkspaceValidationFingerprint(latestRun: LatestIssueRun): string | return readNonEmptyString(payload?.fingerprint); } -function readConfigurationIncompleteFingerprint(latestRun: LatestIssueRun): string | null { +function readConfigurationIncompletePayload(latestRun: LatestIssueRun): Record | null { const payload = parseObject(parseObject(latestRun?.resultJson).configurationIncomplete); - return readNonEmptyString(payload?.fingerprint); + return Object.keys(payload).length > 0 ? payload : null; +} + +function readConfigurationIncompleteFingerprint(latestRun: LatestIssueRun): string | null { + return readNonEmptyString(readConfigurationIncompletePayload(latestRun)?.fingerprint); } export type { RunOutputSilenceSummary, WatchdogDecisionActor }; @@ -1479,7 +1485,11 @@ export function recoveryService( ? "Board operator: repair the project workspace repository URL or clone access, or configure a local checkout cwd, then explicitly retry or reassign." : "Board operator: repair the source task workspace link, project workspace cwd, or git checkout, then explicitly retry or reassign." : recoveryCause === "configuration_incomplete" - ? "Board operator: bind the missing secret(s) named in the run failure, then explicitly retry the original owner or reassign." + ? readConfigurationIncompletePayload(input.latestRun)?.reason === SANDBOX_PROVIDER_PLUGIN_NOT_READY_REASON + ? `Board operator: the sandbox provider plugin named in the run failure is not ready; ${sandboxProviderPluginRemedy( + readNonEmptyString(readConfigurationIncompletePayload(input.latestRun)?.pluginStatus) ?? "error", + )}, then explicitly retry the original owner or reassign.` + : "Board operator: bind the missing secret(s) named in the run failure, then explicitly retry the original owner or reassign." : recoveryCause === "execution_review_participant_recovery" ? "Board operator: repair the failed review participant path, restore a live reviewer, explicitly reassign, or record an intentional resolution." : "Board operator: inspect the evidence, repair the runtime if appropriate, then explicitly retry the original owner, reassign, or intentionally resolve the task.", diff --git a/server/src/services/recovery/stranded-notice.test.ts b/server/src/services/recovery/stranded-notice.test.ts index e26776496c..66aebf5755 100644 --- a/server/src/services/recovery/stranded-notice.test.ts +++ b/server/src/services/recovery/stranded-notice.test.ts @@ -31,6 +31,42 @@ describe("stranded recovery notice seeds", () => { expect(seed.body).not.toContain("Recovery action:"); }); + it("names the sandbox provider plugin and its status when that is the configuration gap", () => { + const seed = buildConfigurationIncompleteRecoveryNoticeSeed({ + reason: "sandbox_provider_plugin_not_ready", + pluginKey: "paperclip.kubernetes-sandbox-provider", + pluginStatus: "error", + }); + expect(seed.title).toBe("Configuration incomplete"); + expect(seed.tone).toBe("danger"); + expect(seed.body).toContain("`paperclip.kubernetes-sandbox-provider`"); + expect(seed.body).toContain("`error`"); + expect(seed.body).toContain("enable the plugin"); + expect(seed.body).not.toContain("secret/env bindings"); + }); + + it("asks for a capability review before enabling an upgrade_pending plugin, and names an operator disable", () => { + const upgrade = buildConfigurationIncompleteRecoveryNoticeSeed({ + reason: "sandbox_provider_plugin_not_ready", + pluginKey: "paperclip.daytona-sandbox-provider", + pluginStatus: "upgrade_pending", + }); + expect(upgrade.body).toContain("review and approve the upgraded plugin's capabilities"); + const disabled = buildConfigurationIncompleteRecoveryNoticeSeed({ + reason: "sandbox_provider_plugin_not_ready", + pluginKey: "paperclip.daytona-sandbox-provider", + pluginStatus: "disabled", + }); + expect(disabled.body).toContain("an operator disabled it"); + }); + + it("keeps the secret-binding copy for other configuration gaps", () => { + expect(buildConfigurationIncompleteRecoveryNoticeSeed({ reason: "secret_binding_missing" }).body).toContain( + "secret/env bindings", + ); + expect(buildConfigurationIncompleteRecoveryNoticeSeed(null).body).toContain("secret/env bindings"); + }); + it("distinguishes todo dispatch from in_progress continuation copy", () => { expect(buildImmediateExecutionPathRecoveryNoticeSeed({ status: "todo" }).body).toContain("retried dispatch"); expect(buildImmediateExecutionPathRecoveryNoticeSeed({ status: "in_progress" }).body).toContain( diff --git a/server/src/services/recovery/stranded-notice.ts b/server/src/services/recovery/stranded-notice.ts index 75cbaace6f..4b3bc4ed49 100644 --- a/server/src/services/recovery/stranded-notice.ts +++ b/server/src/services/recovery/stranded-notice.ts @@ -71,7 +71,52 @@ export function buildWorkspaceValidationRecoveryNoticeSeed(): StrandedRecoveryNo }; } -export function buildConfigurationIncompleteRecoveryNoticeSeed(): StrandedRecoveryNoticeSeed { +export const SANDBOX_PROVIDER_PLUGIN_NOT_READY_REASON = "sandbox_provider_plugin_not_ready"; + +function readNonEmptyStringField(payload: Record | null | undefined, key: string): string | null { + const value = payload?.[key]; + return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; +} + +/** + * What the operator must do to bring a sandbox provider plugin back to + * `ready`, by the status the run observed. Enabling an `upgrade_pending` + * plugin also approves the capabilities the upgrade added, so that case asks + * for a review first. + */ +export function sandboxProviderPluginRemedy(pluginStatus: string): string { + switch (pluginStatus) { + case "upgrade_pending": + return "review and approve the upgraded plugin's capabilities, then enable it (Plugins → Enable)"; + case "disabled": + return "enable the plugin again (Plugins → Enable); an operator disabled it"; + default: + return "enable the plugin (Plugins → Enable); a server restart also re-activates a bundled plugin"; + } +} + +/** + * Seed for a `configuration_incomplete` escalation. `configurationIncomplete` + * is the structured payload the failed run recorded in `resultJson`; the body + * names the specific gap for the reasons this notice knows, and falls back to + * the secret/env-binding wording (the original and most common reason). + */ +export function buildConfigurationIncompleteRecoveryNoticeSeed( + configurationIncomplete?: Record | null, +): StrandedRecoveryNoticeSeed { + if (readNonEmptyStringField(configurationIncomplete, "reason") === SANDBOX_PROVIDER_PLUGIN_NOT_READY_REASON) { + const pluginKey = readNonEmptyStringField(configurationIncomplete, "pluginKey") ?? "the sandbox provider plugin"; + const pluginStatus = readNonEmptyStringField(configurationIncomplete, "pluginStatus") ?? "not ready"; + return { + body: + `Paperclip stopped before dispatching the adapter because the sandbox provider plugin \`${pluginKey}\` ` + + `is in status \`${pluginStatus}\` and cannot lease a sandbox. Runs will keep failing the same way until the ` + + `plugin is \`ready\` again. Moving it to \`blocked\` so an operator can ${sandboxProviderPluginRemedy(pluginStatus)} ` + + "before resuming.", + title: "Configuration incomplete", + tone: "danger", + }; + } return { body: "Paperclip stopped before dispatching the adapter because required secret/env bindings are missing. " +