diff --git a/server/src/__tests__/environment-service.test.ts b/server/src/__tests__/environment-service.test.ts index eff92293fd..4b3ef48810 100644 --- a/server/src/__tests__/environment-service.test.ts +++ b/server/src/__tests__/environment-service.test.ts @@ -630,6 +630,147 @@ describeEmbeddedPostgres("environmentService leases", () => { expect((rows[0]?.metadata as Record)?.managedKubernetesSandbox).toBe(true); }); + it("ensures and refreshes a managed sandbox environment for an arbitrary provider", async () => { + const created = await svc.ensureManagedSandboxEnvironment({ + name: "Daytona", + description: "Managed Daytona sandbox environment.", + provider: "daytona", + config: { target: "us" }, + }); + + expect(created.driver).toBe("sandbox"); + expect(created.name).toBe("Daytona"); + expect(created.config.provider).toBe("daytona"); + expect(created.config.target).toBe("us"); + expect(created.metadata?.managedByPaperclip).toBe(true); + expect(created.metadata?.managedSandboxProvider).toBe("daytona"); + + // Idempotent: a second call refreshes config and name in place, and a + // description omitted from the spec is cleared, not pinned forever. + const refreshed = await svc.ensureManagedSandboxEnvironment({ + name: "Daytona (EU)", + provider: "daytona", + config: { target: "eu" }, + }); + expect(refreshed.id).toBe(created.id); + expect(refreshed.name).toBe("Daytona (EU)"); + expect(refreshed.config.target).toBe("eu"); + expect(refreshed.description).toBeNull(); + + const rows = await db + .select() + .from(environments) + .where(eq(environments.driver, "sandbox")); + expect(rows).toHaveLength(1); + }); + + it("adopts the managed slot on a provider switch and drops the stale kubernetes marker", async () => { + const kubernetes = await svc.ensureKubernetesEnvironment({ inCluster: true, backend: "job" }); + expect(kubernetes.metadata?.managedKubernetesSandbox).toBe(true); + + const daytona = await svc.ensureManagedSandboxEnvironment({ + name: "Daytona", + provider: "daytona", + config: { target: "us" }, + }); + + expect(daytona.id).toBe(kubernetes.id); + expect(daytona.name).toBe("Daytona"); + expect(daytona.config.provider).toBe("daytona"); + expect(daytona.config.backend).toBeUndefined(); + expect(daytona.metadata?.managedSandboxProvider).toBe("daytona"); + expect(daytona.metadata?.managedKubernetesSandbox).toBeUndefined(); + expect(await svc.findKubernetesEnvironment()).toBeNull(); + + // And back: the kubernetes wrapper re-adopts the same row. + const restored = await svc.ensureKubernetesEnvironment({ inCluster: true, backend: "job" }); + expect(restored.id).toBe(kubernetes.id); + expect(restored.metadata?.managedKubernetesSandbox).toBe(true); + }); + + it("archives the managed sandbox row only for its own provider and reactivates on ensure", async () => { + // Nothing provisioned yet: archiving is a no-op. + expect(await svc.archiveManagedSandboxEnvironment({ provider: "daytona" })).toBeNull(); + + const created = await svc.ensureManagedSandboxEnvironment({ + name: "Daytona", + provider: "daytona", + config: { target: "us" }, + }); + expect(created.status).toBe("active"); + + // Another provider's unavailability leaves this provider's row alone. + expect(await svc.archiveManagedSandboxEnvironment({ provider: "kubernetes" })).toBeNull(); + + const archived = await svc.archiveManagedSandboxEnvironment({ provider: "daytona" }); + expect(archived?.id).toBe(created.id); + expect(archived?.status).toBe("archived"); + + // Already archived: a repeat call is a no-op. + expect(await svc.archiveManagedSandboxEnvironment({ provider: "daytona" })).toBeNull(); + + // The next healthy boot's ensure re-activates the same row. + const restored = await svc.ensureManagedSandboxEnvironment({ + name: "Daytona", + provider: "daytona", + config: { target: "us" }, + }); + expect(restored.id).toBe(created.id); + expect(restored.status).toBe("active"); + }); + + it("adopts an existing unmanaged sandbox row holding the desired name", async () => { + const handMade = await svc.create({ + name: "Daytona", + driver: "sandbox", + status: "active", + config: { provider: "daytona", target: "us" }, + }); + expect(handMade.metadata?.managedByPaperclip).toBeUndefined(); + + const adopted = await svc.ensureManagedSandboxEnvironment({ + name: "Daytona", + provider: "daytona", + config: { target: "eu" }, + }); + expect(adopted.id).toBe(handMade.id); + expect(adopted.config.target).toBe("eu"); + expect(adopted.metadata?.managedByPaperclip).toBe(true); + expect(adopted.metadata?.managedSandboxProvider).toBe("daytona"); + + const rows = await db + .select() + .from(environments) + .where(eq(environments.driver, "sandbox")); + expect(rows).toHaveLength(1); + }); + + it("keeps the current name when the desired name belongs to another row", async () => { + await svc.create({ + name: "Daytona", + driver: "ssh", + status: "active", + config: { + host: "fixture.example.test", + port: 22, + username: "fixture", + remoteWorkspacePath: "/srv/paperclip", + }, + }); + const kubernetes = await svc.ensureKubernetesEnvironment({ inCluster: true }); + + // The managed slot is adopted, but the rename would collide with the ssh + // row on environments_name_idx; the ensure keeps the existing name. + const adopted = await svc.ensureManagedSandboxEnvironment({ + name: "Daytona", + provider: "daytona", + config: { target: "us" }, + }); + expect(adopted.id).toBe(kubernetes.id); + expect(adopted.name).toBe(kubernetes.name); + expect(adopted.config.provider).toBe("daytona"); + }); + it("returns a conflict when creating a second environment with the same name", async () => { await seedEnvironment(); diff --git a/server/src/__tests__/managed-config.test.ts b/server/src/__tests__/managed-config.test.ts index 1717b33288..370999e348 100644 --- a/server/src/__tests__/managed-config.test.ts +++ b/server/src/__tests__/managed-config.test.ts @@ -53,6 +53,7 @@ describe("parseManagedConfigEnv", () => { catalogVersion: "2026.720.0", features: { enableApps: false, enablePipelines: true }, plugins: { autoInstall: ["daytona", "kubernetes"] }, + environments: [], }); }); @@ -66,6 +67,7 @@ describe("parseManagedConfigEnv", () => { catalogVersion: "2026.720.0", features: {}, plugins: { autoInstall: [] }, + environments: [], }); }); @@ -223,6 +225,115 @@ describe("parseManagedConfigEnv", () => { }); }); +describe("parseManagedConfigEnv environments section", () => { + const entry = (overrides: Record = {}) => ({ + name: "Daytona", + provider: "daytona", + config: { target: "us" }, + ...overrides, + }); + + it("defaults to an empty list when the section is absent (pre-section documents keep booting)", () => { + const config = parseManagedConfigEnv(envWith(validDoc())); + expect(config?.environments).toEqual([]); + }); + + it("parses a declared environment and freezes it", () => { + const config = parseManagedConfigEnv( + envWith(validDoc({ environments: [entry({ description: "Managed Daytona sandbox." })] })), + ); + expect(config?.environments).toHaveLength(1); + const spec = config?.environments[0]; + expect(spec?.name).toBe("Daytona"); + expect(spec?.description).toBe("Managed Daytona sandbox."); + expect(spec?.provider).toBe("daytona"); + expect(spec?.config).toEqual({ target: "us" }); + expect(Object.isFrozen(config?.environments)).toBe(true); + expect(Object.isFrozen(spec)).toBe(true); + expect(Object.isFrozen(spec?.config)).toBe(true); + }); + + it("treats config and description as optional", () => { + const config = parseManagedConfigEnv( + envWith(validDoc({ environments: [{ name: "Daytona", provider: "daytona" }] })), + ); + expect(config?.environments[0]?.config).toEqual({}); + expect(config?.environments[0]?.description).toBeUndefined(); + }); + + it("rejects a non-array section and non-object entries", () => { + expect(() => + parseManagedConfigEnv(envWith(validDoc({ environments: {} }))), + ).toThrow(/"environments" must be an array/); + expect(() => + parseManagedConfigEnv(envWith(validDoc({ environments: ["daytona"] }))), + ).toThrow(/"environments\[0\]" must be an object/); + }); + + it("rejects more than one entry (single managed sandbox slot)", () => { + expect(() => + parseManagedConfigEnv( + envWith(validDoc({ environments: [entry(), entry({ name: "Other", provider: "kubernetes" })] })), + ), + ).toThrow(/at most one entry/); + }); + + it("rejects unknown entry keys", () => { + expect(() => + parseManagedConfigEnv(envWith(validDoc({ environments: [entry({ envVars: {} })] }))), + ).toThrow(/unknown key "envVars"/); + }); + + it("rejects malformed names, descriptions, and providers", () => { + expect(() => + parseManagedConfigEnv(envWith(validDoc({ environments: [entry({ name: "" })] }))), + ).toThrow(/"environments\[0\].name"/); + expect(() => + parseManagedConfigEnv(envWith(validDoc({ environments: [entry({ name: " Daytona" })] }))), + ).toThrow(/"environments\[0\].name"/); + expect(() => + parseManagedConfigEnv(envWith(validDoc({ environments: [entry({ description: " " })] }))), + ).toThrow(/"environments\[0\].description"/); + expect(() => + parseManagedConfigEnv(envWith(validDoc({ environments: [entry({ provider: 7 })] }))), + ).toThrow(/"environments\[0\].provider"/); + }); + + it("rejects a provider that plugins.autoInstall does not provision", () => { + expect(() => + parseManagedConfigEnv(envWith(validDoc({ environments: [entry({ provider: "modal" })] }))), + ).toThrow(/not in "plugins.autoInstall"/); + }); + + it("rejects a config that sets provider", () => { + expect(() => + parseManagedConfigEnv( + envWith(validDoc({ environments: [entry({ config: { provider: "daytona" } })] })), + ), + ).toThrow(/must not set "provider"/); + }); + + it("rejects secret-bearing config keys at any depth (secrets travel as env vars)", () => { + expect(() => + parseManagedConfigEnv( + envWith(validDoc({ environments: [entry({ config: { apiKey: "not-a-real-key" } })] })), + ), + ).toThrow(/looks secret-bearing/); + expect(() => + parseManagedConfigEnv( + envWith(validDoc({ environments: [entry({ config: { auth: { accessToken: "t" } } })] })), + ), + ).toThrow(/auth.accessToken/); + expect(() => + parseManagedConfigEnv( + envWith( + validDoc({ environments: [entry({ config: { adapters: [{ clientSecret: "s" }] } })] }), + ), + ), + ).toThrow(/adapters\[0\].clientSecret/); + }); +}); + describe("getManagedInstanceConfig", () => { it("caches by raw env value and reparses when it changes", () => { const raw = validDoc(); diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index f7a277fd71..d309533dde 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -206,6 +206,7 @@ vi.mock("../services/index.js", () => ({ })), feedbackService: feedbackServiceFactoryMock, bootstrapExecutionPolicyFromEnv: vi.fn(async () => null), + applyManagedEnvironments: vi.fn(async () => null), environmentCustomImageService: environmentCustomImagesServiceFactoryMock, heartbeatService: heartbeatServiceFactoryMock, issueService: vi.fn(() => ({ update: vi.fn(async () => null) })), diff --git a/server/src/app.ts b/server/src/app.ts index d6f6829684..5eeeb751c0 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -590,7 +590,13 @@ export async function createApp( // swallowed per plugin so the server ALWAYS finishes booting. A degraded // boot (a provider unavailable, some agents cannot run) is strictly // preferable to a crash loop. - void ensureBundledPlugins( + // + // The chain is not awaited here (createApp stays fast), but the settled + // promise is exposed via `app.locals.bundledPluginsStartup` so boot steps + // that must not outrun plugin availability — managed sandbox environments + // (`applyManagedEnvironments`) run before the heartbeat resumes queued + // runs — can sequence on it. It never rejects. + const bundledPluginsStartup = ensureBundledPlugins( bundledPluginInstalls, { registry: pluginRegistry, loader, lifecycle, logger }, // Managed mode reinstalls soft-uninstalled bundles (the control plane @@ -609,6 +615,7 @@ export async function createApp( }).catch((err) => { logger.error({ err }, "Failed to load ready plugins on startup"); }); + app.locals.bundledPluginsStartup = bundledPluginsStartup; let appServicesShutdown = false; const shutdownAppServices = () => { if (appServicesShutdown) return; diff --git a/server/src/index.ts b/server/src/index.ts index 8161c2cd37..3908d1cfc1 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -41,6 +41,7 @@ import { setupEnvironmentCustomImageTerminalWebSocketServer } from "./realtime/e import { setupLiveEventsWebSocketServer } from "./realtime/live-events-ws.js"; import { feedbackService, + applyManagedEnvironments, backfillPrincipalAccessCompatibility, backfillLegacyToolOAuthTokens, bootstrapExecutionPolicyFromEnv, @@ -860,6 +861,31 @@ export async function startServer(): Promise { throw err; } + // Ensure sandbox environments declared in the managed-config document + // (`environments` section) before the heartbeat resumes queued runs. The + // document already parsed fail-closed above; the ensure step itself is + // fail-safe per entry (a degraded boot beats a fleet-wide crash loop), but + // a contradictory deployment that also forces PAPERCLIP_EXECUTION_MODE + // throws here and fails startup. `pluginsReady` sequences the ensure after + // the bundled-plugin install/load pass so a declared environment never + // activates before its provider driver is registered; the worker manager + // additionally gates each entry on a live plugin worker (and archives the + // row of a provider that did not come up). + try { + const bundledPluginsStartup = (app as { locals?: { bundledPluginsStartup?: Promise } }) + .locals?.bundledPluginsStartup; + const managedEnvironmentsResult = await applyManagedEnvironments(db as any, managedConfig, { + pluginsReady: bundledPluginsStartup, + workerManager: pluginWorkerManager, + }); + if (managedEnvironmentsResult) { + logger.warn(managedEnvironmentsResult, "managed sandbox environments ensured from managed config"); + } + } catch (err) { + logger.error({ err }, "failed to apply managed environments from managed config"); + throw err; + } + let drainHeartbeatRunsForShutdown: ((signal: "SIGINT" | "SIGTERM") => Promise) | null = null; let prepareHotRestartShutdown: ((signal: "SIGINT" | "SIGTERM") => Promise<{ skipDrain: boolean }>) | null = null; let heartbeatSchedulerStopped = false; diff --git a/server/src/services/environments.ts b/server/src/services/environments.ts index 6487f7a1b9..d9a4466168 100644 --- a/server/src/services/environments.ts +++ b/server/src/services/environments.ts @@ -73,6 +73,26 @@ export interface KubernetesEnvironmentConfigInput { [key: string]: unknown; } +/** + * Input to `ensureManagedSandboxEnvironment`. Provider-agnostic: `provider` + * is the sandbox plugin's driver key and is forced into `config.provider`; + * the rest of `config` is stored verbatim for the plugin to validate at + * lease time. + */ +export interface ManagedSandboxEnvironmentInput { + name: string; + description?: string; + /** Sandbox provider key (the plugin's driverKey, e.g. "kubernetes", "daytona"). */ + provider: string; + config?: Record; + /** + * Extra metadata markers stamped on the managed row (e.g. the legacy + * kubernetes marker `managedKubernetesSandbox` that + * `findKubernetesEnvironment` keys on). + */ + extraMetadata?: Record; +} + function cloneRecord(value: unknown, fallback: Record | null = null): Record | null { if (!value || typeof value !== "object" || Array.isArray(value)) return fallback; return { ...(value as Record) }; @@ -181,6 +201,177 @@ function countFromRows(rows: Array<{ count: number | string | null | undefined } } export function environmentService(db: Db) { + /** The single Paperclip-managed sandbox row (`environments_managed_sandbox_idx`), if present. */ + const findManagedSandboxRow = () => + db + .select() + .from(environments) + .where(eq(environments.driver, "sandbox")) + .then( + (rows) => + rows.find( + (row) => + (row.metadata as Record | null)?.managedByPaperclip === true, + ) ?? null, + ); + + /** + * Idempotently ensure THE Paperclip-managed sandbox environment for this + * instance, configured for an arbitrary sandbox provider plugin. Mirrors + * `ensureLocalEnvironment`; the partial unique index + * `environments_managed_sandbox_idx` enforces at most one managed sandbox + * row per instance, so this function owns that single slot regardless of + * provider: + * + * - An existing managed row is adopted and refreshed (name, description, + * config, provider) on every call, so operator/control-plane changes flow + * via redeploy without recreating the row — including a provider switch, + * which also drops a stale provider-specific metadata marker. + * - An existing UNmanaged sandbox row holding the desired name is adopted + * and stamped as managed, so a row created by hand before the instance + * became config-managed converges instead of colliding on + * `environments_name_idx` on every boot. + */ + const ensureManagedSandboxEnvironment = async ( + input: ManagedSandboxEnvironmentInput, + ): Promise => { + const desiredConfig: Record = { + ...(input.config ?? {}), + provider: input.provider, + }; + const desiredMetadata: Record = { + managedByPaperclip: true, + managedSandboxProvider: input.provider, + ...(input.extraMetadata ?? {}), + }; + + const adopt = async (row: EnvironmentRow): Promise => { + const metadata: Record = { ...(row.metadata ?? {}), ...desiredMetadata }; + // A provider switch must not leave the previous provider's marker + // behind (`findKubernetesEnvironment` keys on it). + if (desiredMetadata[KUBERNETES_MANAGED_MARKER] !== true) { + delete metadata[KUBERNETES_MANAGED_MARKER]; + } + const now = new Date(); + const runUpdate = (values: { name?: string }) => + db + .update(environments) + .set({ + ...values, + // The row mirrors the managed spec: omitting `description` clears + // a previously configured one rather than pinning it forever. + description: input.description ?? null, + config: desiredConfig, + metadata, + status: "active", + updatedAt: now, + }) + .where(eq(environments.id, row.id)) + .returning() + .then((rows) => rows[0] ?? row); + const updated = await runUpdate({ name: input.name }).catch((error: unknown) => { + // Another row already holds the desired name; keep the current name + // rather than failing a boot-time ensure over a display label. + if (hasConstraintName(error, "environments_name_idx")) { + return runUpdate({}); + } + throw error; + }); + return toEnvironment(updated); + }; + + const existing = await findManagedSandboxRow(); + if (existing) return adopt(existing); + + // The partial unique index `environments_managed_sandbox_idx` enforces + // "at most one Paperclip-managed sandbox row per instance" at the DB + // level. Use ON CONFLICT DO NOTHING keyed on that index so concurrent + // callers can race the INSERT; losers re-read the surviving row. + const now = new Date(); + const inserted = await db + .insert(environments) + .values({ + name: input.name, + description: input.description ?? null, + driver: "sandbox", + status: "active", + config: desiredConfig, + envVars: {}, + metadata: desiredMetadata, + createdAt: now, + updatedAt: now, + }) + .onConflictDoNothing({ + target: [environments.driver], + where: + sql`${environments.driver} = 'sandbox' AND (${environments.metadata} ->> 'managedByPaperclip')::boolean = true`, + }) + .returning() + .then((rows) => rows[0] ?? null) + .catch((error) => { + if ( + hasConstraintName(error, "environments_name_idx") + || hasConstraintName(error, "environments_managed_sandbox_idx") + ) { + return null; + } + throw error; + }); + if (inserted) return toEnvironment(inserted); + + // Either a concurrent caller won the managed slot, or an unmanaged row + // holds the desired name. Adopt whichever exists. + const winner = await findManagedSandboxRow(); + if (winner) return adopt(winner); + const sameName = await db + .select() + .from(environments) + .where(eq(environments.name, input.name)) + .then((rows) => rows[0] ?? null); + if (sameName) { + if (sameName.driver !== "sandbox") { + throw new Error( + `Failed to ensure managed sandbox environment: environment "${input.name}" already exists with driver "${sameName.driver}"`, + ); + } + return adopt(sameName); + } + throw new Error("Failed to ensure managed sandbox environment"); + }; + + /** + * Archive the Paperclip-managed sandbox row when its provider became + * unavailable (plugin missing, not ready, or its worker not running), so + * run scheduling stops selecting an environment whose lease acquisition + * cannot succeed (`resolveEnvironment` rejects non-active rows). + * + * Scoped to the row provisioned for the SAME provider: a row that a + * provider switch left on a different provider is not touched (the ensure + * path adopts it once the new provider is healthy). Reactivation is + * automatic — the next successful `ensureManagedSandboxEnvironment` stamps + * the row `active` again. + * + * Returns the archived environment, or null when there is no active + * managed row for this provider. + */ + const archiveManagedSandboxEnvironment = async ( + input: { provider: string }, + ): Promise => { + const existing = await findManagedSandboxRow(); + if (!existing || existing.status !== "active") return null; + const rowProvider = (existing.metadata as Record | null) + ?.managedSandboxProvider; + if (rowProvider !== input.provider) return null; + const archived = await db + .update(environments) + .set({ status: "archived", updatedAt: new Date() }) + // Guarded on status so a concurrent re-activation is not clobbered. + .where(and(eq(environments.id, existing.id), eq(environments.status, "active"))) + .returning() + .then((rows) => rows[0] ?? null); + return archived ? toEnvironment(archived) : null; + }; + return { list: async ( companyIdOrFilters?: string | EnvironmentListFilters, @@ -256,15 +447,16 @@ export function environmentService(db: Db) { return toEnvironment(existing); }, + ensureManagedSandboxEnvironment, + + archiveManagedSandboxEnvironment, + /** - * Idempotently ensure a managed Kubernetes sandbox environment exists for a - * instance, configured from instance/operator-supplied config. Mirrors - * `ensureLocalEnvironment`, but there is no DB unique index for sandbox - * drivers, so idempotency is by metadata marker + driver lookup. - * - * The environment is `driver: "sandbox"` with `config.provider: - * "kubernetes"` so it resolves to the first-party Kubernetes sandbox - * provider. On subsequent calls the config is refreshed (so operators can + * Idempotently ensure a managed Kubernetes sandbox environment exists for + * an instance, configured from instance/operator-supplied config. A thin + * wrapper over `ensureManagedSandboxEnvironment` that pins the provider to + * "kubernetes" and stamps the legacy marker `findKubernetesEnvironment` + * keys on. On subsequent calls the config is refreshed (so operators can * update egress/runtimeClass via gitops without recreating the row). */ ensureKubernetesEnvironment: async ( @@ -272,94 +464,13 @@ export function environmentService(db: Db) { maybeConfig?: KubernetesEnvironmentConfigInput, ): Promise => { const config = resolveKubernetesConfig(companyIdOrConfig, maybeConfig); - const desiredConfig: Record = { - ...config, + return ensureManagedSandboxEnvironment({ + name: DEFAULT_KUBERNETES_ENVIRONMENT_NAME, + description: DEFAULT_KUBERNETES_ENVIRONMENT_DESCRIPTION, provider: KUBERNETES_PROVIDER_KEY, - }; - const desiredMetadata: Record = { - managedByPaperclip: true, - [KUBERNETES_MANAGED_MARKER]: true, - }; - - const existing = await db - .select() - .from(environments) - .where(eq(environments.driver, "sandbox")) - .then((rows) => - rows.find( - (row) => - (row.metadata as Record | null)?.[KUBERNETES_MANAGED_MARKER] === true, - ) ?? null, - ); - - const now = new Date(); - if (existing) { - const updated = await db - .update(environments) - .set({ - config: desiredConfig, - metadata: { ...(existing.metadata ?? {}), ...desiredMetadata }, - status: "active", - updatedAt: now, - }) - .where(eq(environments.id, existing.id)) - .returning() - .then((rows) => rows[0] ?? existing); - return toEnvironment(updated); - } - - // The partial unique index `environments_managed_sandbox_idx` enforces - // "at most one Paperclip-managed sandbox row per instance" at the DB - // level. Use ON CONFLICT DO NOTHING keyed on that index so concurrent - // callers can race the INSERT; losers re-read the surviving row. - const inserted = await db - .insert(environments) - .values({ - name: DEFAULT_KUBERNETES_ENVIRONMENT_NAME, - description: DEFAULT_KUBERNETES_ENVIRONMENT_DESCRIPTION, - driver: "sandbox", - status: "active", - config: desiredConfig, - envVars: {}, - metadata: desiredMetadata, - createdAt: now, - updatedAt: now, - }) - .onConflictDoNothing({ - target: [environments.driver], - where: - sql`${environments.driver} = 'sandbox' AND (${environments.metadata} ->> 'managedByPaperclip')::boolean = true`, - }) - .returning() - .then((rows) => rows[0] ?? null) - .catch((error) => { - if ( - hasConstraintName(error, "environments_name_idx") - || hasConstraintName(error, "environments_managed_sandbox_idx") - ) { - return null; - } - throw error; - }); - if (inserted) return toEnvironment(inserted); - - const winner = await db - .select() - .from(environments) - .where(eq(environments.driver, "sandbox")) - .then( - (rows) => - rows.find( - (candidate) => - (candidate.metadata as Record | null)?.[ - KUBERNETES_MANAGED_MARKER - ] === true, - ) ?? null, - ); - if (!winner) { - throw new Error("Failed to ensure kubernetes environment"); - } - return toEnvironment(winner); + config, + extraMetadata: { [KUBERNETES_MANAGED_MARKER]: true }, + }); }, /** diff --git a/server/src/services/index.ts b/server/src/services/index.ts index 5f05bcf00d..08e0386c6d 100644 --- a/server/src/services/index.ts +++ b/server/src/services/index.ts @@ -116,9 +116,11 @@ export { managedFeatureKeySet, parseManagedConfigEnv, MANAGED_CONFIG_ENV_KEY, + type ManagedEnvironmentSpec, type ManagedInstanceConfig, } from "./managed-config.js"; export { bootstrapExecutionPolicyFromEnv } from "./execution-policy-bootstrap.js"; +export { applyManagedEnvironments } from "./managed-environments.js"; export { cloudUpstreamService, reconcileCloudUpstreamRunsOnStartup } from "./cloud-upstreams.js"; export { companyPortabilityService } from "./company-portability.js"; export { teamsCatalogService } from "./teams-catalog.js"; diff --git a/server/src/services/managed-config.ts b/server/src/services/managed-config.ts index 87c37b5332..b723930201 100644 --- a/server/src/services/managed-config.ts +++ b/server/src/services/managed-config.ts @@ -9,7 +9,10 @@ * "mode": "cloud", * "catalogVersion": "2026.720.0", * "features": { "": true | false, ... }, - * "plugins": { "autoInstall": ["daytona", "kubernetes"] } + * "plugins": { "autoInstall": ["daytona", "kubernetes"] }, + * "environments": [ + * { "name": "Daytona", "provider": "daytona", "config": { "target": "us" } } + * ] * } * * Parsing follows the `execution-policy-bootstrap.ts` doctrine: a pure @@ -47,6 +50,52 @@ export interface ManagedInstanceConfig { catalogVersion: string; features: Readonly>>; plugins: { readonly autoInstall: readonly string[] }; + /** Sandbox environments the control plane provisions at boot (empty when the section is absent). */ + environments: readonly ManagedEnvironmentSpec[]; +} + +/** + * One declared managed sandbox environment. Provider-agnostic: `provider` is + * the sandbox plugin's driver key (== its `plugins.autoInstall` catalog key), + * and `config` is stored verbatim in the environment row for the plugin to + * validate at lease time — the same split `ensureKubernetesEnvironment` has + * always used, generalized to any bundled sandbox provider. + */ +export interface ManagedEnvironmentSpec { + /** Display name of the instance-level environment row (unique per instance). */ + name: string; + description?: string; + /** Sandbox provider key (the plugin's driverKey, e.g. "daytona"). */ + provider: string; + /** Provider config stored in `environment.config`; never carries secrets. */ + config: Readonly>; +} + +/** + * Managed-config documents deliver NO secrets, ever. Provider credentials + * reach a managed instance as process environment variables (every bundled + * sandbox provider falls back to its env var when `config` omits the key, + * e.g. `DAYTONA_API_KEY`), so any secret-looking config key in the document + * is a misrouted credential and fails startup. + */ +const SECRET_LIKE_CONFIG_KEY_PATTERN = /(api[-_]?key|token|secret|password|credential)/i; + +function findSecretLikeConfigKey(value: Record, path: string): string | null { + for (const [key, child] of Object.entries(value)) { + const childPath = path.length > 0 ? `${path}.${key}` : key; + if (SECRET_LIKE_CONFIG_KEY_PATTERN.test(key)) return childPath; + if (isPlainObject(child)) { + const nested = findSecretLikeConfigKey(child, childPath); + if (nested) return nested; + } else if (Array.isArray(child)) { + for (const [index, element] of child.entries()) { + if (!isPlainObject(element)) continue; + const nested = findSecretLikeConfigKey(element, `${childPath}[${index}]`); + if (nested) return nested; + } + } + } + return null; } let cachedFeatureKeys: ReadonlySet | null = null; @@ -106,10 +155,10 @@ export function parseManagedConfigEnv(env: ManagedConfigEnv): ManagedInstanceCon fail(`must be a JSON object (got ${describeJsonValue(doc)})`); } - const allowedTopLevelKeys = new Set(["v", "mode", "catalogVersion", "features", "plugins"]); + const allowedTopLevelKeys = new Set(["v", "mode", "catalogVersion", "features", "plugins", "environments"]); for (const key of Object.keys(doc)) { if (!allowedTopLevelKeys.has(key)) { - fail(`has unknown top-level key "${key}" (allowed: v, mode, catalogVersion, features, plugins)`); + fail(`has unknown top-level key "${key}" (allowed: v, mode, catalogVersion, features, plugins, environments)`); } } @@ -195,12 +244,93 @@ export function parseManagedConfigEnv(env: ManagedConfigEnv): ManagedInstanceCon autoInstall.push(entry); } + // `environments` is OPTIONAL, unlike `features` and `plugins`: documents + // delivered before the section existed must keep booting newer builds (a + // fleet image roll cannot be lockstepped with a config re-delivery), and + // absence is not a dropped security control — it simply declares no managed + // environments. When present, the section is validated fail-closed like + // everything else. + const environmentSpecs: ManagedEnvironmentSpec[] = []; + if (doc.environments !== undefined) { + if (!Array.isArray(doc.environments)) { + fail(`"environments" must be an array of environment objects (got ${describeJsonValue(doc.environments)})`); + } + // The DB enforces at most ONE Paperclip-managed sandbox row per instance + // (partial unique index `environments_managed_sandbox_idx`); every entry + // here provisions that row, so a longer list can never be satisfied. + if (doc.environments.length > 1) { + fail( + `"environments" supports at most one entry: each entry provisions the single Paperclip-managed sandbox environment (DB invariant environments_managed_sandbox_idx)`, + ); + } + for (const [index, entry] of doc.environments.entries()) { + if (!isPlainObject(entry)) { + fail(`"environments[${index}]" must be an object (got ${describeJsonValue(entry)})`); + } + const allowedEntryKeys = new Set(["name", "description", "provider", "config"]); + for (const key of Object.keys(entry)) { + if (!allowedEntryKeys.has(key)) { + fail(`"environments[${index}]" has unknown key "${key}" (allowed: name, description, provider, config)`); + } + } + if (typeof entry.name !== "string" || entry.name.length === 0 || entry.name.trim() !== entry.name) { + fail( + `"environments[${index}].name" must be a non-empty string without surrounding whitespace (got ${describeJsonValue(entry.name)})`, + ); + } + if ( + entry.description !== undefined + && (typeof entry.description !== "string" || entry.description.trim().length === 0) + ) { + fail(`"environments[${index}].description" must be a non-empty string when present (got ${describeJsonValue(entry.description)})`); + } + if (typeof entry.provider !== "string" || entry.provider.length === 0 || entry.provider.trim() !== entry.provider) { + fail( + `"environments[${index}].provider" must be a non-empty string without surrounding whitespace (got ${describeJsonValue(entry.provider)})`, + ); + } + // Coherence: on a managed instance the control plane is the only plugin + // install path, so an environment whose provider plugin is not + // auto-installed could never serve a lease. Catch the skew at parse time. + if (!autoInstall.includes(entry.provider)) { + fail( + `"environments[${index}].provider" is "${entry.provider}", which is not in "plugins.autoInstall"; a managed environment requires its provider plugin to be provisioned`, + ); + } + const config: Record = {}; + if (entry.config !== undefined) { + if (!isPlainObject(entry.config)) { + fail(`"environments[${index}].config" must be an object (got ${describeJsonValue(entry.config)})`); + } + if (entry.config.provider !== undefined) { + fail(`"environments[${index}].config" must not set "provider"; it is forced from the entry's provider key`); + } + const secretLikeKey = findSecretLikeConfigKey(entry.config, ""); + if (secretLikeKey) { + fail( + `"environments[${index}].config" key "${secretLikeKey}" looks secret-bearing; credentials are delivered to managed instances as process environment variables (the provider's documented env fallback), never in the managed-config document`, + ); + } + Object.assign(config, entry.config); + } + environmentSpecs.push( + Object.freeze({ + name: entry.name, + ...(entry.description !== undefined ? { description: entry.description } : {}), + provider: entry.provider, + config: Object.freeze(config), + }) as ManagedEnvironmentSpec, + ); + } + } + return Object.freeze({ v: SUPPORTED_MANAGED_CONFIG_VERSION, mode: "cloud", catalogVersion: doc.catalogVersion, features: Object.freeze(features), plugins: Object.freeze({ autoInstall: Object.freeze(autoInstall) }), + environments: Object.freeze(environmentSpecs), }) as ManagedInstanceConfig; } diff --git a/server/src/services/managed-environments.test.ts b/server/src/services/managed-environments.test.ts new file mode 100644 index 0000000000..4857c26a8b --- /dev/null +++ b/server/src/services/managed-environments.test.ts @@ -0,0 +1,415 @@ +import { describe, expect, it, vi } from "vitest"; +import type { Db } from "@paperclipai/db"; +import { MANAGED_CONFIG_ENV_KEY, parseManagedConfigEnv } from "./managed-config.js"; +import { + applyManagedEnvironments, + type ApplyManagedEnvironmentsOptions, +} from "./managed-environments.js"; + +const noDb = null as unknown as Db; + +function parsedConfig(overrides: Record = {}) { + const config = parseManagedConfigEnv({ + [MANAGED_CONFIG_ENV_KEY]: JSON.stringify({ + v: 1, + mode: "cloud", + catalogVersion: "2026.720.0", + features: {}, + plugins: { autoInstall: ["daytona"] }, + ...overrides, + }), + }); + if (!config) throw new Error("expected a parsed managed config"); + return config; +} + +function environmentRow(overrides: Record = {}) { + const now = new Date(); + return { + id: "env-1", + name: "Daytona", + description: null, + driver: "sandbox", + status: "active", + config: {}, + envVars: {}, + metadata: {}, + createdAt: now, + updatedAt: now, + ...overrides, + }; +} + +function readyDriverResolver(status = "ready") { + return vi.fn(async () => ({ + plugin: { id: "plugin-1", pluginKey: "sandbox-providers/daytona", status }, + })); +} + +function runningWorkerManager(running = true) { + return { isRunning: vi.fn(() => running), getWorker: vi.fn(() => undefined) }; +} + +type WorkerManagerSeam = NonNullable; +type RecoveryHandle = NonNullable>; + +/** A worker handle fake that records `ready` listeners so tests can fire them. */ +function recoveryHandle() { + const readyListeners: Array<(payload: { pluginId: string }) => void> = []; + const handle: RecoveryHandle = { + on: vi.fn((_event, listener) => { + readyListeners.push(listener); + }), + off: vi.fn((_event, listener) => { + const at = readyListeners.indexOf(listener); + if (at !== -1) readyListeners.splice(at, 1); + }), + }; + return { + handle, + fireReady: () => { + for (const listener of [...readyListeners]) listener({ pluginId: "plugin-1" }); + }, + }; +} + +/** Lets the fire-and-forget recovery ensure settle. */ +function tick() { + return new Promise((resolve) => setImmediate(resolve)); +} + +type EnvironmentsSeam = NonNullable; + +function environmentsSeam(overrides: Partial = {}): EnvironmentsSeam { + return { + ensureManagedSandboxEnvironment: + overrides.ensureManagedSandboxEnvironment ?? vi.fn().mockResolvedValue(environmentRow()), + archiveManagedSandboxEnvironment: + overrides.archiveManagedSandboxEnvironment ?? vi.fn().mockResolvedValue(null), + }; +} + +describe("applyManagedEnvironments", () => { + it("no-ops for self-hosted instances and for documents without environments", async () => { + expect(await applyManagedEnvironments(noDb, null)).toBeNull(); + expect(await applyManagedEnvironments(noDb, parsedConfig())).toBeNull(); + }); + + it("refuses startup when a forced execution mode also claims the managed sandbox slot", async () => { + const config = parsedConfig({ + environments: [{ name: "Daytona", provider: "daytona" }], + }); + await expect( + applyManagedEnvironments(noDb, config, { + env: { PAPERCLIP_EXECUTION_MODE: "kubernetes" }, + }), + ).rejects.toThrow(/mutually exclusive/); + }); + + it("ensures each declared environment through the provider-agnostic service call", async () => { + const ensureManagedSandboxEnvironment = vi + .fn() + .mockResolvedValue(environmentRow()); + const config = parsedConfig({ + environments: [ + { + name: "Daytona", + description: "Managed Daytona sandbox.", + provider: "daytona", + config: { target: "us" }, + }, + ], + }); + + const resolveSandboxProviderDriver = readyDriverResolver(); + const workerManager = runningWorkerManager(); + const result = await applyManagedEnvironments(noDb, config, { + env: {}, + workerManager, + environments: environmentsSeam({ ensureManagedSandboxEnvironment }), + resolveSandboxProviderDriver, + }); + + expect(result).toEqual({ ensured: 1, failed: 0 }); + expect(resolveSandboxProviderDriver).toHaveBeenCalledWith({ db: noDb, driverKey: "daytona" }); + expect(workerManager.isRunning).toHaveBeenCalledWith("plugin-1"); + expect(ensureManagedSandboxEnvironment).toHaveBeenCalledTimes(1); + expect(ensureManagedSandboxEnvironment).toHaveBeenCalledWith({ + name: "Daytona", + description: "Managed Daytona sandbox.", + provider: "daytona", + config: { target: "us" }, + }); + // The frozen parsed config must not leak into the service (the row's + // config is mutated downstream when the provider key is forced in). + const passedConfig = ensureManagedSandboxEnvironment.mock.calls[0]?.[0]?.config; + expect(Object.isFrozen(passedConfig)).toBe(false); + }); + + it("waits for the bundled-plugin startup pass before ensuring anything", async () => { + const ensureManagedSandboxEnvironment = vi + .fn() + .mockResolvedValue(environmentRow()); + const config = parsedConfig({ + environments: [{ name: "Daytona", provider: "daytona" }], + }); + + let releasePlugins!: () => void; + const pluginsReady = new Promise((resolve) => { + releasePlugins = resolve; + }); + + const pending = applyManagedEnvironments(noDb, config, { + env: {}, + pluginsReady, + workerManager: runningWorkerManager(), + environments: environmentsSeam({ ensureManagedSandboxEnvironment }), + resolveSandboxProviderDriver: readyDriverResolver(), + }); + + // Give the ensure every chance to (incorrectly) run early. + await new Promise((resolve) => setImmediate(resolve)); + expect(ensureManagedSandboxEnvironment).not.toHaveBeenCalled(); + + releasePlugins(); + expect(await pending).toEqual({ ensured: 1, failed: 0 }); + expect(ensureManagedSandboxEnvironment).toHaveBeenCalledTimes(1); + }); + + it("skips an entry whose provider plugin is missing and archives its stale row", async () => { + const environments = environmentsSeam({ + archiveManagedSandboxEnvironment: vi.fn().mockResolvedValue(environmentRow({ status: "archived" })), + }); + const config = parsedConfig({ + environments: [{ name: "Daytona", provider: "daytona" }], + }); + + const result = await applyManagedEnvironments(noDb, config, { + env: {}, + workerManager: runningWorkerManager(), + environments, + resolveSandboxProviderDriver: vi.fn(async () => null), + }); + + expect(result).toEqual({ ensured: 0, failed: 1 }); + expect(environments.ensureManagedSandboxEnvironment).not.toHaveBeenCalled(); + expect(environments.archiveManagedSandboxEnvironment).toHaveBeenCalledWith({ + provider: "daytona", + }); + }); + + it("skips (and counts failed) an entry whose provider plugin is not ready", async () => { + const environments = environmentsSeam(); + const config = parsedConfig({ + environments: [{ name: "Daytona", provider: "daytona" }], + }); + + const result = await applyManagedEnvironments(noDb, config, { + env: {}, + workerManager: runningWorkerManager(), + environments, + resolveSandboxProviderDriver: readyDriverResolver("disabled"), + }); + + expect(result).toEqual({ ensured: 0, failed: 1 }); + expect(environments.ensureManagedSandboxEnvironment).not.toHaveBeenCalled(); + expect(environments.archiveManagedSandboxEnvironment).toHaveBeenCalledWith({ + provider: "daytona", + }); + }); + + it("skips an entry whose plugin record is ready but whose worker is not running", async () => { + const environments = environmentsSeam(); + const config = parsedConfig({ + environments: [{ name: "Daytona", provider: "daytona" }], + }); + + const workerManager = runningWorkerManager(false); + const result = await applyManagedEnvironments(noDb, config, { + env: {}, + workerManager, + environments, + resolveSandboxProviderDriver: readyDriverResolver(), + }); + + expect(result).toEqual({ ensured: 0, failed: 1 }); + expect(workerManager.isRunning).toHaveBeenCalledWith("plugin-1"); + expect(environments.ensureManagedSandboxEnvironment).not.toHaveBeenCalled(); + expect(environments.archiveManagedSandboxEnvironment).toHaveBeenCalledWith({ + provider: "daytona", + }); + // No registered handle → no respawn is coming; degraded until next boot. + expect(workerManager.getWorker).toHaveBeenCalledWith("plugin-1"); + }); + + it("reactivates the archived environment once when the provider worker recovers", async () => { + const environments = environmentsSeam(); + const config = parsedConfig({ + environments: [ + { name: "Daytona", provider: "daytona", config: { target: "us" } }, + ], + }); + + const { handle, fireReady } = recoveryHandle(); + const workerManager = { + isRunning: vi.fn(() => false), + getWorker: vi.fn(() => handle), + }; + const result = await applyManagedEnvironments(noDb, config, { + env: {}, + workerManager, + environments, + resolveSandboxProviderDriver: readyDriverResolver(), + }); + + // The boot pass itself stays degraded: archived, counted failed. + expect(result).toEqual({ ensured: 0, failed: 1 }); + expect(environments.archiveManagedSandboxEnvironment).toHaveBeenCalledWith({ + provider: "daytona", + }); + expect(environments.ensureManagedSandboxEnvironment).not.toHaveBeenCalled(); + + // Worker manager respawns the worker → the handle re-emits `ready`. + fireReady(); + fireReady(); + await tick(); + + expect(environments.ensureManagedSandboxEnvironment).toHaveBeenCalledTimes(1); + expect(environments.ensureManagedSandboxEnvironment).toHaveBeenCalledWith({ + name: "Daytona", + description: undefined, + provider: "daytona", + config: { target: "us" }, + }); + expect(handle.off).toHaveBeenCalledTimes(1); + }); + + it("re-checks isRunning after subscribing so a recovery during the archive is not missed", async () => { + const environments = environmentsSeam(); + const config = parsedConfig({ + environments: [{ name: "Daytona", provider: "daytona" }], + }); + + const { handle } = recoveryHandle(); + // Down at the gate check, already back up by the time the listener is + // registered — the `ready` event has been missed. + const workerManager = { + isRunning: vi + .fn() + .mockReturnValueOnce(false) + .mockReturnValue(true), + getWorker: vi.fn(() => handle), + }; + const result = await applyManagedEnvironments(noDb, config, { + env: {}, + workerManager, + environments, + resolveSandboxProviderDriver: readyDriverResolver(), + }); + await tick(); + + expect(result).toEqual({ ensured: 0, failed: 1 }); + expect(environments.ensureManagedSandboxEnvironment).toHaveBeenCalledTimes(1); + }); + + it("does not subscribe for recovery when the plugin record is missing or not ready", async () => { + const config = parsedConfig({ + environments: [{ name: "Daytona", provider: "daytona" }], + }); + + for (const resolveSandboxProviderDriver of [ + vi.fn(async () => null), + readyDriverResolver("disabled"), + ]) { + const workerManager = runningWorkerManager(false); + await applyManagedEnvironments(noDb, config, { + env: {}, + workerManager, + environments: environmentsSeam(), + resolveSandboxProviderDriver, + }); + expect(workerManager.getWorker).not.toHaveBeenCalled(); + } + }); + + it("logs, not throws, when the recovery re-ensure fails", async () => { + const environments = environmentsSeam({ + ensureManagedSandboxEnvironment: vi.fn().mockRejectedValue(new Error("db exploded")), + }); + const config = parsedConfig({ + environments: [{ name: "Daytona", provider: "daytona" }], + }); + + const { handle, fireReady } = recoveryHandle(); + const workerManager = { + isRunning: vi.fn(() => false), + getWorker: vi.fn(() => handle), + }; + await applyManagedEnvironments(noDb, config, { + env: {}, + workerManager, + environments, + resolveSandboxProviderDriver: readyDriverResolver(), + }); + + fireReady(); + await tick(); + // The rejection is swallowed into a log line; reaching this point without + // an unhandled rejection is the assertion. + expect(environments.ensureManagedSandboxEnvironment).toHaveBeenCalledTimes(1); + }); + + it("fails closed when no worker manager is provided", async () => { + const environments = environmentsSeam(); + const config = parsedConfig({ + environments: [{ name: "Daytona", provider: "daytona" }], + }); + + const result = await applyManagedEnvironments(noDb, config, { + env: {}, + environments, + resolveSandboxProviderDriver: readyDriverResolver(), + }); + + expect(result).toEqual({ ensured: 0, failed: 1 }); + expect(environments.ensureManagedSandboxEnvironment).not.toHaveBeenCalled(); + }); + + it("counts a skipped entry once even when archiving its stale row fails", async () => { + const environments = environmentsSeam({ + archiveManagedSandboxEnvironment: vi.fn().mockRejectedValue(new Error("db exploded")), + }); + const config = parsedConfig({ + environments: [{ name: "Daytona", provider: "daytona" }], + }); + + const result = await applyManagedEnvironments(noDb, config, { + env: {}, + workerManager: runningWorkerManager(), + environments, + resolveSandboxProviderDriver: vi.fn(async () => null), + }); + + expect(result).toEqual({ ensured: 0, failed: 1 }); + expect(environments.archiveManagedSandboxEnvironment).toHaveBeenCalledTimes(1); + }); + + it("is fail-safe per entry: an ensure failure is counted, not thrown", async () => { + const environments = environmentsSeam({ + ensureManagedSandboxEnvironment: vi.fn().mockRejectedValue(new Error("db exploded")), + }); + const config = parsedConfig({ + environments: [{ name: "Daytona", provider: "daytona" }], + }); + + const result = await applyManagedEnvironments(noDb, config, { + env: {}, + workerManager: runningWorkerManager(), + environments, + resolveSandboxProviderDriver: readyDriverResolver(), + }); + + expect(result).toEqual({ ensured: 0, failed: 1 }); + expect(environments.archiveManagedSandboxEnvironment).not.toHaveBeenCalled(); + }); +}); diff --git a/server/src/services/managed-environments.ts b/server/src/services/managed-environments.ts new file mode 100644 index 0000000000..fbdf2c6985 --- /dev/null +++ b/server/src/services/managed-environments.ts @@ -0,0 +1,239 @@ +/** + * Managed-environment bootstrap (harness → app contract). + * + * Managed-cloud instances may declare sandbox environments in the + * `environments` section of `PAPERCLIP_MANAGED_CONFIG` (parsed fail-closed in + * `managed-config.ts`). On boot, each declared environment is idempotently + * ensured as the instance-level Paperclip-managed sandbox row via the + * provider-agnostic `ensureManagedSandboxEnvironment` — the control plane + * provisions, tenants use, for any bundled sandbox provider plugin. + * + * The failure posture mirrors bundled plugin provisioning + * (`bundled-plugins.ts`), deliberately split: + * + * 1. **Validation fails closed at parse time** (`managed-config.ts`): a + * malformed section refuses startup with a precise error. + * 2. **The DB ensure step is fail-safe per entry**: an ensure failure is + * logged and boot continues degraded (environment unavailable) rather + * than crash-looping a fleet. + * + * Ensuring is additionally synchronized with provider availability: the + * caller's `pluginsReady` promise (the bundled-plugin install/load pass) is + * awaited first, and an entry whose provider plugin is not installed, `ready`, + * AND running a live worker (`workerManager.isRunning`; a `ready` record whose + * activation failed has no worker and cannot serve leases) afterwards is + * skipped (counted failed) instead of being written as an active row — + * otherwise the heartbeat would resume queued runs against an environment + * whose lease acquisition cannot succeed yet. When such an entry was + * provisioned by an earlier boot, its still-active row is archived + * (`archiveManagedSandboxEnvironment`) for the same reason. Re-activation + * happens on the next healthy boot's ensure, or earlier: when the plugin + * record is `ready` and only the worker is down (a crash in restart-backoff), + * a one-shot `ready` listener on the worker handle re-runs the ensure as soon + * as the worker manager respawns the worker, so a transient crash does not + * leave the environment archived until someone restarts the server. + * + * Removing an entry from the document stops future refreshes but never + * deletes or archives the row — there is intentionally no unprovision path + * here, matching `plugins.autoInstall` semantics (leases may still reference + * the row; withdrawal is an explicit operator action). Archiving above is + * scoped to a declared-but-unavailable provider, not to document removal. + * + * Provider credentials are never part of the declared config: every bundled + * sandbox provider falls back to its documented process environment variable + * (e.g. `DAYTONA_API_KEY`) when `config` omits the key, so the deployment + * delivers secrets as env vars and the managed document stays secret-free. + */ + +import type { Db } from "@paperclipai/db"; +import { logger } from "../middleware/logger.js"; +import { environmentService } from "./environments.js"; +import type { ManagedInstanceConfig } from "./managed-config.js"; +import { parseExecutionPolicyBootstrapEnv } from "./execution-policy-bootstrap.js"; +import { resolvePluginSandboxProviderDriverByKey } from "./plugin-environment-driver.js"; +import type { PluginWorkerManager } from "./plugin-worker-manager.js"; + +export interface ApplyManagedEnvironmentsOptions { + env?: Record; + /** + * Resolves when the bundled-plugin startup pass (install + load; the + * `ensureBundledPlugins` chain in `createApp`) has finished. Awaited before + * any environment is ensured so an active row never precedes its provider + * driver. The promise never rejects (the chain catches internally). + */ + pluginsReady?: Promise; + /** + * The app's plugin worker manager, consulted per entry so an environment is + * only ensured while its provider plugin has a live worker (a `ready` + * registry record whose activation failed cannot serve leases). Fails + * closed: without a worker manager every entry is treated as unavailable. + * + * `getWorker` is `PluginWorkerManager.getWorker` narrowed to the `ready` + * subscription used for post-archive recovery: a worker in crash-restart + * backoff re-emits `ready` when the manager respawns it, and the listener + * re-ensures the archived environment (see `applyManagedEnvironments`). + */ + workerManager?: Pick & { + getWorker(pluginId: string): + | { + on(event: "ready", listener: (payload: { pluginId: string }) => void): void; + off(event: "ready", listener: (payload: { pluginId: string }) => void): void; + } + | undefined; + }; + /** Test seam: overrides the environment service built from `db`. */ + environments?: Pick< + ReturnType, + "ensureManagedSandboxEnvironment" | "archiveManagedSandboxEnvironment" + >; + /** Test seam: overrides the sandbox-provider plugin driver lookup. */ + resolveSandboxProviderDriver?: (input: { + db: Db; + driverKey: string; + }) => Promise<{ plugin: { id: string; pluginKey: string; status: string } } | null>; +} + +/** + * Ensure every environment declared in the managed-config document. Returns + * null when there is nothing to do (self-hosted, or no `environments` + * section); otherwise the ensured/failed counts. Idempotent; safe to call on + * every boot. + */ +export async function applyManagedEnvironments( + db: Db, + managedConfig: ManagedInstanceConfig | null, + opts: ApplyManagedEnvironmentsOptions = {}, +): Promise<{ ensured: number; failed: number } | null> { + if (!managedConfig || managedConfig.environments.length === 0) return null; + + // The forced-execution-mode bootstrap (`PAPERCLIP_EXECUTION_MODE=kubernetes`) + // and this one both own the single Paperclip-managed sandbox row + // (`environments_managed_sandbox_idx`). Configuring both is contradictory; + // refuse startup rather than let bootstrap ordering pick a winner. + const env = opts.env ?? process.env; + if (parseExecutionPolicyBootstrapEnv(env)) { + throw new Error( + `PAPERCLIP_EXECUTION_MODE and the PAPERCLIP_MANAGED_CONFIG "environments" section are mutually exclusive: both manage the single instance sandbox environment`, + ); + } + + // The heartbeat resumes queued runs right after this bootstrap step, and + // lease acquisition fails hard on a provider whose plugin is missing or not + // ready. Wait for the bundled-plugin startup pass to finish, then refuse to + // ensure (and in particular to re-activate) a row whose provider driver did + // not come up — a degraded boot without the row beats an active environment + // that fails every lease until the plugin recovers. + await opts.pluginsReady; + + const resolveDriver = opts.resolveSandboxProviderDriver ?? resolvePluginSandboxProviderDriverByKey; + const environments = opts.environments ?? environmentService(db); + + // Recovery path for a `ready` plugin whose worker was down at check time: + // that shape is usually a crash in restart-backoff, and the manager's + // respawn re-emits `ready` on the same handle. A one-shot listener re-runs + // the idempotent ensure so the recovered provider becomes selectable again + // without waiting for the next boot (this pass has already archived the + // row). The post-subscribe `isRunning` re-check closes the race where the + // worker recovered between the gate check and the subscription. No handle + // means no respawn is coming (activation never started a worker), so the + // degraded-until-next-boot posture stands. + const scheduleRecoveryReactivation = ( + spec: ManagedInstanceConfig["environments"][number], + pluginId: string, + ): void => { + const manager = opts.workerManager; + const handle = manager?.getWorker(pluginId); + if (!handle) return; + let reactivated = false; + const reactivate = (): void => { + if (reactivated) return; + reactivated = true; + handle.off("ready", reactivate); + void environments + .ensureManagedSandboxEnvironment({ + name: spec.name, + description: spec.description, + provider: spec.provider, + config: { ...spec.config }, + }) + .then((environment) => { + logger.info( + { environmentId: environment.id, name: spec.name, provider: spec.provider }, + "managed sandbox environment reactivated after provider worker recovery", + ); + }) + .catch((err: unknown) => { + logger.error( + { err, name: spec.name, provider: spec.provider }, + "failed to reactivate managed sandbox environment after provider worker recovery (degraded: environment unavailable until next boot)", + ); + }); + }; + handle.on("ready", reactivate); + if (manager?.isRunning(pluginId)) reactivate(); + }; + + let ensured = 0; + let failed = 0; + for (const spec of managedConfig.environments) { + try { + const resolved = await resolveDriver({ db, driverKey: spec.provider }); + // `ready` in the registry is necessary but not sufficient: activation + // can fail after install leaves the record `ready`, in which case no + // worker is running and the driver cannot serve leases. + const workerRunning = + resolved != null && (opts.workerManager?.isRunning(resolved.plugin.id) ?? false); + if (!resolved || resolved.plugin.status !== "ready" || !workerRunning) { + failed += 1; + // A row provisioned by an earlier boot must not stay active either — + // archive it so run scheduling stops selecting it. Best-effort: an + // archive failure is logged, and the entry is already counted failed. + const archived = await environments + .archiveManagedSandboxEnvironment({ provider: spec.provider }) + .catch((archiveErr: unknown) => { + logger.error( + { err: archiveErr, name: spec.name, provider: spec.provider }, + "failed to archive the managed sandbox environment of an unavailable provider", + ); + return null; + }); + logger.error( + { + name: spec.name, + provider: spec.provider, + pluginKey: resolved?.plugin.pluginKey ?? null, + pluginStatus: resolved?.plugin.status ?? null, + workerRunning, + archivedEnvironmentId: archived?.id ?? null, + }, + "managed sandbox environment provider plugin is not installed, ready, and running; skipping ensure and archiving any previously provisioned row (degraded: environment unavailable)", + ); + // Only a `ready` record can recover without another boot: the worker + // manager restarts crashed workers, but nothing (re)installs a missing + // or non-ready plugin at runtime. + if (resolved && resolved.plugin.status === "ready") { + scheduleRecoveryReactivation(spec, resolved.plugin.id); + } + continue; + } + const environment = await environments.ensureManagedSandboxEnvironment({ + name: spec.name, + description: spec.description, + provider: spec.provider, + config: { ...spec.config }, + }); + ensured += 1; + logger.info( + { environmentId: environment.id, name: environment.name, provider: spec.provider }, + "managed sandbox environment ensured", + ); + } catch (err) { + failed += 1; + logger.error( + { err, name: spec.name, provider: spec.provider }, + "failed to ensure managed sandbox environment; continuing boot (degraded: environment unavailable)", + ); + } + } + return { ensured, failed }; +}