From f6c6452b25bdd61f0f8d98a19f4b2b907ea4f7ac Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 7 Aug 2026 00:41:42 -0500 Subject: [PATCH] fix(server): preserve managed environment drift on boot (#10979) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Paperclip creates a managed sandbox environment for each company during boot > - Operators can change environment fields after Paperclip creates the environment > - The boot reconciler replaced those changes without checking for drift > - This pull request adds stock hashes and transactional drift reconciliation > - The benefit is that Paperclip can update untouched stock fields without losing operator work ## Linked Issues or Issue Description **What happened?** The managed sandbox boot reconciler rewrote the stock description, configuration, metadata, and status on every start. It did not detect operator changes first. A restart could therefore remove an operator's changes. **Expected behavior** Paperclip must preserve operator changes by default. It must update an untouched stock environment when Paperclip ships new stock values. It must perform each row update and stock-hash update atomically. **Steps to reproduce** 1. Start Paperclip and let it create the managed sandbox environment. 2. Change one Paperclip-owned stock field on that environment. 3. Restart Paperclip. 4. Observe that the previous reconciler replaced the change. **Paperclip version or commit** This bug reproduces on `master` before this change. **Deployment mode** Local development and self-hosted server boot are affected. ## What Changed - Add a shared deterministic stock-hash and drift classifier for built-in resources. - Track the managed sandbox stock hash with the company-scoped built-in resource binding. - Reconcile the environment and its stock metadata in one transaction with a row lock. - Preserve operator-modified and unmanaged rows and report their skipped update state. - Use archive ownership tokens so provider recovery reactivates only Paperclip-archived rows and preserves later operator archive decisions. - Keep operator-owned environment variables and unrelated metadata out of the stock fingerprint. - Add activity records for managed environment creation, updates, skipped drift, tracking initialization, and archive changes. - Add regression tests for current stock, available stock updates, operator drift, unmanaged rows, archive and reactivation, user-owned fields, and concurrent reconciliation. ## Verification - `pnpm -r typecheck` - `pnpm build` - `env -u AWS_ACCESS_KEY_ID -u AWS_SECRET_ACCESS_KEY pnpm test:run -- --mode serialized` (128 suites passed) - Repository general server, UI, CLI, shared, skills catalog, database, adapter, plugin SDK, and plugin creator projects passed. Two embedded-Postgres tests exceeded the host's five-second default under the aggregate run and passed in the complete database project with `--testTimeout=20000`. One timing-sensitive sandbox stream test passed on its focused retry. - Focused managed-environment unit and integration coverage passed: 49 tests across the drift classifier, boot report, and environment service suites. ## Risks - The main risk is an incorrect ownership boundary in the stock fingerprint. The fingerprint includes only Paperclip-owned stock fields. Tests confirm that environment variables and unrelated metadata survive reconciliation. - Concurrent reconciliation could otherwise overwrite a late operator edit. The implementation locks the environment row and updates the row and hash binding in one transaction. A concurrency test covers this path. - There is no schema migration. Existing managed rows initialize tracking without replacing their current values. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used OpenAI GPT-5 Codex. The exact serving snapshot and context-window size were not exposed. The model used reasoning, repository tools, code execution, and test execution. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- .../src/__tests__/environment-service.test.ts | 318 +++++++- server/src/services/built-in-agents.ts | 42 +- server/src/services/environments.ts | 683 ++++++++++++++---- .../src/services/managed-environments.test.ts | 74 +- server/src/services/managed-environments.ts | 95 ++- .../services/managed-resource-drift.test.ts | 43 ++ server/src/services/managed-resource-drift.ts | 36 + 7 files changed, 1088 insertions(+), 203 deletions(-) create mode 100644 server/src/services/managed-resource-drift.test.ts create mode 100644 server/src/services/managed-resource-drift.ts diff --git a/server/src/__tests__/environment-service.test.ts b/server/src/__tests__/environment-service.test.ts index 4ce7c8f825..9e81c9cf70 100644 --- a/server/src/__tests__/environment-service.test.ts +++ b/server/src/__tests__/environment-service.test.ts @@ -2,7 +2,9 @@ import { randomUUID } from "node:crypto"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; import { and, eq } from "drizzle-orm"; import { + activityLog, agents, + builtInManagedResources, companies, companySecretBindings, companySecrets, @@ -44,6 +46,7 @@ describeEmbeddedPostgres("environmentService leases", () => { }); afterEach(async () => { + await db.delete(activityLog); await db.delete(companySecretBindings); await db.delete(environmentCustomImageSetupSessions); await db.delete(environmentLeases); @@ -115,6 +118,18 @@ describeEmbeddedPostgres("environmentService leases", () => { return { companyId, agentId, environmentId, runId }; } + async function seedCompany(name = "Acme") { + const companyId = randomUUID(); + await db.insert(companies).values({ + id: companyId, + name, + status: "active", + createdAt: new Date(), + updatedAt: new Date(), + }); + return companyId; + } + it("acquires and releases a lease for a run", async () => { const { companyId, environmentId, runId } = await seedEnvironment(); @@ -704,12 +719,16 @@ describeEmbeddedPostgres("environmentService leases", () => { }); it("ensures and refreshes a managed sandbox environment for an arbitrary provider", async () => { - const created = await svc.ensureManagedSandboxEnvironment({ + const companyId = await seedCompany(); + const createdResult = await svc.ensureManagedSandboxEnvironment({ + companyId, name: "Daytona", description: "Managed Daytona sandbox environment.", provider: "daytona", config: { target: "us" }, }); + const created = createdResult.environment; + expect(createdResult).toMatchObject({ action: "added", stockStatus: "missing" }); expect(created.driver).toBe("sandbox"); expect(created.name).toBe("Daytona"); @@ -718,18 +737,45 @@ describeEmbeddedPostgres("environmentService leases", () => { expect(created.metadata?.managedByPaperclip).toBe(true); expect(created.metadata?.managedSandboxProvider).toBe("daytona"); - // Idempotent: a second call refreshes config and name in place, and a - // description omitted from the spec is cleared, not pinned forever. - const refreshed = await svc.ensureManagedSandboxEnvironment({ + // A stock update advances config and name in place, and a description + // omitted from the spec is cleared rather than pinned forever. + const refreshedResult = await svc.ensureManagedSandboxEnvironment({ + companyId, name: "Daytona (EU)", provider: "daytona", config: { target: "eu" }, }); + const refreshed = refreshedResult.environment; + expect(refreshedResult).toMatchObject({ + action: "updated", + stockStatus: "stock_update_available", + }); expect(refreshed.id).toBe(created.id); expect(refreshed.name).toBe("Daytona (EU)"); expect(refreshed.config.target).toBe("eu"); expect(refreshed.description).toBeNull(); + const [binding] = await db + .select() + .from(builtInManagedResources) + .where(and( + eq(builtInManagedResources.companyId, companyId), + eq(builtInManagedResources.resourceKind, "environment"), + )); + expect(binding).toMatchObject({ + resourceId: created.id, + stockHash: refreshedResult.stockHash, + }); + const activity = await db + .select({ action: activityLog.action }) + .from(activityLog) + .where(eq(activityLog.companyId, companyId)) + .orderBy(activityLog.createdAt); + expect(activity.map((entry) => entry.action)).toEqual([ + "environment.managed_stock_added", + "environment.managed_stock_updated", + ]); + const rows = await db .select() .from(environments) @@ -737,15 +783,136 @@ describeEmbeddedPostgres("environmentService leases", () => { expect(rows).toHaveLength(1); }); + it("treats stock_current as an environment-row no-op and preserves user-owned fields", async () => { + const companyId = await seedCompany(); + const created = await svc.ensureManagedSandboxEnvironment({ + companyId, + name: "Daytona", + description: "Managed stock", + provider: "daytona", + config: { target: "us" }, + stockVersion: "v1", + }); + const operatorUpdatedAt = new Date("2026-08-06T12:00:00.000Z"); + await db + .update(environments) + .set({ + envVars: { OPERATOR_FLAG: "kept" }, + metadata: { + ...created.environment.metadata, + operatorNote: "keep me", + }, + updatedAt: operatorUpdatedAt, + }) + .where(eq(environments.id, created.environment.id)); + const [bindingBefore] = await db + .select() + .from(builtInManagedResources) + .where(eq(builtInManagedResources.companyId, companyId)); + const activityBefore = await db + .select({ id: activityLog.id }) + .from(activityLog) + .where(eq(activityLog.companyId, companyId)); + + const reconciled = await svc.ensureManagedSandboxEnvironment({ + companyId, + name: "Daytona", + description: "Managed stock", + provider: "daytona", + config: { target: "us" }, + stockVersion: "v1", + }); + const [bindingAfter] = await db + .select() + .from(builtInManagedResources) + .where(eq(builtInManagedResources.companyId, companyId)); + + expect(reconciled).toMatchObject({ + action: "unchanged", + stockStatus: "stock_current", + updateAvailable: false, + }); + expect(reconciled.environment.updatedAt).toEqual(operatorUpdatedAt); + expect(reconciled.environment.envVars).toEqual({ OPERATOR_FLAG: "kept" }); + expect(reconciled.environment.metadata?.operatorNote).toBe("keep me"); + expect(bindingAfter?.updatedAt).toEqual(bindingBefore?.updatedAt); + const activityAfter = await db + .select({ id: activityLog.id }) + .from(activityLog) + .where(eq(activityLog.companyId, companyId)); + expect(activityAfter).toHaveLength(activityBefore.length); + }); + + it("classifies operator drift, preserves the row, and exposes the pending stock update", async () => { + const companyId = await seedCompany(); + const created = await svc.ensureManagedSandboxEnvironment({ + companyId, + name: "Daytona", + description: "Managed stock", + provider: "daytona", + config: { target: "us" }, + stockVersion: "v1", + }); + const [bindingBefore] = await db + .select() + .from(builtInManagedResources) + .where(eq(builtInManagedResources.companyId, companyId)); + const operatorUpdatedAt = new Date("2026-08-06T12:01:00.000Z"); + await db + .update(environments) + .set({ + description: "Operator description", + config: { provider: "daytona", target: "operator" }, + updatedAt: operatorUpdatedAt, + }) + .where(eq(environments.id, created.environment.id)); + + const reconciled = await svc.ensureManagedSandboxEnvironment({ + companyId, + name: "Daytona v2", + description: "Managed stock v2", + provider: "daytona", + config: { target: "eu" }, + stockVersion: "v2", + }); + const [bindingAfter] = await db + .select() + .from(builtInManagedResources) + .where(eq(builtInManagedResources.companyId, companyId)); + + expect(reconciled).toMatchObject({ + action: "skipped", + stockStatus: "operator_modified", + updateAvailable: true, + }); + expect(reconciled.environment).toMatchObject({ + name: "Daytona", + description: "Operator description", + config: { provider: "daytona", target: "operator" }, + updatedAt: operatorUpdatedAt, + }); + expect(bindingAfter?.stockHash).toBe(bindingBefore?.stockHash); + expect(bindingAfter?.stockVersion).toBe("v1"); + expect(reconciled.stockHash).not.toBe(bindingAfter?.stockHash); + const activity = await db + .select({ action: activityLog.action }) + .from(activityLog) + .where(eq(activityLog.companyId, companyId)) + .orderBy(activityLog.createdAt); + expect(activity.at(-1)?.action).toBe("environment.managed_stock_skipped"); + }); + it("adopts the managed slot on a provider switch and drops the stale kubernetes marker", async () => { - const kubernetes = await svc.ensureKubernetesEnvironment({ inCluster: true, backend: "job" }); + const companyId = await seedCompany(); + const kubernetes = await svc.ensureKubernetesEnvironment(companyId, { inCluster: true, backend: "job" }); expect(kubernetes.metadata?.managedKubernetesSandbox).toBe(true); - const daytona = await svc.ensureManagedSandboxEnvironment({ + const daytona = (await svc.ensureManagedSandboxEnvironment({ + companyId, name: "Daytona", provider: "daytona", config: { target: "us" }, - }); + })).environment; expect(daytona.id).toBe(kubernetes.id); expect(daytona.name).toBe("Daytona"); @@ -756,20 +923,22 @@ describeEmbeddedPostgres("environmentService leases", () => { expect(await svc.findKubernetesEnvironment()).toBeNull(); // And back: the kubernetes wrapper re-adopts the same row. - const restored = await svc.ensureKubernetesEnvironment({ inCluster: true, backend: "job" }); + const restored = await svc.ensureKubernetesEnvironment(companyId, { inCluster: true, backend: "job" }); expect(restored.id).toBe(kubernetes.id); expect(restored.metadata?.managedKubernetesSandbox).toBe(true); }); it("archives the managed sandbox row only for its own provider and reactivates on ensure", async () => { + const companyId = await seedCompany(); // Nothing provisioned yet: archiving is a no-op. expect(await svc.archiveManagedSandboxEnvironment({ provider: "daytona" })).toBeNull(); - const created = await svc.ensureManagedSandboxEnvironment({ + const created = (await svc.ensureManagedSandboxEnvironment({ + companyId, name: "Daytona", provider: "daytona", config: { target: "us" }, - }); + })).environment; expect(created.status).toBe("active"); // Another provider's unavailability leaves this provider's row alone. @@ -778,21 +947,121 @@ describeEmbeddedPostgres("environmentService leases", () => { const archived = await svc.archiveManagedSandboxEnvironment({ provider: "daytona" }); expect(archived?.id).toBe(created.id); expect(archived?.status).toBe("archived"); + const archiveActivity = await db + .select({ action: activityLog.action }) + .from(activityLog) + .where(and( + eq(activityLog.companyId, companyId), + eq(activityLog.entityId, created.id), + )) + .orderBy(activityLog.createdAt); + expect(archiveActivity.at(-1)?.action).toBe( + "environment.managed_provider_unavailable_archived", + ); // Already archived: a repeat call is a no-op. expect(await svc.archiveManagedSandboxEnvironment({ provider: "daytona" })).toBeNull(); // The next healthy boot's ensure re-activates the same row. - const restored = await svc.ensureManagedSandboxEnvironment({ + const restored = (await svc.ensureManagedSandboxEnvironment({ + companyId, name: "Daytona", provider: "daytona", config: { target: "us" }, - }); + })).environment; expect(restored.id).toBe(created.id); expect(restored.status).toBe("active"); }); - it("adopts an existing unmanaged sandbox row holding the desired name", async () => { + it("reactivates a reconciler-archived row without replacing operator drift", async () => { + const companyId = await seedCompany(); + const created = await svc.ensureManagedSandboxEnvironment({ + companyId, + name: "Daytona", + description: "Managed stock", + provider: "daytona", + config: { target: "us" }, + stockVersion: "v1", + }); + await db + .update(environments) + .set({ + description: "Operator description", + config: { provider: "daytona", target: "operator" }, + }) + .where(eq(environments.id, created.environment.id)); + + expect((await svc.archiveManagedSandboxEnvironment({ provider: "daytona" }))?.status) + .toBe("archived"); + const [archivedBinding] = await db + .select() + .from(builtInManagedResources) + .where(eq(builtInManagedResources.companyId, companyId)); + expect(archivedBinding?.defaultsJson).toMatchObject({ + description: "Managed stock", + status: "archived", + }); + + const restored = await svc.ensureManagedSandboxEnvironment({ + companyId, + name: "Daytona v2", + description: "Managed stock v2", + provider: "daytona", + config: { target: "eu" }, + stockVersion: "v2", + }); + expect(restored).toMatchObject({ + action: "skipped", + stockStatus: "operator_modified", + updateAvailable: true, + environment: { + status: "active", + name: "Daytona", + description: "Operator description", + config: { provider: "daytona", target: "operator" }, + }, + }); + const [reactivatedBinding] = await db + .select() + .from(builtInManagedResources) + .where(eq(builtInManagedResources.companyId, companyId)); + expect(reactivatedBinding?.defaultsJson).toMatchObject({ + description: "Managed stock", + status: "active", + }); + expect(reactivatedBinding?.stockVersion).toBe("v1"); + expect(restored.stockHash).not.toBe(reactivatedBinding?.stockHash); + }); + + it("preserves an operator archive decision made after provider archival", async () => { + const companyId = await seedCompany(); + const created = await svc.ensureManagedSandboxEnvironment({ + companyId, + name: "Daytona", + provider: "daytona", + config: { target: "us" }, + }); + expect((await svc.archiveManagedSandboxEnvironment({ provider: "daytona" }))?.status) + .toBe("archived"); + expect((await svc.update(created.environment.id, { status: "archived" }))?.status) + .toBe("archived"); + + const reconciled = await svc.ensureManagedSandboxEnvironment({ + companyId, + name: "Daytona", + provider: "daytona", + config: { target: "us" }, + }); + expect(reconciled).toMatchObject({ + action: "skipped", + stockStatus: "operator_modified", + updateAvailable: true, + environment: { status: "archived" }, + }); + }); + + it("preserves an existing unmanaged sandbox row holding the desired name", async () => { + const companyId = await seedCompany(); const handMade = await svc.create({ name: "Daytona", driver: "sandbox", @@ -801,15 +1070,20 @@ describeEmbeddedPostgres("environmentService leases", () => { }); expect(handMade.metadata?.managedByPaperclip).toBeUndefined(); - const adopted = await svc.ensureManagedSandboxEnvironment({ + const reconciliation = await svc.ensureManagedSandboxEnvironment({ + companyId, name: "Daytona", provider: "daytona", config: { target: "eu" }, }); - expect(adopted.id).toBe(handMade.id); - expect(adopted.config.target).toBe("eu"); - expect(adopted.metadata?.managedByPaperclip).toBe(true); - expect(adopted.metadata?.managedSandboxProvider).toBe("daytona"); + expect(reconciliation).toMatchObject({ + action: "skipped", + stockStatus: "operator_modified", + updateAvailable: true, + }); + expect(reconciliation.environment.id).toBe(handMade.id); + expect(reconciliation.environment.config.target).toBe("us"); + expect(reconciliation.environment.metadata?.managedByPaperclip).toBeUndefined(); const rows = await db .select() @@ -819,6 +1093,7 @@ describeEmbeddedPostgres("environmentService leases", () => { }); it("keeps the current name when the desired name belongs to another row", async () => { + const companyId = await seedCompany(); await svc.create({ name: "Daytona", driver: "ssh", @@ -830,15 +1105,16 @@ describeEmbeddedPostgres("environmentService leases", () => { remoteWorkspacePath: "/srv/paperclip", }, }); - const kubernetes = await svc.ensureKubernetesEnvironment({ inCluster: true }); + const kubernetes = await svc.ensureKubernetesEnvironment(companyId, { inCluster: true }); // The managed slot is adopted, but the rename would collide with the ssh // row on environments_name_idx; the ensure keeps the existing name. - const adopted = await svc.ensureManagedSandboxEnvironment({ + const adopted = (await svc.ensureManagedSandboxEnvironment({ + companyId, name: "Daytona", provider: "daytona", config: { target: "us" }, - }); + })).environment; expect(adopted.id).toBe(kubernetes.id); expect(adopted.name).toBe(kubernetes.name); expect(adopted.config.provider).toBe("daytona"); diff --git a/server/src/services/built-in-agents.ts b/server/src/services/built-in-agents.ts index 253686f7ba..b8a25460c7 100644 --- a/server/src/services/built-in-agents.ts +++ b/server/src/services/built-in-agents.ts @@ -1,4 +1,3 @@ -import { createHash } from "node:crypto"; import { readFileSync } from "node:fs"; import { createRequire } from "node:module"; import path from "node:path"; @@ -22,6 +21,12 @@ import { companySkillService } from "./company-skills.js"; import { routineService } from "./routines.js"; import { accessService } from "./access.js"; import { listAdapterModels } from "../adapters/registry.js"; +import { + resourceStatus, + stableJson, + stockHash, + type ManagedResourceStockStatus, +} from "./managed-resource-drift.js"; export type BuiltInAgentStatus = "not_provisioned" | "pending_approval" | "needs_setup" | "ready" | "paused"; @@ -72,11 +77,7 @@ export interface BuiltInAgentProvisionResult { } export type BuiltInManagedResourceKind = "instructions" | "skill" | "routine"; -export type BuiltInManagedResourceStockStatus = - | "missing" - | "stock_current" - | "stock_update_available" - | "operator_modified"; +export type BuiltInManagedResourceStockStatus = ManagedResourceStockStatus; export interface BuiltInManagedResourceState { resourceKind: BuiltInManagedResourceKind; @@ -495,21 +496,6 @@ function uniqueNonEmptyStrings(values: string[]) { return result; } -function stableJson(value: unknown): string { - if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`; - if (value && typeof value === "object") { - return `{${Object.entries(value as Record) - .sort(([left], [right]) => left.localeCompare(right)) - .map(([key, entry]) => `${JSON.stringify(key)}:${stableJson(entry)}`) - .join(",")}}`; - } - return JSON.stringify(value); -} - -function stockHash(value: unknown) { - return `sha256:${createHash("sha256").update(stableJson(value)).digest("hex")}`; -} - function changedFileList(currentFiles: Record, stockFiles: Record) { const paths = new Set([...Object.keys(currentFiles), ...Object.keys(stockFiles)]); return [...paths] @@ -517,20 +503,6 @@ function changedFileList(currentFiles: Record, stockFiles .sort((left, right) => left.localeCompare(right)); } -function resourceStatus(input: { - resourceId: string | null; - currentHash: string | null; - bindingStockHash: string | null; - latestStockHash: string; -}): BuiltInManagedResourceStockStatus { - if (!input.resourceId || !input.currentHash) return "missing"; - if (input.currentHash === input.latestStockHash) return "stock_current"; - if (input.bindingStockHash && input.currentHash === input.bindingStockHash) { - return "stock_update_available"; - } - return "operator_modified"; -} - function stockState(input: { resourceKind: BuiltInManagedResourceKind; resourceKey: string; diff --git a/server/src/services/environments.ts b/server/src/services/environments.ts index 3cfb7b38f2..657d366a02 100644 --- a/server/src/services/environments.ts +++ b/server/src/services/environments.ts @@ -1,7 +1,10 @@ +import { randomUUID } from "node:crypto"; import { and, desc, eq, inArray, ne, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { agents, + builtInManagedResources, + companies, companySecretBindings, environmentCustomImageSetupSessions, environmentLeases, @@ -28,7 +31,13 @@ import { type UpdateEnvironment, } from "@paperclipai/shared"; import { conflict } from "../errors.js"; +import { logActivity } from "./activity-log.js"; import { isCloudManagedInstance } from "./cloud-instance.js"; +import { + resourceStatus, + stockHash, + type ManagedResourceStockStatus, +} from "./managed-resource-drift.js"; type EnvironmentRow = typeof environments.$inferSelect; type EnvironmentLeaseRow = typeof environmentLeases.$inferSelect; @@ -81,6 +90,8 @@ export interface KubernetesEnvironmentConfigInput { * lease time. */ export interface ManagedSandboxEnvironmentInput { + /** Company whose managed-resource binding should be reconciled. Omit at instance boot to bind every company. */ + companyId?: string; name: string; description?: string; /** Sandbox provider key (the plugin's driverKey, e.g. "kubernetes", "daytona"). */ @@ -92,6 +103,24 @@ export interface ManagedSandboxEnvironmentInput { * `findKubernetesEnvironment` keys on). */ extraMetadata?: Record; + /** Version label recorded with the stock binding; hashes remain the drift authority. */ + stockVersion?: string; +} + +export type ManagedSandboxEnvironmentReconcileAction = + | "added" + | "updated" + | "unchanged" + | "skipped"; + +export interface ManagedSandboxEnvironmentReconcileResult { + environment: Environment; + action: ManagedSandboxEnvironmentReconcileAction; + /** Classification observed before this reconciliation wrote anything. */ + stockStatus: ManagedResourceStockStatus; + /** True only when operator drift prevented the available stock update. */ + updateAvailable: boolean; + stockHash: string; } function cloneRecord(value: unknown, fallback: Record | null = null): Record | null { @@ -204,21 +233,81 @@ function countFromRows(rows: Array<{ count: number | string | null | undefined } type DbTransaction = Parameters[0]>[0]; type EnvironmentWriteDb = Pick; -export function environmentService(db: Db) { - /** The single Paperclip-managed sandbox row (`environments_managed_sandbox_idx`), if present. */ - const findManagedSandboxRow = () => - db - .select() - .from(environments) - .where(eq(environments.driver, "sandbox")) - .then( - (rows) => - rows.find( - (row) => - (row.metadata as Record | null)?.managedByPaperclip === true, - ) ?? null, - ); +const MANAGED_ENVIRONMENT_BUNDLE_KEY = "managed-sandbox-environment"; +const MANAGED_ENVIRONMENT_RESOURCE_KIND = "environment"; +const MANAGED_ENVIRONMENT_RESOURCE_KEY = "managed-sandbox"; +const MANAGED_ENVIRONMENT_STOCK_VERSION = "managed-environment-v1"; +const MANAGED_ENVIRONMENT_ARCHIVE_TOKEN_METADATA_KEY = "_paperclipManagedArchiveToken"; +const MANAGED_ENVIRONMENT_ARCHIVE_TOKEN_DEFAULTS_KEY = "_paperclipManagedArchiveToken"; +function managedEnvironmentBaselineDefaults( + defaultsJson: Record, +): Record { + const baseline = { ...defaultsJson }; + delete baseline[MANAGED_ENVIRONMENT_ARCHIVE_TOKEN_DEFAULTS_KEY]; + return baseline; +} + +function withoutManagedEnvironmentArchiveToken( + metadata: Record | null, +): Record | null { + if (!metadata) return null; + const next = { ...metadata }; + delete next[MANAGED_ENVIRONMENT_ARCHIVE_TOKEN_METADATA_KEY]; + return next; +} + +function managedMetadataKeys( + desiredMetadata: Record, + bindings: Array<{ defaultsJson: Record }>, +): string[] { + const keys = new Set([ + "managedByPaperclip", + "managedSandboxProvider", + KUBERNETES_MANAGED_MARKER, + ...Object.keys(desiredMetadata), + ]); + for (const binding of bindings) { + const metadata = cloneRecord(binding.defaultsJson.metadata); + for (const key of Object.keys(metadata ?? {})) keys.add(key); + } + return [...keys].sort((left, right) => left.localeCompare(right)); +} + +function managedEnvironmentStock(input: { + name: string; + description: string | null; + config: Record; + metadata: Record | null; + status: string; +}, metadataKeys: readonly string[]): Record { + const metadata = input.metadata ?? {}; + return { + name: input.name, + description: input.description, + config: input.config, + metadata: Object.fromEntries( + metadataKeys.map((key) => [key, Object.prototype.hasOwnProperty.call(metadata, key) ? metadata[key] : null]), + ), + status: input.status, + }; +} + +function mergeManagedEnvironmentMetadata( + current: Record | null, + desired: Record, + keys: readonly string[], +): Record { + const merged = { ...(current ?? {}) }; + for (const key of keys) { + const value = Object.prototype.hasOwnProperty.call(desired, key) ? desired[key] : null; + if (value === null || value === undefined) delete merged[key]; + else merged[key] = value; + } + return merged; +} + +export function environmentService(db: Db) { /** * Idempotently ensure THE Paperclip-managed sandbox environment for this * instance, configured for an arbitrary sandbox provider plugin. Mirrors @@ -227,18 +316,15 @@ export function environmentService(db: Db) { * row per instance, so this function owns that single slot regardless of * provider: * - * - An existing managed row is adopted and refreshed (name, description, - * config, provider) on every call, so operator/control-plane changes flow - * via redeploy without recreating the row — including a provider switch, - * which also drops a stale provider-specific metadata marker. - * - An existing UNmanaged sandbox row holding the desired name is adopted - * and stamped as managed, so a row created by hand before the instance - * became config-managed converges instead of colliding on - * `environments_name_idx` on every boot. + * - A stock-controlled managed row advances to a new stock hash in the same + * transaction as its managed fields, including provider switches. + * - A stock-current row is returned without an environment write. + * - An operator-modified or previously unmanaged row is preserved and + * reported as skipped; its user-owned fields are never folded into stock. */ const ensureManagedSandboxEnvironment = async ( input: ManagedSandboxEnvironmentInput, - ): Promise => { + ): Promise => { const desiredConfig: Record = { ...(input.config ?? {}), provider: input.provider, @@ -248,99 +334,363 @@ export function environmentService(db: Db) { managedSandboxProvider: input.provider, ...(input.extraMetadata ?? {}), }; + if (desiredMetadata[KUBERNETES_MANAGED_MARKER] !== true) { + desiredMetadata[KUBERNETES_MANAGED_MARKER] = null; + } - const adopt = async (row: EnvironmentRow): Promise => { - const metadata: Record = { ...(row.metadata ?? {}), ...desiredMetadata }; - // A provider switch must not leave the previous provider's marker - // behind (`findKubernetesEnvironment` keys on it). - if (desiredMetadata[KUBERNETES_MANAGED_MARKER] !== true) { - delete metadata[KUBERNETES_MANAGED_MARKER]; - } - const now = new Date(); - const runUpdate = (values: { name?: string }) => - db + let activityCompanyIds: string[] = []; + let trackingInitialized = false; + let providerReactivated = false; + const reconciliation = await db.transaction( + async (tx): Promise => { + const companyIds = input.companyId + ? [input.companyId] + : await tx.select({ id: companies.id }).from(companies).then((rows) => rows.map((row) => row.id)); + activityCompanyIds = companyIds; + const bindingConditions = and( + eq(builtInManagedResources.bundleKey, MANAGED_ENVIRONMENT_BUNDLE_KEY), + eq(builtInManagedResources.resourceKind, MANAGED_ENVIRONMENT_RESOURCE_KIND), + eq(builtInManagedResources.resourceKey, MANAGED_ENVIRONMENT_RESOURCE_KEY), + ...(companyIds.length > 0 ? [inArray(builtInManagedResources.companyId, companyIds)] : []), + ); + const bindings = companyIds.length > 0 + ? await tx.select().from(builtInManagedResources).where(bindingConditions) + : []; + trackingInitialized = bindings.length < companyIds.length; + const keys = managedMetadataKeys(desiredMetadata, bindings); + + const sandboxRows = await tx + .select() + .from(environments) + .where(eq(environments.driver, "sandbox")) + .for("update"); + let row = sandboxRows.find( + (candidate) => (candidate.metadata as Record | null)?.managedByPaperclip === true, + ) ?? sandboxRows.find((candidate) => candidate.name === input.name) ?? null; + + const writeBindings = async ( + environmentId: string, + stockVersion: string, + installedStockHash: string, + defaultsJson: Record, + replace: boolean, + ) => { + const targetCompanyIds = replace + ? companyIds + : companyIds.filter( + (companyId) => !bindings.some((binding) => binding.companyId === companyId), + ); + if (targetCompanyIds.length === 0) return; + const values = targetCompanyIds.map((companyId) => ({ + companyId, + bundleKey: MANAGED_ENVIRONMENT_BUNDLE_KEY, + resourceKind: MANAGED_ENVIRONMENT_RESOURCE_KIND, + resourceKey: MANAGED_ENVIRONMENT_RESOURCE_KEY, + resourceId: environmentId, + stockVersion, + stockHash: installedStockHash, + defaultsJson, + })); + const insert = tx.insert(builtInManagedResources).values(values); + if (!replace) { + await insert.onConflictDoNothing({ + target: [ + builtInManagedResources.companyId, + builtInManagedResources.bundleKey, + builtInManagedResources.resourceKind, + builtInManagedResources.resourceKey, + ], + }); + return; + } + await insert.onConflictDoUpdate({ + target: [ + builtInManagedResources.companyId, + builtInManagedResources.bundleKey, + builtInManagedResources.resourceKind, + builtInManagedResources.resourceKey, + ], + set: { + resourceId: environmentId, + stockVersion, + stockHash: installedStockHash, + defaultsJson, + updatedAt: new Date(), + }, + }); + }; + + if (!row) { + const nameOwner = await tx + .select() + .from(environments) + .where(eq(environments.name, input.name)) + .then((rows) => rows[0] ?? null); + if (nameOwner && nameOwner.driver !== "sandbox") { + throw new Error( + `Failed to ensure managed sandbox environment: environment "${input.name}" already exists with driver "${nameOwner.driver}"`, + ); + } + const now = new Date(); + const inserted = await tx + .insert(environments) + .values({ + name: input.name, + description: input.description ?? null, + driver: "sandbox", + status: "active", + config: desiredConfig, + envVars: {}, + metadata: mergeManagedEnvironmentMetadata(null, desiredMetadata, keys), + createdAt: now, + updatedAt: now, + }) + // Either the managed-slot partial index or the global name index + // can select the concurrent winner. Treat both as convergence and + // reselect that winner under lock below. + .onConflictDoNothing() + .returning() + .then((rows) => rows[0] ?? null); + if (inserted) { + const stock = managedEnvironmentStock({ + name: inserted.name, + description: inserted.description ?? null, + config: inserted.config, + metadata: inserted.metadata, + status: inserted.status, + }, keys); + const latestStockHash = stockHash(stock); + await writeBindings( + inserted.id, + input.stockVersion ?? MANAGED_ENVIRONMENT_STOCK_VERSION, + latestStockHash, + stock, + true, + ); + return { + environment: toEnvironment(inserted), + action: "added", + stockStatus: "missing", + updateAvailable: false, + stockHash: latestStockHash, + }; + } + row = await tx + .select() + .from(environments) + .where(eq(environments.driver, "sandbox")) + .for("update") + .then( + (rows) => rows.find( + (candidate) => (candidate.metadata as Record | null)?.managedByPaperclip === true, + ) ?? null, + ); + if (!row) throw new Error("Failed to ensure managed sandbox environment"); + } + + const nameOwner = await tx + .select({ id: environments.id }) + .from(environments) + .where(eq(environments.name, input.name)) + .then((rows) => rows[0] ?? null); + const desiredName = nameOwner && nameOwner.id !== row.id ? row.name : input.name; + const desiredStock = managedEnvironmentStock({ + name: desiredName, + description: input.description ?? null, + config: desiredConfig, + metadata: desiredMetadata, + status: "active", + }, keys); + const latestStockHash = stockHash(desiredStock); + const currentStock = managedEnvironmentStock({ + name: row.name, + description: row.description ?? null, + config: row.config, + metadata: row.metadata, + status: row.status, + }, keys); + const currentHash = stockHash(currentStock); + const matchingBindings = bindings.filter((binding) => binding.resourceId === row!.id); + const rowArchiveToken = (row.metadata as Record | null) + ?.[MANAGED_ENVIRONMENT_ARCHIVE_TOKEN_METADATA_KEY]; + let stockStatus = resourceStatus({ + resourceId: row.id, + currentHash, + bindingStockHash: matchingBindings[0]?.stockHash ?? null, + latestStockHash, + }); + if (stockStatus === "operator_modified") { + const stockControlledBinding = matchingBindings.find( + (binding) => resourceStatus({ + resourceId: row!.id, + currentHash, + bindingStockHash: binding.stockHash, + latestStockHash, + }) === "stock_update_available", + ); + if (stockControlledBinding) stockStatus = "stock_update_available"; + } + const operatorReaffirmedArchive = row.status === "archived" && matchingBindings.some( + (binding) => { + const bindingToken = binding.defaultsJson[MANAGED_ENVIRONMENT_ARCHIVE_TOKEN_DEFAULTS_KEY]; + return binding.defaultsJson.status === "archived" && + typeof bindingToken === "string" && + bindingToken !== rowArchiveToken; + }, + ); + if (operatorReaffirmedArchive) stockStatus = "operator_modified"; + + if (stockStatus === "operator_modified") { + const baseline = matchingBindings[0]; + let baselineDefaults = baseline + ? managedEnvironmentBaselineDefaults(baseline.defaultsJson) + : desiredStock; + let baselineHash = baseline?.stockHash ?? latestStockHash; + + // Provider unavailability is an operational state transition, not + // an operator edit. If the binding records that Paperclip archived + // this row, restore only its availability status. Keep every other + // operator-modified field intact and leave the stock update pending. + // A manually archived row still has an active binding baseline, so + // it remains operator_modified and is not reactivated here. + const archivedByReconciler = row.status === "archived" && + typeof rowArchiveToken === "string" && + matchingBindings.some((binding) => { + const bindingDefaults = binding.defaultsJson; + return bindingDefaults.status === "archived" && + bindingDefaults[MANAGED_ENVIRONMENT_ARCHIVE_TOKEN_DEFAULTS_KEY] === rowArchiveToken; + }); + if (archivedByReconciler) { + const reactivated = await tx + .update(environments) + .set({ + status: "active", + metadata: withoutManagedEnvironmentArchiveToken(row.metadata), + updatedAt: new Date(), + }) + .where(and(eq(environments.id, row.id), eq(environments.status, "archived"))) + .returning() + .then((rows) => rows[0] ?? null); + if (!reactivated) { + throw new Error("Managed sandbox environment changed during reactivation"); + } + row = reactivated; + providerReactivated = true; + + for (const binding of matchingBindings) { + const reactivatedDefaults = { + ...managedEnvironmentBaselineDefaults(binding.defaultsJson), + status: "active", + }; + const reactivatedHash = stockHash(reactivatedDefaults); + await tx + .update(builtInManagedResources) + .set({ + stockHash: reactivatedHash, + defaultsJson: reactivatedDefaults, + updatedAt: new Date(), + }) + .where(and( + eq(builtInManagedResources.id, binding.id), + eq(builtInManagedResources.resourceId, row.id), + )); + if (binding.id === baseline?.id) { + baselineDefaults = reactivatedDefaults; + baselineHash = reactivatedHash; + } + } + } + await writeBindings( + row.id, + baseline?.stockVersion ?? input.stockVersion ?? MANAGED_ENVIRONMENT_STOCK_VERSION, + baselineHash, + baselineDefaults, + false, + ); + return { + environment: toEnvironment(row), + action: "skipped", + stockStatus, + updateAvailable: true, + stockHash: latestStockHash, + }; + } + + if (stockStatus === "stock_current") { + await writeBindings( + row.id, + input.stockVersion ?? MANAGED_ENVIRONMENT_STOCK_VERSION, + latestStockHash, + desiredStock, + false, + ); + return { + environment: toEnvironment(row), + action: "unchanged", + stockStatus, + updateAvailable: false, + stockHash: latestStockHash, + }; + } + + const updated = await tx .update(environments) .set({ - ...values, - // The row mirrors the managed spec: omitting `description` clears - // a previously configured one rather than pinning it forever. + name: desiredName, description: input.description ?? null, config: desiredConfig, - metadata, + metadata: withoutManagedEnvironmentArchiveToken( + mergeManagedEnvironmentMetadata(row.metadata, desiredMetadata, keys), + ), status: "active", - updatedAt: now, + updatedAt: new Date(), }) .where(eq(environments.id, row.id)) .returning() - .then((rows) => rows[0] ?? row); - const updated = await runUpdate({ name: input.name }).catch((error: unknown) => { - // Another row already holds the desired name; keep the current name - // rather than failing a boot-time ensure over a display label. - if (hasConstraintName(error, "environments_name_idx")) { - return runUpdate({}); - } - throw error; - }); - return toEnvironment(updated); - }; - - const existing = await findManagedSandboxRow(); - if (existing) return adopt(existing); - - // The partial unique index `environments_managed_sandbox_idx` enforces - // "at most one Paperclip-managed sandbox row per instance" at the DB - // level. Use ON CONFLICT DO NOTHING keyed on that index so concurrent - // callers can race the INSERT; losers re-read the surviving row. - const now = new Date(); - const inserted = await db - .insert(environments) - .values({ - name: input.name, - description: input.description ?? null, - driver: "sandbox", - status: "active", - config: desiredConfig, - envVars: {}, - metadata: desiredMetadata, - createdAt: now, - updatedAt: now, - }) - .onConflictDoNothing({ - target: [environments.driver], - where: - sql`${environments.driver} = 'sandbox' AND (${environments.metadata} ->> 'managedByPaperclip')::boolean = true`, - }) - .returning() - .then((rows) => rows[0] ?? null) - .catch((error) => { - if ( - hasConstraintName(error, "environments_name_idx") - || hasConstraintName(error, "environments_managed_sandbox_idx") - ) { - return null; - } - throw error; - }); - if (inserted) return toEnvironment(inserted); - - // Either a concurrent caller won the managed slot, or an unmanaged row - // holds the desired name. Adopt whichever exists. - const winner = await findManagedSandboxRow(); - if (winner) return adopt(winner); - const sameName = await db - .select() - .from(environments) - .where(eq(environments.name, input.name)) - .then((rows) => rows[0] ?? null); - if (sameName) { - if (sameName.driver !== "sandbox") { - throw new Error( - `Failed to ensure managed sandbox environment: environment "${input.name}" already exists with driver "${sameName.driver}"`, + .then((rows) => rows[0] ?? null); + if (!updated) throw new Error("Managed sandbox environment changed during reconciliation"); + await writeBindings( + updated.id, + input.stockVersion ?? MANAGED_ENVIRONMENT_STOCK_VERSION, + latestStockHash, + desiredStock, + true, ); - } - return adopt(sameName); + return { + environment: toEnvironment(updated), + action: "updated", + stockStatus, + updateAvailable: false, + stockHash: latestStockHash, + }; + }, + ); + if (reconciliation.action !== "unchanged" || trackingInitialized) { + const action = reconciliation.action === "added" + ? "environment.managed_stock_added" + : reconciliation.action === "updated" + ? "environment.managed_stock_updated" + : reconciliation.action === "skipped" + ? "environment.managed_stock_skipped" + : "environment.managed_stock_tracking_initialized"; + await Promise.all(activityCompanyIds.map((companyId) => logActivity(db, { + companyId, + actorType: "system", + actorId: "managed-environment-reconciler", + action, + entityType: "environment", + entityId: reconciliation.environment.id, + details: { + provider: input.provider, + reconciliationAction: reconciliation.action, + stockStatus: reconciliation.stockStatus, + updateAvailable: reconciliation.updateAvailable, + stockHash: reconciliation.stockHash, + providerReactivated, + }, + }))); } - throw new Error("Failed to ensure managed sandbox environment"); + return reconciliation; }; /** @@ -359,21 +709,95 @@ export function environmentService(db: Db) { * managed row for this provider. */ const archiveManagedSandboxEnvironment = async ( - input: { provider: string }, + input: { provider: string; companyId?: string }, ): Promise => { - const existing = await findManagedSandboxRow(); - if (!existing || existing.status !== "active") return null; - const rowProvider = (existing.metadata as Record | null) - ?.managedSandboxProvider; - if (rowProvider !== input.provider) return null; - const archived = await db - .update(environments) - .set({ status: "archived", updatedAt: new Date() }) - // Guarded on status so a concurrent re-activation is not clobbered. - .where(and(eq(environments.id, existing.id), eq(environments.status, "active"))) - .returning() - .then((rows) => rows[0] ?? null); - return archived ? toEnvironment(archived) : null; + let activityCompanyIds: string[] = []; + const archived = await db.transaction(async (tx) => { + const existing = await tx + .select() + .from(environments) + .where(eq(environments.driver, "sandbox")) + .for("update") + .then( + (rows) => rows.find( + (row) => (row.metadata as Record | null)?.managedByPaperclip === true, + ) ?? null, + ); + if (!existing || existing.status !== "active") return null; + const rowProvider = (existing.metadata as Record | null) + ?.managedSandboxProvider; + if (rowProvider !== input.provider) return null; + + const companyIds = input.companyId + ? [input.companyId] + : await tx.select({ id: companies.id }).from(companies).then((rows) => rows.map((row) => row.id)); + activityCompanyIds = companyIds; + const bindings = companyIds.length > 0 + ? await tx + .select() + .from(builtInManagedResources) + .where(and( + inArray(builtInManagedResources.companyId, companyIds), + eq(builtInManagedResources.bundleKey, MANAGED_ENVIRONMENT_BUNDLE_KEY), + eq(builtInManagedResources.resourceKind, MANAGED_ENVIRONMENT_RESOURCE_KIND), + eq(builtInManagedResources.resourceKey, MANAGED_ENVIRONMENT_RESOURCE_KEY), + eq(builtInManagedResources.resourceId, existing.id), + )) + : []; + const archiveToken = randomUUID(); + const archivedMetadata = { + ...(existing.metadata ?? {}), + [MANAGED_ENVIRONMENT_ARCHIVE_TOKEN_METADATA_KEY]: archiveToken, + }; + const archived = await tx + .update(environments) + .set({ + status: "archived", + metadata: archivedMetadata, + updatedAt: new Date(), + }) + .where(and(eq(environments.id, existing.id), eq(environments.status, "active"))) + .returning() + .then((rows) => rows[0] ?? null); + if (!archived) return null; + + // Archival is a Paperclip-owned availability transition. Record only + // that status change in each installed baseline. Deriving the new hash + // from defaultsJson keeps operator-modified row fields out of stock. + for (const binding of bindings) { + const archivedStock = { + ...managedEnvironmentBaselineDefaults(binding.defaultsJson), + status: "archived", + }; + await tx + .update(builtInManagedResources) + .set({ + stockHash: stockHash(archivedStock), + defaultsJson: { + ...archivedStock, + [MANAGED_ENVIRONMENT_ARCHIVE_TOKEN_DEFAULTS_KEY]: archiveToken, + }, + updatedAt: new Date(), + }) + .where(and( + eq(builtInManagedResources.id, binding.id), + eq(builtInManagedResources.resourceId, existing.id), + )); + } + return toEnvironment(archived); + }); + if (archived) { + await Promise.all(activityCompanyIds.map((companyId) => logActivity(db, { + companyId, + actorType: "system", + actorId: "managed-environment-reconciler", + action: "environment.managed_provider_unavailable_archived", + entityType: "environment", + entityId: archived.id, + details: { provider: input.provider }, + }))); + } + return archived; }; return { @@ -488,8 +912,8 @@ export function environmentService(db: Db) { * an instance, configured from instance/operator-supplied config. A thin * wrapper over `ensureManagedSandboxEnvironment` that pins the provider to * "kubernetes" and stamps the legacy marker `findKubernetesEnvironment` - * keys on. On subsequent calls the config is refreshed (so operators can - * update egress/runtimeClass via gitops without recreating the row). + * keys on. On subsequent calls stock-controlled config advances in place; + * operator modifications remain untouched for explicit review. */ ensureKubernetesEnvironment: async ( companyIdOrConfig: string | KubernetesEnvironmentConfigInput, @@ -497,12 +921,13 @@ export function environmentService(db: Db) { ): Promise => { const config = resolveKubernetesConfig(companyIdOrConfig, maybeConfig); return ensureManagedSandboxEnvironment({ + companyId: typeof companyIdOrConfig === "string" ? companyIdOrConfig : undefined, name: DEFAULT_KUBERNETES_ENVIRONMENT_NAME, description: DEFAULT_KUBERNETES_ENVIRONMENT_DESCRIPTION, provider: KUBERNETES_PROVIDER_KEY, config, extraMetadata: { [KUBERNETES_MANAGED_MARKER]: true }, - }); + }).then((result) => result.environment); }, /** @@ -570,6 +995,7 @@ export function environmentService(db: Db) { patch: UpdateEnvironment, options?: { db?: EnvironmentWriteDb }, ): Promise => { + const writeDb = options?.db ?? db; const values: Partial = { updatedAt: new Date(), }; @@ -581,9 +1007,18 @@ export function environmentService(db: Db) { if ("envVars" in patch && patch.envVars !== undefined) { values.envVars = (patch.envVars ?? {}) as Record; } - if (patch.metadata !== undefined) values.metadata = patch.metadata ?? null; + if (patch.metadata !== undefined) { + values.metadata = withoutManagedEnvironmentArchiveToken(patch.metadata ?? null); + } else if (patch.status !== undefined) { + const existingMetadata = await writeDb + .select({ metadata: environments.metadata }) + .from(environments) + .where(eq(environments.id, id)) + .then((rows) => rows[0]?.metadata ?? null); + values.metadata = withoutManagedEnvironmentArchiveToken(existingMetadata); + } - const row = await (options?.db ?? db) + const row = await writeDb .update(environments) .set(values) .where(eq(environments.id, id)) diff --git a/server/src/services/managed-environments.test.ts b/server/src/services/managed-environments.test.ts index 4857c26a8b..11b4d4bb8b 100644 --- a/server/src/services/managed-environments.test.ts +++ b/server/src/services/managed-environments.test.ts @@ -40,6 +40,19 @@ function environmentRow(overrides: Record = {}) { }; } +function reconciliationResult( + overrides: Record = {}, +) { + return { + environment: environmentRow(), + action: "added" as const, + stockStatus: "missing" as const, + updateAvailable: false, + stockHash: "sha256:stock", + ...overrides, + }; +} + function readyDriverResolver(status = "ready") { return vi.fn(async () => ({ plugin: { id: "plugin-1", pluginKey: "sandbox-providers/daytona", status }, @@ -83,7 +96,7 @@ type EnvironmentsSeam = NonNullable = {}): EnvironmentsSeam { return { ensureManagedSandboxEnvironment: - overrides.ensureManagedSandboxEnvironment ?? vi.fn().mockResolvedValue(environmentRow()), + overrides.ensureManagedSandboxEnvironment ?? vi.fn().mockResolvedValue(reconciliationResult()), archiveManagedSandboxEnvironment: overrides.archiveManagedSandboxEnvironment ?? vi.fn().mockResolvedValue(null), }; @@ -109,7 +122,7 @@ describe("applyManagedEnvironments", () => { it("ensures each declared environment through the provider-agnostic service call", async () => { const ensureManagedSandboxEnvironment = vi .fn() - .mockResolvedValue(environmentRow()); + .mockResolvedValue(reconciliationResult()); const config = parsedConfig({ environments: [ { @@ -130,7 +143,8 @@ describe("applyManagedEnvironments", () => { resolveSandboxProviderDriver, }); - expect(result).toEqual({ ensured: 1, failed: 0 }); + expect(result).toMatchObject({ ensured: 1, failed: 0, added: 1, skipped: 0 }); + expect(result).toMatchObject({ removed: 0, backedUp: 0 }); expect(resolveSandboxProviderDriver).toHaveBeenCalledWith({ db: noDb, driverKey: "daytona" }); expect(workerManager.isRunning).toHaveBeenCalledWith("plugin-1"); expect(ensureManagedSandboxEnvironment).toHaveBeenCalledTimes(1); @@ -139,6 +153,7 @@ describe("applyManagedEnvironments", () => { description: "Managed Daytona sandbox.", provider: "daytona", config: { target: "us" }, + stockVersion: "2026.720.0", }); // The frozen parsed config must not leak into the service (the row's // config is mutated downstream when the provider key is forced in). @@ -149,7 +164,7 @@ describe("applyManagedEnvironments", () => { it("waits for the bundled-plugin startup pass before ensuring anything", async () => { const ensureManagedSandboxEnvironment = vi .fn() - .mockResolvedValue(environmentRow()); + .mockResolvedValue(reconciliationResult()); const config = parsedConfig({ environments: [{ name: "Daytona", provider: "daytona" }], }); @@ -172,10 +187,42 @@ describe("applyManagedEnvironments", () => { expect(ensureManagedSandboxEnvironment).not.toHaveBeenCalled(); releasePlugins(); - expect(await pending).toEqual({ ensured: 1, failed: 0 }); + expect(await pending).toMatchObject({ ensured: 1, failed: 0, added: 1 }); expect(ensureManagedSandboxEnvironment).toHaveBeenCalledTimes(1); }); + it("reports operator-modified rows as skipped with a stock update still available", async () => { + const environments = environmentsSeam({ + ensureManagedSandboxEnvironment: vi.fn().mockResolvedValue(reconciliationResult({ + action: "skipped", + stockStatus: "operator_modified", + updateAvailable: true, + })), + }); + const result = await applyManagedEnvironments(noDb, parsedConfig({ + environments: [{ name: "Daytona", provider: "daytona" }], + }), { + env: {}, + workerManager: runningWorkerManager(), + environments, + resolveSandboxProviderDriver: readyDriverResolver(), + }); + + expect(result).toMatchObject({ + ensured: 0, + failed: 0, + added: 0, + updated: 0, + unchanged: 0, + skipped: 1, + outcomes: [{ + action: "skipped", + stockStatus: "operator_modified", + updateAvailable: true, + }], + }); + }); + it("skips an entry whose provider plugin is missing and archives its stale row", async () => { const environments = environmentsSeam({ archiveManagedSandboxEnvironment: vi.fn().mockResolvedValue(environmentRow({ status: "archived" })), @@ -191,7 +238,7 @@ describe("applyManagedEnvironments", () => { resolveSandboxProviderDriver: vi.fn(async () => null), }); - expect(result).toEqual({ ensured: 0, failed: 1 }); + expect(result).toMatchObject({ ensured: 0, failed: 1 }); expect(environments.ensureManagedSandboxEnvironment).not.toHaveBeenCalled(); expect(environments.archiveManagedSandboxEnvironment).toHaveBeenCalledWith({ provider: "daytona", @@ -211,7 +258,7 @@ describe("applyManagedEnvironments", () => { resolveSandboxProviderDriver: readyDriverResolver("disabled"), }); - expect(result).toEqual({ ensured: 0, failed: 1 }); + expect(result).toMatchObject({ ensured: 0, failed: 1 }); expect(environments.ensureManagedSandboxEnvironment).not.toHaveBeenCalled(); expect(environments.archiveManagedSandboxEnvironment).toHaveBeenCalledWith({ provider: "daytona", @@ -232,7 +279,7 @@ describe("applyManagedEnvironments", () => { resolveSandboxProviderDriver: readyDriverResolver(), }); - expect(result).toEqual({ ensured: 0, failed: 1 }); + expect(result).toMatchObject({ ensured: 0, failed: 1 }); expect(workerManager.isRunning).toHaveBeenCalledWith("plugin-1"); expect(environments.ensureManagedSandboxEnvironment).not.toHaveBeenCalled(); expect(environments.archiveManagedSandboxEnvironment).toHaveBeenCalledWith({ @@ -263,7 +310,7 @@ describe("applyManagedEnvironments", () => { }); // The boot pass itself stays degraded: archived, counted failed. - expect(result).toEqual({ ensured: 0, failed: 1 }); + expect(result).toMatchObject({ ensured: 0, failed: 1 }); expect(environments.archiveManagedSandboxEnvironment).toHaveBeenCalledWith({ provider: "daytona", }); @@ -280,6 +327,7 @@ describe("applyManagedEnvironments", () => { description: undefined, provider: "daytona", config: { target: "us" }, + stockVersion: "2026.720.0", }); expect(handle.off).toHaveBeenCalledTimes(1); }); @@ -308,7 +356,7 @@ describe("applyManagedEnvironments", () => { }); await tick(); - expect(result).toEqual({ ensured: 0, failed: 1 }); + expect(result).toMatchObject({ ensured: 0, failed: 1 }); expect(environments.ensureManagedSandboxEnvironment).toHaveBeenCalledTimes(1); }); @@ -371,7 +419,7 @@ describe("applyManagedEnvironments", () => { resolveSandboxProviderDriver: readyDriverResolver(), }); - expect(result).toEqual({ ensured: 0, failed: 1 }); + expect(result).toMatchObject({ ensured: 0, failed: 1 }); expect(environments.ensureManagedSandboxEnvironment).not.toHaveBeenCalled(); }); @@ -390,7 +438,7 @@ describe("applyManagedEnvironments", () => { resolveSandboxProviderDriver: vi.fn(async () => null), }); - expect(result).toEqual({ ensured: 0, failed: 1 }); + expect(result).toMatchObject({ ensured: 0, failed: 1 }); expect(environments.archiveManagedSandboxEnvironment).toHaveBeenCalledTimes(1); }); @@ -409,7 +457,7 @@ describe("applyManagedEnvironments", () => { resolveSandboxProviderDriver: readyDriverResolver(), }); - expect(result).toEqual({ ensured: 0, failed: 1 }); + expect(result).toMatchObject({ ensured: 0, failed: 1 }); expect(environments.archiveManagedSandboxEnvironment).not.toHaveBeenCalled(); }); }); diff --git a/server/src/services/managed-environments.ts b/server/src/services/managed-environments.ts index fbdf2c6985..8efb75dea7 100644 --- a/server/src/services/managed-environments.ts +++ b/server/src/services/managed-environments.ts @@ -48,7 +48,9 @@ import type { Db } from "@paperclipai/db"; import { logger } from "../middleware/logger.js"; import { environmentService } from "./environments.js"; +import type { ManagedSandboxEnvironmentReconcileAction } from "./environments.js"; import type { ManagedInstanceConfig } from "./managed-config.js"; +import type { ManagedResourceStockStatus } from "./managed-resource-drift.js"; import { parseExecutionPolicyBootstrapEnv } from "./execution-policy-bootstrap.js"; import { resolvePluginSandboxProviderDriverByKey } from "./plugin-environment-driver.js"; import type { PluginWorkerManager } from "./plugin-worker-manager.js"; @@ -93,6 +95,29 @@ export interface ApplyManagedEnvironmentsOptions { }) => Promise<{ plugin: { id: string; pluginKey: string; status: string } } | null>; } +export interface ManagedEnvironmentReconciliationOutcome { + environmentId: string; + name: string; + provider: string; + action: ManagedSandboxEnvironmentReconcileAction; + stockStatus: ManagedResourceStockStatus; + updateAvailable: boolean; +} + +export interface ApplyManagedEnvironmentsResult { + ensured: number; + failed: number; + added: number; + updated: number; + unchanged: number; + skipped: number; + /** Managed reconciliation never removes rows in preserve-only mode. */ + removed: number; + /** Preserve-only reconciliation never creates replacement backups. */ + backedUp: number; + outcomes: ManagedEnvironmentReconciliationOutcome[]; +} + /** * Ensure every environment declared in the managed-config document. Returns * null when there is nothing to do (self-hosted, or no `environments` @@ -103,7 +128,7 @@ export async function applyManagedEnvironments( db: Db, managedConfig: ManagedInstanceConfig | null, opts: ApplyManagedEnvironmentsOptions = {}, -): Promise<{ ensured: number; failed: number } | null> { +): Promise { if (!managedConfig || managedConfig.environments.length === 0) return null; // The forced-execution-mode bootstrap (`PAPERCLIP_EXECUTION_MODE=kubernetes`) @@ -155,10 +180,18 @@ export async function applyManagedEnvironments( description: spec.description, provider: spec.provider, config: { ...spec.config }, + stockVersion: managedConfig.catalogVersion, }) - .then((environment) => { + .then((result) => { logger.info( - { environmentId: environment.id, name: spec.name, provider: spec.provider }, + { + environmentId: result.environment.id, + name: spec.name, + provider: spec.provider, + action: result.action, + stockStatus: result.stockStatus, + updateAvailable: result.updateAvailable, + }, "managed sandbox environment reactivated after provider worker recovery", ); }) @@ -175,6 +208,11 @@ export async function applyManagedEnvironments( let ensured = 0; let failed = 0; + let added = 0; + let updated = 0; + let unchanged = 0; + let skipped = 0; + const outcomes: ManagedEnvironmentReconciliationOutcome[] = []; for (const spec of managedConfig.environments) { try { const resolved = await resolveDriver({ db, driverKey: spec.provider }); @@ -216,17 +254,44 @@ export async function applyManagedEnvironments( } continue; } - const environment = await environments.ensureManagedSandboxEnvironment({ + const reconciliation = await environments.ensureManagedSandboxEnvironment({ name: spec.name, description: spec.description, provider: spec.provider, config: { ...spec.config }, + stockVersion: managedConfig.catalogVersion, }); - ensured += 1; - logger.info( - { environmentId: environment.id, name: environment.name, provider: spec.provider }, - "managed sandbox environment ensured", - ); + if (reconciliation.action === "skipped") skipped += 1; + else { + ensured += 1; + if (reconciliation.action === "added") added += 1; + else if (reconciliation.action === "updated") updated += 1; + else unchanged += 1; + } + outcomes.push({ + environmentId: reconciliation.environment.id, + name: reconciliation.environment.name, + provider: spec.provider, + action: reconciliation.action, + stockStatus: reconciliation.stockStatus, + updateAvailable: reconciliation.updateAvailable, + }); + const logContext = { + environmentId: reconciliation.environment.id, + name: reconciliation.environment.name, + provider: spec.provider, + action: reconciliation.action, + stockStatus: reconciliation.stockStatus, + updateAvailable: reconciliation.updateAvailable, + }; + if (reconciliation.action === "skipped") { + logger.warn( + logContext, + "managed sandbox environment has operator modifications; preserving row and skipping stock update", + ); + } else { + logger.info(logContext, "managed sandbox environment reconciled"); + } } catch (err) { failed += 1; logger.error( @@ -235,5 +300,15 @@ export async function applyManagedEnvironments( ); } } - return { ensured, failed }; + return { + ensured, + failed, + added, + updated, + unchanged, + skipped, + removed: 0, + backedUp: 0, + outcomes, + }; } diff --git a/server/src/services/managed-resource-drift.test.ts b/server/src/services/managed-resource-drift.test.ts new file mode 100644 index 0000000000..5c1408748d --- /dev/null +++ b/server/src/services/managed-resource-drift.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from "vitest"; +import { resourceStatus, stockHash } from "./managed-resource-drift.js"; + +describe("managed resource drift", () => { + it("uses deterministic object-key ordering for stock hashes", () => { + expect(stockHash({ a: 1, b: { c: 2 } })).toBe( + stockHash({ b: { c: 2 }, a: 1 }), + ); + }); + + it.each([ + { + expected: "missing", + resourceId: null, + currentHash: null, + bindingStockHash: null, + latestStockHash: "latest", + }, + { + expected: "stock_current", + resourceId: "resource-1", + currentHash: "latest", + bindingStockHash: "previous", + latestStockHash: "latest", + }, + { + expected: "stock_update_available", + resourceId: "resource-1", + currentHash: "previous", + bindingStockHash: "previous", + latestStockHash: "latest", + }, + { + expected: "operator_modified", + resourceId: "resource-1", + currentHash: "operator", + bindingStockHash: "previous", + latestStockHash: "latest", + }, + ] as const)("classifies $expected", ({ expected, ...input }) => { + expect(resourceStatus(input)).toBe(expected); + }); +}); diff --git a/server/src/services/managed-resource-drift.ts b/server/src/services/managed-resource-drift.ts new file mode 100644 index 0000000000..addce116bc --- /dev/null +++ b/server/src/services/managed-resource-drift.ts @@ -0,0 +1,36 @@ +import { createHash } from "node:crypto"; + +export type ManagedResourceStockStatus = + | "missing" + | "stock_current" + | "stock_update_available" + | "operator_modified"; + +export function stableJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`; + if (value && typeof value === "object") { + return `{${Object.entries(value as Record) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, entry]) => `${JSON.stringify(key)}:${stableJson(entry)}`) + .join(",")}}`; + } + return JSON.stringify(value); +} + +export function stockHash(value: unknown) { + return `sha256:${createHash("sha256").update(stableJson(value)).digest("hex")}`; +} + +export function resourceStatus(input: { + resourceId: string | null; + currentHash: string | null; + bindingStockHash: string | null; + latestStockHash: string; +}): ManagedResourceStockStatus { + if (!input.resourceId || !input.currentHash) return "missing"; + if (input.currentHash === input.latestStockHash) return "stock_current"; + if (input.bindingStockHash && input.currentHash === input.bindingStockHash) { + return "stock_update_available"; + } + return "operator_modified"; +}