import { createHash, randomUUID } from "node:crypto"; import { mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { eq } from "drizzle-orm"; import { buildSshEnvLabFixtureConfig, getSshEnvLabSupport, startSshEnvLabFixture, stopSshEnvLabFixture, } from "@paperclipai/adapter-utils/ssh"; import { agents, builtInManagedResources, companies, companySecretVersions, companySecrets, createDb, environmentLeases, environments, executionWorkspaces, heartbeatRuns, plugins, projects, } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; import { resolveEnvironmentDriverConfigForRuntime } from "../services/environment-config.ts"; import { SANDBOX_CAPABILITY_KEYS, environmentRuntimeService, findReusableSandboxLeaseId, SandboxOrphanCleanupWriteError, } from "../services/environment-runtime.ts"; import * as sandboxProviderRuntime from "../services/sandbox-provider-runtime.ts"; import * as environmentsModule from "../services/environments.ts"; import { logger } from "../middleware/logger.ts"; import { environmentService } from "../services/environments.ts"; import { heartbeatService } from "../services/heartbeat.ts"; import { secretService } from "../services/secrets.ts"; import type { PluginWorkerManager } from "../services/plugin-worker-manager.ts"; import { getActiveStepContext, runWithRuntimeParent, type StartupSpanContext, } from "@paperclipai/adapter-utils/acpx-engine/startup-timing"; import { traceparentFromContextToken } from "../instrumentation.ts"; import { ROOT_CONTEXT, trace } from "@opentelemetry/api"; import { buildNativeHarnessBackupManifest } from "../services/native-runtime/native-session-executor.ts"; import { createNativeHarnessBackupStamp } from "../services/native-runtime/native-harness-backup-stamp.ts"; const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; const sshFixtureSupport = await getSshEnvLabSupport(); if (!embeddedPostgresSupport.supported) { console.warn( `Skipping embedded Postgres environment runtime tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`, ); } function stableStringify(value: unknown): string { if (Array.isArray(value)) { return `[${value.map((entry) => stableStringify(entry)).join(",")}]`; } if (value && typeof value === "object") { const record = value as Record; return `{${Object.keys(record).sort().map((key) => `${JSON.stringify(key)}:${stableStringify(record[key])}`).join(",")}}`; } return JSON.stringify(value) ?? "null"; } function reusableRuntimeFingerprint(input: { provider: string; adapterType: string | null; config: Record; }) { return createHash("sha256").update(stableStringify(input)).digest("hex"); } describe("findReusableSandboxLeaseId", () => { it("matches reusable plugin-backed sandbox leases by provider", () => { const selected = findReusableSandboxLeaseId({ config: { provider: "fake-plugin", image: "template-b", timeoutMs: 300000, reuseLease: true, }, leases: [ { providerLeaseId: "sandbox-template-a", metadata: { provider: "fake-plugin", image: "template-a", timeoutMs: 300000, reuseLease: true, }, }, { providerLeaseId: "sandbox-template-b", metadata: { provider: "fake-plugin", image: "template-b", timeoutMs: 300000, reuseLease: true, }, }, ], }); expect(selected).toBe("sandbox-template-b"); }); it("requires image identity for reusable fake sandbox leases", () => { const selected = findReusableSandboxLeaseId({ config: { provider: "fake", image: "ubuntu:24.04", reuseLease: true, }, leases: [ { providerLeaseId: "sandbox-image-a", metadata: { provider: "fake", image: "debian:12", reuseLease: true, }, }, { providerLeaseId: "sandbox-image-b", metadata: { provider: "fake", image: "ubuntu:24.04", reuseLease: true, }, }, ], }); expect(selected).toBe("sandbox-image-b"); }); }); describeEmbeddedPostgres("environmentRuntimeService", () => { let stopDb: (() => Promise) | null = null; let db!: ReturnType; let runtime!: ReturnType; const fixtureRoots: string[] = []; // Give each test its own orphan-cleanup spool directory, so a spool file one // test writes never leaks into another test's flush. The default runtime reads // this env override when a test does not pass an explicit spool directory. let orphanCleanupSpoolDir: string | null = null; const previousOrphanCleanupSpoolDir = process.env.SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR; beforeAll(async () => { const started = await startEmbeddedPostgresTestDatabase("environment-runtime"); stopDb = started.stop; db = createDb(started.connectionString); runtime = environmentRuntimeService(db); }); beforeEach(async () => { orphanCleanupSpoolDir = await mkdtemp(path.join(os.tmpdir(), "orphan-cleanup-spool-")); process.env.SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR = orphanCleanupSpoolDir; }); afterEach(async () => { if (orphanCleanupSpoolDir) { await rm(orphanCleanupSpoolDir, { recursive: true, force: true }).catch(() => undefined); orphanCleanupSpoolDir = null; } if (previousOrphanCleanupSpoolDir === undefined) { delete process.env.SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR; } else { process.env.SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR = previousOrphanCleanupSpoolDir; } while (fixtureRoots.length > 0) { const root = fixtureRoots.pop(); if (!root) continue; await stopSshEnvLabFixture(path.join(root, "state.json")).catch(() => undefined); await rm(root, { recursive: true, force: true }).catch(() => undefined); } await db.delete(environmentLeases); await db.delete(heartbeatRuns); await db.delete(agents); await db.delete(environments); await db.delete(executionWorkspaces); await db.delete(plugins); await db.delete(companySecretVersions); await db.delete(companySecrets); await db.delete(projects); await db.delete(companies); }); afterAll(async () => { await stopDb?.(); }); async function seedEnvironment(input: { driver?: string; name?: string; status?: "active" | "disabled"; config?: Record; } = {}) { const companyId = randomUUID(); const agentId = randomUUID(); const environmentId = randomUUID(); const runId = randomUUID(); const driver = input.driver ?? "local"; const environmentName = input.name ?? `${driver}-${environmentId.slice(0, 8)}`; let config = input.config ?? {}; await db.insert(companies).values({ id: companyId, name: "Acme", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await db.insert(agents).values({ id: agentId, companyId, name: "CodexCoder", role: "engineer", status: "active", adapterType: "codex_local", adapterConfig: {}, runtimeConfig: {}, permissions: {}, createdAt: new Date(), updatedAt: new Date(), }); if (typeof config.privateKey === "string" && config.privateKey.length > 0) { const secret = await secretService(db).create(companyId, { name: `environment-runtime-private-key-${randomUUID()}`, provider: "local_encrypted", value: config.privateKey, }); await secretService(db).createBinding({ companyId, secretId: secret.id, targetType: "environment", targetId: environmentId, configPath: "privateKeySecretRef", }); config = { ...config, privateKey: null, privateKeySecretRef: { type: "secret_ref", secretId: secret.id, version: "latest", }, }; } const existingLocalEnvironment = driver === "local" ? await db .select() .from(environments) .where(eq(environments.driver, "local")) .then((rows) => rows[0] ?? null) : null; const environmentRecord = existingLocalEnvironment ?? { id: environmentId, name: environmentName, description: null, driver, status: input.status ?? "active", config, metadata: null, createdAt: new Date(), updatedAt: new Date(), }; if (!existingLocalEnvironment) { await db.insert(environments).values({ id: environmentRecord.id, name: environmentRecord.name, driver: environmentRecord.driver, status: environmentRecord.status, config: environmentRecord.config, createdAt: environmentRecord.createdAt, updatedAt: environmentRecord.updatedAt, }); } await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "manual", status: "running", createdAt: new Date(), updatedAt: new Date(), }); return { companyId, agentId, environment: { id: environmentRecord.id, companyId, name: environmentRecord.name, description: environmentRecord.description, driver: environmentRecord.driver, status: environmentRecord.status, config: environmentRecord.config, metadata: environmentRecord.metadata, createdAt: environmentRecord.createdAt, updatedAt: environmentRecord.updatedAt, } as const, runId, }; } async function seedReusablePluginSandboxLease(adapterType: string | null = null) { const pluginId = randomUUID(); const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment(); const providerConfig = { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, }; const environment = { ...baseEnvironment, name: "Reusable Plugin Sandbox", driver: "sandbox", config: providerConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: providerConfig, }); await db.insert(plugins).values({ id: pluginId, pluginKey: "acme.reusable-sandbox-provider", packageName: "@acme/reusable-sandbox-provider", version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: { id: "acme.reusable-sandbox-provider", apiVersion: 1, version: "1.0.0", displayName: "Reusable Sandbox Provider", description: "Test provider with reusable lease support", author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", supportsReusableLeases: true, configSchema: { type: "object", properties: { image: { type: "string" }, timeoutMs: { type: "number" }, reuseLease: { type: "boolean" }, }, }, }, ], }, status: "ready", installOrder: 1, updatedAt: new Date(), } as any); const executionWorkspaceId = randomUUID(); const projectId = randomUUID(); await db.insert(projects).values({ id: projectId, companyId, name: `Workspace ${projectId.slice(0, 8)}`, status: "active", createdAt: new Date(), updatedAt: new Date(), }); await db.insert(executionWorkspaces).values({ id: executionWorkspaceId, companyId, projectId, mode: "shared_workspace", strategyType: "project_primary", name: "Reusable workspace", status: "active", providerType: "local_fs", createdAt: new Date(), updatedAt: new Date(), }); const reusableLease = await environmentService(db).acquireLease({ companyId, environmentId: environment.id, executionWorkspaceId, heartbeatRunId: runId, leasePolicy: "reuse_by_environment", provider: "fake-plugin", providerLeaseId: "reusable-plugin-lease", metadata: { agentId, driver: "sandbox", pluginId, pluginKey: "acme.reusable-sandbox-provider", sandboxProviderPlugin: true, provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, reusableSandboxLease: { version: 1, companyId, environmentId: environment.id, executionWorkspaceId, agentId, adapterType, provider: "fake-plugin", runtimeFingerprint: reusableRuntimeFingerprint({ provider: "fake-plugin", adapterType, config: providerConfig, }), }, }, }); return { pluginId, companyId, agentId, environment, runId, executionWorkspaceId, reusableLease }; } it("retains a successful reusable sandbox lease without stopping the provider resource", async () => { const { pluginId, runId, reusableLease } = await seedReusablePluginSandboxLease(); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { throw new Error(`Unexpected plugin method while retaining warm lease: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: [ "environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease", ], })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const released = await runtimeWithPlugin.releaseRunLeases( runId, "released", undefined, "keep_running", ); expect(released).toHaveLength(1); expect(released[0]?.lease).toMatchObject({ id: reusableLease.id, status: "retained", cleanupStatus: "success", }); await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({ status: "retained", cleanupStatus: "success", }); expect(workerManager.call).not.toHaveBeenCalled(); }); it("stops a reusable sandbox and keeps its lease resumable", async () => { const { pluginId, runId, reusableLease } = await seedReusablePluginSandboxLease(); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentReleaseLease") return undefined; throw new Error(`Unexpected plugin method while stopping lease: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: [ "environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease", ], })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const released = await runtimeWithPlugin.releaseRunLeases( runId, "failed", undefined, "stop_and_retain", ); expect(released[0]?.lease).toMatchObject({ id: reusableLease.id, status: "released", }); expect(workerManager.call).toHaveBeenCalledWith( pluginId, "environmentReleaseLease", expect.objectContaining({ providerLeaseId: reusableLease.providerLeaseId }), expect.any(Number), ); }); it("resumes the same provider lease on the next per-turn run", async () => { const seeded = await seedReusablePluginSandboxLease(); const workerManager = { isRunning: vi.fn((id: string) => id === seeded.pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "sandbox-exact-resume", metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, remoteCwd: "/workspace", }, }; } if (method === "environmentReleaseLease") return undefined; if (method === "environmentResumeLease") { return { providerLeaseId: "sandbox-exact-resume", metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, remoteCwd: "/workspace", }, }; } throw new Error(`Unexpected plugin method during exact resume: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: [ "environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease", ], })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); await environmentService(db).releaseLease(seeded.reusableLease.id, "expired"); const first = await runtimeWithPlugin.acquireRunLease({ companyId: seeded.companyId, environment: seeded.environment, issueId: null, agentId: seeded.agentId, heartbeatRunId: seeded.runId, persistedExecutionWorkspace: { id: seeded.executionWorkspaceId, mode: "shared_workspace", }, }); const workspaceSyncStamp = { schema: "paperclip.native-workspace-stamp/v1", workspaceId: seeded.executionWorkspaceId, providerLeaseId: "sandbox-exact-resume", remoteCwd: "/workspace", hostSha256: "a".repeat(64), finalizedRunId: seeded.runId, }; await environmentService(db).updateLeaseMetadata(first.lease.id, { ...(first.lease.metadata ?? {}), nativeWorkspaceSync: workspaceSyncStamp, }); await runtimeWithPlugin.releaseRunLeases( seeded.runId, "released", undefined, "stop_and_retain", ); const nextRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: nextRunId, companyId: seeded.companyId, agentId: seeded.agentId, invocationSource: "manual", status: "running", createdAt: new Date(), updatedAt: new Date(), }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId: seeded.companyId, environment: seeded.environment, issueId: null, agentId: seeded.agentId, heartbeatRunId: nextRunId, persistedExecutionWorkspace: { id: seeded.executionWorkspaceId, mode: "shared_workspace", }, }); expect(first.lease.metadata?.sandboxLeaseAcquisition).toEqual({ outcome: "created" }); expect(acquired.lease.providerLeaseId).toBe("sandbox-exact-resume"); expect(acquired.lease.metadata?.sandboxLeaseAcquisition).toEqual({ outcome: "resumed", }); expect(acquired.lease.metadata?.nativeWorkspaceSync).toEqual( workspaceSyncStamp, ); expect( workerManager.call.mock.calls.filter( (call) => call[1] === "environmentAcquireLease", ), ).toHaveLength(1); }); it("destroys a disposable paperclip_runner sandbox after the turn", async () => { const { pluginId, runId, reusableLease } = await seedReusablePluginSandboxLease(); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentDestroyLease") return undefined; throw new Error(`Unexpected plugin method while destroying lease: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: [ "environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease", ], })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const released = await runtimeWithPlugin.releaseRunLeases( runId, "released", undefined, "destroy", ); expect(released[0]?.lease).toMatchObject({ id: reusableLease.id, status: "expired", }); expect(workerManager.call).toHaveBeenCalledWith( pluginId, "environmentDestroyLease", expect.objectContaining({ providerLeaseId: reusableLease.providerLeaseId }), expect.any(Number), ); }); it("does not destroy native sandbox state when its verified backup is missing", async () => { const { pluginId, runId, reusableLease } = await seedReusablePluginSandboxLease(); await environmentService(db).updateLeaseMetadata(reusableLease.id, { ...(reusableLease.metadata ?? {}), sandboxLeaseAcquisition: { outcome: "created" }, }); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { throw new Error(`Provider teardown must not run without a backup: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: [ "environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease", ], })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const errors: unknown[] = []; const released = await runtimeWithPlugin.releaseRunLeases( runId, "released", (_leaseId, error) => errors.push(error), "destroy", ); expect(released).toEqual([]); expect(errors).toHaveLength(1); expect(String(errors[0])).toContain("runner_harness_backup_unavailable"); expect(workerManager.call).not.toHaveBeenCalled(); await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({ status: "active", }); }); it("acquires and releases a local run lease through the runtime seam", async () => { const { companyId, environment, runId } = await seedEnvironment(); const acquired = await runtime.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); expect(acquired.lease.status).toBe("active"); expect(acquired.lease.metadata).toMatchObject({ driver: "local", executionWorkspaceMode: null, }); expect(acquired.leaseContext).toEqual({ executionWorkspaceId: null, executionWorkspaceMode: null, }); const released = await runtime.releaseRunLeases(runId); expect(released).toHaveLength(1); expect(released[0]?.environment.driver).toBe("local"); expect(released[0]?.lease.status).toBe("released"); const rows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.id, acquired.lease.id)); expect(rows[0]?.status).toBe("released"); }); it("allows projectless runs through the runtime seam", async () => { const { companyId, environment, runId } = await seedEnvironment(); const acquired = await runtime.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); expect(acquired.lease.executionWorkspaceId).toBeNull(); expect(acquired.leaseContext.executionWorkspaceId).toBeNull(); expect(acquired.leaseContext.executionWorkspaceMode).toBeNull(); }); it("rejects truly unsupported drivers before acquiring a lease", async () => { const { companyId, agentId, environment, runId } = await seedEnvironment({ driver: "ssh", name: "Fixture SSH", config: { host: "ssh.example.test", port: 22, username: "ssh-user", remoteWorkspacePath: "/srv/paperclip/workspace", privateKey: null, knownHosts: null, strictHostKeyChecking: true, }, }); const runtimeWithoutSsh = environmentRuntimeService(db, { drivers: [ { driver: "local", acquireRunLease: async () => { throw new Error("should not acquire"); }, releaseRunLease: async () => null, }, ], }); await expect( runtimeWithoutSsh.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }), ).rejects.toThrow('Environment driver "ssh" is not registered in the environment runtime yet.'); const rows = await db.select().from(environmentLeases); expect(rows).toHaveLength(0); }); it("acquires and releases an SSH run lease through the runtime seam", async () => { if (!sshFixtureSupport.supported) { console.warn( `Skipping SSH runtime fixture test: ${sshFixtureSupport.reason ?? "unsupported environment"}`, ); return; } const fixtureRoot = await mkdtemp(path.join(os.tmpdir(), "paperclip-environment-runtime-ssh-")); fixtureRoots.push(fixtureRoot); const statePath = path.join(fixtureRoot, "state.json"); const fixture = await startSshEnvLabFixture({ statePath }); const sshConfig = await buildSshEnvLabFixtureConfig(fixture); const { companyId, environment, runId } = await seedEnvironment({ driver: "ssh", name: "Fixture SSH", config: sshConfig, }); try { const acquired = await runtime.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); expect(acquired.lease.status).toBe("active"); expect(acquired.lease.providerLeaseId).toContain(`ssh://${sshConfig.username}@${sshConfig.host}:${sshConfig.port}`); expect(acquired.lease.metadata).toMatchObject({ driver: "ssh", host: sshConfig.host, port: sshConfig.port, username: sshConfig.username, remoteWorkspacePath: sshConfig.remoteWorkspacePath, remoteCwd: sshConfig.remoteWorkspacePath, }); const released = await runtime.releaseRunLeases(runId); expect(released).toHaveLength(1); expect(released[0]?.environment.driver).toBe("ssh"); expect(released[0]?.lease.status).toBe("released"); } finally { } }); it("acquires and releases a fake sandbox run lease through the runtime seam", async () => { const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Fake Sandbox", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: true, }, }); const acquired = await runtime.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); expect(acquired.lease.status).toBe("active"); expect(acquired.lease.providerLeaseId).toMatch(new RegExp(`^sandbox://fake/${runId}/[0-9a-f-]{36}$`)); expect(acquired.lease.leasePolicy).toBe("ephemeral"); expect(acquired.lease.metadata).toMatchObject({ driver: "sandbox", provider: "fake", image: "ubuntu:24.04", reuseLease: true, }); const released = await runtime.releaseRunLeases(runId); expect(released).toHaveLength(1); expect(released[0]?.environment.driver).toBe("sandbox"); expect(released[0]?.lease.status).toBe("released"); }); it("releases the remote sandbox when the lease insert rejects a foreign-company binding", async () => { const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Foreign-bound Fake Sandbox", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false, }, }); // A managed reconciliation binds the environment to another company after the // route guard read an empty binding list. The lease insert then rejects the // foreign-company binding with the 403 `environment_company_mismatch`. const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co", issuePrefix: "OTA", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await db.insert(builtInManagedResources).values({ companyId: otherCompanyId, bundleKey: "managed-environment", resourceKind: "environment", resourceKey: "managed-sandbox", resourceId: environment.id, stockVersion: "1", stockHash: "hash", }); const destroySpy = vi.spyOn(sandboxProviderRuntime, "destroySandboxProviderLease"); try { await expect( runtime.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }), ).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" }, }); // The acquire records the durable pending-cleanup row before the teardown, // so the row lands while the database is proven reachable. The successful // teardown then releases the row to the terminal `expired` state, so no // active or pending_cleanup row remains for the orphan. const leaseRows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(leaseRows).toHaveLength(1); expect(leaseRows[0]?.status).toBe("expired"); expect(leaseRows[0]?.cleanupStatus).toBe("success"); // The acquire already provisioned the remote sandbox, so it releases the // sandbox on the rejection. Without this teardown the rejected insert leaks // a live sandbox that no lease row tracks. The teardown carries the // provisioned provider lease id. expect(destroySpy).toHaveBeenCalledTimes(1); expect(destroySpy.mock.calls[0]?.[0]?.providerLeaseId).toMatch( new RegExp(`^sandbox://fake/${runId}/[0-9a-f-]{36}$`), ); } finally { destroySpy.mockRestore(); } }); it("destroys the remote plugin sandbox when the lease insert rejects a foreign-company binding", async () => { // The Claude login runs on a plugin-backed sandbox provider (Daytona), so this // path is the one the login uses. It must also release the remote sandbox when // the conditional lease insert rejects a foreign-company binding. const pluginId = randomUUID(); const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); const pluginConfig = { provider: "fake-plugin", image: "fake:test", reuseLease: false }; const environment = { ...baseEnvironment, name: "Foreign-bound Plugin Sandbox", driver: "sandbox", config: pluginConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: pluginConfig, }); await db.insert(plugins).values({ id: pluginId, pluginKey: "paperclip.fake-plugin-sandbox-provider", packageName: "@paperclipai/plugin-fake-sandbox", version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: { id: "paperclip.fake-plugin-sandbox-provider", apiVersion: 1, version: "1.0.0", displayName: "Fake Plugin Sandbox Provider", description: "Test fake plugin provider", author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", configSchema: { type: "object" }, }, ], }, status: "ready", installOrder: 1, updatedAt: new Date(), } as any); // A managed reconciliation binds the environment to another company after the // route guard read an empty binding list. The lease insert then rejects the // foreign-company binding with the 403 `environment_company_mismatch`. const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co", issuePrefix: "OTB", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await db.insert(builtInManagedResources).values({ companyId: otherCompanyId, bundleKey: "managed-environment", resourceKind: "environment", resourceKey: "managed-sandbox", resourceId: environment.id, stockVersion: "1", stockHash: "hash", }); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), getWorker: vi.fn(() => ({ supportedMethods: ["environmentDestroyLease"] })), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "plugin-lease-1", metadata: { provider: "fake-plugin", image: "fake:test", reuseLease: false }, }; } if (method === "environmentDestroyLease") { return undefined; } throw new Error(`Unexpected plugin method: ${method}`); }), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); await expect( runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }), ).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" }, }); // The acquire records the durable pending-cleanup row before the teardown, // so the row lands while the database is proven reachable. The successful // teardown then releases the row to the terminal `expired` state, so no // active or pending_cleanup row remains for the orphan. const leaseRows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(leaseRows).toHaveLength(1); expect(leaseRows[0]?.status).toBe("expired"); expect(leaseRows[0]?.cleanupStatus).toBe("success"); // The acquire provisioned the remote plugin sandbox, so it destroys the // sandbox on the rejection. Without this teardown the rejected insert leaks a // live sandbox that no lease row tracks. const destroyCalls = (workerManager.call as unknown as ReturnType).mock.calls.filter( (callArgs) => callArgs[1] === "environmentDestroyLease", ); expect(destroyCalls).toHaveLength(1); expect(destroyCalls[0]?.[2]).toMatchObject({ providerLeaseId: "plugin-lease-1" }); await environmentService(db).update(environment.id, { driver: "local", config: {} }); }); it("records a durable pending-cleanup lease when the built-in teardown fails after a foreign-company rejection", async () => { const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Foreign-bound Fake Sandbox Teardown Fail", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, }); const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co Built-in", issuePrefix: "OTC", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await db.insert(builtInManagedResources).values({ companyId: otherCompanyId, bundleKey: "managed-environment", resourceKind: "environment", resourceKey: "managed-sandbox", resourceId: environment.id, stockVersion: "1", stockHash: "hash", }); // The remote teardown fails, so the acquire cannot release the sandbox // directly. It must record a durable pending-cleanup row instead. const destroySpy = vi .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") .mockRejectedValue(new Error("teardown failed")); try { await expect( runtime.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }), ).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } }); expect(destroySpy).toHaveBeenCalledTimes(1); // The failed teardown left a durable pending-cleanup row that carries the // provider lease id for a later sweep. const rows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rows).toHaveLength(1); expect(rows[0]?.status).toBe("pending_cleanup"); expect(rows[0]?.cleanupStatus).toBe("failed"); expect(rows[0]?.providerLeaseId).toMatch(new RegExp(`^sandbox://fake/${runId}/[0-9a-f-]{36}$`)); } finally { destroySpy.mockRestore(); } }); it("records the durable pending-cleanup row before it runs the inline teardown", async () => { const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Foreign-bound Fake Sandbox Write First", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, }); const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co Write First", issuePrefix: "OWF", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await db.insert(builtInManagedResources).values({ companyId: otherCompanyId, bundleKey: "managed-environment", resourceKind: "environment", resourceKey: "managed-sandbox", resourceId: environment.id, stockVersion: "1", stockHash: "hash", }); // The teardown reads the lease table when it runs. A durable pending_cleanup // row must already exist, which proves the acquire records the row before the // teardown. This ordering closes the window the finding describes: the durable // write lands while the database is proven reachable, before the teardown RPC // that a database outage could otherwise interrupt. let pendingCleanupRowsAtTeardown = -1; const destroySpy = vi .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") .mockImplementation(async () => { const rows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); pendingCleanupRowsAtTeardown = rows.filter((row) => row.status === "pending_cleanup").length; }); try { await expect( runtime.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }), ).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } }); // The durable row already tracked the orphan when the teardown ran. expect(pendingCleanupRowsAtTeardown).toBe(1); expect(destroySpy).toHaveBeenCalledTimes(1); // The teardown succeeded, so the acquire released the row to the terminal // `expired` state and left no pending_cleanup row. const rows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rows).toHaveLength(1); expect(rows[0]?.status).toBe("expired"); expect(rows[0]?.cleanupStatus).toBe("success"); } finally { destroySpy.mockRestore(); } }); it("leaves no orphan and no error when the durable write fails but the teardown succeeds", async () => { const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Foreign-bound Fake Sandbox Write Fail Teardown Ok", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, }); const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co Write Fail", issuePrefix: "OWK", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await db.insert(builtInManagedResources).values({ companyId: otherCompanyId, bundleKey: "managed-environment", resourceKind: "environment", resourceKey: "managed-sandbox", resourceId: environment.id, stockVersion: "1", stockHash: "hash", }); // Force every durable pending-cleanup write to fail, but let the teardown // succeed. A successful teardown removes the orphan, so the acquire needs no // durable row and raises no orphan-cleanup write error. The write-first order // never turns a clean teardown into a failure. const realEnvironmentService = environmentsModule.environmentService; const factorySpy = vi .spyOn(environmentsModule, "environmentService") .mockImplementation((database: Parameters[0]) => { const real = realEnvironmentService(database); return { ...real, insertPendingCleanupLease: () => Promise.reject(new Error("pending-cleanup write failed; database down")), }; }); const destroySpy = vi .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") .mockResolvedValue(undefined as never); try { // Set the retry backoff to zero, so the failed write retries never slow the // test. const runtime = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0 }); const rejection = await runtime .acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }) .then( () => { throw new Error("acquireRunLease resolved but must reject"); }, (error: unknown) => error, ); // The rejection is the original company mismatch, not an orphan-cleanup // write error, because the teardown removed the orphan. expect(rejection).not.toBeInstanceOf(SandboxOrphanCleanupWriteError); expect(rejection).toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } }); expect(destroySpy).toHaveBeenCalledTimes(1); // No orphan remains and no durable row was needed, so the lease table is // empty for the environment. const rows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rows).toHaveLength(0); // The buffer holds no orphan, so a flush recovers nothing. const flushed = await runtime.flushDeferredOrphanCleanups(); expect(flushed).toEqual({ recovered: 0, pending: 0 }); } finally { destroySpy.mockRestore(); factorySpy.mockRestore(); } }); it("records a durable pending-cleanup lease when the plugin teardown fails after a foreign-company rejection", async () => { const pluginId = randomUUID(); const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); const pluginConfig = { provider: "fake-plugin", image: "fake:test", reuseLease: false }; const environment = { ...baseEnvironment, name: "Foreign-bound Plugin Sandbox Teardown Fail", driver: "sandbox", config: pluginConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: pluginConfig, }); await db.insert(plugins).values({ id: pluginId, pluginKey: "paperclip.fake-plugin-sandbox-provider", packageName: "@paperclipai/plugin-fake-sandbox", version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: { id: "paperclip.fake-plugin-sandbox-provider", apiVersion: 1, version: "1.0.0", displayName: "Fake Plugin Sandbox Provider", description: "Test fake plugin provider", author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", configSchema: { type: "object" }, }, ], }, status: "ready", installOrder: 1, updatedAt: new Date(), } as any); const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co Plugin", issuePrefix: "OTD", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await db.insert(builtInManagedResources).values({ companyId: otherCompanyId, bundleKey: "managed-environment", resourceKind: "environment", resourceKey: "managed-sandbox", resourceId: environment.id, stockVersion: "1", stockHash: "hash", }); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), getWorker: vi.fn(() => ({ supportedMethods: ["environmentDestroyLease"] })), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "plugin-lease-2", metadata: { provider: "fake-plugin", image: "fake:test", reuseLease: false }, }; } if (method === "environmentDestroyLease") { throw new Error("destroy failed"); } throw new Error(`Unexpected plugin method: ${method}`); }), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); await expect( runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }), ).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } }); // The failed plugin teardown left a durable pending-cleanup row that carries // the provider lease id for a later sweep. const rows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rows).toHaveLength(1); expect(rows[0]?.status).toBe("pending_cleanup"); expect(rows[0]?.cleanupStatus).toBe("failed"); expect(rows[0]?.providerLeaseId).toBe("plugin-lease-2"); await environmentService(db).update(environment.id, { driver: "local", config: {} }); }); it("throws a SandboxOrphanCleanupWriteError when the built-in durable pending-cleanup write also fails", async () => { const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Foreign-bound Fake Sandbox Cleanup Write Fail", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, }); const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co Cleanup Write", issuePrefix: "OCW", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await db.insert(builtInManagedResources).values({ companyId: otherCompanyId, bundleKey: "managed-environment", resourceKind: "environment", resourceKey: "managed-sandbox", resourceId: environment.id, stockVersion: "1", stockHash: "hash", }); // The remote teardown fails, so the acquire tries to record a durable // pending-cleanup row instead. const destroySpy = vi .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") .mockRejectedValue(new Error("teardown failed")); // Force the durable pending-cleanup write to fail on every attempt. The // write retries a few times, so reject the insert each time and let the // primary company-bound acquire reject for real. const failingInsert = vi .fn["insertPendingCleanupLease"]>, Promise>() .mockRejectedValue(new Error("pending-cleanup write failed")); const realEnvironmentService = environmentsModule.environmentService; const factorySpy = vi .spyOn(environmentsModule, "environmentService") .mockImplementation((database: Parameters[0]) => { const real = realEnvironmentService(database); return { ...real, insertPendingCleanupLease: failingInsert, }; }); // The durable database write fails, so the only durable handle left is the // error log. Capture it to prove the leaked sandbox stays discoverable. const logSpy = vi.spyOn(logger, "error").mockImplementation(() => logger); try { // Set the retry backoff to zero, so the retries never slow the test. const runtimeWithFailingCleanup = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0, }); const rejection = await runtimeWithFailingCleanup .acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }) .then( () => { throw new Error("acquireRunLease resolved but must reject"); }, (error: unknown) => error, ); // The failed durable write is not swallowed: the acquire throws a // SandboxOrphanCleanupWriteError that keeps the original 403 rejection as // its cause. expect(rejection).toBeInstanceOf(SandboxOrphanCleanupWriteError); expect(rejection).toMatchObject({ provider: "fake" }); expect((rejection as SandboxOrphanCleanupWriteError).cause).toMatchObject({ status: 403, details: { code: "environment_company_mismatch" }, }); expect(destroySpy).toHaveBeenCalledTimes(1); // The write retried the durable insert before it gave up, so more than one // attempt ran. expect(failingInsert.mock.calls.length).toBeGreaterThan(1); // The durable write failed before it created a row, so no lease row exists. const rows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rows).toHaveLength(0); // The durable log preserves the provider identifiers, so an operator keeps // a handle to find and tear the leaked sandbox down by hand. const cleanupLog = logSpy.mock.calls.find( ([fields]) => (fields as { errorKind?: string } | undefined)?.errorKind === "sandbox_orphan_cleanup_write_failed", ); expect(cleanupLog).toBeDefined(); expect(cleanupLog?.[0]).toMatchObject({ provider: "fake", companyId, environmentId: environment.id, }); // The log never carries the caught exception, so no credential leaks. expect(cleanupLog?.[0]).not.toHaveProperty("cause"); expect(cleanupLog?.[0]).not.toHaveProperty("cleanupWriteError"); } finally { logSpy.mockRestore(); factorySpy.mockRestore(); destroySpy.mockRestore(); } }); it("recovers cleanup state when a retry of the durable pending-cleanup write succeeds", async () => { const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Foreign-bound Fake Sandbox Cleanup Write Retry", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, }); const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co Cleanup Retry", issuePrefix: "OCR", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await db.insert(builtInManagedResources).values({ companyId: otherCompanyId, bundleKey: "managed-environment", resourceKind: "environment", resourceKey: "managed-sandbox", resourceId: environment.id, stockVersion: "1", stockHash: "hash", }); // The remote teardown fails, so the acquire tries to record a durable // pending-cleanup row instead. const destroySpy = vi .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") .mockRejectedValue(new Error("teardown failed")); // The first durable write attempt fails, so the acquire retries. The second // attempt delegates to the real insert, so a durable pending_cleanup row // lands and a later sweep can find the orphan. let insertAttempts = 0; const realEnvironmentService = environmentsModule.environmentService; const factorySpy = vi .spyOn(environmentsModule, "environmentService") .mockImplementation((database: Parameters[0]) => { const real = realEnvironmentService(database); return { ...real, insertPendingCleanupLease: ( input: Parameters[0], ) => { insertAttempts += 1; if (insertAttempts === 1) { return Promise.reject(new Error("pending-cleanup write failed once")); } return real.insertPendingCleanupLease(input); }, }; }); try { // Set the retry backoff to zero, so the retry never slows the test. const runtimeWithRetry = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0, }); const rejection = await runtimeWithRetry .acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }) .then( () => { throw new Error("acquireRunLease resolved but must reject"); }, (error: unknown) => error, ); // The retry landed the durable row, so the acquire rejects with the // original company-mismatch rejection, not a SandboxOrphanCleanupWriteError. expect(rejection).not.toBeInstanceOf(SandboxOrphanCleanupWriteError); expect(rejection).toMatchObject({ status: 403, details: { code: "environment_company_mismatch" }, }); expect(destroySpy).toHaveBeenCalledTimes(1); // The first attempt failed and the second succeeded, so the write ran twice. expect(insertAttempts).toBe(2); // A durable pending_cleanup lease row now tracks the orphan, so a sweep can // find and release the leaked sandbox. const rows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rows).toHaveLength(1); expect(rows[0]?.status).toBe("pending_cleanup"); expect(rows[0]?.cleanupStatus).toBe("failed"); expect(rows[0]?.failureReason).toBe("acquire_rejected_teardown_failed"); } finally { factorySpy.mockRestore(); destroySpy.mockRestore(); } }); it("buffers the orphan in-process and a later sweep flush lands the durable row after the database recovers", async () => { const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Foreign-bound Fake Sandbox Cleanup Buffer Flush", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, }); const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co Cleanup Buffer", issuePrefix: "OCB", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await db.insert(builtInManagedResources).values({ companyId: otherCompanyId, bundleKey: "managed-environment", resourceKind: "environment", resourceKey: "managed-sandbox", resourceId: environment.id, stockVersion: "1", stockHash: "hash", }); // The remote teardown fails, so the acquire tries to record a durable // pending-cleanup row instead. const destroySpy = vi .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") .mockRejectedValue(new Error("teardown failed")); // The database is down, so every synchronous pending-cleanup write rejects. // The flag flips to false when the database recovers, so a later flush lands // the durable row. let databaseDown = true; const realEnvironmentService = environmentsModule.environmentService; const factorySpy = vi .spyOn(environmentsModule, "environmentService") .mockImplementation((database: Parameters[0]) => { const real = realEnvironmentService(database); return { ...real, insertPendingCleanupLease: ( input: Parameters[0], ) => { if (databaseDown) { return Promise.reject(new Error("pending-cleanup write failed; database down")); } return real.insertPendingCleanupLease(input); }, }; }); const logSpy = vi.spyOn(logger, "error").mockImplementation(() => logger); try { // Set the retry backoff to zero, so the retries never slow the test. const runtime = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0 }); const rejection = await runtime .acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }) .then( () => { throw new Error("acquireRunLease resolved but must reject"); }, (error: unknown) => error, ); // The synchronous write failed on every attempt, so the acquire still // throws the orphan-cleanup write error that keeps the original rejection. expect(rejection).toBeInstanceOf(SandboxOrphanCleanupWriteError); // The database was down, so no durable row exists yet. let rows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rows).toHaveLength(0); // The acquire buffered the orphan in-process, so the log records the buffer. const bufferedLog = logSpy.mock.calls.find( ([fields]) => (fields as { errorKind?: string } | undefined)?.errorKind === "sandbox_orphan_cleanup_write_failed", ); expect(bufferedLog?.[0]).toMatchObject({ buffered: true }); // The database recovers, so the next cleanup-sweep flush lands the row. databaseDown = false; const flushed = await runtime.flushDeferredOrphanCleanups(); expect(flushed).toEqual({ recovered: 1, pending: 0 }); // A durable pending_cleanup row now tracks the orphan, so a sweep finds and // releases the leaked sandbox. rows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rows).toHaveLength(1); expect(rows[0]?.status).toBe("pending_cleanup"); expect(rows[0]?.cleanupStatus).toBe("failed"); expect(rows[0]?.failureReason).toBe("acquire_rejected_teardown_failed"); expect(rows[0]?.providerLeaseId).toBeTruthy(); // The buffer is empty now, so a second flush inserts nothing. const second = await runtime.flushDeferredOrphanCleanups(); expect(second).toEqual({ recovered: 0, pending: 0 }); const rowsAfter = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rowsAfter).toHaveLength(1); } finally { logSpy.mockRestore(); factorySpy.mockRestore(); destroySpy.mockRestore(); } }); it("persists the orphan to the durable spool so a restart recovers and lands the durable row", async () => { const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Foreign-bound Fake Sandbox Cleanup Spool Restart", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, }); const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co Cleanup Spool Restart", issuePrefix: "OCS", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await db.insert(builtInManagedResources).values({ companyId: otherCompanyId, bundleKey: "managed-environment", resourceKind: "environment", resourceKey: "managed-sandbox", resourceId: environment.id, stockVersion: "1", stockHash: "hash", }); // The remote teardown fails, so the acquire tries to record a durable // pending-cleanup row instead. const destroySpy = vi .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") .mockRejectedValue(new Error("teardown failed")); // The database is down, so every synchronous pending-cleanup write rejects. // The flag flips to false to simulate the database recovering after a // restart. let databaseDown = true; const realEnvironmentService = environmentsModule.environmentService; const factorySpy = vi .spyOn(environmentsModule, "environmentService") .mockImplementation((database: Parameters[0]) => { const real = realEnvironmentService(database); return { ...real, insertPendingCleanupLease: ( input: Parameters[0], ) => { if (databaseDown) { return Promise.reject(new Error("pending-cleanup write failed; database down")); } return real.insertPendingCleanupLease(input); }, }; }); const logSpy = vi.spyOn(logger, "error").mockImplementation(() => logger); const spoolDir = process.env.SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR!; try { // The first process acquires, fails every synchronous write, and persists // the orphan to the durable spool. The retry backoff is zero, so the test // never waits. const firstProcessRuntime = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0 }); const rejection = await firstProcessRuntime .acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }) .then( () => { throw new Error("acquireRunLease resolved but must reject"); }, (error: unknown) => error, ); expect(rejection).toBeInstanceOf(SandboxOrphanCleanupWriteError); // The database was down, so no durable row exists yet. let rows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rows).toHaveLength(0); // The acquire persisted the orphan to the durable spool, so a file waits on // the disk and the log records the durable persist. const spoolFiles = (await readdir(spoolDir)).filter((name) => name.endsWith(".json")); expect(spoolFiles).toHaveLength(1); const persistedLog = logSpy.mock.calls.find( ([fields]) => (fields as { errorKind?: string } | undefined)?.errorKind === "sandbox_orphan_cleanup_write_failed", ); expect(persistedLog?.[0]).toMatchObject({ persisted: true }); // The process restarts. A fresh runtime keeps no in-process buffer, so only // the durable spool carries the orphan. The database recovers, so the first // cleanup-sweep flush loads the spool and lands the durable row. databaseDown = false; const restartedRuntime = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0 }); const flushed = await restartedRuntime.flushDeferredOrphanCleanups(); expect(flushed).toEqual({ recovered: 1, pending: 0 }); rows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rows).toHaveLength(1); expect(rows[0]?.status).toBe("pending_cleanup"); expect(rows[0]?.failureReason).toBe("acquire_rejected_teardown_failed"); expect(rows[0]?.providerLeaseId).toBeTruthy(); // The flush removed the spooled copy after the row landed, so a second // restart never re-inserts a duplicate. const spoolFilesAfter = (await readdir(spoolDir)).filter((name) => name.endsWith(".json")); expect(spoolFilesAfter).toHaveLength(0); const secondRestartRuntime = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0 }); const secondFlush = await secondRestartRuntime.flushDeferredOrphanCleanups(); expect(secondFlush).toEqual({ recovered: 0, pending: 0 }); const rowsAfter = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rowsAfter).toHaveLength(1); } finally { logSpy.mockRestore(); factorySpy.mockRestore(); destroySpy.mockRestore(); } }); it("buffers the orphan in-process when the durable spool write also fails", async () => { const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Foreign-bound Fake Sandbox Cleanup Spool Unwritable", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, }); const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co Cleanup Spool Unwritable", issuePrefix: "OCU", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await db.insert(builtInManagedResources).values({ companyId: otherCompanyId, bundleKey: "managed-environment", resourceKind: "environment", resourceKey: "managed-sandbox", resourceId: environment.id, stockVersion: "1", stockHash: "hash", }); const destroySpy = vi .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") .mockRejectedValue(new Error("teardown failed")); let databaseDown = true; const realEnvironmentService = environmentsModule.environmentService; const factorySpy = vi .spyOn(environmentsModule, "environmentService") .mockImplementation((database: Parameters[0]) => { const real = realEnvironmentService(database); return { ...real, insertPendingCleanupLease: ( input: Parameters[0], ) => { if (databaseDown) { return Promise.reject(new Error("pending-cleanup write failed; database down")); } return real.insertPendingCleanupLease(input); }, }; }); const logSpy = vi.spyOn(logger, "error").mockImplementation(() => logger); try { // Point the spool at an unwritable path, so the durable spool write fails // like a full or read-only disk. The acquire must fall back to the // in-process buffer and never lose the orphan silently. const unwritableSpool = { append: () => Promise.resolve(false), load: () => Promise.resolve([]), remove: () => Promise.resolve(), }; const runtime = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0, orphanCleanupSpool: unwritableSpool, }); await runtime .acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }) .then( () => { throw new Error("acquireRunLease resolved but must reject"); }, () => undefined, ); // The spool write failed, so the log records the in-process-only fallback. const fallbackLog = logSpy.mock.calls.find( ([fields]) => (fields as { errorKind?: string } | undefined)?.errorKind === "sandbox_orphan_cleanup_write_failed", ); expect(fallbackLog?.[0]).toMatchObject({ persisted: false, buffered: true }); // The same runtime still keeps the orphan in-process, so a flush after the // database recovers lands the durable row. databaseDown = false; const flushed = await runtime.flushDeferredOrphanCleanups(); expect(flushed).toEqual({ recovered: 1, pending: 0 }); const rows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rows).toHaveLength(1); expect(rows[0]?.status).toBe("pending_cleanup"); } finally { logSpy.mockRestore(); factorySpy.mockRestore(); destroySpy.mockRestore(); } }); it("keeps the buffered orphan when the flush write still fails, then recovers on a later flush", async () => { const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Foreign-bound Fake Sandbox Cleanup Buffer Requeue", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, }); const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co Cleanup Requeue", issuePrefix: "OCQ", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await db.insert(builtInManagedResources).values({ companyId: otherCompanyId, bundleKey: "managed-environment", resourceKind: "environment", resourceKey: "managed-sandbox", resourceId: environment.id, stockVersion: "1", stockHash: "hash", }); const destroySpy = vi .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") .mockRejectedValue(new Error("teardown failed")); let databaseDown = true; const realEnvironmentService = environmentsModule.environmentService; const factorySpy = vi .spyOn(environmentsModule, "environmentService") .mockImplementation((database: Parameters[0]) => { const real = realEnvironmentService(database); return { ...real, insertPendingCleanupLease: ( input: Parameters[0], ) => { if (databaseDown) { return Promise.reject(new Error("pending-cleanup write failed; database down")); } return real.insertPendingCleanupLease(input); }, }; }); const errorLogSpy = vi.spyOn(logger, "error").mockImplementation(() => logger); const warnLogSpy = vi.spyOn(logger, "warn").mockImplementation(() => logger); try { const runtime = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0 }); await runtime .acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }) .then( () => { throw new Error("acquireRunLease resolved but must reject"); }, () => undefined, ); // The database is still down, so the flush re-queues the orphan instead of // losing it. The buffer keeps the record for a later tick. const firstFlush = await runtime.flushDeferredOrphanCleanups(); expect(firstFlush).toEqual({ recovered: 0, pending: 1 }); const rowsWhileDown = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rowsWhileDown).toHaveLength(0); // The flush logs the re-queue and never carries the caught write exception. // The sync-retry path also warns with the same error kind, so match on the // `requeued` field that only the flush warn carries. const requeueLog = warnLogSpy.mock.calls.find( ([fields]) => (fields as { requeued?: boolean } | undefined)?.requeued === true, ); expect(requeueLog).toBeDefined(); expect(requeueLog?.[0]).toMatchObject({ errorKind: "sandbox_orphan_cleanup_write_failed", requeued: true, }); expect(requeueLog?.[0]).not.toHaveProperty("cause"); expect(requeueLog?.[0]).not.toHaveProperty("cleanupWriteError"); // The database recovers, so the next flush lands the durable row exactly // once from the still-buffered record. databaseDown = false; const secondFlush = await runtime.flushDeferredOrphanCleanups(); expect(secondFlush).toEqual({ recovered: 1, pending: 0 }); const rowsAfter = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rowsAfter).toHaveLength(1); expect(rowsAfter[0]?.status).toBe("pending_cleanup"); } finally { warnLogSpy.mockRestore(); errorLogSpy.mockRestore(); factorySpy.mockRestore(); destroySpy.mockRestore(); } }); it("throws a SandboxOrphanCleanupWriteError when the plugin durable pending-cleanup write also fails", async () => { const pluginId = randomUUID(); const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); const pluginConfig = { provider: "fake-plugin", image: "fake:test", reuseLease: false }; const environment = { ...baseEnvironment, name: "Foreign-bound Plugin Sandbox Cleanup Write Fail", driver: "sandbox", config: pluginConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: pluginConfig, }); await db.insert(plugins).values({ id: pluginId, pluginKey: "paperclip.fake-plugin-sandbox-provider", packageName: "@paperclipai/plugin-fake-sandbox", version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: { id: "paperclip.fake-plugin-sandbox-provider", apiVersion: 1, version: "1.0.0", displayName: "Fake Plugin Sandbox Provider", description: "Test fake plugin provider", author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", configSchema: { type: "object" }, }, ], }, status: "ready", installOrder: 1, updatedAt: new Date(), } as any); const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co Plugin Cleanup Write", issuePrefix: "OPW", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await db.insert(builtInManagedResources).values({ companyId: otherCompanyId, bundleKey: "managed-environment", resourceKind: "environment", resourceKey: "managed-sandbox", resourceId: environment.id, stockVersion: "1", stockHash: "hash", }); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), getWorker: vi.fn(() => ({ supportedMethods: ["environmentDestroyLease"] })), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "plugin-lease-write-fail", metadata: { provider: "fake-plugin", image: "fake:test", reuseLease: false }, }; } if (method === "environmentDestroyLease") { throw new Error("destroy failed"); } throw new Error(`Unexpected plugin method: ${method}`); }), } as unknown as PluginWorkerManager; const realEnvironmentService = environmentsModule.environmentService; const factorySpy = vi .spyOn(environmentsModule, "environmentService") .mockImplementation((database: Parameters[0]) => { const real = realEnvironmentService(database); return { ...real, insertPendingCleanupLease: () => Promise.reject(new Error("pending-cleanup write failed")), }; }); try { const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager, pendingCleanupWriteBackoffMs: 0, }); const rejection = await runtimeWithPlugin .acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }) .then( () => { throw new Error("acquireRunLease resolved but must reject"); }, (error: unknown) => error, ); expect(rejection).toBeInstanceOf(SandboxOrphanCleanupWriteError); expect(rejection).toMatchObject({ provider: "fake-plugin", providerLeaseId: "plugin-lease-write-fail" }); expect((rejection as SandboxOrphanCleanupWriteError).cause).toMatchObject({ status: 403, details: { code: "environment_company_mismatch" }, }); // The durable write failed before it created a row, so no lease row exists. const rows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rows).toHaveLength(0); } finally { factorySpy.mockRestore(); } await environmentService(db).update(environment.id, { driver: "local", config: {} }); }); it("records the orphan directly in the pending_cleanup state with one atomic insert", async () => { const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Atomic Pending Cleanup Insert", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, }); const lease = await environmentService(db).insertPendingCleanupLease({ companyId, environmentId: environment.id, executionWorkspaceId: null, issueId: null, heartbeatRunId: runId, provider: "fake", providerLeaseId: "sandbox://fake/atomic-orphan", metadata: { provider: "fake", reuseLease: false }, failureReason: "acquire_rejected_teardown_failed", }); // The returned lease is already in the terminal recovery state. It never // passes through the `active` state, so a crash cannot strand the orphan. expect(lease.status).toBe("pending_cleanup"); expect(lease.cleanupStatus).toBe("failed"); expect(lease.leasePolicy).toBe("ephemeral"); expect(lease.failureReason).toBe("acquire_rejected_teardown_failed"); expect(lease.providerLeaseId).toBe("sandbox://fake/atomic-orphan"); expect(lease.releasedAt).not.toBeNull(); // One insert created exactly one row, and the sweep reads that row directly. const rows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rows).toHaveLength(1); expect(rows[0]?.status).toBe("pending_cleanup"); expect(rows[0]?.cleanupStatus).toBe("failed"); }); it("tears the recorded built-in provider lease down for a pending-cleanup orphan", async () => { const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Cleanup Retry Built-in", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, }); const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co Retry Built-in", issuePrefix: "OTE", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await db.insert(builtInManagedResources).values({ companyId: otherCompanyId, bundleKey: "managed-environment", resourceKind: "environment", resourceKey: "managed-sandbox", resourceId: environment.id, stockVersion: "1", stockHash: "hash", }); // The first teardown fails, so the acquire records a pending-cleanup orphan. // The sweep then retries the teardown, which succeeds this time. const destroySpy = vi .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") .mockRejectedValueOnce(new Error("teardown failed")) .mockResolvedValue(undefined); try { await expect( runtime.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }), ).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } }); const orphan = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)) .then((r) => r[0]!); expect(orphan.status).toBe("pending_cleanup"); const providerLeaseId = orphan.providerLeaseId; // The teardown accepts a null environment and reads the recorded provider // lease, so a delete or a provider change never strands it. const lease = await environmentService(db).getLeaseById(orphan.id); await runtime.retryPendingSandboxTeardown({ environment: null, lease: lease! }); // The retry called the provider teardown a second time with the orphan // provider lease id. expect(destroySpy).toHaveBeenCalledTimes(2); expect(destroySpy).toHaveBeenLastCalledWith(expect.objectContaining({ providerLeaseId })); } finally { destroySpy.mockRestore(); } }); it("throws and keeps the pending-cleanup orphan when the teardown retry fails", async () => { const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Cleanup Retry Scope", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, }); const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co Retry Scope", issuePrefix: "OTF", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await db.insert(builtInManagedResources).values({ companyId: otherCompanyId, bundleKey: "managed-environment", resourceKind: "environment", resourceKey: "managed-sandbox", resourceId: environment.id, stockVersion: "1", stockHash: "hash", }); const destroySpy = vi .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") .mockRejectedValue(new Error("teardown failed")); try { await expect( runtime.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }), ).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } }); const orphan = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)) .then((r) => r[0]!); expect(orphan.status).toBe("pending_cleanup"); // The teardown retries, but the provider destroy still fails, so the // teardown throws. The caller keeps the orphan pending for a later sweep. const lease = await environmentService(db).getLeaseById(orphan.id); await expect( runtime.retryPendingSandboxTeardown({ environment: null, lease: lease! }), ).rejects.toThrow(); const still = await db .select() .from(environmentLeases) .where(eq(environmentLeases.id, orphan.id)) .then((r) => r[0]!); expect(still.status).toBe("pending_cleanup"); expect(still.cleanupStatus).toBe("failed"); } finally { destroySpy.mockRestore(); } }); it("tears the recorded provider down after the environment provider changes", async () => { // Ordering: an acquire provisions a sandbox with provider A, the lease // insert rejects a foreign-company binding, and the compensating teardown // fails, so the acquire records an orphan for provider A. A provider change // then re-points the environment before the sweep runs. The sweep must tear // the orphan down from the recorded provider metadata, not the current // environment provider, so the change cannot strand the teardown. const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Provider Change Vs Orphan", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, }); const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co Provider Change", issuePrefix: "OTP", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await db.insert(builtInManagedResources).values({ companyId: otherCompanyId, bundleKey: "managed-environment", resourceKind: "environment", resourceKey: "managed-sandbox", resourceId: environment.id, stockVersion: "1", stockHash: "hash", }); const destroySpy = vi .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") .mockRejectedValueOnce(new Error("teardown failed")) .mockResolvedValue(undefined); try { await expect( runtime.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }), ).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } }); const orphan = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)) .then((r) => r[0]!); expect(orphan.status).toBe("pending_cleanup"); expect(orphan.provider).toBe("fake"); const providerLeaseId = orphan.providerLeaseId; // The provider-target mutation re-points the environment to a different // driver. The orphan teardown must not depend on this current config. await environmentService(db).update(environment.id, { driver: "local", config: {} }); // The teardown tears the recorded provider lease down, so it never reads // the re-pointed environment provider. const lease = await environmentService(db).getLeaseById(orphan.id); await runtime.retryPendingSandboxTeardown({ environment: null, lease: lease! }); expect(destroySpy).toHaveBeenLastCalledWith(expect.objectContaining({ providerLeaseId })); } finally { destroySpy.mockRestore(); } }); it("reports the plugin cleanup worker not ready while its worker is down, then ready after it recovers", async () => { // A pending_cleanup orphan targets a plugin-backed provider. The sweep must // not consume a finite retry attempt while the plugin worker is briefly down. // So the readiness probe reports "not ready" for a down worker and "ready" // after the worker recovers. A built-in provider has no worker, so it is // always ready. const pluginId = randomUUID(); const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Cleanup Worker Readiness", config: { provider: "fake-plugin-ready", image: "fake:test", reuseLease: false }, }); await db.insert(plugins).values({ id: pluginId, pluginKey: "paperclip.fake-plugin-ready-sandbox-provider", packageName: "@paperclipai/plugin-fake-ready-sandbox", version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: { id: "paperclip.fake-plugin-ready-sandbox-provider", apiVersion: 1, version: "1.0.0", displayName: "Fake Plugin Ready Sandbox Provider", description: "Test fake plugin provider readiness", author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "fake-plugin-ready", kind: "sandbox_provider", displayName: "Fake Plugin Ready", configSchema: { type: "object" }, }, ], }, status: "ready", installOrder: 1, updatedAt: new Date(), } as any); // The worker is down at first, then recovers on the second probe. let workerRunning = false; const workerManager = { isRunning: vi.fn((id: string) => id === pluginId && workerRunning), call: vi.fn(async () => { throw new Error("call must not run during a readiness probe"); }), } as unknown as PluginWorkerManager; const orphan = await environmentService(db).insertPendingCleanupLease({ companyId, environmentId: environment.id, executionWorkspaceId: null, issueId: null, heartbeatRunId: runId, provider: "fake-plugin-ready", providerLeaseId: "plugin-lease-readiness", metadata: { provider: "fake-plugin-ready", driver: "sandbox", reuseLease: false }, failureReason: "acquire_rejected_teardown_failed", }); const lease = (await environmentService(db).getLeaseById(orphan.id))!; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); // A down plugin worker is the transient restart window, so the probe reports // not ready. The sweep skips the lease without a claim, so no attempt burns. await expect( runtimeWithPlugin.isPendingCleanupWorkerReady({ environment, lease }), ).resolves.toBe(false); // The probe never calls the worker; it only checks the live worker state. expect((workerManager.call as ReturnType)).not.toHaveBeenCalled(); // The worker recovers, so the probe now reports ready and the sweep proceeds. workerRunning = true; await expect( runtimeWithPlugin.isPendingCleanupWorkerReady({ environment, lease }), ).resolves.toBe(true); // A built-in provider has no plugin worker, so its orphan is always ready. const builtinOrphan = await environmentService(db).insertPendingCleanupLease({ companyId, environmentId: environment.id, executionWorkspaceId: null, issueId: null, heartbeatRunId: runId, provider: "fake", providerLeaseId: "builtin-lease-readiness", metadata: { provider: "fake", driver: "sandbox", reuseLease: false }, failureReason: "acquire_rejected_teardown_failed", }); const builtinLease = (await environmentService(db).getLeaseById(builtinOrphan.id))!; await expect( runtimeWithPlugin.isPendingCleanupWorkerReady({ environment, lease: builtinLease }), ).resolves.toBe(true); }); it("reports the plugin cleanup provider not ready while the plugin is not ready, then ready after it recovers", async () => { // A pending_cleanup orphan targets a plugin-backed provider. A plugin reload // or a plugin reinstall moves the plugin through a "not ready" status. The // sweep must not consume a finite retry attempt in that transient window. So // the readiness probe reports "not ready" while the plugin status is not // "ready", even when its worker runs, and "ready" after the plugin recovers. const pluginId = randomUUID(); const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Cleanup Plugin Readiness", config: { provider: "fake-plugin-reload", image: "fake:test", reuseLease: false }, }); await db.insert(plugins).values({ id: pluginId, pluginKey: "paperclip.fake-plugin-reload-sandbox-provider", packageName: "@paperclipai/plugin-fake-reload-sandbox", version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: { id: "paperclip.fake-plugin-reload-sandbox-provider", apiVersion: 1, version: "1.0.0", displayName: "Fake Plugin Reload Sandbox Provider", description: "Test fake plugin provider reload readiness", author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "fake-plugin-reload", kind: "sandbox_provider", displayName: "Fake Plugin Reload", configSchema: { type: "object" }, }, ], }, // The plugin starts in a not-ready status, as during a reload. status: "installing", installOrder: 1, updatedAt: new Date(), } as any); // The worker runs the whole time, so the probe result depends on the plugin // status alone, not on a down worker. const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async () => { throw new Error("call must not run during a readiness probe"); }), } as unknown as PluginWorkerManager; const orphan = await environmentService(db).insertPendingCleanupLease({ companyId, environmentId: environment.id, executionWorkspaceId: null, issueId: null, heartbeatRunId: runId, provider: "fake-plugin-reload", providerLeaseId: "plugin-lease-reload", metadata: { provider: "fake-plugin-reload", driver: "sandbox", reuseLease: false }, failureReason: "acquire_rejected_teardown_failed", }); const lease = (await environmentService(db).getLeaseById(orphan.id))!; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); // A not-ready plugin is the transient reload window, so the probe reports // not ready. The sweep skips the lease without a claim, so no attempt burns. await expect( runtimeWithPlugin.isPendingCleanupWorkerReady({ environment, lease }), ).resolves.toBe(false); expect((workerManager.call as ReturnType)).not.toHaveBeenCalled(); // The plugin becomes ready, so the probe now reports ready and the sweep // proceeds. await db.update(plugins).set({ status: "ready" }).where(eq(plugins.id, pluginId)); await expect( runtimeWithPlugin.isPendingCleanupWorkerReady({ environment, lease }), ).resolves.toBe(true); }); it("reports the plugin cleanup provider not ready while the plugin is missing, then ready after it returns", async () => { // A pending_cleanup orphan targets a plugin-backed provider whose plugin row // is gone. A plugin reinstall removes and re-adds the plugin row, so the row // is missing for a short window. The sweep must not consume a finite retry // attempt in that window. So the readiness probe reports "not ready" while no // plugin row resolves the provider, and "ready" after the plugin returns. const pluginId = randomUUID(); const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Cleanup Plugin Missing", config: { provider: "fake-plugin-missing", image: "fake:test", reuseLease: false }, }); // The worker manager reports the plugin worker as running, so the probe // result depends on the missing plugin row alone, not on a down worker. const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async () => { throw new Error("call must not run during a readiness probe"); }), } as unknown as PluginWorkerManager; const orphan = await environmentService(db).insertPendingCleanupLease({ companyId, environmentId: environment.id, executionWorkspaceId: null, issueId: null, heartbeatRunId: runId, provider: "fake-plugin-missing", providerLeaseId: "plugin-lease-missing", metadata: { provider: "fake-plugin-missing", driver: "sandbox", reuseLease: false }, failureReason: "acquire_rejected_teardown_failed", }); const lease = (await environmentService(db).getLeaseById(orphan.id))!; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); // No plugin row resolves the provider, so the probe reports not ready. The // sweep skips the lease without a claim, so no attempt burns while the plugin // reinstall is in flight. await expect( runtimeWithPlugin.isPendingCleanupWorkerReady({ environment, lease }), ).resolves.toBe(false); expect((workerManager.call as ReturnType)).not.toHaveBeenCalled(); // The plugin reinstall completes, so a ready plugin row now resolves the // provider and the probe reports ready. await db.insert(plugins).values({ id: pluginId, pluginKey: "paperclip.fake-plugin-missing-sandbox-provider", packageName: "@paperclipai/plugin-fake-missing-sandbox", version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: { id: "paperclip.fake-plugin-missing-sandbox-provider", apiVersion: 1, version: "1.0.0", displayName: "Fake Plugin Missing Sandbox Provider", description: "Test fake plugin provider missing readiness", author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "fake-plugin-missing", kind: "sandbox_provider", displayName: "Fake Plugin Missing", configSchema: { type: "object" }, }, ], }, status: "ready", installOrder: 1, updatedAt: new Date(), } as any); await expect( runtimeWithPlugin.isPendingCleanupWorkerReady({ environment, lease }), ).resolves.toBe(true); }); it("records an orphan with a null reference when a delete already removed the environment", async () => { // Ordering: a delete removes the environment before the acquire records the // orphan. The environment foreign key must not fail the insert. The record // persists with a null reference and its immutable provider metadata, so a // later sweep still tears the orphan down. const { companyId, environment } = await seedEnvironment({ driver: "sandbox", name: "Delete Before Orphan Record", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, }); // Remove the environment row first, so the record happens after the delete. await environmentService(db).remove(environment.id); const orphan = await environmentService(db).insertPendingCleanupLease({ companyId, environmentId: environment.id, provider: "fake", providerLeaseId: "sandbox://fake/delete-before-record", metadata: { provider: "fake", image: "ubuntu:24.04", driver: "sandbox" }, failureReason: "acquire_rejected_teardown_failed", }); // The insert kept no environment reference, so the delete did not fail it. expect(orphan.environmentId).toBeNull(); expect(orphan.status).toBe("pending_cleanup"); const destroySpy = vi .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") .mockResolvedValue(undefined); try { // The recorded metadata drives the teardown, so a null environment tears // the orphan down. const lease = await environmentService(db).getLeaseById(orphan.id); await runtime.retryPendingSandboxTeardown({ environment: null, lease: lease! }); expect(destroySpy).toHaveBeenCalledWith( expect.objectContaining({ providerLeaseId: "sandbox://fake/delete-before-record" }), ); } finally { destroySpy.mockRestore(); } }); it("keeps the orphan and tears it down after the environment is deleted", async () => { // Ordering: an orphan is recorded, then the environment is deleted. The // foreign key sets the reference null instead of cascading the row away, so // the orphan survives. The sweep tears it down from the recorded metadata. const { companyId, environment } = await seedEnvironment({ driver: "sandbox", name: "Delete After Orphan Record", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, }); const orphan = await environmentService(db).insertPendingCleanupLease({ companyId, environmentId: environment.id, provider: "fake", providerLeaseId: "sandbox://fake/delete-after-record", metadata: { provider: "fake", image: "ubuntu:24.04", driver: "sandbox" }, failureReason: "acquire_rejected_teardown_failed", }); expect(orphan.environmentId).toBe(environment.id); // Delete the environment row directly. The foreign key sets the lease // reference null, so the orphan survives instead of being cascaded away. await environmentService(db).remove(environment.id); const survived = await db .select() .from(environmentLeases) .where(eq(environmentLeases.id, orphan.id)) .then((r) => r[0]!); expect(survived.environmentId).toBeNull(); expect(survived.status).toBe("pending_cleanup"); const destroySpy = vi .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") .mockResolvedValue(undefined); try { // The recorded metadata drives the teardown, so a null environment tears // the orphan down after the delete set the reference null. const lease = await environmentService(db).getLeaseById(orphan.id); await runtime.retryPendingSandboxTeardown({ environment: null, lease: lease! }); expect(destroySpy).toHaveBeenCalledWith( expect.objectContaining({ providerLeaseId: "sandbox://fake/delete-after-record" }), ); } finally { destroySpy.mockRestore(); } }); // A plugin provider whose config declares an `apiKey` secret-ref field. The // orphan teardown must resolve that recorded ref even when the environment // binding is gone, so these tests register a provider with a real secret-ref // field, unlike the plain `fake-plugin` tests above. const SECRET_REF_PLUGIN_KEY = "paperclip.secret-plugin-sandbox-provider"; const SECRET_REF_PROVIDER = "secret-plugin"; async function registerSecretRefPluginProvider(): Promise { const pluginId = randomUUID(); await db.insert(plugins).values({ id: pluginId, pluginKey: SECRET_REF_PLUGIN_KEY, packageName: "@paperclipai/plugin-secret-sandbox", version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: { id: SECRET_REF_PLUGIN_KEY, apiVersion: 1, version: "1.0.0", displayName: "Secret Plugin Sandbox Provider", description: "Test plugin provider with a secret-ref config field", author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: SECRET_REF_PROVIDER, kind: "sandbox_provider", displayName: "Secret Plugin", configSchema: { type: "object", properties: { apiKey: { type: "string", format: "secret-ref" }, }, }, }, ], }, status: "ready", installOrder: 1, updatedAt: new Date(), } as any); return pluginId; } // A worker manager mock that records the config of each destroy call, so a // test can assert the resolved API key reached the provider teardown. function createDestroyRecordingWorkerManager(pluginId: string): { workerManager: PluginWorkerManager; destroyConfigs: Array | undefined>; } { const destroyConfigs: Array | undefined> = []; const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string, args: any) => { if (method === "environmentDestroyLease") { destroyConfigs.push(args?.config as Record | undefined); return {}; } throw new Error(`Unexpected plugin method: ${method}`); }), } as unknown as PluginWorkerManager; return { workerManager, destroyConfigs }; } async function createApiKeySecret(companyId: string, value: string): Promise { const secret = await secretService(db).create(companyId, { name: `sandbox-cleanup-api-key-${randomUUID()}`, provider: "local_encrypted", value, }); return secret.id; } it("tears down a secret-backed plugin orphan when a delete already removed the environment", async () => { // Ordering: a delete removed the environment (and its secret binding) before // the acquire recorded the orphan. The recorded config keeps the API-key // secret ref, so the teardown must resolve it without the environment // binding and send the resolved value to the provider destroy call. const pluginId = await registerSecretRefPluginProvider(); const { companyId, environment } = await seedEnvironment({ driver: "sandbox", name: "Secret Plugin Delete Before Record", config: { provider: SECRET_REF_PROVIDER, reuseLease: false }, }); const secretId = await createApiKeySecret(companyId, "old-provider-api-key"); await secretService(db).createBinding({ companyId, secretId, targetType: "environment", targetId: environment.id, configPath: "apiKey", }); // Remove the environment first, so the record happens after the delete. await environmentService(db).remove(environment.id); const orphan = await environmentService(db).insertPendingCleanupLease({ companyId, environmentId: environment.id, provider: SECRET_REF_PROVIDER, providerLeaseId: "secret-plugin-lease-delete-before", metadata: { provider: SECRET_REF_PROVIDER, driver: "sandbox", sandboxProviderPlugin: true, pluginId, pluginKey: SECRET_REF_PLUGIN_KEY, apiKey: secretId, reuseLease: false, }, failureReason: "acquire_rejected_teardown_failed", }); expect(orphan.environmentId).toBeNull(); expect(orphan.status).toBe("pending_cleanup"); const { workerManager, destroyConfigs } = createDestroyRecordingWorkerManager(pluginId); const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const lease = await environmentService(db).getLeaseById(orphan.id); await runtimeWithPlugin.retryPendingSandboxTeardown({ environment: null, lease: lease! }); expect(destroyConfigs).toHaveLength(1); // The recorded secret ref resolved to the old credential, and the resolved // value reached the provider teardown instead of the secret ref. expect(destroyConfigs[0]).toMatchObject({ apiKey: "old-provider-api-key" }); expect(destroyConfigs[0]?.apiKey).not.toBe(secretId); }); it("tears down a secret-backed plugin orphan after the environment is deleted", async () => { // Ordering: the orphan is recorded, then a delete removes the environment. // The foreign key sets the lease reference null, so the orphan survives. The // teardown resolves the recorded API-key ref without the environment. const pluginId = await registerSecretRefPluginProvider(); const { companyId, environment } = await seedEnvironment({ driver: "sandbox", name: "Secret Plugin Delete After Record", config: { provider: SECRET_REF_PROVIDER, reuseLease: false }, }); const secretId = await createApiKeySecret(companyId, "old-provider-api-key"); await secretService(db).createBinding({ companyId, secretId, targetType: "environment", targetId: environment.id, configPath: "apiKey", }); const orphan = await environmentService(db).insertPendingCleanupLease({ companyId, environmentId: environment.id, provider: SECRET_REF_PROVIDER, providerLeaseId: "secret-plugin-lease-delete-after", metadata: { provider: SECRET_REF_PROVIDER, driver: "sandbox", sandboxProviderPlugin: true, pluginId, pluginKey: SECRET_REF_PLUGIN_KEY, apiKey: secretId, reuseLease: false, }, failureReason: "acquire_rejected_teardown_failed", }); expect(orphan.environmentId).toBe(environment.id); // Delete the environment. The foreign key sets the lease reference null, so // the orphan survives and the current binding is gone. await environmentService(db).remove(environment.id); const survived = await db .select() .from(environmentLeases) .where(eq(environmentLeases.id, orphan.id)) .then((r) => r[0]!); expect(survived.environmentId).toBeNull(); const { workerManager, destroyConfigs } = createDestroyRecordingWorkerManager(pluginId); const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const lease = await environmentService(db).getLeaseById(orphan.id); await runtimeWithPlugin.retryPendingSandboxTeardown({ environment: null, lease: lease! }); expect(destroyConfigs).toHaveLength(1); expect(destroyConfigs[0]).toMatchObject({ apiKey: "old-provider-api-key" }); }); it("tears down a secret-backed plugin orphan with the recorded credential after the provider is reconfigured", async () => { // Ordering: the orphan is recorded for the old credential, then the // environment is reconfigured to a new credential. The environment binding // now points to a different secret at the same path. The teardown must // resolve the OLD recorded ref, not the new binding, so it destroys the // sandbox the old credential provisioned. const pluginId = await registerSecretRefPluginProvider(); const { companyId, environment } = await seedEnvironment({ driver: "sandbox", name: "Secret Plugin Reconfigured", config: { provider: SECRET_REF_PROVIDER, reuseLease: false }, }); const oldSecretId = await createApiKeySecret(companyId, "old-provider-api-key"); const orphan = await environmentService(db).insertPendingCleanupLease({ companyId, environmentId: environment.id, provider: SECRET_REF_PROVIDER, providerLeaseId: "secret-plugin-lease-reconfigured", metadata: { provider: SECRET_REF_PROVIDER, driver: "sandbox", sandboxProviderPlugin: true, pluginId, pluginKey: SECRET_REF_PLUGIN_KEY, apiKey: oldSecretId, reuseLease: false, }, failureReason: "acquire_rejected_teardown_failed", }); expect(orphan.environmentId).toBe(environment.id); // Reconfigure the environment: a new secret replaces the binding at the same // path. The old recorded ref is no longer bound to the environment. const newSecretId = await createApiKeySecret(companyId, "new-provider-api-key"); await secretService(db).createBinding({ companyId, secretId: newSecretId, targetType: "environment", targetId: environment.id, configPath: "apiKey", }); await environmentService(db).update(environment.id, { driver: "sandbox", config: { provider: SECRET_REF_PROVIDER, apiKey: newSecretId, reuseLease: false }, }); const { workerManager, destroyConfigs } = createDestroyRecordingWorkerManager(pluginId); const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const lease = await environmentService(db).getLeaseById(orphan.id); await runtimeWithPlugin.retryPendingSandboxTeardown({ environment: null, lease: lease! }); expect(destroyConfigs).toHaveLength(1); // The teardown resolved the OLD recorded credential, not the new binding. expect(destroyConfigs[0]).toMatchObject({ apiKey: "old-provider-api-key" }); expect(destroyConfigs[0]?.apiKey).not.toBe("new-provider-api-key"); await environmentService(db).update(environment.id, { driver: "local", config: {} }); }); it("tears the recorded plugin provider lease down for a pending-cleanup orphan", async () => { const pluginId = randomUUID(); const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); const pluginConfig = { provider: "fake-plugin", image: "fake:test", reuseLease: false }; const environment = { ...baseEnvironment, name: "Cleanup Retry Plugin", driver: "sandbox", config: pluginConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: pluginConfig, }); await db.insert(plugins).values({ id: pluginId, pluginKey: "paperclip.fake-plugin-sandbox-provider", packageName: "@paperclipai/plugin-fake-sandbox", version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: { id: "paperclip.fake-plugin-sandbox-provider", apiVersion: 1, version: "1.0.0", displayName: "Fake Plugin Sandbox Provider", description: "Test fake plugin provider", author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", configSchema: { type: "object" }, }, ], }, status: "ready", installOrder: 1, updatedAt: new Date(), } as any); const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co Retry Plugin", issuePrefix: "OTG", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await db.insert(builtInManagedResources).values({ companyId: otherCompanyId, bundleKey: "managed-environment", resourceKind: "environment", resourceKey: "managed-sandbox", resourceId: environment.id, stockVersion: "1", stockHash: "hash", }); // The first destroy fails, so the acquire records a pending-cleanup orphan. // The sweep retries the destroy, which succeeds this time. let destroyAttempts = 0; const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), getWorker: vi.fn(() => ({ supportedMethods: ["environmentDestroyLease"] })), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "plugin-lease-3", metadata: { provider: "fake-plugin", image: "fake:test", reuseLease: false }, }; } if (method === "environmentDestroyLease") { destroyAttempts += 1; if (destroyAttempts === 1) { throw new Error("destroy failed"); } return {}; } throw new Error(`Unexpected plugin method: ${method}`); }), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); await expect( runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }), ).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } }); const orphan = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)) .then((r) => r[0]!); expect(orphan.status).toBe("pending_cleanup"); expect(orphan.providerLeaseId).toBe("plugin-lease-3"); const lease = await environmentService(db).getLeaseById(orphan.id); await runtimeWithPlugin.retryPendingSandboxTeardown({ environment: null, lease: lease! }); expect(destroyAttempts).toBe(2); const destroyCall = (workerManager.call as unknown as ReturnType).mock.calls .filter((callArgs) => callArgs[1] === "environmentDestroyLease") .at(-1); expect(destroyCall?.[2]).toMatchObject({ providerLeaseId: "plugin-lease-3" }); await environmentService(db).update(environment.id, { driver: "local", config: {} }); }); it("uses plugin-backed sandbox config for execute and release", async () => { const pluginId = randomUUID(); const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); const fakePluginConfig = { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: false, }; const environment = { ...baseEnvironment, name: "Fake Plugin Sandbox", driver: "sandbox", config: fakePluginConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: fakePluginConfig, }); await db.insert(plugins).values({ id: pluginId, pluginKey: "paperclip.fake-plugin-sandbox-provider", packageName: "@paperclipai/plugin-fake-sandbox", version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: { id: "paperclip.fake-plugin-sandbox-provider", apiVersion: 1, version: "1.0.0", displayName: "Fake Plugin Sandbox Provider", description: "Test fake plugin provider", author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", configSchema: { type: "object" }, }, ], }, status: "ready", installOrder: 1, updatedAt: new Date(), } as any); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string, params: any) => { expect(params.config).toEqual(expect.objectContaining({ image: "fake:test", timeoutMs: 1234, reuseLease: false, })); expect(params.config).not.toHaveProperty("provider"); if (method === "environmentAcquireLease") { return { providerLeaseId: "sandbox-1", metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: false, remoteCwd: "/workspace", }, }; } if (method === "environmentExecute") { return { exitCode: 0, signal: null, timedOut: false, stdout: "ok\n", stderr: "", }; } if (method === "environmentReleaseLease") { expect(params.config).toEqual({ image: "fake:test", timeoutMs: 1234, reuseLease: false, }); expect(params.config).not.toHaveProperty("driver"); expect(params.config).not.toHaveProperty("executionWorkspaceMode"); expect(params.config).not.toHaveProperty("pluginId"); expect(params.config).not.toHaveProperty("pluginKey"); expect(params.config).not.toHaveProperty("providerMetadata"); expect(params.config).not.toHaveProperty("provider"); expect(params.config).not.toHaveProperty("sandboxProviderPlugin"); return undefined; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); const executed = await runtimeWithPlugin.execute({ environment, lease: acquired.lease, command: "printf", args: ["ok"], cwd: "/workspace", env: {}, timeoutMs: 1000, }); await environmentService(db).update(environment.id, { driver: "local", config: {}, }); const released = await runtimeWithPlugin.releaseRunLeases(runId); expect(executed.stdout).toBe("ok\n"); expect(released).toHaveLength(1); expect(released[0]?.lease.status).toBe("released"); // The execute call carries the optional log sink as the fifth argument; it // is undefined when the caller passes no sink. expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentExecute", expect.anything(), 31000, undefined); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentReleaseLease", expect.anything(), 31234); }); // Build the fake plugin fixture for the run-parent release tests below. A // plugin sandbox provider can open a persistent session on the first command // and delete it on lease release; the delete emits a provider // `session.close` span. The host mints that span's parent from the active // step context at the release RPC. So the release must run under the run // parent, or the span loses its traceparent and the backend drops it. async function seedFakePluginSandbox() { const pluginId = randomUUID(); const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); const fakePluginConfig = { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: false, }; const environment = { ...baseEnvironment, name: "Fake Plugin Sandbox", driver: "sandbox", config: fakePluginConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: fakePluginConfig, }); await db.insert(plugins).values({ id: pluginId, pluginKey: "paperclip.fake-plugin-sandbox-provider", packageName: "@paperclipai/plugin-fake-sandbox", version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: { id: "paperclip.fake-plugin-sandbox-provider", apiVersion: 1, version: "1.0.0", displayName: "Fake Plugin Sandbox Provider", description: "Test fake plugin provider", author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", configSchema: { type: "object" }, }, ], }, status: "ready", installOrder: 1, updatedAt: new Date(), } as any); return { companyId, environment, runId, pluginId }; } it("runs a plugin-backed lease release under the run-time exec parent, so the teardown span keeps a valid traceparent", async () => { const { companyId, environment, runId, pluginId } = await seedFakePluginSandbox(); // The active step context observed at the moment the host issues the release // RPC. The real worker manager mints the provider-span traceparent from // exactly this value, so a non-null context with a valid parent proves the // teardown span keeps a host-minted parent. let releaseStepContext: ReturnType = null; const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "sandbox-1", metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: false, remoteCwd: "/workspace", }, }; } if (method === "environmentExecute") { return { exitCode: 0, signal: null, timedOut: false, stdout: "ok\n", stderr: "" }; } if (method === "environmentReleaseLease") { releaseStepContext = getActiveStepContext(); return undefined; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); // Build a real host run-parent context and run the first command under it, // exactly as the run drives an exec. The driver records this context for the // lease and replays it around the later release RPC. const runParent: StartupSpanContext = trace.setSpanContext(ROOT_CONTEXT, { traceId: "0af7651916cd43dd8448eb211c80319c", spanId: "b7ad6b7169203331", traceFlags: 1, isRemote: false, }); await runWithRuntimeParent(runParent, () => runtimeWithPlugin.execute({ environment, lease: acquired.lease, command: "printf", args: ["ok"], cwd: "/workspace", env: {}, timeoutMs: 1000, }), ); await environmentService(db).update(environment.id, { driver: "local", config: {} }); const released = await runtimeWithPlugin.releaseRunLeases(runId); expect(released).toHaveLength(1); // The release RPC ran under the recorded run parent, so the host mints a // valid W3C traceparent from it. A dropped span would show a null context. expect(releaseStepContext).not.toBeNull(); expect(releaseStepContext?.parentContext).toBe(runParent); expect(traceparentFromContextToken(releaseStepContext?.parentContext)).toBe( "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01", ); }); it("releases a plugin-backed lease with no active step context when the run drove no exec", async () => { const { companyId, environment, runId, pluginId } = await seedFakePluginSandbox(); let releaseStepContext: ReturnType = null; let releaseCalled = false; const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "sandbox-1", metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: false, remoteCwd: "/workspace", }, }; } if (method === "environmentReleaseLease") { releaseCalled = true; releaseStepContext = getActiveStepContext(); return undefined; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); await environmentService(db).update(environment.id, { driver: "local", config: {} }); const released = await runtimeWithPlugin.releaseRunLeases(runId); expect(released).toHaveLength(1); // No command ran, so no session opened and no teardown span is emitted. The // release must not invent a parent from an unrelated ambient context; it // runs unwrapped, exactly as before the fix. expect(releaseCalled).toBe(true); expect(releaseStepContext).toBeNull(); }); it("forwards the bypassSession flag to the plugin execute RPC so a pre-run command skips the session", async () => { const { companyId, environment, runId, pluginId } = await seedFakePluginSandbox(); // Capture the params of each environmentExecute RPC, so the test can assert // the host forwards `bypassSession` to the provider unchanged. const executeParams: Array> = []; const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string, params: Record) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "sandbox-1", metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: false, remoteCwd: "/workspace", }, }; } if (method === "environmentExecute") { executeParams.push(params); return { exitCode: 0, signal: null, timedOut: false, stdout: "ok\n", stderr: "" }; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); // A pre-run command (the provision command) sets bypassSession explicitly. await runtimeWithPlugin.execute({ environment, lease: acquired.lease, command: "bash", args: ["-lc", "true"], cwd: "/workspace", env: {}, timeoutMs: 1000, bypassSession: true, }); // An in-run command runs under the run-parent context, so the host does not // bypass the session and the provider opens/uses the session. const runParent: StartupSpanContext = trace.setSpanContext(ROOT_CONTEXT, { traceId: "0af7651916cd43dd8448eb211c80319c", spanId: "b7ad6b7169203331", traceFlags: 1, isRemote: false, }); await runWithRuntimeParent(runParent, () => runtimeWithPlugin.execute({ environment, lease: acquired.lease, command: "printf", args: ["ok"], cwd: "/workspace", env: {}, timeoutMs: 1000, }), ); expect(executeParams).toHaveLength(2); expect(executeParams[0]?.bypassSession).toBe(true); // An in-run command carries a run parent, so it does not bypass the session. expect(executeParams[1]?.bypassSession).toBe(false); }); it("bypasses the session for a context-less command so the setup span keeps a run parent", async () => { const { companyId, environment, runId, pluginId } = await seedFakePluginSandbox(); // Capture the params of each environmentExecute RPC, so the test can assert // the host derives `bypassSession` from the active run-parent context. const executeParams: Array> = []; const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string, params: Record) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "sandbox-1", metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: false, remoteCwd: "/workspace", }, }; } if (method === "environmentExecute") { executeParams.push(params); return { exitCode: 0, signal: null, timedOut: false, stdout: "ok\n", stderr: "" }; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); // A command with no active run-parent context (a pre-run install/probe or // the agent process launch) runs before the run trace is active. The host // mints no plugin RPC traceparent for it, so a session opened here would drop // its setup span. The host bypasses the session for such a command. await runtimeWithPlugin.execute({ environment, lease: acquired.lease, command: "sh", args: ["-c", "command -v claude"], cwd: "/workspace", env: {}, timeoutMs: 1000, }); expect(executeParams).toHaveLength(1); expect(executeParams[0]?.bypassSession).toBe(true); }); it("builds the workspace-realization record with referenced sources for a plugin-backed sandbox realize", async () => { // A provider plugin realize handler returns only its realized cwd and provider metadata; it does // not build the workspace-realization record. The server must build that record from the run // request, so the referenced (mentioned) project sources reach the adapter through // `realization.additional`. Without the record the sandbox agent never receives the mentioned // projects. This test drives the plugin-backed sandbox realize path and asserts the referenced // source survives into the returned record. const pluginId = randomUUID(); const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); const fakePluginConfig = { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: false, }; const environment = { ...baseEnvironment, name: "Fake Plugin Sandbox Realize", driver: "sandbox", config: fakePluginConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: fakePluginConfig, }); await db.insert(plugins).values({ id: pluginId, pluginKey: "paperclip.fake-plugin-sandbox-provider", packageName: "@paperclipai/plugin-fake-sandbox", version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: { id: "paperclip.fake-plugin-sandbox-provider", apiVersion: 1, version: "1.0.0", displayName: "Fake Plugin Sandbox Provider", description: "Test fake plugin provider", author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", configSchema: { type: "object" }, }, ], }, status: "ready", installOrder: 1, updatedAt: new Date(), } as any); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "sandbox-realize-1", metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: false, remoteCwd: "/workspace", }, }; } if (method === "environmentRealizeWorkspace") { // Mimic a real provider (for example Daytona): return only the realized cwd and provider // metadata, never a `workspaceRealization` record. return { cwd: "/workspace/project", metadata: { provider: "fake-plugin", remoteCwd: "/workspace/project", }, }; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); const workspaceRealizationRequest = { version: 1, adapterType: "codex_local", companyId, environmentId: environment.id, executionWorkspaceId: null, issueId: null, heartbeatRunId: runId, requestedMode: "ephemeral", source: { kind: "project_primary", localPath: "/tmp/anchor", projectId: "anchor-project", projectWorkspaceId: "anchor-workspace", repoUrl: null, repoRef: null, strategy: "project_primary", branchName: null, worktreePath: null, }, additionalSources: [ { localPath: "/tmp/referenced-project", projectId: "referenced-project-1", projectWorkspaceId: "referenced-workspace-1", repoUrl: null, repoRef: null, }, ], }; const realized = await runtimeWithPlugin.realizeWorkspace({ environment, lease: acquired.lease, workspace: { localPath: "/tmp/anchor", mode: "ephemeral", metadata: { workspaceRealizationRequest }, }, }); // The provider realized cwd and provider metadata survive. expect(realized.cwd).toBe("/workspace/project"); expect(realized.metadata?.provider).toBe("fake-plugin"); // The server-built record carries the referenced source through `additional`, so the adapter can // stage the mentioned project into the sandbox. const realization = realized.metadata?.workspaceRealization as Record | undefined; expect(realization).toBeDefined(); expect(realization?.additional).toEqual([ expect.objectContaining({ path: "/tmp/referenced-project", projectId: "referenced-project-1", }), ]); }); it("builds the workspace-realization record with referenced sources for a built-in sandbox realize", async () => { // The sandbox driver `realizeWorkspace` has two exits: a plugin-backed provider and a // built-in provider. Both must build the same workspace-realization record, so the // referenced (mentioned) project sources reach the adapter through `realization.additional`. // The test above covers the plugin exit. This test covers the built-in exit (no provider // plugin), so a regression on either exit fails a test. const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Fake Sandbox Realize", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: true, }, }); const acquired = await runtime.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); const workspaceRealizationRequest = { version: 1, adapterType: "codex_local", companyId, environmentId: environment.id, executionWorkspaceId: null, issueId: null, heartbeatRunId: runId, requestedMode: "ephemeral", source: { kind: "project_primary", localPath: "/tmp/anchor", projectId: "anchor-project", projectWorkspaceId: "anchor-workspace", repoUrl: null, repoRef: null, strategy: "project_primary", branchName: null, worktreePath: null, }, additionalSources: [ { localPath: "/tmp/referenced-project", projectId: "referenced-project-1", projectWorkspaceId: "referenced-workspace-1", repoUrl: null, repoRef: null, }, ], }; const realized = await runtime.realizeWorkspace({ environment, lease: acquired.lease, workspace: { localPath: "/tmp/anchor", mode: "ephemeral", metadata: { workspaceRealizationRequest }, }, }); // The built-in exit builds the record and carries the referenced source through `additional`, // so the adapter can stage the mentioned project into the sandbox. const realization = realized.metadata?.workspaceRealization as Record | undefined; expect(realization).toBeDefined(); expect(realization?.additional).toEqual([ expect.objectContaining({ path: "/tmp/referenced-project", projectId: "referenced-project-1", }), ]); }); it("uses resolved secret-ref config for plugin-backed sandbox execute and release", async () => { const pluginId = randomUUID(); const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); const apiSecret = await secretService(db).create(companyId, { name: `secure-plugin-api-key-${randomUUID()}`, provider: "local_encrypted", value: "resolved-provider-key", }); const providerConfig = { provider: "secure-plugin", template: "base", apiKey: apiSecret.id, timeoutMs: 1234, reuseLease: false, }; const environment = { ...baseEnvironment, name: "Secure Plugin Sandbox", driver: "sandbox", config: providerConfig, }; await secretService(db).createBinding({ companyId, secretId: apiSecret.id, targetType: "environment", targetId: environment.id, configPath: "apiKey", }); await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: providerConfig, }); await db.insert(plugins).values({ id: pluginId, pluginKey: "acme.secure-sandbox-provider", packageName: "@acme/secure-sandbox-provider", version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: { id: "acme.secure-sandbox-provider", apiVersion: 1, version: "1.0.0", displayName: "Secure Sandbox Provider", description: "Test schema-driven provider", author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "secure-plugin", kind: "sandbox_provider", displayName: "Secure Sandbox", configSchema: { type: "object", properties: { template: { type: "string" }, apiKey: { type: "string", format: "secret-ref" }, timeoutMs: { type: "number" }, reuseLease: { type: "boolean" }, }, }, }, ], }, status: "ready", installOrder: 1, updatedAt: new Date(), } as any); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string, params: any) => { expect(params.config.apiKey).toBe("resolved-provider-key"); expect(params.config).not.toHaveProperty("provider"); if (method === "environmentAcquireLease") { return { providerLeaseId: "sandbox-1", metadata: { provider: "secure-plugin", template: "base", apiKey: "resolved-provider-key", timeoutMs: 1234, reuseLease: false, sandboxId: "sandbox-1", remoteCwd: "/workspace", }, }; } if (method === "environmentExecute") { return { exitCode: 0, signal: null, timedOut: false, stdout: "ok\n", stderr: "", }; } if (method === "environmentReleaseLease") { return undefined; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); expect(acquired.lease.metadata).toMatchObject({ provider: "secure-plugin", template: "base", apiKey: apiSecret.id, timeoutMs: 1234, sandboxId: "sandbox-1", }); const executed = await runtimeWithPlugin.execute({ environment, lease: acquired.lease, command: "printf", args: ["ok"], cwd: "/workspace", env: {}, }); await environmentService(db).update(environment.id, { driver: "local", config: {}, }); const released = await runtimeWithPlugin.releaseRunLeases(runId); expect(executed.stdout).toBe("ok\n"); expect(released).toHaveLength(1); expect(released[0]?.lease.status).toBe("released"); // The execute call carries the optional log sink as the fifth argument; it // is undefined when the caller passes no sink. expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentExecute", expect.objectContaining({ config: expect.objectContaining({ apiKey: "resolved-provider-key", }), }), 31234, undefined); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentReleaseLease", expect.objectContaining({ config: expect.objectContaining({ apiKey: "resolved-provider-key", }), }), 31234); }); it("resolves persisted secret_ref binding objects in sandbox provider config at runtime", async () => { const pluginId = randomUUID(); const { companyId, environment: baseEnvironment } = await seedEnvironment(); const apiSecret = await secretService(db).create(companyId, { name: `secure-plugin-api-key-${randomUUID()}`, provider: "local_encrypted", value: "resolved-provider-key", }); const providerConfig = { provider: "secure-plugin", template: "base", apiKey: { type: "secret_ref", secretId: apiSecret.id, version: "latest" }, timeoutMs: 1234, reuseLease: false, }; const environment = { ...baseEnvironment, name: "Secure Plugin Sandbox", driver: "sandbox", config: providerConfig, }; await secretService(db).createBinding({ companyId, secretId: apiSecret.id, targetType: "environment", targetId: environment.id, configPath: "apiKey", }); await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: providerConfig, }); await db.insert(plugins).values({ id: pluginId, pluginKey: "acme.secure-sandbox-provider", packageName: "@acme/secure-sandbox-provider", version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: { id: "acme.secure-sandbox-provider", apiVersion: 1, version: "1.0.0", displayName: "Secure Sandbox Provider", description: "Test schema-driven provider", author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "secure-plugin", kind: "sandbox_provider", displayName: "Secure Sandbox", configSchema: { type: "object", properties: { template: { type: "string" }, apiKey: { type: "string", format: "secret-ref" }, timeoutMs: { type: "number" }, reuseLease: { type: "boolean" }, }, }, }, ], }, status: "ready", installOrder: 1, updatedAt: new Date(), } as any); const resolved = await resolveEnvironmentDriverConfigForRuntime(db, companyId, environment); expect(resolved.driver).toBe("sandbox"); expect(resolved.config).toMatchObject({ provider: "secure-plugin", template: "base", apiKey: "resolved-provider-key", }); }); it("waits briefly for a ready sandbox provider plugin worker to come online", async () => { const pluginId = randomUUID(); const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); const providerConfig = { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: false, }; const environment = { ...baseEnvironment, name: "Eventually Running Plugin Sandbox", driver: "sandbox", config: providerConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: providerConfig, }); await db.insert(plugins).values({ id: pluginId, pluginKey: "acme.eventually-running-sandbox-provider", packageName: "@acme/eventually-running-sandbox-provider", version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: { id: "acme.eventually-running-sandbox-provider", apiVersion: 1, version: "1.0.0", displayName: "Eventually Running Sandbox Provider", description: "Test plugin worker startup grace period", author: "Acme", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", configSchema: { type: "object" }, }, ], }, status: "ready", installOrder: 1, updatedAt: new Date(), } as any); let runningChecks = 0; const workerManager = { isRunning: vi.fn((id: string) => { if (id !== pluginId) return false; runningChecks += 1; return runningChecks >= 3; }), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "sandbox-1", metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: false, }, }; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager, pluginWorkerReadyTimeoutMs: 25, pluginWorkerReadyPollMs: 1, }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); expect(acquired.lease.providerLeaseId).toBe("sandbox-1"); expect(workerManager.isRunning).toHaveBeenCalledTimes(3); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", expect.anything(), 31234); }); it("throws a worker-not-running error once the readiness deadline is exhausted", async () => { const pluginId = randomUUID(); const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); const providerConfig = { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: false, }; const environment = { ...baseEnvironment, name: "Never Running Plugin Sandbox", driver: "sandbox", config: providerConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: providerConfig, }); await db.insert(plugins).values({ id: pluginId, pluginKey: "acme.never-running-sandbox-provider", packageName: "@acme/never-running-sandbox-provider", version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: { id: "acme.never-running-sandbox-provider", apiVersion: 1, version: "1.0.0", displayName: "Never Running Sandbox Provider", description: "Test plugin worker that never comes online before the readiness deadline", author: "Acme", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", configSchema: { type: "object" }, }, ], }, status: "ready", installOrder: 1, updatedAt: new Date(), } as any); const workerManager = { isRunning: vi.fn(() => false), call: vi.fn(async (_pluginId: string, method: string) => { throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager, pluginWorkerReadyTimeoutMs: 25, pluginWorkerReadyPollMs: 1, }); await expect( runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }), ).rejects.toThrow(/worker is not running/); // Confirms the loop actually polled repeatedly across the deadline window // instead of giving up after a single check. expect(workerManager.isRunning.mock.calls.length).toBeGreaterThanOrEqual(2); }); it("keeps polling across a longer worker restart window and succeeds once the handle registers late", async () => { const pluginId = randomUUID(); const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); const providerConfig = { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: false, }; const environment = { ...baseEnvironment, name: "Registered Late Plugin Sandbox", driver: "sandbox", config: providerConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: providerConfig, }); await db.insert(plugins).values({ id: pluginId, pluginKey: "acme.registered-late-sandbox-provider", packageName: "@acme/registered-late-sandbox-provider", version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: { id: "acme.registered-late-sandbox-provider", apiVersion: 1, version: "1.0.0", displayName: "Registered Late Sandbox Provider", description: "Test plugin worker handle that stays unregistered/starting for most of the readiness window", author: "Acme", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", configSchema: { type: "object" }, }, ], }, status: "ready", installOrder: 1, updatedAt: new Date(), } as any); // Simulates a worker process restart: the handle is absent (or "starting") // for most of the readiness window and only reports running with several // checks left before the deadline. A larger attempt count than the // existing "waits briefly" sanity test, to prove the loop is bound by the // deadline rather than by a small fixed number of attempts. const readyAfterChecks = 8; let checks = 0; const workerManager = { isRunning: vi.fn((id: string) => { if (id !== pluginId) return false; checks += 1; return checks >= readyAfterChecks; }), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "sandbox-registered-late", metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: false, }, }; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager, pluginWorkerReadyTimeoutMs: 200, pluginWorkerReadyPollMs: 1, }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); expect(acquired.lease.providerLeaseId).toBe("sandbox-registered-late"); expect(checks).toBe(readyAfterChecks); }); it("extends plugin-backed sandbox lease RPC timeouts from provider config", async () => { const pluginId = randomUUID(); const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); const providerConfig = { provider: "fake-plugin", image: "fake:test", timeoutMs: 1_234, bridgeRequestTimeoutMs: 40_000, reuseLease: false, }; const environment = { ...baseEnvironment, name: "Long Lease Plugin Sandbox", driver: "sandbox", config: providerConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: providerConfig, }); await db.insert(plugins).values({ id: pluginId, pluginKey: "acme.long-lease-sandbox-provider", packageName: "@acme/long-lease-sandbox-provider", version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: { id: "acme.long-lease-sandbox-provider", apiVersion: 1, version: "1.0.0", displayName: "Long Lease Sandbox Provider", description: "Test plugin worker acquire timeout", author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", configSchema: { type: "object" }, }, ], }, status: "ready", installOrder: 1, updatedAt: new Date(), } as any); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "sandbox-1", metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1_234, bridgeRequestTimeoutMs: 40_000, reuseLease: false, }, }; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); expect(acquired.lease.providerLeaseId).toBe("sandbox-1"); expect(workerManager.call).toHaveBeenCalledWith( pluginId, "environmentAcquireLease", expect.objectContaining({ driverKey: "fake-plugin", config: { image: "fake:test", timeoutMs: 1_234, bridgeRequestTimeoutMs: 40_000, reuseLease: false, }, }), 70_000, ); }); it("preserves the exact lease when plugin-backed sandbox resume throws", async () => { const pluginId = randomUUID(); const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment(); const providerConfig = { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, }; const environment = { ...baseEnvironment, name: "Reusable Plugin Sandbox", driver: "sandbox", config: providerConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: providerConfig, }); await db.insert(plugins).values({ id: pluginId, pluginKey: "acme.fake-sandbox-provider", packageName: "@acme/fake-sandbox-provider", version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: { id: "acme.fake-sandbox-provider", apiVersion: 1, version: "1.0.0", displayName: "Fake Sandbox Provider", description: "Test schema-driven provider", author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", supportsReusableLeases: true, configSchema: { type: "object", properties: { image: { type: "string" }, timeoutMs: { type: "number" }, reuseLease: { type: "boolean" }, }, }, }, ], }, status: "ready", installOrder: 1, updatedAt: new Date(), } as any); const executionWorkspaceId = randomUUID(); const projectId = randomUUID(); await db.insert(projects).values({ id: projectId, companyId, name: `Workspace ${projectId.slice(0, 8)}`, status: "active", createdAt: new Date(), updatedAt: new Date(), }); await db.insert(executionWorkspaces).values({ id: executionWorkspaceId, companyId, projectId, mode: "shared_workspace", strategyType: "project_primary", name: "Reusable workspace", status: "active", providerType: "local_fs", createdAt: new Date(), updatedAt: new Date(), }); const staleLease = await environmentService(db).acquireLease({ companyId, environmentId: environment.id, executionWorkspaceId, heartbeatRunId: runId, leasePolicy: "reuse_by_environment", provider: "fake-plugin", providerLeaseId: "stale-plugin-lease", metadata: { agentId, driver: "sandbox", pluginId, pluginKey: "acme.fake-sandbox-provider", sandboxProviderPlugin: true, provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, reusableSandboxLease: { version: 1, companyId, environmentId: environment.id, executionWorkspaceId, agentId, adapterType: null, provider: "fake-plugin", runtimeFingerprint: reusableRuntimeFingerprint({ provider: "fake-plugin", adapterType: null, config: providerConfig, }), }, }, }); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentResumeLease") { throw new Error("stale sandbox"); } if (method === "environmentDestroyLease") { return undefined; } if (method === "environmentAcquireLease") { return { providerLeaseId: "fresh-plugin-lease", metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, remoteCwd: "/workspace", }, }; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); await expect(runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, agentId, heartbeatRunId: runId, persistedExecutionWorkspace: { id: executionWorkspaceId, mode: "shared_workspace", }, })).rejects.toThrow("the lease was preserved and no replacement was created"); expect(workerManager.call).toHaveBeenNthCalledWith(1, pluginId, "environmentResumeLease", expect.objectContaining({ driverKey: "fake-plugin", providerLeaseId: "stale-plugin-lease", }), 31234); expect(workerManager.call).not.toHaveBeenCalledWith( pluginId, "environmentDestroyLease", expect.anything(), expect.anything(), ); expect(workerManager.call).not.toHaveBeenCalledWith( pluginId, "environmentAcquireLease", expect.anything(), expect.anything(), ); await expect(environmentService(db).getLeaseById(staleLease.id)).resolves.toMatchObject({ status: "active", }); }); it("does not allocate a native-runner replacement without a verified backup", async () => { const seeded = await seedReusablePluginSandboxLease("paperclip_runner"); const workerManager = { isRunning: vi.fn((id: string) => id === seeded.pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentResumeLease") { return { providerLeaseId: null, metadata: { expired: true } }; } if (method === "environmentDestroyLease") return undefined; if (method === "environmentAcquireLease") { throw new Error("replacement must not be allocated"); } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: [ "environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease", ], })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); await expect(runtimeWithPlugin.acquireRunLease({ companyId: seeded.companyId, environment: seeded.environment, issueId: null, agentId: seeded.agentId, adapterType: "paperclip_runner", heartbeatRunId: seeded.runId, persistedExecutionWorkspace: { id: seeded.executionWorkspaceId, mode: "shared_workspace", }, })).rejects.toThrow("runner_harness_backup_unavailable"); expect(workerManager.call).not.toHaveBeenCalledWith( seeded.pluginId, "environmentAcquireLease", expect.anything(), expect.anything(), ); expect(workerManager.call).not.toHaveBeenCalledWith( seeded.pluginId, "environmentDestroyLease", expect.anything(), expect.anything(), ); }); it("permits native-runner replacement only after verifying the stamped backup", async () => { const seeded = await seedReusablePluginSandboxLease("paperclip_runner"); const backupBase = await mkdtemp(path.join(os.tmpdir(), "paperclip-runtime-replacement-")); const previousStateDirectory = process.env.PAPERCLIP_RUNNER_STATE_DIR; process.env.PAPERCLIP_RUNNER_STATE_DIR = backupBase; try { const normalizedSessionId = "native-replacement-session"; const runnerInstanceId = "native-replacement-runner"; const sessionScopeId = "native-replacement-session-scope-v2"; const sessionRoot = path.join( backupBase, createHash("sha256").update(sessionScopeId).digest("hex"), ); const current = path.join(sessionRoot, "failover-backups", "current"); await mkdir(path.join(current, "runner"), { recursive: true }); await mkdir(path.join(current, "codex-home", "sessions"), { recursive: true }); await writeFile(path.join(current, "runner", "runner-state.json"), "runner-state"); await writeFile(path.join(current, "codex-home", "sessions", "thread.jsonl"), "thread-state"); const execution = { provider: { kind: "codex", model: null, approvalPolicy: "never" }, binding: { companyId: seeded.companyId, runId: seeded.runId, issueId: "issue", agentId: seeded.agentId, executionWorkspaceId: seeded.executionWorkspaceId, }, workspace: { cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null }, session: { normalizedSessionId, driverKind: "codex_app_server", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, }, } as never; const manifest = buildNativeHarnessBackupManifest({ backupRoot: current, execution, runnerInstanceId, providerSessionIdentity: { providerSessionId: "thread-1", providerBackendSessionId: "session-1", providerSessionIdentity: null, }, sourceProviderLeaseId: seeded.reusableLease.providerLeaseId!, }); const manifestPath = path.join(current, "manifest.json"); await writeFile(manifestPath, JSON.stringify(manifest)); const stamp = createNativeHarnessBackupStamp({ manifestPath, sessionScopeId, authorizedProviderLeaseId: seeded.reusableLease.providerLeaseId!, normalizedSessionId, runnerInstanceId, completedAt: manifest.completedAt, }); await environmentService(db).updateLeaseMetadata(seeded.reusableLease.id, { ...(seeded.reusableLease.metadata ?? {}), nativeHarnessBackup: stamp, }); const workerManager = { isRunning: vi.fn((id: string) => id === seeded.pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentResumeLease") { return { providerLeaseId: null, metadata: { expired: true } }; } if (method === "environmentDestroyLease") return undefined; if (method === "environmentAcquireLease") { return { providerLeaseId: "replacement-plugin-lease", metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, remoteCwd: "/workspace", }, }; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: [ "environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease", ], })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId: seeded.companyId, environment: seeded.environment, issueId: null, agentId: seeded.agentId, adapterType: "paperclip_runner", heartbeatRunId: seeded.runId, persistedExecutionWorkspace: { id: seeded.executionWorkspaceId, mode: "shared_workspace", }, }); expect(acquired.lease.providerLeaseId).toBe("replacement-plugin-lease"); expect(acquired.lease.metadata?.sandboxLeaseAcquisition).toEqual({ outcome: "replacement", reason: "not_found", }); } finally { if (previousStateDirectory === undefined) { delete process.env.PAPERCLIP_RUNNER_STATE_DIR; } else { process.env.PAPERCLIP_RUNNER_STATE_DIR = previousStateDirectory; } await rm(backupBase, { recursive: true, force: true }); } }); it("fails closed and does not resume when a worker restart drops the resume method after the capability snapshot", async () => { const pluginId = randomUUID(); const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment(); const providerConfig = { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, }; const environment = { ...baseEnvironment, name: "Reusable Plugin Sandbox", driver: "sandbox", config: providerConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: providerConfig, }); await db.insert(plugins).values({ id: pluginId, pluginKey: "acme.fake-sandbox-provider", packageName: "@acme/fake-sandbox-provider", version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: { id: "acme.fake-sandbox-provider", apiVersion: 1, version: "1.0.0", displayName: "Fake Sandbox Provider", description: "Test schema-driven provider", author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", supportsReusableLeases: true, configSchema: { type: "object", properties: { image: { type: "string" }, timeoutMs: { type: "number" }, reuseLease: { type: "boolean" }, }, }, }, ], }, status: "ready", installOrder: 1, updatedAt: new Date(), } as any); const executionWorkspaceId = randomUUID(); const projectId = randomUUID(); await db.insert(projects).values({ id: projectId, companyId, name: `Workspace ${projectId.slice(0, 8)}`, status: "active", createdAt: new Date(), updatedAt: new Date(), }); await db.insert(executionWorkspaces).values({ id: executionWorkspaceId, companyId, projectId, mode: "shared_workspace", strategyType: "project_primary", name: "Reusable workspace", status: "active", providerType: "local_fs", createdAt: new Date(), updatedAt: new Date(), }); const staleLease = await environmentService(db).acquireLease({ companyId, environmentId: environment.id, executionWorkspaceId, heartbeatRunId: runId, leasePolicy: "reuse_by_environment", provider: "fake-plugin", providerLeaseId: "stale-plugin-lease", metadata: { agentId, driver: "sandbox", pluginId, pluginKey: "acme.fake-sandbox-provider", sandboxProviderPlugin: true, provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, reusableSandboxLease: { version: 1, companyId, environmentId: environment.id, executionWorkspaceId, agentId, adapterType: null, provider: "fake-plugin", runtimeFingerprint: reusableRuntimeFingerprint({ provider: "fake-plugin", adapterType: null, config: providerConfig, }), }, }, }); // The runtime reads the worker methods once to decide reuse, then does // asynchronous database work before the resume dispatch. A worker restart // in that window drops `environmentResumeLease`. The first read returns the // reuse verbs, so the runtime treats the lease as resumable. Every later // read returns the restarted worker's methods, which no longer include // `environmentResumeLease`. let getWorkerReads = 0; const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentResumeLease") { // A regression that reads the stale snapshot dispatches the resume // RPC and fails here. The live worker cannot serve the method. throw new Error("worker no longer advertises environmentResumeLease"); } if (method === "environmentDestroyLease") { return undefined; } if (method === "environmentAcquireLease") { return { providerLeaseId: "fresh-plugin-lease", metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, remoteCwd: "/workspace", }, }; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => { getWorkerReads += 1; return { supportedMethods: getWorkerReads === 1 ? ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] : ["environmentReleaseLease", "environmentDestroyLease"], }; }), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); await expect(runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, agentId, heartbeatRunId: runId, persistedExecutionWorkspace: { id: executionWorkspaceId, mode: "shared_workspace", }, })).rejects.toThrow("the lease was preserved and no replacement was created"); // The runtime re-checks the live worker before the resume dispatch. The // restarted worker no longer advertises `environmentResumeLease`, so the // runtime must not dispatch the resume RPC. expect(workerManager.call).not.toHaveBeenCalledWith( pluginId, "environmentResumeLease", expect.anything(), expect.anything(), ); // It preserves the stale reusable lease and does not allocate a replacement. expect(workerManager.call).not.toHaveBeenCalledWith( pluginId, "environmentDestroyLease", expect.anything(), expect.anything(), ); expect(workerManager.call).not.toHaveBeenCalledWith( pluginId, "environmentAcquireLease", expect.anything(), expect.anything(), ); await expect(environmentService(db).getLeaseById(staleLease.id)).resolves.toMatchObject({ status: "active", }); }); // Seed a reusable plugin sandbox lease that a worker created under an earlier // capability set. The worker restarts and no longer advertises the lifecycle // methods. The lease lifecycle paths must verify the live worker before they // dispatch a lifecycle RPC, so the runtime fails closed instead of a doomed // dispatch. async function seedStaleLifecycleReusableLease( leasePolicy: "reuse_by_environment" | "retain_on_failure" = "reuse_by_environment", ) { const pluginId = randomUUID(); const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment(); const providerConfig = { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, }; const environment = { ...baseEnvironment, name: "Reusable Plugin Sandbox", driver: "sandbox", config: providerConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: providerConfig, }); await db.insert(plugins).values({ id: pluginId, pluginKey: "acme.fake-sandbox-provider", packageName: "@acme/fake-sandbox-provider", version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: { id: "acme.fake-sandbox-provider", apiVersion: 1, version: "1.0.0", displayName: "Fake Sandbox Provider", description: "Test schema-driven provider", author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", supportsReusableLeases: true, configSchema: { type: "object" }, }, ], }, status: "ready", installOrder: 1, updatedAt: new Date(), } as any); const executionWorkspaceId = randomUUID(); const projectId = randomUUID(); await db.insert(projects).values({ id: projectId, companyId, name: `Workspace ${projectId.slice(0, 8)}`, status: "active", createdAt: new Date(), updatedAt: new Date(), }); await db.insert(executionWorkspaces).values({ id: executionWorkspaceId, companyId, projectId, mode: "shared_workspace", strategyType: "project_primary", name: "Reusable workspace", status: "active", providerType: "local_fs", createdAt: new Date(), updatedAt: new Date(), }); const lease = await environmentService(db).acquireLease({ companyId, environmentId: environment.id, executionWorkspaceId, heartbeatRunId: runId, leasePolicy, provider: "fake-plugin", providerLeaseId: "stale-lifecycle-lease", metadata: { agentId, driver: "sandbox", pluginId, pluginKey: "acme.fake-sandbox-provider", sandboxProviderPlugin: true, provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, }, }); // The worker is running, but its discovery list dropped the reusable-lease // lifecycle methods. `call` throws on any lifecycle RPC so a regression that // dispatches one fails the test through the `not.toHaveBeenCalledWith` check. const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentAcquireLease", "environmentExecute"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); return { pluginId, environment, runId, lease, workerManager, runtimeWithPlugin }; } it("routes release to pending_cleanup when the worker no longer advertises the release lifecycle method", async () => { const { pluginId, lease, workerManager, runtimeWithPlugin } = await seedStaleLifecycleReusableLease(); const released = await runtimeWithPlugin.releaseRunLeases(lease.heartbeatRunId!); expect(released).toHaveLength(1); expect(workerManager.call).not.toHaveBeenCalledWith( pluginId, "environmentReleaseLease", expect.anything(), expect.anything(), ); // The failed release verification must enter the pending-cleanup retry flow. // The reaper sweeps only `pending_cleanup` leases, so a `released` status // here would strand the still-active provider resource. await expect(environmentService(db).getLeaseById(lease.id)).resolves.toMatchObject({ status: "pending_cleanup", cleanupStatus: "failed", failureReason: "release_cleanup_failed", }); }); it("retains a retain_on_failure lease on failed release instead of routing to pending_cleanup", async () => { const { lease, runtimeWithPlugin } = await seedStaleLifecycleReusableLease("retain_on_failure"); const released = await runtimeWithPlugin.releaseRunLeases(lease.heartbeatRunId!, "failed"); expect(released).toHaveLength(1); // A retain_on_failure lease keeps the provider resource for reuse. The // reaper destroys `pending_cleanup` leases, so the retained lease must not // enter that flow even when the release verification fails. await expect(environmentService(db).getLeaseById(lease.id)).resolves.toMatchObject({ status: "retained", cleanupStatus: "failed", }); }); it("fails closed on expiry destruction when the worker no longer advertises the destroy lifecycle method", async () => { const { pluginId, lease, workerManager, runtimeWithPlugin } = await seedStaleLifecycleReusableLease(); await runtimeWithPlugin.releaseRunLeases(lease.heartbeatRunId!, "expired"); expect(workerManager.call).not.toHaveBeenCalledWith( pluginId, "environmentDestroyLease", expect.anything(), expect.anything(), ); await expect(environmentService(db).getLeaseById(lease.id)).resolves.toMatchObject({ status: "pending_cleanup", cleanupStatus: "failed", }); }); it("does not resume released reusable plugin sandbox leases after provider config drift", async () => { const pluginId = randomUUID(); const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment(); const providerConfig = { provider: "fake-plugin", image: "template-a", timeoutMs: 1234, reuseLease: true, }; const environment = { ...baseEnvironment, name: "Reusable Plugin Sandbox", driver: "sandbox", config: providerConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: providerConfig, }); await db.insert(plugins).values({ id: pluginId, pluginKey: "acme.fake-sandbox-provider", packageName: "@acme/fake-sandbox-provider", version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: { id: "acme.fake-sandbox-provider", apiVersion: 1, version: "1.0.0", displayName: "Fake Sandbox Provider", description: "Test schema-driven provider", author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", supportsReusableLeases: true, configSchema: { type: "object", properties: { image: { type: "string" }, timeoutMs: { type: "number" }, reuseLease: { type: "boolean" }, }, }, }, ], }, status: "ready", installOrder: 1, updatedAt: new Date(), } as any); const executionWorkspaceId = randomUUID(); const projectId = randomUUID(); await db.insert(projects).values({ id: projectId, companyId, name: `Workspace ${projectId.slice(0, 8)}`, status: "active", createdAt: new Date(), updatedAt: new Date(), }); await db.insert(executionWorkspaces).values({ id: executionWorkspaceId, companyId, projectId, mode: "shared_workspace", strategyType: "project_primary", name: "Reusable workspace", status: "active", providerType: "local_fs", createdAt: new Date(), updatedAt: new Date(), }); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string, params: any) => { if (method === "environmentAcquireLease") { return { providerLeaseId: `lease-${params.config.image}`, metadata: { provider: "fake-plugin", image: params.config.image, timeoutMs: params.config.timeoutMs, reuseLease: true, remoteCwd: "/workspace", }, }; } if (method === "environmentReleaseLease" || method === "environmentDestroyLease") { return undefined; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const first = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, agentId, heartbeatRunId: runId, persistedExecutionWorkspace: { id: executionWorkspaceId, mode: "shared_workspace", }, }); expect(first.lease.metadata?.reusableSandboxLease).toMatchObject({ provider: "fake-plugin", leaseFingerprint: expect.objectContaining({ category: "lease", fingerprint: expect.stringMatching(/^v1:sha256:[a-f0-9]{64}$/), }), }); await runtimeWithPlugin.releaseRunLeases(runId); const nextRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: nextRunId, companyId, agentId, invocationSource: "manual", status: "running", createdAt: new Date(), updatedAt: new Date(), }); const updatedEnvironment = { ...environment, config: { ...providerConfig, image: "template-b", }, }; await environmentService(db).update(environment.id, { config: updatedEnvironment.config, }); const second = await runtimeWithPlugin.acquireRunLease({ companyId, environment: updatedEnvironment, issueId: null, agentId, heartbeatRunId: nextRunId, persistedExecutionWorkspace: { id: executionWorkspaceId, mode: "shared_workspace", }, }); expect(second.lease.providerLeaseId).toBe("lease-template-b"); expect(workerManager.call).not.toHaveBeenCalledWith( pluginId, "environmentResumeLease", expect.anything(), expect.anything(), ); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentDestroyLease", expect.objectContaining({ providerLeaseId: "lease-template-a", }), 31234); await expect(environmentService(db).getLeaseById(first.lease.id)).resolves.toMatchObject({ status: "expired", cleanupStatus: "success", failureReason: "lease_fingerprint_mismatch", }); }); it("does not resume released reusable plugin sandbox leases after secret version drift", async () => { const pluginId = randomUUID(); const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment(); const apiSecret = await secretService(db).create(companyId, { name: `secure-plugin-api-key-${randomUUID()}`, provider: "local_encrypted", value: "resolved-provider-key", }); const providerConfig = { provider: "secure-plugin", template: "base", apiKey: apiSecret.id, timeoutMs: 1234, reuseLease: true, }; const environment = { ...baseEnvironment, name: "Secure Plugin Sandbox", driver: "sandbox", config: providerConfig, }; await secretService(db).createBinding({ companyId, secretId: apiSecret.id, targetType: "environment", targetId: environment.id, configPath: "apiKey", }); await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: providerConfig, }); await db.insert(plugins).values({ id: pluginId, pluginKey: "acme.secure-sandbox-provider", packageName: "@acme/secure-sandbox-provider", version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: { id: "acme.secure-sandbox-provider", apiVersion: 1, version: "1.0.0", displayName: "Secure Sandbox Provider", description: "Test schema-driven provider", author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "secure-plugin", kind: "sandbox_provider", displayName: "Secure Sandbox", supportsReusableLeases: true, configSchema: { type: "object", properties: { template: { type: "string" }, apiKey: { type: "string", format: "secret-ref" }, timeoutMs: { type: "number" }, reuseLease: { type: "boolean" }, }, }, }, ], }, status: "ready", installOrder: 1, updatedAt: new Date(), } as any); const executionWorkspaceId = randomUUID(); const projectId = randomUUID(); await db.insert(projects).values({ id: projectId, companyId, name: `Workspace ${projectId.slice(0, 8)}`, status: "active", createdAt: new Date(), updatedAt: new Date(), }); await db.insert(executionWorkspaces).values({ id: executionWorkspaceId, companyId, projectId, mode: "shared_workspace", strategyType: "project_primary", name: "Reusable workspace", status: "active", providerType: "local_fs", createdAt: new Date(), updatedAt: new Date(), }); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string, params: any) => { if (method === "environmentAcquireLease") { return { providerLeaseId: `lease-${params.config.apiKey}`, metadata: { provider: "secure-plugin", template: params.config.template, apiKey: params.config.apiKey, timeoutMs: params.config.timeoutMs, reuseLease: true, remoteCwd: "/workspace", }, }; } if (method === "environmentReleaseLease" || method === "environmentDestroyLease") { return undefined; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const first = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, agentId, heartbeatRunId: runId, persistedExecutionWorkspace: { id: executionWorkspaceId, mode: "shared_workspace", }, }); await runtimeWithPlugin.releaseRunLeases(runId); await secretService(db).rotate(apiSecret.id, { value: "rotated-provider-key" }); const nextRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: nextRunId, companyId, agentId, invocationSource: "manual", status: "running", createdAt: new Date(), updatedAt: new Date(), }); const second = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, agentId, heartbeatRunId: nextRunId, persistedExecutionWorkspace: { id: executionWorkspaceId, mode: "shared_workspace", }, }); expect(second.lease.providerLeaseId).toBe("lease-rotated-provider-key"); expect(workerManager.call).not.toHaveBeenCalledWith( pluginId, "environmentResumeLease", expect.anything(), expect.anything(), ); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentDestroyLease", expect.objectContaining({ providerLeaseId: "lease-resolved-provider-key", }), 31234); await expect(environmentService(db).getLeaseById(first.lease.id)).resolves.toMatchObject({ status: "expired", cleanupStatus: "success", failureReason: "lease_fingerprint_mismatch", }); const firstMetadata = JSON.stringify(first.lease.metadata); expect(firstMetadata).not.toContain("resolved-provider-key"); expect(firstMetadata).not.toContain("rotated-provider-key"); }); it("preserves active reusable sandbox leases held by another running run", async () => { const { pluginId, companyId, agentId, environment, executionWorkspaceId, reusableLease } = await seedReusablePluginSandboxLease(); const nextRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: nextRunId, companyId, agentId, invocationSource: "manual", status: "running", createdAt: new Date(), updatedAt: new Date(), }); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "fresh-plugin-lease", metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, remoteCwd: "/workspace", }, }; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, agentId, heartbeatRunId: nextRunId, persistedExecutionWorkspace: { id: executionWorkspaceId, mode: "shared_workspace", }, }); expect(acquired.lease.providerLeaseId).toBe("fresh-plugin-lease"); expect(workerManager.call).toHaveBeenCalledOnce(); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", expect.objectContaining({ agentId, executionWorkspaceId, runId: nextRunId, }), 31234); await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({ status: "active", cleanupStatus: null, }); }); it("does not retain or resume plugin-backed sandbox leases unless the provider opts in", async () => { const pluginId = randomUUID(); const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment(); const providerConfig = { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, }; const environment = { ...baseEnvironment, name: "Non-reusable Plugin Sandbox", driver: "sandbox", config: providerConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: providerConfig, }); await db.insert(plugins).values({ id: pluginId, pluginKey: "acme.non-reusable-sandbox-provider", packageName: "@acme/non-reusable-sandbox-provider", version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: { id: "acme.non-reusable-sandbox-provider", apiVersion: 1, version: "1.0.0", displayName: "Non-reusable Sandbox Provider", description: "Test provider without reusable lease support", author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", configSchema: { type: "object" }, }, ], }, status: "ready", installOrder: 1, updatedAt: new Date(), } as any); const executionWorkspaceId = randomUUID(); const projectId = randomUUID(); await db.insert(projects).values({ id: projectId, companyId, name: `Workspace ${projectId.slice(0, 8)}`, status: "active", createdAt: new Date(), updatedAt: new Date(), }); await db.insert(executionWorkspaces).values({ id: executionWorkspaceId, companyId, projectId, mode: "shared_workspace", strategyType: "project_primary", name: "Non-reusable workspace", status: "active", providerType: "local_fs", createdAt: new Date(), updatedAt: new Date(), }); await environmentService(db).acquireLease({ companyId, environmentId: environment.id, executionWorkspaceId, heartbeatRunId: runId, leasePolicy: "reuse_by_environment", provider: "fake-plugin", providerLeaseId: "old-plugin-lease", metadata: { agentId, provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, reusableSandboxLease: { version: 1, companyId, environmentId: environment.id, executionWorkspaceId, agentId, adapterType: null, provider: "fake-plugin", runtimeFingerprint: reusableRuntimeFingerprint({ provider: "fake-plugin", adapterType: null, config: providerConfig, }), }, }, }); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "fresh-plugin-lease", metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, remoteCwd: "/workspace", }, }; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, agentId, heartbeatRunId: runId, persistedExecutionWorkspace: { id: executionWorkspaceId, mode: "shared_workspace", }, }); expect(acquired.lease.providerLeaseId).toBe("fresh-plugin-lease"); expect(acquired.lease.leasePolicy).toBe("ephemeral"); expect(workerManager.call).toHaveBeenCalledTimes(1); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", expect.anything(), 31234); }); it("does not resume a lease when the nested capability disables reusable leases", async () => { // The provider declares the legacy `supportsReusableLeases: true` flag but // the nested `sandboxCapabilities.reusableLeases: false`. The nested value // wins through the capability contract, so acquisition must acquire a fresh // lease and must never resume the existing reusable lease. const pluginId = randomUUID(); const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment(); const providerConfig = { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, }; const environment = { ...baseEnvironment, name: "Nested-disabled Plugin Sandbox", driver: "sandbox", config: providerConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: providerConfig, }); await db.insert(plugins).values({ id: pluginId, pluginKey: "acme.nested-disabled-sandbox-provider", packageName: "@acme/nested-disabled-sandbox-provider", version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: { id: "acme.nested-disabled-sandbox-provider", apiVersion: 1, version: "1.0.0", displayName: "Nested-disabled Sandbox Provider", description: "Test provider with a legacy flag and a disabled nested capability", author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", supportsReusableLeases: true, sandboxCapabilities: { reusableLeases: false }, configSchema: { type: "object", properties: { image: { type: "string" }, timeoutMs: { type: "number" }, reuseLease: { type: "boolean" }, }, }, }, ], }, status: "ready", installOrder: 1, updatedAt: new Date(), } as any); const executionWorkspaceId = randomUUID(); const projectId = randomUUID(); await db.insert(projects).values({ id: projectId, companyId, name: `Workspace ${projectId.slice(0, 8)}`, status: "active", createdAt: new Date(), updatedAt: new Date(), }); await db.insert(executionWorkspaces).values({ id: executionWorkspaceId, companyId, projectId, mode: "shared_workspace", strategyType: "project_primary", name: "Nested-disabled workspace", status: "active", providerType: "local_fs", createdAt: new Date(), updatedAt: new Date(), }); await environmentService(db).acquireLease({ companyId, environmentId: environment.id, executionWorkspaceId, heartbeatRunId: runId, leasePolicy: "reuse_by_environment", provider: "fake-plugin", providerLeaseId: "old-plugin-lease", metadata: { agentId, provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, reusableSandboxLease: { version: 1, companyId, environmentId: environment.id, executionWorkspaceId, agentId, adapterType: null, provider: "fake-plugin", runtimeFingerprint: reusableRuntimeFingerprint({ provider: "fake-plugin", adapterType: null, config: providerConfig, }), }, }, }); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "fresh-plugin-lease", metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, remoteCwd: "/workspace", }, }; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, agentId, heartbeatRunId: runId, persistedExecutionWorkspace: { id: executionWorkspaceId, mode: "shared_workspace", }, }); expect(acquired.lease.providerLeaseId).toBe("fresh-plugin-lease"); expect(acquired.lease.leasePolicy).toBe("ephemeral"); expect(workerManager.call).toHaveBeenCalledTimes(1); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", expect.anything(), 31234); expect(workerManager.call).not.toHaveBeenCalledWith( pluginId, "environmentResumeLease", expect.anything(), expect.anything(), ); }); it("fails closed and does not resume when the worker does not verify the reuse methods", async () => { // The provider declares `reusableLeases: true`, but its worker advertises // neither `environmentResumeLease` nor `environmentReleaseLease`. The runtime // must not resume or reuse the lease, because it cannot dispatch a resume or // a release the worker does not serve. It acquires a fresh ephemeral lease // and leaves the old reusable lease untouched. const pluginId = randomUUID(); const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment(); const providerConfig = { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, }; const environment = { ...baseEnvironment, name: "Unverified-worker Plugin Sandbox", driver: "sandbox", config: providerConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: providerConfig, }); await db.insert(plugins).values({ id: pluginId, pluginKey: "acme.unverified-worker-sandbox-provider", packageName: "@acme/unverified-worker-sandbox-provider", version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: { id: "acme.unverified-worker-sandbox-provider", apiVersion: 1, version: "1.0.0", displayName: "Unverified-worker Sandbox Provider", description: "Test provider that declares reusable leases but whose worker lacks the reuse methods", author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", sandboxCapabilities: { reusableLeases: true }, configSchema: { type: "object", properties: { image: { type: "string" }, timeoutMs: { type: "number" }, reuseLease: { type: "boolean" }, }, }, }, ], }, status: "ready", installOrder: 1, updatedAt: new Date(), } as any); const executionWorkspaceId = randomUUID(); const projectId = randomUUID(); await db.insert(projects).values({ id: projectId, companyId, name: `Workspace ${projectId.slice(0, 8)}`, status: "active", createdAt: new Date(), updatedAt: new Date(), }); await db.insert(executionWorkspaces).values({ id: executionWorkspaceId, companyId, projectId, mode: "shared_workspace", strategyType: "project_primary", name: "Unverified-worker workspace", status: "active", providerType: "local_fs", createdAt: new Date(), updatedAt: new Date(), }); const existingLease = await environmentService(db).acquireLease({ companyId, environmentId: environment.id, executionWorkspaceId, heartbeatRunId: runId, leasePolicy: "reuse_by_environment", provider: "fake-plugin", providerLeaseId: "old-plugin-lease", metadata: { agentId, provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, reusableSandboxLease: { version: 1, companyId, environmentId: environment.id, executionWorkspaceId, agentId, adapterType: null, provider: "fake-plugin", runtimeFingerprint: reusableRuntimeFingerprint({ provider: "fake-plugin", adapterType: null, config: providerConfig, }), }, }, }); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "fresh-plugin-lease", metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, remoteCwd: "/workspace", }, }; } throw new Error(`Unexpected plugin method: ${method}`); }), // The worker verifies only `environmentExecute`. It advertises neither // reuse verb, so the reusable-lease capability fails closed. getWorker: vi.fn(() => ({ supportedMethods: ["environmentExecute"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, agentId, heartbeatRunId: runId, persistedExecutionWorkspace: { id: executionWorkspaceId, mode: "shared_workspace", }, }); expect(acquired.lease.providerLeaseId).toBe("fresh-plugin-lease"); expect(acquired.lease.leasePolicy).toBe("ephemeral"); expect(workerManager.call).toHaveBeenCalledTimes(1); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", expect.anything(), 31234); expect(workerManager.call).not.toHaveBeenCalledWith( pluginId, "environmentResumeLease", expect.anything(), expect.anything(), ); // The runtime leaves the old reusable lease untouched: it neither resumes // nor destroys a lease it cannot serve. await expect(environmentService(db).getLeaseById(existingLease.id)).resolves.toMatchObject({ status: "active", leasePolicy: "reuse_by_environment", }); }); it("destroys scoped reusable plugin-backed sandbox leases", async () => { const { pluginId, companyId, executionWorkspaceId, reusableLease } = await seedReusablePluginSandboxLease(); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentDestroyLease") { return undefined; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const destroyed = await runtimeWithPlugin.destroyReusableSandboxLeases({ companyId, executionWorkspaceId, failureReason: "execution_workspace_closed", }); expect(destroyed).toHaveLength(1); expect(destroyed[0]?.lease.id).toBe(reusableLease.id); expect(destroyed[0]?.lease.status).toBe("expired"); expect(workerManager.call).toHaveBeenCalledWith( pluginId, "environmentDestroyLease", expect.objectContaining({ driverKey: "fake-plugin", providerLeaseId: "reusable-plugin-lease", }), 31234, ); await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({ status: "expired", failureReason: "execution_workspace_closed", cleanupStatus: "success", }); }); it("destroys reusable plugin-backed sandbox leases scoped to an environment", async () => { const { pluginId, runId, reusableLease } = await seedReusablePluginSandboxLease(); // The holding run is finished, so the reservation is stale and destroyable. await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, runId)); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentDestroyLease") { return undefined; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const result = await runtimeWithPlugin.destroyReusableSandboxLeasesForEnvironment({ environmentId: reusableLease.environmentId!, failureReason: "environment_deleted", }); expect(result).toEqual({ destroyed: 1, failed: 0, skippedLiveRun: 0 }); expect(workerManager.call).toHaveBeenCalledWith( pluginId, "environmentDestroyLease", expect.objectContaining({ driverKey: "fake-plugin", providerLeaseId: "reusable-plugin-lease", }), 31234, ); await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({ status: "expired", failureReason: "environment_deleted", cleanupStatus: "success", }); }); it("keeps a reusable lease held by an in-flight run out of the environment-scoped destroy", async () => { const { pluginId, reusableLease } = await seedReusablePluginSandboxLease(); // seedEnvironment leaves the holding run in `running` status. const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async () => undefined), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const result = await runtimeWithPlugin.destroyReusableSandboxLeasesForEnvironment({ environmentId: reusableLease.environmentId!, failureReason: "environment_deleted", }); expect(result).toEqual({ destroyed: 0, failed: 0, skippedLiveRun: 1 }); expect(workerManager.call).not.toHaveBeenCalled(); // The lease keeps its reusable status, so the delete guard still blocks. await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({ status: "active", leasePolicy: "reuse_by_environment", }); }); it("sweeps reusable cleanup with the configuration recorded on the lease", async () => { const { pluginId, environment, reusableLease } = await seedReusablePluginSandboxLease(); const environmentsSvc = environmentService(db); await environmentsSvc.releaseLease(reusableLease.id, "pending_cleanup", { failureReason: "release_cleanup_failed", cleanupStatus: "failed", }); // Re-point the environment after the reusable sandbox was provisioned. // The pending-cleanup sweep still enters destroyRunLease because the // environment exists, but destroy must target the provider and config // captured on the lease rather than this current configuration. await environmentsSvc.update(environment.id, { config: { provider: "replacement-provider", image: "replacement:test", timeoutMs: 9999, reuseLease: true, }, }); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentDestroyLease") return undefined; throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"], })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const heartbeat = heartbeatService(db, { environmentRuntime: runtimeWithPlugin }); const result = await heartbeat.sweepPendingCleanupLeases({ backoffMs: 0 }); expect(result).toEqual({ swept: 1, destroyed: 1, capped: 0 }); expect(workerManager.call).toHaveBeenCalledWith( pluginId, "environmentDestroyLease", expect.objectContaining({ driverKey: "fake-plugin", environmentId: environment.id, providerLeaseId: "reusable-plugin-lease", config: expect.objectContaining({ image: "fake:test", timeoutMs: 1234, reuseLease: true, }), }), 31234, ); const destroyInput = vi.mocked(workerManager.call).mock.calls[0]?.[2] as { config?: Record; }; expect(destroyInput.config).not.toMatchObject({ image: "replacement:test", timeoutMs: 9999, }); await expect(environmentsSvc.getLeaseById(reusableLease.id)).resolves.toMatchObject({ status: "expired", cleanupStatus: "success", }); }); it("retries reusable plugin-backed sandbox destroy when the worker is unavailable", async () => { const { pluginId, companyId, executionWorkspaceId, reusableLease } = await seedReusablePluginSandboxLease(); const offlineWorkerManager = { isRunning: vi.fn(() => false), call: vi.fn(), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithOfflinePlugin = environmentRuntimeService(db, { pluginWorkerManager: offlineWorkerManager, }); const pending = await runtimeWithOfflinePlugin.destroyReusableSandboxLeases({ companyId, executionWorkspaceId, failureReason: "execution_workspace_closed", }); expect(pending).toHaveLength(1); expect(pending[0]?.lease.id).toBe(reusableLease.id); expect(pending[0]?.lease.status).toBe("pending_cleanup"); expect(offlineWorkerManager.call).not.toHaveBeenCalled(); await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({ status: "pending_cleanup", failureReason: "execution_workspace_closed", cleanupStatus: "failed", }); const recoveredWorkerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentDestroyLease") { return undefined; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithRecoveredPlugin = environmentRuntimeService(db, { pluginWorkerManager: recoveredWorkerManager, }); const retried = await runtimeWithRecoveredPlugin.destroyReusableSandboxLeases({ companyId, executionWorkspaceId, failureReason: "cleanup_retry", }); expect(retried).toHaveLength(1); expect(retried[0]?.lease.id).toBe(reusableLease.id); expect(retried[0]?.lease.status).toBe("expired"); expect(recoveredWorkerManager.call).toHaveBeenCalledWith( pluginId, "environmentDestroyLease", expect.objectContaining({ driverKey: "fake-plugin", providerLeaseId: "reusable-plugin-lease", }), 31234, ); await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({ status: "expired", failureReason: "cleanup_retry", cleanupStatus: "success", }); }); it("resolves effective capabilities from the lease's exact plugin, not an earlier plugin that shares the driver key", async () => { // The helper seeds the plugin that owns the lease. Pin it to `pluginId` // through the lease metadata and give it a lower-priority sibling. const { pluginId, environment, reusableLease } = await seedReusablePluginSandboxLease(); // Rewrite the owner plugin so it DENIES reusable leases through the nested // capability declaration. await db .update(plugins) .set({ manifestJson: { id: "acme.reusable-sandbox-provider", apiVersion: 1, version: "1.0.0", displayName: "Reusable Sandbox Provider", description: "Owner plugin that denies reusable leases", author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", sandboxCapabilities: { reusableLeases: false }, configSchema: { type: "object", properties: {} }, }, ], }, updatedAt: new Date(), } as any) .where(eq(plugins.id, pluginId)); // Install an EARLIER plugin that shares the driver key and grants reusable // leases. It is not ready and never acquired this lease. A resolver keyed by // driver key alone would read this declaration and grant a capability the // owner plugin denied. const collidingPluginId = randomUUID(); await db.insert(plugins).values({ id: collidingPluginId, pluginKey: "acme.colliding-sandbox-provider", packageName: "@acme/colliding-sandbox-provider", version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: { id: "acme.colliding-sandbox-provider", apiVersion: 1, version: "1.0.0", displayName: "Colliding Sandbox Provider", description: "Earlier plugin that shares the driver key", author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", supportsReusableLeases: true, configSchema: { type: "object", properties: {} }, }, ], }, status: "installed", installOrder: 0, updatedAt: new Date(), } as any); // The owner worker verifies the reuse verbs and the sync verbs, so every // capability is verified. Only the owner's declaration can narrow one. const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(), getWorker: vi.fn((id: string) => id === pluginId ? { supportedMethods: [ "environmentResumeLease", "environmentReleaseLease", "environmentSyncIn", "environmentSyncOut", ], } : undefined, ), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); expect(reusableLease.metadata?.pluginId).toBe(pluginId); const effective = await runtimeWithPlugin.resolveCapabilities({ environment, lease: reusableLease, }); // The runtime read the owner plugin's declaration, so it denies reusable // leases even though the earlier plugin grants them. expect(effective?.reusableLeases).toBe(false); // The owner plugin does not restrict native sync, and its worker verifies // the sync verbs, so those stay granted. This proves the resolver read the // owner declaration and did not fail every capability closed. expect(effective?.nativeSyncIn).toBe(true); expect(effective?.nativeSyncOut).toBe(true); }); it("fails every effective capability closed when the pinned plugin id is absent from the registry", async () => { // The lease pins a plugin id, but that plugin record is gone. A stale worker // entry still advertises every method. The runtime must not read the stale // methods; it must fail closed because the exact-plugin identity is gone. const { pluginId, environment, reusableLease } = await seedReusablePluginSandboxLease(); await db.delete(plugins).where(eq(plugins.id, pluginId)); const workerManager = { isRunning: vi.fn(() => true), call: vi.fn(), getWorker: vi.fn(() => ({ supportedMethods: [ "environmentResumeLease", "environmentReleaseLease", "environmentExecute", "environmentSyncIn", "environmentSyncOut", ], })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const effective = await runtimeWithPlugin.resolveCapabilities({ environment, lease: reusableLease, }); for (const key of SANDBOX_CAPABILITY_KEYS) { expect(effective?.[key]).toBe(false); } }); it("fails every effective capability closed when the pinned plugin no longer declares this provider key", async () => { // The pinned plugin still exists, but it no longer declares a // `sandbox_provider` driver with this key (here it changed the driver kind). // A running worker still advertises every method. The runtime must fail // closed because the exact-plugin declaration is gone. const { pluginId, environment, reusableLease } = await seedReusablePluginSandboxLease(); await db .update(plugins) .set({ manifestJson: { id: "acme.reusable-sandbox-provider", apiVersion: 1, version: "1.0.0", displayName: "Reusable Sandbox Provider", description: "Owner plugin that no longer declares the provider key", author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "fake-plugin", // The key exists, but the kind is now a plain environment driver, // not a sandbox provider. The by-id resolver fails closed. kind: "environment_driver", displayName: "Fake Plugin", configSchema: { type: "object", properties: {} }, }, ], }, updatedAt: new Date(), } as any) .where(eq(plugins.id, pluginId)); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(), getWorker: vi.fn(() => ({ supportedMethods: [ "environmentResumeLease", "environmentReleaseLease", "environmentExecute", "environmentSyncIn", "environmentSyncOut", ], })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const effective = await runtimeWithPlugin.resolveCapabilities({ environment, lease: reusableLease, }); for (const key of SANDBOX_CAPABILITY_KEYS) { expect(effective?.[key]).toBe(false); } }); it("defers to verified worker discovery for a valid pinned plugin that omits sandboxCapabilities", async () => { // A valid, identified plugin whose manifest declares no `sandboxCapabilities` // and no legacy reuse flag. Its worker verifies the sync verbs. An omitted // declaration is NOT an identity failure: the runtime defers to the verified // baseline, so native sync stays granted while unverified capabilities stay // false. const { pluginId, environment, reusableLease } = await seedReusablePluginSandboxLease(); await db .update(plugins) .set({ manifestJson: { id: "acme.reusable-sandbox-provider", apiVersion: 1, version: "1.0.0", displayName: "Reusable Sandbox Provider", description: "Owner plugin that omits the capability declaration", author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", configSchema: { type: "object", properties: {} }, }, ], }, updatedAt: new Date(), } as any) .where(eq(plugins.id, pluginId)); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(), getWorker: vi.fn((id: string) => id === pluginId ? { supportedMethods: ["environmentSyncIn", "environmentSyncOut"] } : undefined, ), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const effective = await runtimeWithPlugin.resolveCapabilities({ environment, lease: reusableLease, }); // The worker verified the sync verbs and the omitted declaration adds no // restriction, so native sync stays granted. expect(effective?.nativeSyncIn).toBe(true); expect(effective?.nativeSyncOut).toBe(true); // The worker did not verify the reuse verbs, so reusable leases stay false. expect(effective?.reusableLeases).toBe(false); }); it("releases a sandbox run lease from metadata after the environment config changes", async () => { const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Fake Sandbox", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: true, }, }); const acquired = await runtime.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); await environmentService(db).update(environment.id, { driver: "local", config: {}, }); const released = await runtime.releaseRunLeases(runId); expect(released).toHaveLength(1); expect(released[0]?.lease.id).toBe(acquired.lease.id); expect(released[0]?.lease.status).toBe("released"); }); it("does not reuse a sandbox lease owned by a different agent for the same execution workspace", async () => { const { companyId, agentId, environment, runId } = await seedEnvironment({ driver: "plugin", name: "Plugin Fake plugin", config: { pluginKey: "acme.environments", driverKey: "fake-plugin", driverConfig: { template: "base", }, }, }); const otherAgentId = randomUUID(); const executionWorkspaceId = randomUUID(); const pluginId = randomUUID(); const projectId = randomUUID(); await db.insert(projects).values({ id: projectId, companyId, name: `Workspace ${projectId.slice(0, 8)}`, status: "active", createdAt: new Date(), updatedAt: new Date(), }); await db.insert(executionWorkspaces).values({ id: executionWorkspaceId, companyId, projectId, mode: "shared_workspace", strategyType: "project_primary", name: "Existing workspace", status: "active", providerType: "local_fs", createdAt: new Date(), updatedAt: new Date(), }); await db.insert(plugins).values({ id: pluginId, pluginKey: "acme.environments", packageName: "@acme/paperclip-environments", version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: { id: "acme.environments", apiVersion: 1, version: "1.0.0", displayName: "Acme Environments", description: "Test plugin environment driver", author: "Acme", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", configSchema: { type: "object" }, }, ], }, status: "ready", installOrder: 1, updatedAt: new Date(), } as any); await environmentService(db).acquireLease({ companyId, environmentId: environment.id, executionWorkspaceId, heartbeatRunId: runId, leasePolicy: "reuse_by_environment", provider: "fake-plugin", providerLeaseId: "other-agent-lease", metadata: { agentId, provider: "fake-plugin", template: "base", reuseLease: true, }, }); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "fresh-agent-lease", metadata: { provider: "fake-plugin", template: "base", reuseLease: true, }, }; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, agentId: otherAgentId, heartbeatRunId: runId, persistedExecutionWorkspace: { id: executionWorkspaceId, mode: "shared_workspace", }, }); expect(acquired.lease.providerLeaseId).toBe("fresh-agent-lease"); expect(workerManager.call).toHaveBeenCalledTimes(1); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", expect.objectContaining({ agentId: otherAgentId, executionWorkspaceId, })); }); it("delegates plugin environment leases through the plugin worker manager", async () => { const pluginId = randomUUID(); const expiresAt = new Date(Date.now() + 60_000).toISOString(); const workerManager = { isRunning: vi.fn(() => true), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "plugin-lease-1", expiresAt, metadata: { driver: "local", pluginId: "provider-plugin-id", pluginKey: "provider.plugin", driverKey: "provider-driver", executionWorkspaceMode: "provider-mode", provider: "test-provider", remoteCwd: "/workspace", }, }; } return undefined; }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager, }); const { companyId, environment, runId } = await seedEnvironment({ driver: "plugin", name: "Plugin Fake plugin", config: { pluginKey: "acme.environments", driverKey: "fake-plugin", driverConfig: { template: "base", }, }, }); await db.insert(plugins).values({ id: pluginId, pluginKey: "acme.environments", packageName: "@acme/paperclip-environments", version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: { id: "acme.environments", apiVersion: 1, version: "1.0.0", displayName: "Acme Environments", description: "Test plugin environment driver", author: "Acme", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "fake-plugin", displayName: "Fake plugin", configSchema: { type: "object" }, }, ], }, status: "ready", installOrder: 1, updatedAt: new Date(), } as any); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", { driverKey: "fake-plugin", companyId, environmentId: environment.id, executionWorkspaceId: undefined, executionWorkspaceSettings: null, issueId: null, config: { template: "base" }, agentId: undefined, adapterType: undefined, runId, workspaceMode: undefined, }); expect(acquired.lease.providerLeaseId).toBe("plugin-lease-1"); expect(acquired.lease.expiresAt?.toISOString()).toBe(expiresAt); expect(acquired.lease.metadata).toMatchObject({ driver: "plugin", pluginId, pluginKey: "acme.environments", driverKey: "fake-plugin", executionWorkspaceMode: null, providerMetadata: { driver: "local", pluginId: "provider-plugin-id", pluginKey: "provider.plugin", driverKey: "provider-driver", executionWorkspaceMode: "provider-mode", provider: "test-provider", remoteCwd: "/workspace", }, }); await environmentService(db).update(environment.id, { driver: "local", config: {}, }); const released = await runtimeWithPlugin.releaseRunLeases(runId); expect(released).toHaveLength(1); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentReleaseLease", { driverKey: "fake-plugin", companyId, environmentId: environment.id, issueId: null, config: {}, providerLeaseId: "plugin-lease-1", leaseMetadata: expect.objectContaining({ driver: "plugin", pluginId, providerMetadata: expect.objectContaining({ driver: "local", }), }), }); expect(released[0]?.lease.status).toBe("released"); }); async function seedPluginDriverEnvironment(pluginId: string) { const seeded = await seedEnvironment({ driver: "plugin", name: "Plugin Fake plugin", config: { pluginKey: "acme.environments", driverKey: "fake-plugin", driverConfig: { template: "base" }, }, }); await db.insert(plugins).values({ id: pluginId, pluginKey: "acme.environments", packageName: "@acme/paperclip-environments", version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: { id: "acme.environments", apiVersion: 1, version: "1.0.0", displayName: "Acme Environments", description: "Test plugin environment driver", author: "Acme", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "fake-plugin", displayName: "Fake plugin", configSchema: { type: "object" } }, ], }, status: "ready", installOrder: 1, updatedAt: new Date(), } as any); return seeded; } it("forwards a requested expiry to the acquire RPC and records only a provider-attested expiry", async () => { const pluginId = randomUUID(); let receivedRequestedExpiresAt: unknown; const workerManager = { isRunning: vi.fn(() => true), call: vi.fn(async (_pluginId: string, method: string, params: Record) => { if (method === "environmentAcquireLease") { receivedRequestedExpiresAt = params.requestedExpiresAt; // The provider grants no expiry. return { providerLeaseId: "plugin-lease-no-expiry", metadata: { remoteCwd: "/workspace" } }; } return undefined; }), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const { companyId, environment, runId } = await seedPluginDriverEnvironment(pluginId); const requestedExpiresAt = new Date(Date.now() + 60_000); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, requestedExpiresAt, }); // The acquire RPC receives the requested expiry as an ISO 8601 string. expect(receivedRequestedExpiresAt).toBe(requestedExpiresAt.toISOString()); // The provider gave no expiry, so the lease row records no expiry. The runtime // never synthesizes the requested deadline as evidence of provider enforcement. expect(acquired.lease.expiresAt ?? null).toBeNull(); }); it("records the real provider expiry when the provider grants one later than the requested deadline", async () => { const pluginId = randomUUID(); const providerExpiresAt = new Date(Date.now() + 120_000).toISOString(); const workerManager = { isRunning: vi.fn(() => true), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "plugin-lease-later", expiresAt: providerExpiresAt, metadata: { remoteCwd: "/workspace" }, }; } return undefined; }), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const { companyId, environment, runId } = await seedPluginDriverEnvironment(pluginId); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, requestedExpiresAt: new Date(Date.now() + 60_000), }); // The lease row records the real provider expiry, not the requested deadline. // The caller then verifies the expiry bounds its deadline and fails closed. expect(acquired.lease.expiresAt?.toISOString()).toBe(providerExpiresAt); }); it("delegates the full plugin environment lifecycle through the worker manager", async () => { const pluginId = randomUUID(); const workerManager = { isRunning: vi.fn(() => true), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "plugin-lease-full", metadata: { remoteCwd: "/workspace", }, }; } if (method === "environmentResumeLease") { return { providerLeaseId: "plugin-lease-full", metadata: { resumed: true, }, }; } if (method === "environmentRealizeWorkspace") { return { cwd: "/workspace/project", metadata: { realized: true, }, }; } if (method === "environmentExecute") { return { exitCode: 0, signal: null, timedOut: false, stdout: "ok\n", stderr: "", metadata: { commandId: "cmd-1", }, }; } return undefined; }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager, }); const { companyId, environment, runId } = await seedEnvironment({ driver: "plugin", name: "Plugin Full Lifecycle", config: { pluginKey: "acme.environments", driverKey: "fake-plugin", driverConfig: { template: "base", }, }, }); await db.insert(plugins).values({ id: pluginId, pluginKey: "acme.environments", packageName: "@acme/paperclip-environments", version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: { id: "acme.environments", apiVersion: 1, version: "1.0.0", displayName: "Acme Environments", description: "Test plugin environment driver", author: "Acme", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "fake-plugin", displayName: "Fake plugin", configSchema: { type: "object" }, }, ], }, status: "ready", installOrder: 1, updatedAt: new Date(), } as any); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); const resumed = await runtimeWithPlugin.resumeRunLease({ environment, lease: acquired.lease, }); const realized = await runtimeWithPlugin.realizeWorkspace({ environment, lease: acquired.lease, workspace: { localPath: "/tmp/project", mode: "ephemeral", }, }); const executed = await runtimeWithPlugin.execute({ environment, lease: acquired.lease, command: "echo", args: ["ok"], cwd: realized.cwd, env: { FOO: "bar" }, stdin: "", timeoutMs: 1000, }); const destroyed = await runtimeWithPlugin.destroyRunLease({ environment, lease: acquired.lease, }); expect(resumed).toMatchObject({ providerLeaseId: "plugin-lease-full", metadata: { resumed: true, }, }); expect(realized).toEqual({ cwd: "/workspace/project", metadata: { realized: true, }, }); expect(executed).toMatchObject({ exitCode: 0, timedOut: false, stdout: "ok\n", }); expect(destroyed?.status).toBe("failed"); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentResumeLease", { driverKey: "fake-plugin", companyId, environmentId: environment.id, issueId: null, config: { template: "base" }, providerLeaseId: "plugin-lease-full", leaseMetadata: expect.objectContaining({ driver: "plugin", pluginId, }), }); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentRealizeWorkspace", expect.objectContaining({ driverKey: "fake-plugin", companyId, environmentId: environment.id, config: { template: "base" }, workspace: { localPath: "/tmp/project", mode: "ephemeral", }, })); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentExecute", expect.objectContaining({ driverKey: "fake-plugin", companyId, environmentId: environment.id, command: "echo", args: ["ok"], cwd: "/workspace/project", env: { FOO: "bar" }, }), 31000); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentDestroyLease", { driverKey: "fake-plugin", companyId, environmentId: environment.id, issueId: null, config: { template: "base" }, providerLeaseId: "plugin-lease-full", leaseMetadata: expect.objectContaining({ driver: "plugin", pluginId, }), }); }); it("releases with the driver captured on the lease even if the environment driver changes later", async () => { const { companyId, environment, runId } = await seedEnvironment(); const environmentsSvc = environmentService(db); const localRelease = vi.fn(async ({ lease, status }: { lease: { id: string }; status: "released" | "expired" | "failed" }) => await environmentsSvc.releaseLease(lease.id, status) ); const sshRelease = vi.fn(async () => { throw new Error("ssh release should not be called"); }); const runtimeWithSpies = environmentRuntimeService(db, { drivers: [ { driver: "local", acquireRunLease: async (input) => await environmentsSvc.acquireLease({ companyId: input.companyId, environmentId: input.environment.id, executionWorkspaceId: input.executionWorkspaceId, issueId: input.issueId, heartbeatRunId: input.heartbeatRunId, metadata: { driver: input.environment.driver, executionWorkspaceMode: input.executionWorkspaceMode, }, }), releaseRunLease: localRelease, }, { driver: "ssh", acquireRunLease: async () => { throw new Error("ssh acquire should not be called"); }, releaseRunLease: sshRelease, }, ], }); const acquired = await runtimeWithSpies.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); await environmentsSvc.update(environment.id, { driver: "ssh" }); const released = await runtimeWithSpies.releaseRunLeases(runId); expect(released).toHaveLength(1); expect(localRelease).toHaveBeenCalledTimes(1); expect(sshRelease).not.toHaveBeenCalled(); expect(acquired.lease.metadata?.driver).toBe("local"); }); it("test_release_run_leases_continues_after_first_release_fails", async () => { const { companyId, environment, runId } = await seedEnvironment(); const environmentsSvc = environmentService(db); // Seed two active leases for one run. The first driver release throws; the // second must still release. const failingLease = await environmentsSvc.acquireLease({ companyId, environmentId: environment.id, heartbeatRunId: runId, provider: "local", providerLeaseId: "fail-release", metadata: { driver: "local" }, }); const healthyLease = await environmentsSvc.acquireLease({ companyId, environmentId: environment.id, heartbeatRunId: runId, provider: "local", providerLeaseId: "healthy-release", metadata: { driver: "local" }, }); const runtimeWithFailingDriver = environmentRuntimeService(db, { drivers: [ { driver: "local", acquireRunLease: async () => { throw new Error("acquire should not be called"); }, releaseRunLease: async ({ lease, status }) => { if (lease.providerLeaseId === "fail-release") { throw new Error("driver release failed"); } return await environmentsSvc.releaseLease(lease.id, status); }, }, ], }); const errors: Array<{ leaseId: string; error: unknown }> = []; const released = await runtimeWithFailingDriver.releaseRunLeases( runId, "released", (leaseId, error) => errors.push({ leaseId, error }), ); // The healthy lease released even though the first release failed. expect(released).toHaveLength(1); expect(released[0]?.lease.id).toBe(healthyLease.id); expect(released[0]?.lease.status).toBe("released"); // The failed release reported one lease-specific error. expect(errors).toHaveLength(1); expect(errors[0]?.leaseId).toBe(failingLease.id); expect((errors[0]?.error as Error).message).toBe("driver release failed"); // The database confirms the isolation. The healthy lease released; the // failing lease stayed active. const rows = await db.select().from(environmentLeases); expect(rows.find((row) => row.id === healthyLease.id)?.status).toBe("released"); expect(rows.find((row) => row.id === failingLease.id)?.status).toBe("active"); }); });