diff --git a/packages/plugins/sandbox-providers/daytona/src/manifest.ts b/packages/plugins/sandbox-providers/daytona/src/manifest.ts index dcc684342e..c78ef2c00f 100644 --- a/packages/plugins/sandbox-providers/daytona/src/manifest.ts +++ b/packages/plugins/sandbox-providers/daytona/src/manifest.ts @@ -23,6 +23,7 @@ const manifest: PaperclipPluginManifestV1 = { displayName: "Daytona Sandbox", description: "Provisions Daytona sandboxes with configurable image or snapshot selection, startup timeouts, and lease reuse.", + supportsReusableLeases: true, configSchema: { type: "object", properties: { diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts index a9f0022989..1620ca96f3 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts @@ -174,6 +174,9 @@ describe("Daytona sandbox provider plugin", () => { companyId: "company-1", environmentId: "env-1", runId: "run-1", + agentId: "agent-1", + executionWorkspaceId: "workspace-1", + adapterType: "codex_local", config: { image: "node:20", timeoutMs: 300000, @@ -189,8 +192,21 @@ describe("Daytona sandbox provider plugin", () => { sandboxId: "sandbox-123", remoteCwd: "/home/daytona/paperclip-workspace", reuseLease: true, + workspaceSentinel: { + path: "/home/daytona/paperclip-workspace/.paperclip-runtime/reusable-sandbox-lease.json", + result: "written", + }, }, }); + expect(sandbox.fs.createFolder).toHaveBeenCalledWith( + "/home/daytona/paperclip-workspace/.paperclip-runtime", + "755", + ); + expect(sandbox.fs.uploadFile).toHaveBeenCalledWith( + expect.any(Buffer), + "/home/daytona/paperclip-workspace/.paperclip-runtime/reusable-sandbox-lease.json", + 300, + ); }); it("deletes the sandbox if lease setup throws after sandbox creation", async () => { @@ -287,6 +303,94 @@ describe("Daytona sandbox provider plugin", () => { }); }); + it("resumes a reusable lease when the workspace sentinel matches", async () => { + process.env.DAYTONA_API_KEY = "host-key"; + const sandbox = createMockSandbox({ id: "sandbox-reuse", state: "stopped" }); + sandbox.process.executeCommand + .mockResolvedValueOnce({ + exitCode: 0, + result: JSON.stringify({ token: "sentinel-token" }), + artifacts: { stdout: JSON.stringify({ token: "sentinel-token" }) }, + }) + .mockResolvedValueOnce({ + exitCode: 0, + result: "bash", + artifacts: { stdout: "bash" }, + }); + mockGet.mockResolvedValue(sandbox); + + const lease = await plugin.definition.onEnvironmentResumeLease?.({ + driverKey: "daytona", + companyId: "company-1", + environmentId: "env-1", + providerLeaseId: "sandbox-reuse", + config: { + timeoutMs: 300000, + reuseLease: true, + }, + leaseMetadata: { + workspaceSentinel: { + path: "/home/daytona/paperclip-workspace/.paperclip-runtime/reusable-sandbox-lease.json", + token: "sentinel-token", + result: "written", + }, + }, + }); + + expect(sandbox.start).toHaveBeenCalledWith(300); + expect(lease).toMatchObject({ + providerLeaseId: "sandbox-reuse", + metadata: { + resumedLease: true, + workspaceSentinel: { + result: "matched", + token: "sentinel-token", + }, + }, + }); + }); + + it("expires a reusable lease when the workspace sentinel does not match", async () => { + process.env.DAYTONA_API_KEY = "host-key"; + const sandbox = createMockSandbox({ id: "sandbox-reuse", state: "stopped" }); + sandbox.process.executeCommand.mockResolvedValueOnce({ + exitCode: 0, + result: JSON.stringify({ token: "other-token" }), + artifacts: { stdout: JSON.stringify({ token: "other-token" }) }, + }); + mockGet.mockResolvedValue(sandbox); + + await expect(plugin.definition.onEnvironmentResumeLease?.({ + driverKey: "daytona", + companyId: "company-1", + environmentId: "env-1", + providerLeaseId: "sandbox-reuse", + config: { + timeoutMs: 300000, + reuseLease: true, + }, + leaseMetadata: { + workspaceSentinel: { + path: "/home/daytona/paperclip-workspace/.paperclip-runtime/reusable-sandbox-lease.json", + token: "sentinel-token", + result: "written", + }, + }, + })).resolves.toEqual({ + providerLeaseId: null, + metadata: { + expired: true, + workspaceSentinel: { + path: "/home/daytona/paperclip-workspace/.paperclip-runtime/reusable-sandbox-lease.json", + token: "sentinel-token", + result: "mismatch", + }, + }, + }); + + expect(sandbox.process.executeCommand).toHaveBeenCalledTimes(1); + }); + it("stops reusable leases and deletes ephemeral leases on release", async () => { process.env.DAYTONA_API_KEY = "host-key"; const reusable = createMockSandbox({ id: "sandbox-reusable" }); diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.ts index 2ea53ba648..cc5b294c1e 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.ts @@ -1,5 +1,5 @@ import path from "node:path"; -import { randomUUID } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { Daytona, DaytonaNotFoundError, DaytonaTimeoutError } from "@daytonaio/sdk"; import type { CreateSandboxBaseParams, @@ -44,6 +44,14 @@ interface DaytonaDriverConfig { reuseLease: boolean; } +type WorkspaceSentinelResult = { + path: string; + token: string | null; + result: "written" | "matched" | "missing" | "mismatch" | "skipped"; +}; + +const WORKSPACE_SENTINEL_RELATIVE_PATH = ".paperclip-runtime/reusable-sandbox-lease.json"; + function parseOptionalString(value: unknown): string | null { return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; } @@ -166,6 +174,20 @@ function formatErrorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function stableStringify(value: unknown): string { + if (Array.isArray(value)) { + return `[${value.map((entry) => stableStringify(entry)).join(",")}]`; + } + if (isRecord(value)) { + return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableStringify(value[key])}`).join(",")}}`; + } + return JSON.stringify(value) ?? "null"; +} + function isValidUrl(value: string): boolean { try { new URL(value); @@ -210,12 +232,109 @@ async function detectSandboxShellCommand(sandbox: Sandbox, timeoutSeconds: numbe } } +function workspaceSentinelToken(input: { + params: Pick; + config: DaytonaDriverConfig; +}): string | null { + if (!input.config.reuseLease || !input.params.agentId || !input.params.executionWorkspaceId) { + return null; + } + return createHash("sha256") + .update(stableStringify({ + provider: "daytona", + companyId: input.params.companyId, + environmentId: input.params.environmentId, + agentId: input.params.agentId, + executionWorkspaceId: input.params.executionWorkspaceId, + adapterType: input.params.adapterType ?? null, + image: input.config.image, + snapshot: input.config.snapshot, + target: input.config.target, + })) + .digest("hex"); +} + +function workspaceSentinelPath(remoteCwd: string): string { + return path.posix.join(remoteCwd, WORKSPACE_SENTINEL_RELATIVE_PATH); +} + +async function writeWorkspaceSentinel(input: { + sandbox: Sandbox; + remoteCwd: string; + params: PluginEnvironmentAcquireLeaseParams; + config: DaytonaDriverConfig; + timeoutSeconds: number; +}): Promise { + const sentinelPath = workspaceSentinelPath(input.remoteCwd); + const token = workspaceSentinelToken({ params: input.params, config: input.config }); + if (!token) { + return { path: sentinelPath, token: null, result: "skipped" }; + } + await input.sandbox.fs.createFolder(path.posix.dirname(sentinelPath), "755"); + await input.sandbox.fs.uploadFile( + Buffer.from(JSON.stringify({ + version: 1, + token, + companyId: input.params.companyId, + environmentId: input.params.environmentId, + agentId: input.params.agentId, + executionWorkspaceId: input.params.executionWorkspaceId, + adapterType: input.params.adapterType ?? null, + provider: "daytona", + writtenAt: new Date().toISOString(), + }, null, 2), "utf8"), + sentinelPath, + input.timeoutSeconds, + ); + return { path: sentinelPath, token, result: "written" }; +} + +async function verifyWorkspaceSentinel(input: { + sandbox: Sandbox; + remoteCwd: string; + leaseMetadata?: Record; + timeoutSeconds: number; +}): Promise { + const metadataSentinel = isRecord(input.leaseMetadata?.workspaceSentinel) + ? input.leaseMetadata.workspaceSentinel + : null; + const sentinelPath = typeof metadataSentinel?.path === "string" + ? metadataSentinel.path + : workspaceSentinelPath(input.remoteCwd); + const expectedToken = typeof metadataSentinel?.token === "string" ? metadataSentinel.token : null; + if (!expectedToken) { + return { path: sentinelPath, token: null, result: "missing" }; + } + + const result = await input.sandbox.process.executeCommand( + `cat ${shellQuote(sentinelPath)}`, + undefined, + undefined, + input.timeoutSeconds, + ); + if (result.exitCode !== 0) { + return { path: sentinelPath, token: expectedToken, result: "missing" }; + } + try { + const parsed = JSON.parse(result.result ?? result.artifacts?.stdout ?? "") as unknown; + const actualToken = isRecord(parsed) && typeof parsed.token === "string" ? parsed.token : null; + return { + path: sentinelPath, + token: expectedToken, + result: actualToken === expectedToken ? "matched" : "mismatch", + }; + } catch { + return { path: sentinelPath, token: expectedToken, result: "mismatch" }; + } +} + function leaseMetadata(input: { config: DaytonaDriverConfig; sandbox: Sandbox; shellCommand: "bash" | "sh"; remoteCwd: string; resumedLease: boolean; + workspaceSentinel?: WorkspaceSentinelResult; }) { return { provider: "daytona", @@ -230,6 +349,7 @@ function leaseMetadata(input: { reuseLease: input.config.reuseLease, remoteCwd: input.remoteCwd, resumedLease: input.resumedLease, + ...(input.workspaceSentinel ? { workspaceSentinel: input.workspaceSentinel } : {}), }; } @@ -499,9 +619,16 @@ const plugin = definePlugin({ try { const remoteCwd = await resolveSandboxWorkingDirectory(sandbox); const shellCommand = await detectSandboxShellCommand(sandbox, toTimeoutSeconds(config.timeoutMs)); + const workspaceSentinel = await writeWorkspaceSentinel({ + sandbox, + remoteCwd, + params, + config, + timeoutSeconds: toTimeoutSeconds(config.timeoutMs), + }); return { providerLeaseId: sandbox.id, - metadata: leaseMetadata({ config, sandbox, shellCommand, remoteCwd, resumedLease: false }), + metadata: leaseMetadata({ config, sandbox, shellCommand, remoteCwd, resumedLease: false, workspaceSentinel }), }; } catch (error) { await sandbox.delete(toTimeoutSeconds(config.timeoutMs)).catch(() => undefined); @@ -521,10 +648,19 @@ const plugin = definePlugin({ await ensureSandboxStarted(sandbox, toTimeoutSeconds(config.timeoutMs)); try { const remoteCwd = await resolveSandboxWorkingDirectory(sandbox); + const workspaceSentinel = await verifyWorkspaceSentinel({ + sandbox, + remoteCwd, + leaseMetadata: params.leaseMetadata, + timeoutSeconds: toTimeoutSeconds(config.timeoutMs), + }); + if (workspaceSentinel.result !== "matched") { + return { providerLeaseId: null, metadata: { expired: true, workspaceSentinel } }; + } const shellCommand = await detectSandboxShellCommand(sandbox, toTimeoutSeconds(config.timeoutMs)); return { providerLeaseId: sandbox.id, - metadata: leaseMetadata({ config, sandbox, shellCommand, remoteCwd, resumedLease: true }), + metadata: leaseMetadata({ config, sandbox, shellCommand, remoteCwd, resumedLease: true, workspaceSentinel }), }; } catch (error) { await sandbox.delete(toTimeoutSeconds(config.timeoutMs)).catch(() => undefined); diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.ts index 5ec4681888..954fa76a9d 100644 --- a/packages/shared/src/constants.ts +++ b/packages/shared/src/constants.ts @@ -421,7 +421,7 @@ export type EnvironmentDriver = (typeof ENVIRONMENT_DRIVERS)[number]; export const ENVIRONMENT_STATUSES = ["active", "archived"] as const; export type EnvironmentStatus = (typeof ENVIRONMENT_STATUSES)[number]; -export const ENVIRONMENT_LEASE_STATUSES = ["active", "released", "expired", "failed", "retained"] as const; +export const ENVIRONMENT_LEASE_STATUSES = ["active", "released", "expired", "failed", "retained", "pending_cleanup"] as const; export type EnvironmentLeaseStatus = (typeof ENVIRONMENT_LEASE_STATUSES)[number]; export const ENVIRONMENT_LEASE_POLICIES = [ diff --git a/packages/shared/src/types/plugin.ts b/packages/shared/src/types/plugin.ts index 351b2dda0e..366a60ba63 100644 --- a/packages/shared/src/types/plugin.ts +++ b/packages/shared/src/types/plugin.ts @@ -138,6 +138,12 @@ export interface PluginEnvironmentDriverDeclaration { displayName: string; /** Optional description for operator-facing docs or UI affordances. */ description?: string; + /** + * Sandbox providers must opt in before the host retains and resumes provider + * leases across runs. Providers without this flag keep per-run acquire/release + * behavior even if their config schema exposes a reuse-like setting. + */ + supportsReusableLeases?: boolean; /** JSON Schema describing the driver's provider-specific configuration. */ configSchema: JsonSchema; } diff --git a/packages/shared/src/validators/plugin.ts b/packages/shared/src/validators/plugin.ts index 0a857b44d8..45a1e3a845 100644 --- a/packages/shared/src/validators/plugin.ts +++ b/packages/shared/src/validators/plugin.ts @@ -124,6 +124,7 @@ export const pluginEnvironmentDriverDeclarationSchema = z.object({ kind: z.enum(["environment_driver", "sandbox_provider"]).optional(), displayName: z.string().min(1).max(100), description: z.string().max(500).optional(), + supportsReusableLeases: z.boolean().optional(), configSchema: jsonSchemaSchema, }); diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index e7ff1f9c98..4ec89cbe1c 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -1,4 +1,4 @@ -import { randomUUID } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { mkdtemp, rm } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; @@ -42,6 +42,25 @@ if (!embeddedPostgresSupport.supported) { ); } +function stableStringify(value: unknown): string { + if (Array.isArray(value)) { + return `[${value.map((entry) => stableStringify(entry)).join(",")}]`; + } + if (value && typeof value === "object") { + const record = value as Record; + return `{${Object.keys(record).sort().map((key) => `${JSON.stringify(key)}:${stableStringify(record[key])}`).join(",")}}`; + } + return JSON.stringify(value) ?? "null"; +} + +function reusableRuntimeFingerprint(input: { + provider: string; + adapterType: string | null; + config: Record; +}) { + return createHash("sha256").update(stableStringify(input)).digest("hex"); +} + describe("findReusableSandboxLeaseId", () => { it("matches reusable plugin-backed sandbox leases by provider", () => { const selected = findReusableSandboxLeaseId({ @@ -258,6 +277,124 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { }; } + async function seedReusablePluginSandboxLease() { + const pluginId = randomUUID(); + const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment(); + const providerConfig = { + provider: "fake-plugin", + image: "fake:test", + timeoutMs: 1234, + reuseLease: true, + }; + const environment = { + ...baseEnvironment, + name: "Reusable Plugin Sandbox", + driver: "sandbox", + config: providerConfig, + }; + await environmentService(db).update(environment.id, { + driver: "sandbox", + name: environment.name, + config: providerConfig, + }); + await db.insert(plugins).values({ + id: pluginId, + pluginKey: "acme.reusable-sandbox-provider", + packageName: "@acme/reusable-sandbox-provider", + version: "1.0.0", + apiVersion: 1, + categories: ["automation"], + manifestJson: { + id: "acme.reusable-sandbox-provider", + apiVersion: 1, + version: "1.0.0", + displayName: "Reusable Sandbox Provider", + description: "Test provider with reusable lease support", + author: "Paperclip", + categories: ["automation"], + capabilities: ["environment.drivers.register"], + entrypoints: { worker: "dist/worker.js" }, + environmentDrivers: [ + { + driverKey: "fake-plugin", + kind: "sandbox_provider", + displayName: "Fake Plugin", + supportsReusableLeases: true, + configSchema: { + type: "object", + properties: { + image: { type: "string" }, + timeoutMs: { type: "number" }, + reuseLease: { type: "boolean" }, + }, + }, + }, + ], + }, + status: "ready", + installOrder: 1, + updatedAt: new Date(), + } as any); + const executionWorkspaceId = randomUUID(); + const projectId = randomUUID(); + await db.insert(projects).values({ + id: projectId, + companyId, + name: `Workspace ${projectId.slice(0, 8)}`, + status: "active", + createdAt: new Date(), + updatedAt: new Date(), + }); + await db.insert(executionWorkspaces).values({ + id: executionWorkspaceId, + companyId, + projectId, + mode: "shared_workspace", + strategyType: "project_primary", + name: "Reusable workspace", + status: "active", + providerType: "local_fs", + createdAt: new Date(), + updatedAt: new Date(), + }); + const reusableLease = await environmentService(db).acquireLease({ + companyId, + environmentId: environment.id, + executionWorkspaceId, + heartbeatRunId: runId, + leasePolicy: "reuse_by_environment", + provider: "fake-plugin", + providerLeaseId: "reusable-plugin-lease", + metadata: { + agentId, + driver: "sandbox", + pluginId, + pluginKey: "acme.reusable-sandbox-provider", + sandboxProviderPlugin: true, + provider: "fake-plugin", + image: "fake:test", + timeoutMs: 1234, + reuseLease: true, + reusableSandboxLease: { + version: 1, + companyId, + environmentId: environment.id, + executionWorkspaceId, + agentId, + adapterType: null, + provider: "fake-plugin", + runtimeFingerprint: reusableRuntimeFingerprint({ + provider: "fake-plugin", + adapterType: null, + config: providerConfig, + }), + }, + }, + }); + + return { pluginId, companyId, executionWorkspaceId, reusableLease }; + } + it("acquires and releases a local run lease through the runtime seam", async () => { const { companyId, environment, runId } = await seedEnvironment(); @@ -415,7 +552,8 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { }); expect(acquired.lease.status).toBe("active"); - expect(acquired.lease.providerLeaseId).toBe(`sandbox://fake/${environment.id}/workspace/agent`); + expect(acquired.lease.providerLeaseId).toMatch(new RegExp(`^sandbox://fake/${runId}/[0-9a-f-]{36}$`)); + expect(acquired.lease.leasePolicy).toBe("ephemeral"); expect(acquired.lease.metadata).toMatchObject({ driver: "sandbox", provider: "fake", @@ -942,6 +1080,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", + supportsReusableLeases: true, configSchema: { type: "object", properties: { @@ -979,7 +1118,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { createdAt: new Date(), updatedAt: new Date(), }); - await environmentService(db).acquireLease({ + const staleLease = await environmentService(db).acquireLease({ companyId, environmentId: environment.id, executionWorkspaceId, @@ -989,10 +1128,28 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { providerLeaseId: "stale-plugin-lease", metadata: { agentId, + driver: "sandbox", + pluginId, + pluginKey: "acme.fake-sandbox-provider", + sandboxProviderPlugin: true, provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, + reusableSandboxLease: { + version: 1, + companyId, + environmentId: environment.id, + executionWorkspaceId, + agentId, + adapterType: null, + provider: "fake-plugin", + runtimeFingerprint: reusableRuntimeFingerprint({ + provider: "fake-plugin", + adapterType: null, + config: providerConfig, + }), + }, }, }); @@ -1002,6 +1159,9 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { if (method === "environmentResumeLease") { throw new Error("stale sandbox"); } + if (method === "environmentDestroyLease") { + return undefined; + } if (method === "environmentAcquireLease") { return { providerLeaseId: "fresh-plugin-lease", @@ -1036,7 +1196,11 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { driverKey: "fake-plugin", providerLeaseId: "stale-plugin-lease", }), 31234); - expect(workerManager.call).toHaveBeenNthCalledWith(2, pluginId, "environmentAcquireLease", expect.objectContaining({ + expect(workerManager.call).toHaveBeenNthCalledWith(2, pluginId, "environmentDestroyLease", expect.objectContaining({ + driverKey: "fake-plugin", + providerLeaseId: "stale-plugin-lease", + }), 31234); + expect(workerManager.call).toHaveBeenNthCalledWith(3, pluginId, "environmentAcquireLease", expect.objectContaining({ driverKey: "fake-plugin", config: { image: "fake:test", @@ -1047,6 +1211,257 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { executionWorkspaceId, runId, }), 31234); + await expect(environmentService(db).getLeaseById(staleLease.id)).resolves.toMatchObject({ + status: "expired", + cleanupStatus: "success", + }); + }); + + it("does not retain or resume plugin-backed sandbox leases unless the provider opts in", async () => { + const pluginId = randomUUID(); + const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment(); + const providerConfig = { + provider: "fake-plugin", + image: "fake:test", + timeoutMs: 1234, + reuseLease: true, + }; + const environment = { + ...baseEnvironment, + name: "Non-reusable Plugin Sandbox", + driver: "sandbox", + config: providerConfig, + }; + await environmentService(db).update(environment.id, { + driver: "sandbox", + name: environment.name, + config: providerConfig, + }); + await db.insert(plugins).values({ + id: pluginId, + pluginKey: "acme.non-reusable-sandbox-provider", + packageName: "@acme/non-reusable-sandbox-provider", + version: "1.0.0", + apiVersion: 1, + categories: ["automation"], + manifestJson: { + id: "acme.non-reusable-sandbox-provider", + apiVersion: 1, + version: "1.0.0", + displayName: "Non-reusable Sandbox Provider", + description: "Test provider without reusable lease support", + author: "Paperclip", + categories: ["automation"], + capabilities: ["environment.drivers.register"], + entrypoints: { worker: "dist/worker.js" }, + environmentDrivers: [ + { + driverKey: "fake-plugin", + kind: "sandbox_provider", + displayName: "Fake Plugin", + configSchema: { type: "object" }, + }, + ], + }, + status: "ready", + installOrder: 1, + updatedAt: new Date(), + } as any); + const executionWorkspaceId = randomUUID(); + const projectId = randomUUID(); + await db.insert(projects).values({ + id: projectId, + companyId, + name: `Workspace ${projectId.slice(0, 8)}`, + status: "active", + createdAt: new Date(), + updatedAt: new Date(), + }); + await db.insert(executionWorkspaces).values({ + id: executionWorkspaceId, + companyId, + projectId, + mode: "shared_workspace", + strategyType: "project_primary", + name: "Non-reusable workspace", + status: "active", + providerType: "local_fs", + createdAt: new Date(), + updatedAt: new Date(), + }); + await environmentService(db).acquireLease({ + companyId, + environmentId: environment.id, + executionWorkspaceId, + heartbeatRunId: runId, + leasePolicy: "reuse_by_environment", + provider: "fake-plugin", + providerLeaseId: "old-plugin-lease", + metadata: { + agentId, + provider: "fake-plugin", + image: "fake:test", + timeoutMs: 1234, + reuseLease: true, + reusableSandboxLease: { + version: 1, + companyId, + environmentId: environment.id, + executionWorkspaceId, + agentId, + adapterType: null, + provider: "fake-plugin", + runtimeFingerprint: reusableRuntimeFingerprint({ + provider: "fake-plugin", + adapterType: null, + config: providerConfig, + }), + }, + }, + }); + + const workerManager = { + isRunning: vi.fn((id: string) => id === pluginId), + call: vi.fn(async (_pluginId: string, method: string) => { + if (method === "environmentAcquireLease") { + return { + providerLeaseId: "fresh-plugin-lease", + metadata: { + provider: "fake-plugin", + image: "fake:test", + timeoutMs: 1234, + reuseLease: true, + remoteCwd: "/workspace", + }, + }; + } + throw new Error(`Unexpected plugin method: ${method}`); + }), + } as unknown as PluginWorkerManager; + const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); + + const acquired = await runtimeWithPlugin.acquireRunLease({ + companyId, + environment, + issueId: null, + agentId, + heartbeatRunId: runId, + persistedExecutionWorkspace: { + id: executionWorkspaceId, + mode: "shared_workspace", + }, + }); + + expect(acquired.lease.providerLeaseId).toBe("fresh-plugin-lease"); + expect(acquired.lease.leasePolicy).toBe("ephemeral"); + expect(workerManager.call).toHaveBeenCalledTimes(1); + expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", expect.anything(), 31234); + }); + + it("destroys scoped reusable plugin-backed sandbox leases", async () => { + const { pluginId, companyId, executionWorkspaceId, reusableLease } = + await seedReusablePluginSandboxLease(); + + const workerManager = { + isRunning: vi.fn((id: string) => id === pluginId), + call: vi.fn(async (_pluginId: string, method: string) => { + if (method === "environmentDestroyLease") { + return undefined; + } + throw new Error(`Unexpected plugin method: ${method}`); + }), + } as unknown as PluginWorkerManager; + const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); + + const destroyed = await runtimeWithPlugin.destroyReusableSandboxLeases({ + companyId, + executionWorkspaceId, + failureReason: "execution_workspace_closed", + }); + + expect(destroyed).toHaveLength(1); + expect(destroyed[0]?.lease.id).toBe(reusableLease.id); + expect(destroyed[0]?.lease.status).toBe("expired"); + expect(workerManager.call).toHaveBeenCalledWith( + pluginId, + "environmentDestroyLease", + expect.objectContaining({ + driverKey: "fake-plugin", + providerLeaseId: "reusable-plugin-lease", + }), + 31234, + ); + await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({ + status: "expired", + failureReason: "execution_workspace_closed", + cleanupStatus: "success", + }); + }); + + it("retries reusable plugin-backed sandbox destroy when the worker is unavailable", async () => { + const { pluginId, companyId, executionWorkspaceId, reusableLease } = + await seedReusablePluginSandboxLease(); + + const offlineWorkerManager = { + isRunning: vi.fn(() => false), + call: vi.fn(), + } as unknown as PluginWorkerManager; + const runtimeWithOfflinePlugin = environmentRuntimeService(db, { + pluginWorkerManager: offlineWorkerManager, + }); + + const pending = await runtimeWithOfflinePlugin.destroyReusableSandboxLeases({ + companyId, + executionWorkspaceId, + failureReason: "execution_workspace_closed", + }); + + expect(pending).toHaveLength(1); + expect(pending[0]?.lease.id).toBe(reusableLease.id); + expect(pending[0]?.lease.status).toBe("pending_cleanup"); + expect(offlineWorkerManager.call).not.toHaveBeenCalled(); + await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({ + status: "pending_cleanup", + failureReason: "execution_workspace_closed", + cleanupStatus: "failed", + }); + + const recoveredWorkerManager = { + isRunning: vi.fn((id: string) => id === pluginId), + call: vi.fn(async (_pluginId: string, method: string) => { + if (method === "environmentDestroyLease") { + return undefined; + } + throw new Error(`Unexpected plugin method: ${method}`); + }), + } as unknown as PluginWorkerManager; + const runtimeWithRecoveredPlugin = environmentRuntimeService(db, { + pluginWorkerManager: recoveredWorkerManager, + }); + + const retried = await runtimeWithRecoveredPlugin.destroyReusableSandboxLeases({ + companyId, + executionWorkspaceId, + failureReason: "cleanup_retry", + }); + + expect(retried).toHaveLength(1); + expect(retried[0]?.lease.id).toBe(reusableLease.id); + expect(retried[0]?.lease.status).toBe("expired"); + expect(recoveredWorkerManager.call).toHaveBeenCalledWith( + pluginId, + "environmentDestroyLease", + expect.objectContaining({ + driverKey: "fake-plugin", + providerLeaseId: "reusable-plugin-lease", + }), + 31234, + ); + await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({ + status: "expired", + failureReason: "cleanup_retry", + cleanupStatus: "success", + }); }); it("releases a sandbox run lease from metadata after the environment config changes", async () => { diff --git a/server/src/__tests__/sandbox-provider-runtime.test.ts b/server/src/__tests__/sandbox-provider-runtime.test.ts index 191975230e..3552f395f7 100644 --- a/server/src/__tests__/sandbox-provider-runtime.test.ts +++ b/server/src/__tests__/sandbox-provider-runtime.test.ts @@ -44,7 +44,7 @@ describe("sandbox provider runtime", () => { })).rejects.toThrow('Sandbox provider "fake-plugin" is not registered as a built-in provider.'); }); - it("acquires and resumes fake leases deterministically", async () => { + it("does not resume fake leases because the built-in fake provider does not opt in", async () => { const lease = await acquireSandboxProviderLease({ config: { provider: "fake", @@ -56,7 +56,7 @@ describe("sandbox provider runtime", () => { issueId: "issue-1", }); - expect(lease.providerLeaseId).toBe("sandbox://fake/env-1/workspace/agent"); + expect(lease.providerLeaseId).toMatch(/^sandbox:\/\/fake\/run-1\/[0-9a-f-]+$/); expect(lease.metadata).toEqual(expect.objectContaining({ provider: "fake", image: "ubuntu:24.04", @@ -75,8 +75,9 @@ describe("sandbox provider runtime", () => { reusableProviderLeaseId: lease.providerLeaseId, }); - expect(resumed.providerLeaseId).toBe(lease.providerLeaseId); - expect(resumed.metadata).toEqual(expect.objectContaining({ resumedLease: true })); + expect(resumed.providerLeaseId).toMatch(/^sandbox:\/\/fake\/run-2\/[0-9a-f-]+$/); + expect(resumed.providerLeaseId).not.toBe(lease.providerLeaseId); + expect(resumed.metadata).not.toEqual(expect.objectContaining({ resumedLease: true })); }); it("matches reusable fake leases through the selected provider implementation", () => { diff --git a/server/src/app.ts b/server/src/app.ts index 6658ecfa2b..efd9a74c9a 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -233,7 +233,7 @@ export async function createApp( api.use(fileResourceRoutes(db)); api.use(routineRoutes(db, { pluginWorkerManager: workerManager })); api.use(environmentRoutes(db, { pluginWorkerManager: workerManager })); - api.use(executionWorkspaceRoutes(db)); + api.use(executionWorkspaceRoutes(db, { pluginWorkerManager: workerManager })); api.use(goalRoutes(db)); api.use(boardChatRoutes(db, { deploymentMode: opts.deploymentMode })); api.use(approvalRoutes(db, { pluginWorkerManager: workerManager })); diff --git a/server/src/routes/execution-workspaces.ts b/server/src/routes/execution-workspaces.ts index bbe80687cb..bf79b44455 100644 --- a/server/src/routes/execution-workspaces.ts +++ b/server/src/routes/execution-workspaces.ts @@ -30,14 +30,19 @@ import { } from "./workspace-command-authz.js"; import { assertCanManageExecutionWorkspaceRuntimeServices } from "./workspace-runtime-service-authz.js"; import { appendWithCap } from "../adapters/utils.js"; +import { environmentRuntimeService } from "../services/environment-runtime.js"; +import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; const WORKSPACE_CONTROL_OUTPUT_MAX_CHARS = 256 * 1024; -export function executionWorkspaceRoutes(db: Db) { +export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: PluginWorkerManager } = {}) { const router = Router(); const svc = executionWorkspaceService(db); const access = accessService(db); const workspaceOperationsSvc = workspaceOperationService(db); + const environmentRuntime = environmentRuntimeService(db, { + pluginWorkerManager: opts.pluginWorkerManager, + }); async function assertExecutionWorkspaceReadAllowed(req: Request, res: Response, companyId: string) { const decision = await access.decide({ @@ -538,6 +543,12 @@ export function executionWorkspaceRoutes(db: Db) { } workspace = archivedWorkspace; + await environmentRuntime.destroyReusableSandboxLeases({ + companyId: existing.companyId, + executionWorkspaceId: existing.id, + failureReason: "execution_workspace_closed", + }); + if (existing.mode === "shared_workspace") { await db .update(issues) diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index fbce62ccd7..21b7d01932 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -121,6 +121,7 @@ import { feedbackService } from "../services/feedback.js"; import { instanceSettingsService } from "../services/instance-settings.js"; import { readAcceptedPlanConfirmationTarget } from "../services/issues.js"; import { environmentService } from "../services/environments.js"; +import { environmentRuntimeService } from "../services/environment-runtime.js"; import { redactSensitiveText } from "../redaction.js"; import { createCompanySearchRateLimiter, @@ -1107,6 +1108,9 @@ export function issueRoutes( const routinesSvc = routineService(db, { pluginWorkerManager: opts.pluginWorkerManager, }); + const environmentRuntime = environmentRuntimeService(db, { + pluginWorkerManager: opts.pluginWorkerManager, + }); const issueTreeControlFactory = Object.prototype.hasOwnProperty.call( serviceIndex, "issueTreeControlService", @@ -2793,6 +2797,27 @@ export function issueRoutes( }); } + async function destroyReusableSandboxLeasesForTerminalIssue(issue: { + id: string; + companyId: string; + status: string; + executionWorkspaceId?: string | null; + }) { + try { + await environmentRuntime.destroyReusableSandboxLeases({ + companyId: issue.companyId, + issueId: issue.id, + executionWorkspaceId: issue.executionWorkspaceId ?? null, + failureReason: `issue_terminal_${issue.status}`, + }); + } catch (err) { + logger.warn( + { err, issueId: issue.id, executionWorkspaceId: issue.executionWorkspaceId ?? null }, + "failed to destroy reusable sandbox leases for terminal issue", + ); + } + } + async function resolveIssueRouteId(rawId: string): Promise { const identifier = normalizeIssueReferenceIdentifier(rawId); if (identifier) { @@ -6492,6 +6517,9 @@ export function issueRoutes( const becameTerminal = !["done", "cancelled"].includes(existing.status) && ["done", "cancelled"].includes(issue.status); + if (becameTerminal) { + await destroyReusableSandboxLeasesForTerminalIssue(issue); + } if (becameTerminal && issue.parentId) { const parent = await svc.getWakeableParentAfterChildCompletion(issue.parentId); if (parent) { @@ -7883,6 +7911,9 @@ export function issueRoutes( const becameTerminal = !["done", "cancelled"].includes(issueBeforeCommentDecision.status) && ["done", "cancelled"].includes(currentIssue.status); + if (becameTerminal) { + await destroyReusableSandboxLeasesForTerminalIssue(currentIssue); + } if (becameTerminal && currentIssue.parentId) { const parent = await svc.getWakeableParentAfterChildCompletion(currentIssue.parentId); if (parent) { diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index 496c85c20b..c5323085e5 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -1,4 +1,4 @@ -import { randomUUID } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { and, eq, inArray } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { environmentLeases } from "@paperclipai/db"; @@ -24,7 +24,9 @@ import { } from "./environment-config.js"; import { acquireSandboxProviderLease, + destroySandboxProviderLease, findReusableSandboxProviderLeaseId, + getSandboxProvider as getBuiltinSandboxProvider, isBuiltinSandboxProvider, releaseSandboxProviderLease, sandboxConfigFromLeaseMetadata, @@ -149,6 +151,7 @@ function resolvePluginSandboxRpcTimeoutMs(config: Record): numb export interface EnvironmentDriverLeaseInput { environment: Environment; lease: EnvironmentLease; + failureReason?: string; } export interface EnvironmentDriverRealizeWorkspaceInput extends EnvironmentDriverLeaseInput { @@ -197,6 +200,118 @@ function getLeaseDriverKey(lease: Pick, environmen return leaseDriver ?? environment.driver; } +function toEnvironmentLeaseSnapshot(row: typeof environmentLeases.$inferSelect): EnvironmentLease { + return { + id: row.id, + companyId: row.companyId, + environmentId: row.environmentId, + executionWorkspaceId: row.executionWorkspaceId ?? null, + issueId: row.issueId ?? null, + heartbeatRunId: row.heartbeatRunId ?? null, + status: row.status as EnvironmentLease["status"], + leasePolicy: row.leasePolicy as EnvironmentLease["leasePolicy"], + provider: row.provider ?? null, + providerLeaseId: row.providerLeaseId ?? null, + acquiredAt: row.acquiredAt, + lastUsedAt: row.lastUsedAt, + expiresAt: row.expiresAt ?? null, + releasedAt: row.releasedAt ?? null, + failureReason: row.failureReason ?? null, + cleanupStatus: row.cleanupStatus as EnvironmentLease["cleanupStatus"], + metadata: (row.metadata as Record | null) ?? null, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + }; +} + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function stableStringify(value: unknown): string { + if (Array.isArray(value)) { + return `[${value.map((entry) => stableStringify(entry)).join(",")}]`; + } + if (isRecord(value)) { + return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableStringify(value[key])}`).join(",")}}`; + } + return JSON.stringify(value) ?? "null"; +} + +function reusableRuntimeFingerprint(input: { + provider: string; + adapterType: string | null; + config: Record; +}): string { + return createHash("sha256") + .update(stableStringify(input)) + .digest("hex"); +} + +function buildReusableSandboxLeaseScope(input: { + companyId: string; + environmentId: string; + executionWorkspaceId: string | null; + agentId: string | null; + adapterType: string | null; + provider: string; + config: Record; + providerMetadata?: Record | null; +}): Record | null { + if (!input.executionWorkspaceId || !input.agentId) return null; + const providerMetadata = input.providerMetadata ?? {}; + const adapterType = input.adapterType ?? null; + const remoteCwd = readString(providerMetadata.remoteCwd); + const workspaceSentinel = isRecord(providerMetadata.workspaceSentinel) + ? { ...providerMetadata.workspaceSentinel } + : null; + return { + version: 1, + companyId: input.companyId, + environmentId: input.environmentId, + executionWorkspaceId: input.executionWorkspaceId, + agentId: input.agentId, + adapterType, + provider: input.provider, + runtimeFingerprint: reusableRuntimeFingerprint({ + provider: input.provider, + adapterType, + config: input.config, + }), + ...(remoteCwd ? { remoteCwd } : {}), + ...(workspaceSentinel ? { workspaceSentinel } : {}), + }; +} + +function reusableSandboxLeaseScopeMatches(input: { + lease: Pick; + companyId: string; + environmentId: string; + executionWorkspaceId: string | null; + agentId: string | null; + adapterType: string | null; + provider: string; + config: Record; +}): boolean { + if (!input.executionWorkspaceId || !input.agentId) return false; + const scope = input.lease.metadata?.reusableSandboxLease; + if (!isRecord(scope)) return false; + const adapterType = input.adapterType ?? null; + return ( + scope.companyId === input.companyId && + scope.environmentId === input.environmentId && + scope.executionWorkspaceId === input.executionWorkspaceId && + scope.agentId === input.agentId && + scope.adapterType === adapterType && + scope.provider === input.provider && + scope.runtimeFingerprint === reusableRuntimeFingerprint({ + provider: input.provider, + adapterType, + config: input.config, + }) + ); +} + export function findReusableSandboxLeaseId(input: { config: SandboxEnvironmentConfig; leases: Array>; @@ -449,13 +564,15 @@ function createSandboxEnvironmentDriver( const workerConfig = stripSandboxProviderEnvelope(parsed.config); const storedConfig = storedParsed.config; + const supportsReusableLeases = pluginProvider.resolved.driver.supportsReusableLeases === true; // Ad-hoc tests (heartbeatRunId === null) must never resume an existing // provider lease. If they did, releasing the test lease at the end of // the probe would tear down the live heartbeat run that owns it. // We also filter out leases whose policy is not reuse_by_environment - // so any non-reusable lease (including ad-hoc test leases that - // landed in the table from older code paths) cannot be matched. + // and whose status is not reusable so non-reusable, cleanup-pending, + // or terminal rows cannot be matched. const reusableExistingLeases = + supportsReusableLeases && parsed.config.reuseLease && input.heartbeatRunId !== null && input.executionWorkspaceId !== null && @@ -463,11 +580,23 @@ function createSandboxEnvironmentDriver( ? (await environmentsSvc.listLeases(input.environment.id)) .filter((lease) => lease.leasePolicy === "reuse_by_environment" && + ["active", "released", "retained"].includes(lease.status) && lease.executionWorkspaceId === input.executionWorkspaceId && - lease.metadata?.agentId === input.agentId, + lease.metadata?.agentId === input.agentId && + reusableSandboxLeaseScopeMatches({ + lease, + companyId: input.companyId, + environmentId: input.environment.id, + executionWorkspaceId: input.executionWorkspaceId, + agentId: input.agentId, + adapterType: input.adapterType, + provider: parsed.config.provider, + config: sandboxConfigForLeaseMetadata(storedConfig), + }), ) : []; const reusableProviderLeaseId = + supportsReusableLeases && parsed.config.reuseLease && input.heartbeatRunId !== null && input.executionWorkspaceId !== null && @@ -478,26 +607,38 @@ function createSandboxEnvironmentDriver( ? reusableExistingLeases.find((lease) => lease.providerLeaseId === reusableProviderLeaseId) : null; - const providerLease = reusableLease?.providerLeaseId - ? await pluginWorkerManager.call( - pluginProvider.resolved.plugin.id, - "environmentResumeLease", - { - driverKey: parsed.config.provider, - companyId: input.companyId, - environmentId: input.environment.id, - issueId: input.issueId, - config: workerConfig, - providerLeaseId: reusableLease.providerLeaseId, - leaseMetadata: reusableLease.metadata ?? undefined, - }, - resolvePluginSandboxRpcTimeoutMs(workerConfig), - ).then((resumed) => + let providerLease: PluginEnvironmentLease | null = null; + if (reusableLease?.providerLeaseId) { + try { + const resumed = await pluginWorkerManager.call( + pluginProvider.resolved.plugin.id, + "environmentResumeLease", + { + driverKey: parsed.config.provider, + companyId: input.companyId, + environmentId: input.environment.id, + issueId: input.issueId, + config: workerConfig, + providerLeaseId: reusableLease.providerLeaseId, + leaseMetadata: reusableLease.metadata ?? undefined, + }, + resolvePluginSandboxRpcTimeoutMs(workerConfig), + ); + providerLease = typeof resumed.providerLeaseId === "string" && resumed.providerLeaseId.length > 0 ? resumed - : null, - ).catch(() => null) - : null; + : null; + } catch { + providerLease = null; + } + if (!providerLease) { + await destroyReusableSandboxLease({ + environment: input.environment, + lease: reusableLease, + failureReason: "resume_failed", + }); + } + } const acquiredLease = providerLease ?? await pluginWorkerManager.call( pluginProvider.resolved.plugin.id, "environmentAcquireLease", @@ -529,9 +670,25 @@ function createSandboxEnvironmentDriver( // Ad-hoc test leases are never publishable for reuse: storing them // as `reuse_by_environment` would let a concurrent heartbeat resume // the test's provider lease and lose its sandbox when the test ends. - const resolvedLeasePolicy = parsed.config.reuseLease && input.heartbeatRunId !== null + const resolvedLeasePolicy = supportsReusableLeases && parsed.config.reuseLease && input.heartbeatRunId !== null ? "reuse_by_environment" : "ephemeral"; + const sanitizedProviderMetadata = stripSecretRefValuesFromPluginLeaseMetadata({ + metadata: acquiredLease.metadata, + schema: pluginProvider.resolved.driver.configSchema as Record | null | undefined, + }); + const reusableScope = resolvedLeasePolicy === "reuse_by_environment" + ? buildReusableSandboxLeaseScope({ + companyId: input.companyId, + environmentId: input.environment.id, + executionWorkspaceId: input.executionWorkspaceId, + agentId: input.agentId, + adapterType: input.adapterType, + provider: parsed.config.provider, + config: sandboxConfigForLeaseMetadata(storedConfig), + providerMetadata: sanitizedProviderMetadata, + }) + : null; return await environmentsSvc.acquireLease({ companyId: input.companyId, @@ -551,10 +708,8 @@ function createSandboxEnvironmentDriver( pluginKey: pluginProvider.resolved.plugin.pluginKey, sandboxProviderPlugin: true, ...sandboxConfigForLeaseMetadata(storedConfig), - ...stripSecretRefValuesFromPluginLeaseMetadata({ - metadata: acquiredLease.metadata, - schema: pluginProvider.resolved.driver.configSchema as Record | null | undefined, - }), + ...sanitizedProviderMetadata, + ...(reusableScope ? { reusableSandboxLease: reusableScope } : {}), }, }); } @@ -562,9 +717,12 @@ function createSandboxEnvironmentDriver( // Built-in sandbox provider path. Same guard as the plugin-backed path: // ad-hoc tests (heartbeatRunId === null) must never resume an existing // provider lease, or releasing the test lease will terminate the live - // heartbeat run that shares it. Filter to leases whose policy is - // reuse_by_environment so non-reusable rows can never be matched. + // heartbeat run that shares it. Filter to reusable policies and statuses + // so non-reusable, cleanup-pending, or terminal rows can never be matched. + const builtinSandboxProvider = getBuiltinSandboxProvider(parsed.config.provider); + const supportsReusableLeases = builtinSandboxProvider?.supportsReusableLeases === true; const reusableProviderLeaseId = + supportsReusableLeases && parsed.config.reuseLease && input.heartbeatRunId !== null && input.executionWorkspaceId !== null && @@ -576,28 +734,73 @@ function createSandboxEnvironmentDriver( config: parsed.config, leases: leases.filter((lease) => lease.leasePolicy === "reuse_by_environment" && + ["active", "released", "retained"].includes(lease.status) && lease.executionWorkspaceId === input.executionWorkspaceId && - lease.metadata?.agentId === input.agentId, + lease.metadata?.agentId === input.agentId && + reusableSandboxLeaseScopeMatches({ + lease, + companyId: input.companyId, + environmentId: input.environment.id, + executionWorkspaceId: input.executionWorkspaceId, + agentId: input.agentId, + adapterType: input.adapterType, + provider: parsed.config.provider, + config: sandboxConfigForLeaseMetadata(parsed.config), + }), ), }), )) : null; + const reusableLease = reusableProviderLeaseId + ? (await environmentsSvc.listLeases(input.environment.id)).find((lease) => lease.providerLeaseId === reusableProviderLeaseId) + : null; - const providerLease = await acquireSandboxProviderLease({ - config: parsed.config, - environmentId: input.environment.id, - heartbeatRunId: input.heartbeatRunId ?? randomUUID(), - issueId: input.issueId, - agentId: input.agentId, - executionWorkspaceId: input.executionWorkspaceId, - reusableProviderLeaseId, - }); + let providerLease; + try { + providerLease = await acquireSandboxProviderLease({ + config: parsed.config, + environmentId: input.environment.id, + heartbeatRunId: input.heartbeatRunId ?? randomUUID(), + issueId: input.issueId, + agentId: input.agentId, + executionWorkspaceId: input.executionWorkspaceId, + reusableProviderLeaseId, + }); + } catch (error) { + if (reusableLease) { + await destroyReusableSandboxLease({ + environment: input.environment, + lease: reusableLease, + failureReason: "resume_failed", + }); + } + throw error; + } + if (reusableLease && providerLease.providerLeaseId !== reusableLease.providerLeaseId) { + await destroyReusableSandboxLease({ + environment: input.environment, + lease: reusableLease, + failureReason: "resume_failed", + }); + } // Same ephemeral-policy-for-tests guard as the plugin-backed path: // ad-hoc test leases must not be publishable for reuse. - const resolvedLeasePolicy = parsed.config.reuseLease && input.heartbeatRunId !== null + const resolvedLeasePolicy = supportsReusableLeases && parsed.config.reuseLease && input.heartbeatRunId !== null ? "reuse_by_environment" : "ephemeral"; + const reusableScope = resolvedLeasePolicy === "reuse_by_environment" + ? buildReusableSandboxLeaseScope({ + companyId: input.companyId, + environmentId: input.environment.id, + executionWorkspaceId: input.executionWorkspaceId, + agentId: input.agentId, + adapterType: input.adapterType, + provider: parsed.config.provider, + config: sandboxConfigForLeaseMetadata(parsed.config), + providerMetadata: providerLease.metadata, + }) + : null; return await environmentsSvc.acquireLease({ companyId: input.companyId, @@ -613,11 +816,20 @@ function createSandboxEnvironmentDriver( driver: input.environment.driver, executionWorkspaceMode: input.executionWorkspaceMode, ...providerLease.metadata, + ...(reusableScope ? { reusableSandboxLease: reusableScope } : {}), }, }); }, async releaseRunLease(input) { + if (input.status === "expired" && input.lease.leasePolicy === "reuse_by_environment") { + return await destroyReusableSandboxLease({ + environment: input.environment, + lease: input.lease, + failureReason: "lease_expired", + }); + } + // Check if this lease was acquired through a plugin. if (input.lease.metadata?.sandboxProviderPlugin) { return await releasePluginBackedSandboxLease(input); @@ -748,6 +960,14 @@ function createSandboxEnvironmentDriver( } throw new Error("Sandbox driver does not support direct command execution for built-in providers."); }, + + async destroyRunLease(input) { + return await destroyReusableSandboxLease({ + environment: input.environment, + lease: input.lease, + failureReason: input.failureReason ?? "lease_destroyed", + }); + }, }; async function releasePluginBackedSandboxLease( @@ -790,6 +1010,68 @@ function createSandboxEnvironmentDriver( cleanupStatus, }); } + + async function destroyReusableSandboxLease(input: { + environment: Environment; + lease: EnvironmentLease; + failureReason: string; + }): Promise { + let cleanupStatus: "success" | "failed" = "success"; + const metadata = input.lease.metadata ?? {}; + + try { + if (metadata.sandboxProviderPlugin) { + const pluginId = readString(metadata.pluginId); + const providerKey = readString(metadata.provider); + if (!pluginId || !providerKey || !pluginWorkerManager?.isRunning(pluginId)) { + cleanupStatus = "failed"; + } else { + const config = await resolvePluginSandboxRuntimeConfig({ + environment: input.environment, + lease: input.lease, + provider: providerKey, + }); + await pluginWorkerManager.call(pluginId, "environmentDestroyLease", { + driverKey: providerKey, + companyId: input.lease.companyId, + environmentId: input.environment.id, + issueId: input.lease.issueId, + config: stripSandboxProviderEnvelope(config as SandboxEnvironmentConfig), + providerLeaseId: input.lease.providerLeaseId, + leaseMetadata: metadata, + }, resolvePluginSandboxRpcTimeoutMs(stripSandboxProviderEnvelope(config as SandboxEnvironmentConfig))); + } + } else { + const metadataConfig = sandboxConfigFromLeaseMetadata(input.lease); + const parsed = metadataConfig + ? await resolveEnvironmentDriverConfigForRuntime(db, input.lease.companyId, { + id: input.environment.id, + driver: "sandbox", + config: metadataConfig as unknown as Record, + }) + : await resolveEnvironmentDriverConfigForRuntime(db, input.lease.companyId, input.environment); + if (parsed.driver !== "sandbox") { + cleanupStatus = "failed"; + } else { + await destroySandboxProviderLease({ + config: parsed.config, + providerLeaseId: input.lease.providerLeaseId, + }); + } + } + } catch { + cleanupStatus = "failed"; + } + + return await environmentsSvc.releaseLease( + input.lease.id, + cleanupStatus === "success" ? "expired" : "pending_cleanup", + { + failureReason: input.failureReason, + cleanupStatus, + }, + ); + } } function parseExpiresAt(value: string | null | undefined): Date | null { @@ -1026,7 +1308,9 @@ function createPluginEnvironmentDriver( providerLeaseId: input.lease.providerLeaseId, leaseMetadata: input.lease.metadata ?? undefined, }); - return await environmentsSvc.releaseLease(input.lease.id, "failed"); + return await environmentsSvc.releaseLease(input.lease.id, "failed", { + failureReason: input.failureReason ?? "lease_destroyed", + }); }, async realizeWorkspace(input) { @@ -1213,27 +1497,7 @@ export function environmentRuntimeService( const environment = await environmentsSvc.getById(leaseRow.environmentId); if (!environment) continue; - const leaseSnapshot: EnvironmentLease = { - id: leaseRow.id, - companyId: leaseRow.companyId, - environmentId: leaseRow.environmentId, - executionWorkspaceId: leaseRow.executionWorkspaceId ?? null, - issueId: leaseRow.issueId ?? null, - heartbeatRunId: leaseRow.heartbeatRunId ?? null, - status: leaseRow.status as EnvironmentLease["status"], - leasePolicy: leaseRow.leasePolicy as EnvironmentLease["leasePolicy"], - provider: leaseRow.provider ?? null, - providerLeaseId: leaseRow.providerLeaseId ?? null, - acquiredAt: leaseRow.acquiredAt, - lastUsedAt: leaseRow.lastUsedAt, - expiresAt: leaseRow.expiresAt ?? null, - releasedAt: leaseRow.releasedAt ?? null, - failureReason: leaseRow.failureReason ?? null, - cleanupStatus: leaseRow.cleanupStatus as EnvironmentLease["cleanupStatus"], - metadata: (leaseRow.metadata as Record | null) ?? null, - createdAt: leaseRow.createdAt, - updatedAt: leaseRow.updatedAt, - }; + const leaseSnapshot = toEnvironmentLeaseSnapshot(leaseRow); const driver = getDriver(getLeaseDriverKey(leaseSnapshot, environment)); const lease = driver ? await driver.releaseRunLease({ @@ -1258,6 +1522,60 @@ export function environmentRuntimeService( return released; }, + async destroyReusableSandboxLeases(input: { + companyId: string; + issueId?: string | null; + executionWorkspaceId?: string | null; + failureReason?: string; + }): Promise { + const scopeConditions = [ + input.issueId ? eq(environmentLeases.issueId, input.issueId) : undefined, + input.executionWorkspaceId ? eq(environmentLeases.executionWorkspaceId, input.executionWorkspaceId) : undefined, + ].filter((condition): condition is NonNullable => Boolean(condition)); + if (scopeConditions.length === 0) return []; + + const leaseRows = await db + .select() + .from(environmentLeases) + .where( + and( + eq(environmentLeases.companyId, input.companyId), + eq(environmentLeases.leasePolicy, "reuse_by_environment"), + inArray(environmentLeases.status, ["active", "released", "retained", "pending_cleanup"]), + ...scopeConditions, + ), + ); + + const destroyed: EnvironmentRuntimeLeaseRecord[] = []; + for (const leaseRow of leaseRows) { + const environment = await environmentsSvc.getById(leaseRow.environmentId); + if (!environment) continue; + const leaseSnapshot = toEnvironmentLeaseSnapshot(leaseRow); + const driver = getDriver(getLeaseDriverKey(leaseSnapshot, environment)); + const lease = driver?.destroyRunLease + ? await driver.destroyRunLease({ + environment, + lease: leaseSnapshot, + failureReason: input.failureReason ?? "reusable_lease_destroyed", + }) + : await environmentsSvc.releaseLease(leaseSnapshot.id, "pending_cleanup", { + failureReason: input.failureReason ?? "reusable_lease_destroyed", + cleanupStatus: "failed", + }); + if (!lease) continue; + destroyed.push({ + environment, + lease, + leaseContext: { + executionWorkspaceId: lease.executionWorkspaceId, + executionWorkspaceMode: + (lease.metadata?.executionWorkspaceMode as ExecutionWorkspace["mode"] | null | undefined) ?? null, + }, + }); + } + return destroyed; + }, + async resumeRunLease(input: EnvironmentDriverLeaseInput): Promise { const driver = requireDriverKey(getLeaseDriverKey(input.lease, input.environment)); if (!driver.resumeRunLease) { diff --git a/server/src/services/environments.ts b/server/src/services/environments.ts index c746389a6c..c68dad20e9 100644 --- a/server/src/services/environments.ts +++ b/server/src/services/environments.ts @@ -499,7 +499,7 @@ export function environmentService(db: Db) { releaseLease: async ( id: string, - status: Extract = "released", + status: Extract = "released", options?: { failureReason?: string; cleanupStatus?: EnvironmentLeaseCleanupStatus; diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 3544d3307d..f097ecff1a 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -841,6 +841,7 @@ async function resolveRunScopedMentionedSkillKeys(input: { function leaseReleaseStatusForRunStatus( status: string | null | undefined, ): Extract { + if (status === "cancelled") return "expired"; return status === "failed" || status === "timed_out" ? "failed" : "released"; } diff --git a/server/src/services/sandbox-provider-runtime.ts b/server/src/services/sandbox-provider-runtime.ts index e664fe00d5..0b089fdd6f 100644 --- a/server/src/services/sandbox-provider-runtime.ts +++ b/server/src/services/sandbox-provider-runtime.ts @@ -78,6 +78,7 @@ export interface SandboxExecuteResult { export interface SandboxProvider { readonly provider: SandboxEnvironmentProvider; + readonly supportsReusableLeases?: boolean; validateConfig(config: SandboxEnvironmentConfig): Promise; probe(config: SandboxEnvironmentConfig): Promise; acquireLease(input: AcquireSandboxLeaseInput): Promise; @@ -117,6 +118,7 @@ function buildFakeSandboxProbe(config: FakeSandboxEnvironmentConfig): Environmen class FakeSandboxProvider implements SandboxProvider { readonly provider = "fake" as const; + readonly supportsReusableLeases = false; async validateConfig(config: SandboxEnvironmentConfig): Promise { assertProviderConfig(this.provider, config); @@ -138,7 +140,7 @@ class FakeSandboxProvider implements SandboxProvider { async acquireLease(input: AcquireSandboxLeaseInput): Promise { assertProviderConfig(this.provider, input.config); - const providerLeaseId = input.config.reuseLease + const providerLeaseId = input.config.reuseLease && this.supportsReusableLeases ? `sandbox://fake/${input.environmentId}/${input.executionWorkspaceId ?? "workspace"}/${input.agentId ?? "agent"}` : `sandbox://fake/${input.heartbeatRunId}/${randomUUID()}`; @@ -336,7 +338,7 @@ export async function acquireSandboxProviderLease(input: { reusableProviderLeaseId?: string | null; }): Promise { const provider = requireSandboxProvider(input.config.provider); - if (input.config.reuseLease && input.reusableProviderLeaseId) { + if (provider.supportsReusableLeases && input.config.reuseLease && input.reusableProviderLeaseId) { const resumedLease = await provider.resumeLease({ config: input.config, providerLeaseId: input.reusableProviderLeaseId,