feat: reuse Daytona sandbox leases (#8513)
Add opt-in reusable Daytona sandbox lease support, including retryable pending cleanup handling.\n\nPR: https://github.com/paperclipai/paperclip/pull/8513
This commit is contained in:
Devin Foley
GitHub
parent
2e2da3bc2f
commit
cd38c150b0
|
|
@ -23,6 +23,7 @@ const manifest: PaperclipPluginManifestV1 = {
|
|||
displayName: "Daytona Sandbox",
|
||||
description:
|
||||
"Provisions Daytona sandboxes with configurable image or snapshot selection, startup timeouts, and lease reuse.",
|
||||
supportsReusableLeases: true,
|
||||
configSchema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
|
|
|
|||
|
|
@ -174,6 +174,9 @@ describe("Daytona sandbox provider plugin", () => {
|
|||
companyId: "company-1",
|
||||
environmentId: "env-1",
|
||||
runId: "run-1",
|
||||
agentId: "agent-1",
|
||||
executionWorkspaceId: "workspace-1",
|
||||
adapterType: "codex_local",
|
||||
config: {
|
||||
image: "node:20",
|
||||
timeoutMs: 300000,
|
||||
|
|
@ -189,8 +192,21 @@ describe("Daytona sandbox provider plugin", () => {
|
|||
sandboxId: "sandbox-123",
|
||||
remoteCwd: "/home/daytona/paperclip-workspace",
|
||||
reuseLease: true,
|
||||
workspaceSentinel: {
|
||||
path: "/home/daytona/paperclip-workspace/.paperclip-runtime/reusable-sandbox-lease.json",
|
||||
result: "written",
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(sandbox.fs.createFolder).toHaveBeenCalledWith(
|
||||
"/home/daytona/paperclip-workspace/.paperclip-runtime",
|
||||
"755",
|
||||
);
|
||||
expect(sandbox.fs.uploadFile).toHaveBeenCalledWith(
|
||||
expect.any(Buffer),
|
||||
"/home/daytona/paperclip-workspace/.paperclip-runtime/reusable-sandbox-lease.json",
|
||||
300,
|
||||
);
|
||||
});
|
||||
|
||||
it("deletes the sandbox if lease setup throws after sandbox creation", async () => {
|
||||
|
|
@ -287,6 +303,94 @@ describe("Daytona sandbox provider plugin", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("resumes a reusable lease when the workspace sentinel matches", async () => {
|
||||
process.env.DAYTONA_API_KEY = "host-key";
|
||||
const sandbox = createMockSandbox({ id: "sandbox-reuse", state: "stopped" });
|
||||
sandbox.process.executeCommand
|
||||
.mockResolvedValueOnce({
|
||||
exitCode: 0,
|
||||
result: JSON.stringify({ token: "sentinel-token" }),
|
||||
artifacts: { stdout: JSON.stringify({ token: "sentinel-token" }) },
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
exitCode: 0,
|
||||
result: "bash",
|
||||
artifacts: { stdout: "bash" },
|
||||
});
|
||||
mockGet.mockResolvedValue(sandbox);
|
||||
|
||||
const lease = await plugin.definition.onEnvironmentResumeLease?.({
|
||||
driverKey: "daytona",
|
||||
companyId: "company-1",
|
||||
environmentId: "env-1",
|
||||
providerLeaseId: "sandbox-reuse",
|
||||
config: {
|
||||
timeoutMs: 300000,
|
||||
reuseLease: true,
|
||||
},
|
||||
leaseMetadata: {
|
||||
workspaceSentinel: {
|
||||
path: "/home/daytona/paperclip-workspace/.paperclip-runtime/reusable-sandbox-lease.json",
|
||||
token: "sentinel-token",
|
||||
result: "written",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(sandbox.start).toHaveBeenCalledWith(300);
|
||||
expect(lease).toMatchObject({
|
||||
providerLeaseId: "sandbox-reuse",
|
||||
metadata: {
|
||||
resumedLease: true,
|
||||
workspaceSentinel: {
|
||||
result: "matched",
|
||||
token: "sentinel-token",
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("expires a reusable lease when the workspace sentinel does not match", async () => {
|
||||
process.env.DAYTONA_API_KEY = "host-key";
|
||||
const sandbox = createMockSandbox({ id: "sandbox-reuse", state: "stopped" });
|
||||
sandbox.process.executeCommand.mockResolvedValueOnce({
|
||||
exitCode: 0,
|
||||
result: JSON.stringify({ token: "other-token" }),
|
||||
artifacts: { stdout: JSON.stringify({ token: "other-token" }) },
|
||||
});
|
||||
mockGet.mockResolvedValue(sandbox);
|
||||
|
||||
await expect(plugin.definition.onEnvironmentResumeLease?.({
|
||||
driverKey: "daytona",
|
||||
companyId: "company-1",
|
||||
environmentId: "env-1",
|
||||
providerLeaseId: "sandbox-reuse",
|
||||
config: {
|
||||
timeoutMs: 300000,
|
||||
reuseLease: true,
|
||||
},
|
||||
leaseMetadata: {
|
||||
workspaceSentinel: {
|
||||
path: "/home/daytona/paperclip-workspace/.paperclip-runtime/reusable-sandbox-lease.json",
|
||||
token: "sentinel-token",
|
||||
result: "written",
|
||||
},
|
||||
},
|
||||
})).resolves.toEqual({
|
||||
providerLeaseId: null,
|
||||
metadata: {
|
||||
expired: true,
|
||||
workspaceSentinel: {
|
||||
path: "/home/daytona/paperclip-workspace/.paperclip-runtime/reusable-sandbox-lease.json",
|
||||
token: "sentinel-token",
|
||||
result: "mismatch",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(sandbox.process.executeCommand).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("stops reusable leases and deletes ephemeral leases on release", async () => {
|
||||
process.env.DAYTONA_API_KEY = "host-key";
|
||||
const reusable = createMockSandbox({ id: "sandbox-reusable" });
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import path from "node:path";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { Daytona, DaytonaNotFoundError, DaytonaTimeoutError } from "@daytonaio/sdk";
|
||||
import type {
|
||||
CreateSandboxBaseParams,
|
||||
|
|
@ -44,6 +44,14 @@ interface DaytonaDriverConfig {
|
|||
reuseLease: boolean;
|
||||
}
|
||||
|
||||
type WorkspaceSentinelResult = {
|
||||
path: string;
|
||||
token: string | null;
|
||||
result: "written" | "matched" | "missing" | "mismatch" | "skipped";
|
||||
};
|
||||
|
||||
const WORKSPACE_SENTINEL_RELATIVE_PATH = ".paperclip-runtime/reusable-sandbox-lease.json";
|
||||
|
||||
function parseOptionalString(value: unknown): string | null {
|
||||
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
|
||||
}
|
||||
|
|
@ -166,6 +174,20 @@ function formatErrorMessage(error: unknown): string {
|
|||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function stableStringify(value: unknown): string {
|
||||
if (Array.isArray(value)) {
|
||||
return `[${value.map((entry) => stableStringify(entry)).join(",")}]`;
|
||||
}
|
||||
if (isRecord(value)) {
|
||||
return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableStringify(value[key])}`).join(",")}}`;
|
||||
}
|
||||
return JSON.stringify(value) ?? "null";
|
||||
}
|
||||
|
||||
function isValidUrl(value: string): boolean {
|
||||
try {
|
||||
new URL(value);
|
||||
|
|
@ -210,12 +232,109 @@ async function detectSandboxShellCommand(sandbox: Sandbox, timeoutSeconds: numbe
|
|||
}
|
||||
}
|
||||
|
||||
function workspaceSentinelToken(input: {
|
||||
params: Pick<PluginEnvironmentAcquireLeaseParams, "companyId" | "environmentId" | "agentId" | "executionWorkspaceId" | "adapterType">;
|
||||
config: DaytonaDriverConfig;
|
||||
}): string | null {
|
||||
if (!input.config.reuseLease || !input.params.agentId || !input.params.executionWorkspaceId) {
|
||||
return null;
|
||||
}
|
||||
return createHash("sha256")
|
||||
.update(stableStringify({
|
||||
provider: "daytona",
|
||||
companyId: input.params.companyId,
|
||||
environmentId: input.params.environmentId,
|
||||
agentId: input.params.agentId,
|
||||
executionWorkspaceId: input.params.executionWorkspaceId,
|
||||
adapterType: input.params.adapterType ?? null,
|
||||
image: input.config.image,
|
||||
snapshot: input.config.snapshot,
|
||||
target: input.config.target,
|
||||
}))
|
||||
.digest("hex");
|
||||
}
|
||||
|
||||
function workspaceSentinelPath(remoteCwd: string): string {
|
||||
return path.posix.join(remoteCwd, WORKSPACE_SENTINEL_RELATIVE_PATH);
|
||||
}
|
||||
|
||||
async function writeWorkspaceSentinel(input: {
|
||||
sandbox: Sandbox;
|
||||
remoteCwd: string;
|
||||
params: PluginEnvironmentAcquireLeaseParams;
|
||||
config: DaytonaDriverConfig;
|
||||
timeoutSeconds: number;
|
||||
}): Promise<WorkspaceSentinelResult> {
|
||||
const sentinelPath = workspaceSentinelPath(input.remoteCwd);
|
||||
const token = workspaceSentinelToken({ params: input.params, config: input.config });
|
||||
if (!token) {
|
||||
return { path: sentinelPath, token: null, result: "skipped" };
|
||||
}
|
||||
await input.sandbox.fs.createFolder(path.posix.dirname(sentinelPath), "755");
|
||||
await input.sandbox.fs.uploadFile(
|
||||
Buffer.from(JSON.stringify({
|
||||
version: 1,
|
||||
token,
|
||||
companyId: input.params.companyId,
|
||||
environmentId: input.params.environmentId,
|
||||
agentId: input.params.agentId,
|
||||
executionWorkspaceId: input.params.executionWorkspaceId,
|
||||
adapterType: input.params.adapterType ?? null,
|
||||
provider: "daytona",
|
||||
writtenAt: new Date().toISOString(),
|
||||
}, null, 2), "utf8"),
|
||||
sentinelPath,
|
||||
input.timeoutSeconds,
|
||||
);
|
||||
return { path: sentinelPath, token, result: "written" };
|
||||
}
|
||||
|
||||
async function verifyWorkspaceSentinel(input: {
|
||||
sandbox: Sandbox;
|
||||
remoteCwd: string;
|
||||
leaseMetadata?: Record<string, unknown>;
|
||||
timeoutSeconds: number;
|
||||
}): Promise<WorkspaceSentinelResult> {
|
||||
const metadataSentinel = isRecord(input.leaseMetadata?.workspaceSentinel)
|
||||
? input.leaseMetadata.workspaceSentinel
|
||||
: null;
|
||||
const sentinelPath = typeof metadataSentinel?.path === "string"
|
||||
? metadataSentinel.path
|
||||
: workspaceSentinelPath(input.remoteCwd);
|
||||
const expectedToken = typeof metadataSentinel?.token === "string" ? metadataSentinel.token : null;
|
||||
if (!expectedToken) {
|
||||
return { path: sentinelPath, token: null, result: "missing" };
|
||||
}
|
||||
|
||||
const result = await input.sandbox.process.executeCommand(
|
||||
`cat ${shellQuote(sentinelPath)}`,
|
||||
undefined,
|
||||
undefined,
|
||||
input.timeoutSeconds,
|
||||
);
|
||||
if (result.exitCode !== 0) {
|
||||
return { path: sentinelPath, token: expectedToken, result: "missing" };
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(result.result ?? result.artifacts?.stdout ?? "") as unknown;
|
||||
const actualToken = isRecord(parsed) && typeof parsed.token === "string" ? parsed.token : null;
|
||||
return {
|
||||
path: sentinelPath,
|
||||
token: expectedToken,
|
||||
result: actualToken === expectedToken ? "matched" : "mismatch",
|
||||
};
|
||||
} catch {
|
||||
return { path: sentinelPath, token: expectedToken, result: "mismatch" };
|
||||
}
|
||||
}
|
||||
|
||||
function leaseMetadata(input: {
|
||||
config: DaytonaDriverConfig;
|
||||
sandbox: Sandbox;
|
||||
shellCommand: "bash" | "sh";
|
||||
remoteCwd: string;
|
||||
resumedLease: boolean;
|
||||
workspaceSentinel?: WorkspaceSentinelResult;
|
||||
}) {
|
||||
return {
|
||||
provider: "daytona",
|
||||
|
|
@ -230,6 +349,7 @@ function leaseMetadata(input: {
|
|||
reuseLease: input.config.reuseLease,
|
||||
remoteCwd: input.remoteCwd,
|
||||
resumedLease: input.resumedLease,
|
||||
...(input.workspaceSentinel ? { workspaceSentinel: input.workspaceSentinel } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -499,9 +619,16 @@ const plugin = definePlugin({
|
|||
try {
|
||||
const remoteCwd = await resolveSandboxWorkingDirectory(sandbox);
|
||||
const shellCommand = await detectSandboxShellCommand(sandbox, toTimeoutSeconds(config.timeoutMs));
|
||||
const workspaceSentinel = await writeWorkspaceSentinel({
|
||||
sandbox,
|
||||
remoteCwd,
|
||||
params,
|
||||
config,
|
||||
timeoutSeconds: toTimeoutSeconds(config.timeoutMs),
|
||||
});
|
||||
return {
|
||||
providerLeaseId: sandbox.id,
|
||||
metadata: leaseMetadata({ config, sandbox, shellCommand, remoteCwd, resumedLease: false }),
|
||||
metadata: leaseMetadata({ config, sandbox, shellCommand, remoteCwd, resumedLease: false, workspaceSentinel }),
|
||||
};
|
||||
} catch (error) {
|
||||
await sandbox.delete(toTimeoutSeconds(config.timeoutMs)).catch(() => undefined);
|
||||
|
|
@ -521,10 +648,19 @@ const plugin = definePlugin({
|
|||
await ensureSandboxStarted(sandbox, toTimeoutSeconds(config.timeoutMs));
|
||||
try {
|
||||
const remoteCwd = await resolveSandboxWorkingDirectory(sandbox);
|
||||
const workspaceSentinel = await verifyWorkspaceSentinel({
|
||||
sandbox,
|
||||
remoteCwd,
|
||||
leaseMetadata: params.leaseMetadata,
|
||||
timeoutSeconds: toTimeoutSeconds(config.timeoutMs),
|
||||
});
|
||||
if (workspaceSentinel.result !== "matched") {
|
||||
return { providerLeaseId: null, metadata: { expired: true, workspaceSentinel } };
|
||||
}
|
||||
const shellCommand = await detectSandboxShellCommand(sandbox, toTimeoutSeconds(config.timeoutMs));
|
||||
return {
|
||||
providerLeaseId: sandbox.id,
|
||||
metadata: leaseMetadata({ config, sandbox, shellCommand, remoteCwd, resumedLease: true }),
|
||||
metadata: leaseMetadata({ config, sandbox, shellCommand, remoteCwd, resumedLease: true, workspaceSentinel }),
|
||||
};
|
||||
} catch (error) {
|
||||
await sandbox.delete(toTimeoutSeconds(config.timeoutMs)).catch(() => undefined);
|
||||
|
|
|
|||
|
|
@ -421,7 +421,7 @@ export type EnvironmentDriver = (typeof ENVIRONMENT_DRIVERS)[number];
|
|||
export const ENVIRONMENT_STATUSES = ["active", "archived"] as const;
|
||||
export type EnvironmentStatus = (typeof ENVIRONMENT_STATUSES)[number];
|
||||
|
||||
export const ENVIRONMENT_LEASE_STATUSES = ["active", "released", "expired", "failed", "retained"] as const;
|
||||
export const ENVIRONMENT_LEASE_STATUSES = ["active", "released", "expired", "failed", "retained", "pending_cleanup"] as const;
|
||||
export type EnvironmentLeaseStatus = (typeof ENVIRONMENT_LEASE_STATUSES)[number];
|
||||
|
||||
export const ENVIRONMENT_LEASE_POLICIES = [
|
||||
|
|
|
|||
|
|
@ -138,6 +138,12 @@ export interface PluginEnvironmentDriverDeclaration {
|
|||
displayName: string;
|
||||
/** Optional description for operator-facing docs or UI affordances. */
|
||||
description?: string;
|
||||
/**
|
||||
* Sandbox providers must opt in before the host retains and resumes provider
|
||||
* leases across runs. Providers without this flag keep per-run acquire/release
|
||||
* behavior even if their config schema exposes a reuse-like setting.
|
||||
*/
|
||||
supportsReusableLeases?: boolean;
|
||||
/** JSON Schema describing the driver's provider-specific configuration. */
|
||||
configSchema: JsonSchema;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -124,6 +124,7 @@ export const pluginEnvironmentDriverDeclarationSchema = z.object({
|
|||
kind: z.enum(["environment_driver", "sandbox_provider"]).optional(),
|
||||
displayName: z.string().min(1).max(100),
|
||||
description: z.string().max(500).optional(),
|
||||
supportsReusableLeases: z.boolean().optional(),
|
||||
configSchema: jsonSchemaSchema,
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
|
@ -42,6 +42,25 @@ if (!embeddedPostgresSupport.supported) {
|
|||
);
|
||||
}
|
||||
|
||||
function stableStringify(value: unknown): string {
|
||||
if (Array.isArray(value)) {
|
||||
return `[${value.map((entry) => stableStringify(entry)).join(",")}]`;
|
||||
}
|
||||
if (value && typeof value === "object") {
|
||||
const record = value as Record<string, unknown>;
|
||||
return `{${Object.keys(record).sort().map((key) => `${JSON.stringify(key)}:${stableStringify(record[key])}`).join(",")}}`;
|
||||
}
|
||||
return JSON.stringify(value) ?? "null";
|
||||
}
|
||||
|
||||
function reusableRuntimeFingerprint(input: {
|
||||
provider: string;
|
||||
adapterType: string | null;
|
||||
config: Record<string, unknown>;
|
||||
}) {
|
||||
return createHash("sha256").update(stableStringify(input)).digest("hex");
|
||||
}
|
||||
|
||||
describe("findReusableSandboxLeaseId", () => {
|
||||
it("matches reusable plugin-backed sandbox leases by provider", () => {
|
||||
const selected = findReusableSandboxLeaseId({
|
||||
|
|
@ -258,6 +277,124 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
|||
};
|
||||
}
|
||||
|
||||
async function seedReusablePluginSandboxLease() {
|
||||
const pluginId = randomUUID();
|
||||
const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment();
|
||||
const providerConfig = {
|
||||
provider: "fake-plugin",
|
||||
image: "fake:test",
|
||||
timeoutMs: 1234,
|
||||
reuseLease: true,
|
||||
};
|
||||
const environment = {
|
||||
...baseEnvironment,
|
||||
name: "Reusable Plugin Sandbox",
|
||||
driver: "sandbox",
|
||||
config: providerConfig,
|
||||
};
|
||||
await environmentService(db).update(environment.id, {
|
||||
driver: "sandbox",
|
||||
name: environment.name,
|
||||
config: providerConfig,
|
||||
});
|
||||
await db.insert(plugins).values({
|
||||
id: pluginId,
|
||||
pluginKey: "acme.reusable-sandbox-provider",
|
||||
packageName: "@acme/reusable-sandbox-provider",
|
||||
version: "1.0.0",
|
||||
apiVersion: 1,
|
||||
categories: ["automation"],
|
||||
manifestJson: {
|
||||
id: "acme.reusable-sandbox-provider",
|
||||
apiVersion: 1,
|
||||
version: "1.0.0",
|
||||
displayName: "Reusable Sandbox Provider",
|
||||
description: "Test provider with reusable lease support",
|
||||
author: "Paperclip",
|
||||
categories: ["automation"],
|
||||
capabilities: ["environment.drivers.register"],
|
||||
entrypoints: { worker: "dist/worker.js" },
|
||||
environmentDrivers: [
|
||||
{
|
||||
driverKey: "fake-plugin",
|
||||
kind: "sandbox_provider",
|
||||
displayName: "Fake Plugin",
|
||||
supportsReusableLeases: true,
|
||||
configSchema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
image: { type: "string" },
|
||||
timeoutMs: { type: "number" },
|
||||
reuseLease: { type: "boolean" },
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
status: "ready",
|
||||
installOrder: 1,
|
||||
updatedAt: new Date(),
|
||||
} as any);
|
||||
const executionWorkspaceId = randomUUID();
|
||||
const projectId = randomUUID();
|
||||
await db.insert(projects).values({
|
||||
id: projectId,
|
||||
companyId,
|
||||
name: `Workspace ${projectId.slice(0, 8)}`,
|
||||
status: "active",
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
});
|
||||
await db.insert(executionWorkspaces).values({
|
||||
id: executionWorkspaceId,
|
||||
companyId,
|
||||
projectId,
|
||||
mode: "shared_workspace",
|
||||
strategyType: "project_primary",
|
||||
name: "Reusable workspace",
|
||||
status: "active",
|
||||
providerType: "local_fs",
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
});
|
||||
const reusableLease = await environmentService(db).acquireLease({
|
||||
companyId,
|
||||
environmentId: environment.id,
|
||||
executionWorkspaceId,
|
||||
heartbeatRunId: runId,
|
||||
leasePolicy: "reuse_by_environment",
|
||||
provider: "fake-plugin",
|
||||
providerLeaseId: "reusable-plugin-lease",
|
||||
metadata: {
|
||||
agentId,
|
||||
driver: "sandbox",
|
||||
pluginId,
|
||||
pluginKey: "acme.reusable-sandbox-provider",
|
||||
sandboxProviderPlugin: true,
|
||||
provider: "fake-plugin",
|
||||
image: "fake:test",
|
||||
timeoutMs: 1234,
|
||||
reuseLease: true,
|
||||
reusableSandboxLease: {
|
||||
version: 1,
|
||||
companyId,
|
||||
environmentId: environment.id,
|
||||
executionWorkspaceId,
|
||||
agentId,
|
||||
adapterType: null,
|
||||
provider: "fake-plugin",
|
||||
runtimeFingerprint: reusableRuntimeFingerprint({
|
||||
provider: "fake-plugin",
|
||||
adapterType: null,
|
||||
config: providerConfig,
|
||||
}),
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
return { pluginId, companyId, executionWorkspaceId, reusableLease };
|
||||
}
|
||||
|
||||
it("acquires and releases a local run lease through the runtime seam", async () => {
|
||||
const { companyId, environment, runId } = await seedEnvironment();
|
||||
|
||||
|
|
@ -415,7 +552,8 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
|||
});
|
||||
|
||||
expect(acquired.lease.status).toBe("active");
|
||||
expect(acquired.lease.providerLeaseId).toBe(`sandbox://fake/${environment.id}/workspace/agent`);
|
||||
expect(acquired.lease.providerLeaseId).toMatch(new RegExp(`^sandbox://fake/${runId}/[0-9a-f-]{36}$`));
|
||||
expect(acquired.lease.leasePolicy).toBe("ephemeral");
|
||||
expect(acquired.lease.metadata).toMatchObject({
|
||||
driver: "sandbox",
|
||||
provider: "fake",
|
||||
|
|
@ -942,6 +1080,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
|||
driverKey: "fake-plugin",
|
||||
kind: "sandbox_provider",
|
||||
displayName: "Fake Plugin",
|
||||
supportsReusableLeases: true,
|
||||
configSchema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
|
|
@ -979,7 +1118,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
|||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
});
|
||||
await environmentService(db).acquireLease({
|
||||
const staleLease = await environmentService(db).acquireLease({
|
||||
companyId,
|
||||
environmentId: environment.id,
|
||||
executionWorkspaceId,
|
||||
|
|
@ -989,10 +1128,28 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
|||
providerLeaseId: "stale-plugin-lease",
|
||||
metadata: {
|
||||
agentId,
|
||||
driver: "sandbox",
|
||||
pluginId,
|
||||
pluginKey: "acme.fake-sandbox-provider",
|
||||
sandboxProviderPlugin: true,
|
||||
provider: "fake-plugin",
|
||||
image: "fake:test",
|
||||
timeoutMs: 1234,
|
||||
reuseLease: true,
|
||||
reusableSandboxLease: {
|
||||
version: 1,
|
||||
companyId,
|
||||
environmentId: environment.id,
|
||||
executionWorkspaceId,
|
||||
agentId,
|
||||
adapterType: null,
|
||||
provider: "fake-plugin",
|
||||
runtimeFingerprint: reusableRuntimeFingerprint({
|
||||
provider: "fake-plugin",
|
||||
adapterType: null,
|
||||
config: providerConfig,
|
||||
}),
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
|
|
@ -1002,6 +1159,9 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
|||
if (method === "environmentResumeLease") {
|
||||
throw new Error("stale sandbox");
|
||||
}
|
||||
if (method === "environmentDestroyLease") {
|
||||
return undefined;
|
||||
}
|
||||
if (method === "environmentAcquireLease") {
|
||||
return {
|
||||
providerLeaseId: "fresh-plugin-lease",
|
||||
|
|
@ -1036,7 +1196,11 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
|||
driverKey: "fake-plugin",
|
||||
providerLeaseId: "stale-plugin-lease",
|
||||
}), 31234);
|
||||
expect(workerManager.call).toHaveBeenNthCalledWith(2, pluginId, "environmentAcquireLease", expect.objectContaining({
|
||||
expect(workerManager.call).toHaveBeenNthCalledWith(2, pluginId, "environmentDestroyLease", expect.objectContaining({
|
||||
driverKey: "fake-plugin",
|
||||
providerLeaseId: "stale-plugin-lease",
|
||||
}), 31234);
|
||||
expect(workerManager.call).toHaveBeenNthCalledWith(3, pluginId, "environmentAcquireLease", expect.objectContaining({
|
||||
driverKey: "fake-plugin",
|
||||
config: {
|
||||
image: "fake:test",
|
||||
|
|
@ -1047,6 +1211,257 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
|||
executionWorkspaceId,
|
||||
runId,
|
||||
}), 31234);
|
||||
await expect(environmentService(db).getLeaseById(staleLease.id)).resolves.toMatchObject({
|
||||
status: "expired",
|
||||
cleanupStatus: "success",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not retain or resume plugin-backed sandbox leases unless the provider opts in", async () => {
|
||||
const pluginId = randomUUID();
|
||||
const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment();
|
||||
const providerConfig = {
|
||||
provider: "fake-plugin",
|
||||
image: "fake:test",
|
||||
timeoutMs: 1234,
|
||||
reuseLease: true,
|
||||
};
|
||||
const environment = {
|
||||
...baseEnvironment,
|
||||
name: "Non-reusable Plugin Sandbox",
|
||||
driver: "sandbox",
|
||||
config: providerConfig,
|
||||
};
|
||||
await environmentService(db).update(environment.id, {
|
||||
driver: "sandbox",
|
||||
name: environment.name,
|
||||
config: providerConfig,
|
||||
});
|
||||
await db.insert(plugins).values({
|
||||
id: pluginId,
|
||||
pluginKey: "acme.non-reusable-sandbox-provider",
|
||||
packageName: "@acme/non-reusable-sandbox-provider",
|
||||
version: "1.0.0",
|
||||
apiVersion: 1,
|
||||
categories: ["automation"],
|
||||
manifestJson: {
|
||||
id: "acme.non-reusable-sandbox-provider",
|
||||
apiVersion: 1,
|
||||
version: "1.0.0",
|
||||
displayName: "Non-reusable Sandbox Provider",
|
||||
description: "Test provider without reusable lease support",
|
||||
author: "Paperclip",
|
||||
categories: ["automation"],
|
||||
capabilities: ["environment.drivers.register"],
|
||||
entrypoints: { worker: "dist/worker.js" },
|
||||
environmentDrivers: [
|
||||
{
|
||||
driverKey: "fake-plugin",
|
||||
kind: "sandbox_provider",
|
||||
displayName: "Fake Plugin",
|
||||
configSchema: { type: "object" },
|
||||
},
|
||||
],
|
||||
},
|
||||
status: "ready",
|
||||
installOrder: 1,
|
||||
updatedAt: new Date(),
|
||||
} as any);
|
||||
const executionWorkspaceId = randomUUID();
|
||||
const projectId = randomUUID();
|
||||
await db.insert(projects).values({
|
||||
id: projectId,
|
||||
companyId,
|
||||
name: `Workspace ${projectId.slice(0, 8)}`,
|
||||
status: "active",
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
});
|
||||
await db.insert(executionWorkspaces).values({
|
||||
id: executionWorkspaceId,
|
||||
companyId,
|
||||
projectId,
|
||||
mode: "shared_workspace",
|
||||
strategyType: "project_primary",
|
||||
name: "Non-reusable workspace",
|
||||
status: "active",
|
||||
providerType: "local_fs",
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
});
|
||||
await environmentService(db).acquireLease({
|
||||
companyId,
|
||||
environmentId: environment.id,
|
||||
executionWorkspaceId,
|
||||
heartbeatRunId: runId,
|
||||
leasePolicy: "reuse_by_environment",
|
||||
provider: "fake-plugin",
|
||||
providerLeaseId: "old-plugin-lease",
|
||||
metadata: {
|
||||
agentId,
|
||||
provider: "fake-plugin",
|
||||
image: "fake:test",
|
||||
timeoutMs: 1234,
|
||||
reuseLease: true,
|
||||
reusableSandboxLease: {
|
||||
version: 1,
|
||||
companyId,
|
||||
environmentId: environment.id,
|
||||
executionWorkspaceId,
|
||||
agentId,
|
||||
adapterType: null,
|
||||
provider: "fake-plugin",
|
||||
runtimeFingerprint: reusableRuntimeFingerprint({
|
||||
provider: "fake-plugin",
|
||||
adapterType: null,
|
||||
config: providerConfig,
|
||||
}),
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const workerManager = {
|
||||
isRunning: vi.fn((id: string) => id === pluginId),
|
||||
call: vi.fn(async (_pluginId: string, method: string) => {
|
||||
if (method === "environmentAcquireLease") {
|
||||
return {
|
||||
providerLeaseId: "fresh-plugin-lease",
|
||||
metadata: {
|
||||
provider: "fake-plugin",
|
||||
image: "fake:test",
|
||||
timeoutMs: 1234,
|
||||
reuseLease: true,
|
||||
remoteCwd: "/workspace",
|
||||
},
|
||||
};
|
||||
}
|
||||
throw new Error(`Unexpected plugin method: ${method}`);
|
||||
}),
|
||||
} as unknown as PluginWorkerManager;
|
||||
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
||||
|
||||
const acquired = await runtimeWithPlugin.acquireRunLease({
|
||||
companyId,
|
||||
environment,
|
||||
issueId: null,
|
||||
agentId,
|
||||
heartbeatRunId: runId,
|
||||
persistedExecutionWorkspace: {
|
||||
id: executionWorkspaceId,
|
||||
mode: "shared_workspace",
|
||||
},
|
||||
});
|
||||
|
||||
expect(acquired.lease.providerLeaseId).toBe("fresh-plugin-lease");
|
||||
expect(acquired.lease.leasePolicy).toBe("ephemeral");
|
||||
expect(workerManager.call).toHaveBeenCalledTimes(1);
|
||||
expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", expect.anything(), 31234);
|
||||
});
|
||||
|
||||
it("destroys scoped reusable plugin-backed sandbox leases", async () => {
|
||||
const { pluginId, companyId, executionWorkspaceId, reusableLease } =
|
||||
await seedReusablePluginSandboxLease();
|
||||
|
||||
const workerManager = {
|
||||
isRunning: vi.fn((id: string) => id === pluginId),
|
||||
call: vi.fn(async (_pluginId: string, method: string) => {
|
||||
if (method === "environmentDestroyLease") {
|
||||
return undefined;
|
||||
}
|
||||
throw new Error(`Unexpected plugin method: ${method}`);
|
||||
}),
|
||||
} as unknown as PluginWorkerManager;
|
||||
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
|
||||
|
||||
const destroyed = await runtimeWithPlugin.destroyReusableSandboxLeases({
|
||||
companyId,
|
||||
executionWorkspaceId,
|
||||
failureReason: "execution_workspace_closed",
|
||||
});
|
||||
|
||||
expect(destroyed).toHaveLength(1);
|
||||
expect(destroyed[0]?.lease.id).toBe(reusableLease.id);
|
||||
expect(destroyed[0]?.lease.status).toBe("expired");
|
||||
expect(workerManager.call).toHaveBeenCalledWith(
|
||||
pluginId,
|
||||
"environmentDestroyLease",
|
||||
expect.objectContaining({
|
||||
driverKey: "fake-plugin",
|
||||
providerLeaseId: "reusable-plugin-lease",
|
||||
}),
|
||||
31234,
|
||||
);
|
||||
await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({
|
||||
status: "expired",
|
||||
failureReason: "execution_workspace_closed",
|
||||
cleanupStatus: "success",
|
||||
});
|
||||
});
|
||||
|
||||
it("retries reusable plugin-backed sandbox destroy when the worker is unavailable", async () => {
|
||||
const { pluginId, companyId, executionWorkspaceId, reusableLease } =
|
||||
await seedReusablePluginSandboxLease();
|
||||
|
||||
const offlineWorkerManager = {
|
||||
isRunning: vi.fn(() => false),
|
||||
call: vi.fn(),
|
||||
} as unknown as PluginWorkerManager;
|
||||
const runtimeWithOfflinePlugin = environmentRuntimeService(db, {
|
||||
pluginWorkerManager: offlineWorkerManager,
|
||||
});
|
||||
|
||||
const pending = await runtimeWithOfflinePlugin.destroyReusableSandboxLeases({
|
||||
companyId,
|
||||
executionWorkspaceId,
|
||||
failureReason: "execution_workspace_closed",
|
||||
});
|
||||
|
||||
expect(pending).toHaveLength(1);
|
||||
expect(pending[0]?.lease.id).toBe(reusableLease.id);
|
||||
expect(pending[0]?.lease.status).toBe("pending_cleanup");
|
||||
expect(offlineWorkerManager.call).not.toHaveBeenCalled();
|
||||
await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({
|
||||
status: "pending_cleanup",
|
||||
failureReason: "execution_workspace_closed",
|
||||
cleanupStatus: "failed",
|
||||
});
|
||||
|
||||
const recoveredWorkerManager = {
|
||||
isRunning: vi.fn((id: string) => id === pluginId),
|
||||
call: vi.fn(async (_pluginId: string, method: string) => {
|
||||
if (method === "environmentDestroyLease") {
|
||||
return undefined;
|
||||
}
|
||||
throw new Error(`Unexpected plugin method: ${method}`);
|
||||
}),
|
||||
} as unknown as PluginWorkerManager;
|
||||
const runtimeWithRecoveredPlugin = environmentRuntimeService(db, {
|
||||
pluginWorkerManager: recoveredWorkerManager,
|
||||
});
|
||||
|
||||
const retried = await runtimeWithRecoveredPlugin.destroyReusableSandboxLeases({
|
||||
companyId,
|
||||
executionWorkspaceId,
|
||||
failureReason: "cleanup_retry",
|
||||
});
|
||||
|
||||
expect(retried).toHaveLength(1);
|
||||
expect(retried[0]?.lease.id).toBe(reusableLease.id);
|
||||
expect(retried[0]?.lease.status).toBe("expired");
|
||||
expect(recoveredWorkerManager.call).toHaveBeenCalledWith(
|
||||
pluginId,
|
||||
"environmentDestroyLease",
|
||||
expect.objectContaining({
|
||||
driverKey: "fake-plugin",
|
||||
providerLeaseId: "reusable-plugin-lease",
|
||||
}),
|
||||
31234,
|
||||
);
|
||||
await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({
|
||||
status: "expired",
|
||||
failureReason: "cleanup_retry",
|
||||
cleanupStatus: "success",
|
||||
});
|
||||
});
|
||||
|
||||
it("releases a sandbox run lease from metadata after the environment config changes", async () => {
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ describe("sandbox provider runtime", () => {
|
|||
})).rejects.toThrow('Sandbox provider "fake-plugin" is not registered as a built-in provider.');
|
||||
});
|
||||
|
||||
it("acquires and resumes fake leases deterministically", async () => {
|
||||
it("does not resume fake leases because the built-in fake provider does not opt in", async () => {
|
||||
const lease = await acquireSandboxProviderLease({
|
||||
config: {
|
||||
provider: "fake",
|
||||
|
|
@ -56,7 +56,7 @@ describe("sandbox provider runtime", () => {
|
|||
issueId: "issue-1",
|
||||
});
|
||||
|
||||
expect(lease.providerLeaseId).toBe("sandbox://fake/env-1/workspace/agent");
|
||||
expect(lease.providerLeaseId).toMatch(/^sandbox:\/\/fake\/run-1\/[0-9a-f-]+$/);
|
||||
expect(lease.metadata).toEqual(expect.objectContaining({
|
||||
provider: "fake",
|
||||
image: "ubuntu:24.04",
|
||||
|
|
@ -75,8 +75,9 @@ describe("sandbox provider runtime", () => {
|
|||
reusableProviderLeaseId: lease.providerLeaseId,
|
||||
});
|
||||
|
||||
expect(resumed.providerLeaseId).toBe(lease.providerLeaseId);
|
||||
expect(resumed.metadata).toEqual(expect.objectContaining({ resumedLease: true }));
|
||||
expect(resumed.providerLeaseId).toMatch(/^sandbox:\/\/fake\/run-2\/[0-9a-f-]+$/);
|
||||
expect(resumed.providerLeaseId).not.toBe(lease.providerLeaseId);
|
||||
expect(resumed.metadata).not.toEqual(expect.objectContaining({ resumedLease: true }));
|
||||
});
|
||||
|
||||
it("matches reusable fake leases through the selected provider implementation", () => {
|
||||
|
|
|
|||
|
|
@ -233,7 +233,7 @@ export async function createApp(
|
|||
api.use(fileResourceRoutes(db));
|
||||
api.use(routineRoutes(db, { pluginWorkerManager: workerManager }));
|
||||
api.use(environmentRoutes(db, { pluginWorkerManager: workerManager }));
|
||||
api.use(executionWorkspaceRoutes(db));
|
||||
api.use(executionWorkspaceRoutes(db, { pluginWorkerManager: workerManager }));
|
||||
api.use(goalRoutes(db));
|
||||
api.use(boardChatRoutes(db, { deploymentMode: opts.deploymentMode }));
|
||||
api.use(approvalRoutes(db, { pluginWorkerManager: workerManager }));
|
||||
|
|
|
|||
|
|
@ -30,14 +30,19 @@ import {
|
|||
} from "./workspace-command-authz.js";
|
||||
import { assertCanManageExecutionWorkspaceRuntimeServices } from "./workspace-runtime-service-authz.js";
|
||||
import { appendWithCap } from "../adapters/utils.js";
|
||||
import { environmentRuntimeService } from "../services/environment-runtime.js";
|
||||
import type { PluginWorkerManager } from "../services/plugin-worker-manager.js";
|
||||
|
||||
const WORKSPACE_CONTROL_OUTPUT_MAX_CHARS = 256 * 1024;
|
||||
|
||||
export function executionWorkspaceRoutes(db: Db) {
|
||||
export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: PluginWorkerManager } = {}) {
|
||||
const router = Router();
|
||||
const svc = executionWorkspaceService(db);
|
||||
const access = accessService(db);
|
||||
const workspaceOperationsSvc = workspaceOperationService(db);
|
||||
const environmentRuntime = environmentRuntimeService(db, {
|
||||
pluginWorkerManager: opts.pluginWorkerManager,
|
||||
});
|
||||
|
||||
async function assertExecutionWorkspaceReadAllowed(req: Request, res: Response, companyId: string) {
|
||||
const decision = await access.decide({
|
||||
|
|
@ -538,6 +543,12 @@ export function executionWorkspaceRoutes(db: Db) {
|
|||
}
|
||||
workspace = archivedWorkspace;
|
||||
|
||||
await environmentRuntime.destroyReusableSandboxLeases({
|
||||
companyId: existing.companyId,
|
||||
executionWorkspaceId: existing.id,
|
||||
failureReason: "execution_workspace_closed",
|
||||
});
|
||||
|
||||
if (existing.mode === "shared_workspace") {
|
||||
await db
|
||||
.update(issues)
|
||||
|
|
|
|||
|
|
@ -121,6 +121,7 @@ import { feedbackService } from "../services/feedback.js";
|
|||
import { instanceSettingsService } from "../services/instance-settings.js";
|
||||
import { readAcceptedPlanConfirmationTarget } from "../services/issues.js";
|
||||
import { environmentService } from "../services/environments.js";
|
||||
import { environmentRuntimeService } from "../services/environment-runtime.js";
|
||||
import { redactSensitiveText } from "../redaction.js";
|
||||
import {
|
||||
createCompanySearchRateLimiter,
|
||||
|
|
@ -1107,6 +1108,9 @@ export function issueRoutes(
|
|||
const routinesSvc = routineService(db, {
|
||||
pluginWorkerManager: opts.pluginWorkerManager,
|
||||
});
|
||||
const environmentRuntime = environmentRuntimeService(db, {
|
||||
pluginWorkerManager: opts.pluginWorkerManager,
|
||||
});
|
||||
const issueTreeControlFactory = Object.prototype.hasOwnProperty.call(
|
||||
serviceIndex,
|
||||
"issueTreeControlService",
|
||||
|
|
@ -2793,6 +2797,27 @@ export function issueRoutes(
|
|||
});
|
||||
}
|
||||
|
||||
async function destroyReusableSandboxLeasesForTerminalIssue(issue: {
|
||||
id: string;
|
||||
companyId: string;
|
||||
status: string;
|
||||
executionWorkspaceId?: string | null;
|
||||
}) {
|
||||
try {
|
||||
await environmentRuntime.destroyReusableSandboxLeases({
|
||||
companyId: issue.companyId,
|
||||
issueId: issue.id,
|
||||
executionWorkspaceId: issue.executionWorkspaceId ?? null,
|
||||
failureReason: `issue_terminal_${issue.status}`,
|
||||
});
|
||||
} catch (err) {
|
||||
logger.warn(
|
||||
{ err, issueId: issue.id, executionWorkspaceId: issue.executionWorkspaceId ?? null },
|
||||
"failed to destroy reusable sandbox leases for terminal issue",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function resolveIssueRouteId(rawId: string): Promise<string> {
|
||||
const identifier = normalizeIssueReferenceIdentifier(rawId);
|
||||
if (identifier) {
|
||||
|
|
@ -6492,6 +6517,9 @@ export function issueRoutes(
|
|||
|
||||
const becameTerminal =
|
||||
!["done", "cancelled"].includes(existing.status) && ["done", "cancelled"].includes(issue.status);
|
||||
if (becameTerminal) {
|
||||
await destroyReusableSandboxLeasesForTerminalIssue(issue);
|
||||
}
|
||||
if (becameTerminal && issue.parentId) {
|
||||
const parent = await svc.getWakeableParentAfterChildCompletion(issue.parentId);
|
||||
if (parent) {
|
||||
|
|
@ -7883,6 +7911,9 @@ export function issueRoutes(
|
|||
const becameTerminal =
|
||||
!["done", "cancelled"].includes(issueBeforeCommentDecision.status) &&
|
||||
["done", "cancelled"].includes(currentIssue.status);
|
||||
if (becameTerminal) {
|
||||
await destroyReusableSandboxLeasesForTerminalIssue(currentIssue);
|
||||
}
|
||||
if (becameTerminal && currentIssue.parentId) {
|
||||
const parent = await svc.getWakeableParentAfterChildCompletion(currentIssue.parentId);
|
||||
if (parent) {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { and, eq, inArray } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { environmentLeases } from "@paperclipai/db";
|
||||
|
|
@ -24,7 +24,9 @@ import {
|
|||
} from "./environment-config.js";
|
||||
import {
|
||||
acquireSandboxProviderLease,
|
||||
destroySandboxProviderLease,
|
||||
findReusableSandboxProviderLeaseId,
|
||||
getSandboxProvider as getBuiltinSandboxProvider,
|
||||
isBuiltinSandboxProvider,
|
||||
releaseSandboxProviderLease,
|
||||
sandboxConfigFromLeaseMetadata,
|
||||
|
|
@ -149,6 +151,7 @@ function resolvePluginSandboxRpcTimeoutMs(config: Record<string, unknown>): numb
|
|||
export interface EnvironmentDriverLeaseInput {
|
||||
environment: Environment;
|
||||
lease: EnvironmentLease;
|
||||
failureReason?: string;
|
||||
}
|
||||
|
||||
export interface EnvironmentDriverRealizeWorkspaceInput extends EnvironmentDriverLeaseInput {
|
||||
|
|
@ -197,6 +200,118 @@ function getLeaseDriverKey(lease: Pick<EnvironmentLease, "metadata">, environmen
|
|||
return leaseDriver ?? environment.driver;
|
||||
}
|
||||
|
||||
function toEnvironmentLeaseSnapshot(row: typeof environmentLeases.$inferSelect): EnvironmentLease {
|
||||
return {
|
||||
id: row.id,
|
||||
companyId: row.companyId,
|
||||
environmentId: row.environmentId,
|
||||
executionWorkspaceId: row.executionWorkspaceId ?? null,
|
||||
issueId: row.issueId ?? null,
|
||||
heartbeatRunId: row.heartbeatRunId ?? null,
|
||||
status: row.status as EnvironmentLease["status"],
|
||||
leasePolicy: row.leasePolicy as EnvironmentLease["leasePolicy"],
|
||||
provider: row.provider ?? null,
|
||||
providerLeaseId: row.providerLeaseId ?? null,
|
||||
acquiredAt: row.acquiredAt,
|
||||
lastUsedAt: row.lastUsedAt,
|
||||
expiresAt: row.expiresAt ?? null,
|
||||
releasedAt: row.releasedAt ?? null,
|
||||
failureReason: row.failureReason ?? null,
|
||||
cleanupStatus: row.cleanupStatus as EnvironmentLease["cleanupStatus"],
|
||||
metadata: (row.metadata as Record<string, unknown> | null) ?? null,
|
||||
createdAt: row.createdAt,
|
||||
updatedAt: row.updatedAt,
|
||||
};
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function stableStringify(value: unknown): string {
|
||||
if (Array.isArray(value)) {
|
||||
return `[${value.map((entry) => stableStringify(entry)).join(",")}]`;
|
||||
}
|
||||
if (isRecord(value)) {
|
||||
return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableStringify(value[key])}`).join(",")}}`;
|
||||
}
|
||||
return JSON.stringify(value) ?? "null";
|
||||
}
|
||||
|
||||
function reusableRuntimeFingerprint(input: {
|
||||
provider: string;
|
||||
adapterType: string | null;
|
||||
config: Record<string, unknown>;
|
||||
}): string {
|
||||
return createHash("sha256")
|
||||
.update(stableStringify(input))
|
||||
.digest("hex");
|
||||
}
|
||||
|
||||
function buildReusableSandboxLeaseScope(input: {
|
||||
companyId: string;
|
||||
environmentId: string;
|
||||
executionWorkspaceId: string | null;
|
||||
agentId: string | null;
|
||||
adapterType: string | null;
|
||||
provider: string;
|
||||
config: Record<string, unknown>;
|
||||
providerMetadata?: Record<string, unknown> | null;
|
||||
}): Record<string, unknown> | null {
|
||||
if (!input.executionWorkspaceId || !input.agentId) return null;
|
||||
const providerMetadata = input.providerMetadata ?? {};
|
||||
const adapterType = input.adapterType ?? null;
|
||||
const remoteCwd = readString(providerMetadata.remoteCwd);
|
||||
const workspaceSentinel = isRecord(providerMetadata.workspaceSentinel)
|
||||
? { ...providerMetadata.workspaceSentinel }
|
||||
: null;
|
||||
return {
|
||||
version: 1,
|
||||
companyId: input.companyId,
|
||||
environmentId: input.environmentId,
|
||||
executionWorkspaceId: input.executionWorkspaceId,
|
||||
agentId: input.agentId,
|
||||
adapterType,
|
||||
provider: input.provider,
|
||||
runtimeFingerprint: reusableRuntimeFingerprint({
|
||||
provider: input.provider,
|
||||
adapterType,
|
||||
config: input.config,
|
||||
}),
|
||||
...(remoteCwd ? { remoteCwd } : {}),
|
||||
...(workspaceSentinel ? { workspaceSentinel } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function reusableSandboxLeaseScopeMatches(input: {
|
||||
lease: Pick<EnvironmentLease, "metadata">;
|
||||
companyId: string;
|
||||
environmentId: string;
|
||||
executionWorkspaceId: string | null;
|
||||
agentId: string | null;
|
||||
adapterType: string | null;
|
||||
provider: string;
|
||||
config: Record<string, unknown>;
|
||||
}): boolean {
|
||||
if (!input.executionWorkspaceId || !input.agentId) return false;
|
||||
const scope = input.lease.metadata?.reusableSandboxLease;
|
||||
if (!isRecord(scope)) return false;
|
||||
const adapterType = input.adapterType ?? null;
|
||||
return (
|
||||
scope.companyId === input.companyId &&
|
||||
scope.environmentId === input.environmentId &&
|
||||
scope.executionWorkspaceId === input.executionWorkspaceId &&
|
||||
scope.agentId === input.agentId &&
|
||||
scope.adapterType === adapterType &&
|
||||
scope.provider === input.provider &&
|
||||
scope.runtimeFingerprint === reusableRuntimeFingerprint({
|
||||
provider: input.provider,
|
||||
adapterType,
|
||||
config: input.config,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
export function findReusableSandboxLeaseId(input: {
|
||||
config: SandboxEnvironmentConfig;
|
||||
leases: Array<Pick<EnvironmentLease, "providerLeaseId" | "metadata">>;
|
||||
|
|
@ -449,13 +564,15 @@ function createSandboxEnvironmentDriver(
|
|||
|
||||
const workerConfig = stripSandboxProviderEnvelope(parsed.config);
|
||||
const storedConfig = storedParsed.config;
|
||||
const supportsReusableLeases = pluginProvider.resolved.driver.supportsReusableLeases === true;
|
||||
// Ad-hoc tests (heartbeatRunId === null) must never resume an existing
|
||||
// provider lease. If they did, releasing the test lease at the end of
|
||||
// the probe would tear down the live heartbeat run that owns it.
|
||||
// We also filter out leases whose policy is not reuse_by_environment
|
||||
// so any non-reusable lease (including ad-hoc test leases that
|
||||
// landed in the table from older code paths) cannot be matched.
|
||||
// and whose status is not reusable so non-reusable, cleanup-pending,
|
||||
// or terminal rows cannot be matched.
|
||||
const reusableExistingLeases =
|
||||
supportsReusableLeases &&
|
||||
parsed.config.reuseLease &&
|
||||
input.heartbeatRunId !== null &&
|
||||
input.executionWorkspaceId !== null &&
|
||||
|
|
@ -463,11 +580,23 @@ function createSandboxEnvironmentDriver(
|
|||
? (await environmentsSvc.listLeases(input.environment.id))
|
||||
.filter((lease) =>
|
||||
lease.leasePolicy === "reuse_by_environment" &&
|
||||
["active", "released", "retained"].includes(lease.status) &&
|
||||
lease.executionWorkspaceId === input.executionWorkspaceId &&
|
||||
lease.metadata?.agentId === input.agentId,
|
||||
lease.metadata?.agentId === input.agentId &&
|
||||
reusableSandboxLeaseScopeMatches({
|
||||
lease,
|
||||
companyId: input.companyId,
|
||||
environmentId: input.environment.id,
|
||||
executionWorkspaceId: input.executionWorkspaceId,
|
||||
agentId: input.agentId,
|
||||
adapterType: input.adapterType,
|
||||
provider: parsed.config.provider,
|
||||
config: sandboxConfigForLeaseMetadata(storedConfig),
|
||||
}),
|
||||
)
|
||||
: [];
|
||||
const reusableProviderLeaseId =
|
||||
supportsReusableLeases &&
|
||||
parsed.config.reuseLease &&
|
||||
input.heartbeatRunId !== null &&
|
||||
input.executionWorkspaceId !== null &&
|
||||
|
|
@ -478,26 +607,38 @@ function createSandboxEnvironmentDriver(
|
|||
? reusableExistingLeases.find((lease) => lease.providerLeaseId === reusableProviderLeaseId)
|
||||
: null;
|
||||
|
||||
const providerLease = reusableLease?.providerLeaseId
|
||||
? await pluginWorkerManager.call(
|
||||
pluginProvider.resolved.plugin.id,
|
||||
"environmentResumeLease",
|
||||
{
|
||||
driverKey: parsed.config.provider,
|
||||
companyId: input.companyId,
|
||||
environmentId: input.environment.id,
|
||||
issueId: input.issueId,
|
||||
config: workerConfig,
|
||||
providerLeaseId: reusableLease.providerLeaseId,
|
||||
leaseMetadata: reusableLease.metadata ?? undefined,
|
||||
},
|
||||
resolvePluginSandboxRpcTimeoutMs(workerConfig),
|
||||
).then((resumed) =>
|
||||
let providerLease: PluginEnvironmentLease | null = null;
|
||||
if (reusableLease?.providerLeaseId) {
|
||||
try {
|
||||
const resumed = await pluginWorkerManager.call(
|
||||
pluginProvider.resolved.plugin.id,
|
||||
"environmentResumeLease",
|
||||
{
|
||||
driverKey: parsed.config.provider,
|
||||
companyId: input.companyId,
|
||||
environmentId: input.environment.id,
|
||||
issueId: input.issueId,
|
||||
config: workerConfig,
|
||||
providerLeaseId: reusableLease.providerLeaseId,
|
||||
leaseMetadata: reusableLease.metadata ?? undefined,
|
||||
},
|
||||
resolvePluginSandboxRpcTimeoutMs(workerConfig),
|
||||
);
|
||||
providerLease =
|
||||
typeof resumed.providerLeaseId === "string" && resumed.providerLeaseId.length > 0
|
||||
? resumed
|
||||
: null,
|
||||
).catch(() => null)
|
||||
: null;
|
||||
: null;
|
||||
} catch {
|
||||
providerLease = null;
|
||||
}
|
||||
if (!providerLease) {
|
||||
await destroyReusableSandboxLease({
|
||||
environment: input.environment,
|
||||
lease: reusableLease,
|
||||
failureReason: "resume_failed",
|
||||
});
|
||||
}
|
||||
}
|
||||
const acquiredLease = providerLease ?? await pluginWorkerManager.call(
|
||||
pluginProvider.resolved.plugin.id,
|
||||
"environmentAcquireLease",
|
||||
|
|
@ -529,9 +670,25 @@ function createSandboxEnvironmentDriver(
|
|||
// Ad-hoc test leases are never publishable for reuse: storing them
|
||||
// as `reuse_by_environment` would let a concurrent heartbeat resume
|
||||
// the test's provider lease and lose its sandbox when the test ends.
|
||||
const resolvedLeasePolicy = parsed.config.reuseLease && input.heartbeatRunId !== null
|
||||
const resolvedLeasePolicy = supportsReusableLeases && parsed.config.reuseLease && input.heartbeatRunId !== null
|
||||
? "reuse_by_environment"
|
||||
: "ephemeral";
|
||||
const sanitizedProviderMetadata = stripSecretRefValuesFromPluginLeaseMetadata({
|
||||
metadata: acquiredLease.metadata,
|
||||
schema: pluginProvider.resolved.driver.configSchema as Record<string, unknown> | null | undefined,
|
||||
});
|
||||
const reusableScope = resolvedLeasePolicy === "reuse_by_environment"
|
||||
? buildReusableSandboxLeaseScope({
|
||||
companyId: input.companyId,
|
||||
environmentId: input.environment.id,
|
||||
executionWorkspaceId: input.executionWorkspaceId,
|
||||
agentId: input.agentId,
|
||||
adapterType: input.adapterType,
|
||||
provider: parsed.config.provider,
|
||||
config: sandboxConfigForLeaseMetadata(storedConfig),
|
||||
providerMetadata: sanitizedProviderMetadata,
|
||||
})
|
||||
: null;
|
||||
|
||||
return await environmentsSvc.acquireLease({
|
||||
companyId: input.companyId,
|
||||
|
|
@ -551,10 +708,8 @@ function createSandboxEnvironmentDriver(
|
|||
pluginKey: pluginProvider.resolved.plugin.pluginKey,
|
||||
sandboxProviderPlugin: true,
|
||||
...sandboxConfigForLeaseMetadata(storedConfig),
|
||||
...stripSecretRefValuesFromPluginLeaseMetadata({
|
||||
metadata: acquiredLease.metadata,
|
||||
schema: pluginProvider.resolved.driver.configSchema as Record<string, unknown> | null | undefined,
|
||||
}),
|
||||
...sanitizedProviderMetadata,
|
||||
...(reusableScope ? { reusableSandboxLease: reusableScope } : {}),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
|
@ -562,9 +717,12 @@ function createSandboxEnvironmentDriver(
|
|||
// Built-in sandbox provider path. Same guard as the plugin-backed path:
|
||||
// ad-hoc tests (heartbeatRunId === null) must never resume an existing
|
||||
// provider lease, or releasing the test lease will terminate the live
|
||||
// heartbeat run that shares it. Filter to leases whose policy is
|
||||
// reuse_by_environment so non-reusable rows can never be matched.
|
||||
// heartbeat run that shares it. Filter to reusable policies and statuses
|
||||
// so non-reusable, cleanup-pending, or terminal rows can never be matched.
|
||||
const builtinSandboxProvider = getBuiltinSandboxProvider(parsed.config.provider);
|
||||
const supportsReusableLeases = builtinSandboxProvider?.supportsReusableLeases === true;
|
||||
const reusableProviderLeaseId =
|
||||
supportsReusableLeases &&
|
||||
parsed.config.reuseLease &&
|
||||
input.heartbeatRunId !== null &&
|
||||
input.executionWorkspaceId !== null &&
|
||||
|
|
@ -576,28 +734,73 @@ function createSandboxEnvironmentDriver(
|
|||
config: parsed.config,
|
||||
leases: leases.filter((lease) =>
|
||||
lease.leasePolicy === "reuse_by_environment" &&
|
||||
["active", "released", "retained"].includes(lease.status) &&
|
||||
lease.executionWorkspaceId === input.executionWorkspaceId &&
|
||||
lease.metadata?.agentId === input.agentId,
|
||||
lease.metadata?.agentId === input.agentId &&
|
||||
reusableSandboxLeaseScopeMatches({
|
||||
lease,
|
||||
companyId: input.companyId,
|
||||
environmentId: input.environment.id,
|
||||
executionWorkspaceId: input.executionWorkspaceId,
|
||||
agentId: input.agentId,
|
||||
adapterType: input.adapterType,
|
||||
provider: parsed.config.provider,
|
||||
config: sandboxConfigForLeaseMetadata(parsed.config),
|
||||
}),
|
||||
),
|
||||
}),
|
||||
))
|
||||
: null;
|
||||
const reusableLease = reusableProviderLeaseId
|
||||
? (await environmentsSvc.listLeases(input.environment.id)).find((lease) => lease.providerLeaseId === reusableProviderLeaseId)
|
||||
: null;
|
||||
|
||||
const providerLease = await acquireSandboxProviderLease({
|
||||
config: parsed.config,
|
||||
environmentId: input.environment.id,
|
||||
heartbeatRunId: input.heartbeatRunId ?? randomUUID(),
|
||||
issueId: input.issueId,
|
||||
agentId: input.agentId,
|
||||
executionWorkspaceId: input.executionWorkspaceId,
|
||||
reusableProviderLeaseId,
|
||||
});
|
||||
let providerLease;
|
||||
try {
|
||||
providerLease = await acquireSandboxProviderLease({
|
||||
config: parsed.config,
|
||||
environmentId: input.environment.id,
|
||||
heartbeatRunId: input.heartbeatRunId ?? randomUUID(),
|
||||
issueId: input.issueId,
|
||||
agentId: input.agentId,
|
||||
executionWorkspaceId: input.executionWorkspaceId,
|
||||
reusableProviderLeaseId,
|
||||
});
|
||||
} catch (error) {
|
||||
if (reusableLease) {
|
||||
await destroyReusableSandboxLease({
|
||||
environment: input.environment,
|
||||
lease: reusableLease,
|
||||
failureReason: "resume_failed",
|
||||
});
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (reusableLease && providerLease.providerLeaseId !== reusableLease.providerLeaseId) {
|
||||
await destroyReusableSandboxLease({
|
||||
environment: input.environment,
|
||||
lease: reusableLease,
|
||||
failureReason: "resume_failed",
|
||||
});
|
||||
}
|
||||
|
||||
// Same ephemeral-policy-for-tests guard as the plugin-backed path:
|
||||
// ad-hoc test leases must not be publishable for reuse.
|
||||
const resolvedLeasePolicy = parsed.config.reuseLease && input.heartbeatRunId !== null
|
||||
const resolvedLeasePolicy = supportsReusableLeases && parsed.config.reuseLease && input.heartbeatRunId !== null
|
||||
? "reuse_by_environment"
|
||||
: "ephemeral";
|
||||
const reusableScope = resolvedLeasePolicy === "reuse_by_environment"
|
||||
? buildReusableSandboxLeaseScope({
|
||||
companyId: input.companyId,
|
||||
environmentId: input.environment.id,
|
||||
executionWorkspaceId: input.executionWorkspaceId,
|
||||
agentId: input.agentId,
|
||||
adapterType: input.adapterType,
|
||||
provider: parsed.config.provider,
|
||||
config: sandboxConfigForLeaseMetadata(parsed.config),
|
||||
providerMetadata: providerLease.metadata,
|
||||
})
|
||||
: null;
|
||||
|
||||
return await environmentsSvc.acquireLease({
|
||||
companyId: input.companyId,
|
||||
|
|
@ -613,11 +816,20 @@ function createSandboxEnvironmentDriver(
|
|||
driver: input.environment.driver,
|
||||
executionWorkspaceMode: input.executionWorkspaceMode,
|
||||
...providerLease.metadata,
|
||||
...(reusableScope ? { reusableSandboxLease: reusableScope } : {}),
|
||||
},
|
||||
});
|
||||
},
|
||||
|
||||
async releaseRunLease(input) {
|
||||
if (input.status === "expired" && input.lease.leasePolicy === "reuse_by_environment") {
|
||||
return await destroyReusableSandboxLease({
|
||||
environment: input.environment,
|
||||
lease: input.lease,
|
||||
failureReason: "lease_expired",
|
||||
});
|
||||
}
|
||||
|
||||
// Check if this lease was acquired through a plugin.
|
||||
if (input.lease.metadata?.sandboxProviderPlugin) {
|
||||
return await releasePluginBackedSandboxLease(input);
|
||||
|
|
@ -748,6 +960,14 @@ function createSandboxEnvironmentDriver(
|
|||
}
|
||||
throw new Error("Sandbox driver does not support direct command execution for built-in providers.");
|
||||
},
|
||||
|
||||
async destroyRunLease(input) {
|
||||
return await destroyReusableSandboxLease({
|
||||
environment: input.environment,
|
||||
lease: input.lease,
|
||||
failureReason: input.failureReason ?? "lease_destroyed",
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
async function releasePluginBackedSandboxLease(
|
||||
|
|
@ -790,6 +1010,68 @@ function createSandboxEnvironmentDriver(
|
|||
cleanupStatus,
|
||||
});
|
||||
}
|
||||
|
||||
async function destroyReusableSandboxLease(input: {
|
||||
environment: Environment;
|
||||
lease: EnvironmentLease;
|
||||
failureReason: string;
|
||||
}): Promise<EnvironmentLease | null> {
|
||||
let cleanupStatus: "success" | "failed" = "success";
|
||||
const metadata = input.lease.metadata ?? {};
|
||||
|
||||
try {
|
||||
if (metadata.sandboxProviderPlugin) {
|
||||
const pluginId = readString(metadata.pluginId);
|
||||
const providerKey = readString(metadata.provider);
|
||||
if (!pluginId || !providerKey || !pluginWorkerManager?.isRunning(pluginId)) {
|
||||
cleanupStatus = "failed";
|
||||
} else {
|
||||
const config = await resolvePluginSandboxRuntimeConfig({
|
||||
environment: input.environment,
|
||||
lease: input.lease,
|
||||
provider: providerKey,
|
||||
});
|
||||
await pluginWorkerManager.call(pluginId, "environmentDestroyLease", {
|
||||
driverKey: providerKey,
|
||||
companyId: input.lease.companyId,
|
||||
environmentId: input.environment.id,
|
||||
issueId: input.lease.issueId,
|
||||
config: stripSandboxProviderEnvelope(config as SandboxEnvironmentConfig),
|
||||
providerLeaseId: input.lease.providerLeaseId,
|
||||
leaseMetadata: metadata,
|
||||
}, resolvePluginSandboxRpcTimeoutMs(stripSandboxProviderEnvelope(config as SandboxEnvironmentConfig)));
|
||||
}
|
||||
} else {
|
||||
const metadataConfig = sandboxConfigFromLeaseMetadata(input.lease);
|
||||
const parsed = metadataConfig
|
||||
? await resolveEnvironmentDriverConfigForRuntime(db, input.lease.companyId, {
|
||||
id: input.environment.id,
|
||||
driver: "sandbox",
|
||||
config: metadataConfig as unknown as Record<string, unknown>,
|
||||
})
|
||||
: await resolveEnvironmentDriverConfigForRuntime(db, input.lease.companyId, input.environment);
|
||||
if (parsed.driver !== "sandbox") {
|
||||
cleanupStatus = "failed";
|
||||
} else {
|
||||
await destroySandboxProviderLease({
|
||||
config: parsed.config,
|
||||
providerLeaseId: input.lease.providerLeaseId,
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
cleanupStatus = "failed";
|
||||
}
|
||||
|
||||
return await environmentsSvc.releaseLease(
|
||||
input.lease.id,
|
||||
cleanupStatus === "success" ? "expired" : "pending_cleanup",
|
||||
{
|
||||
failureReason: input.failureReason,
|
||||
cleanupStatus,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function parseExpiresAt(value: string | null | undefined): Date | null {
|
||||
|
|
@ -1026,7 +1308,9 @@ function createPluginEnvironmentDriver(
|
|||
providerLeaseId: input.lease.providerLeaseId,
|
||||
leaseMetadata: input.lease.metadata ?? undefined,
|
||||
});
|
||||
return await environmentsSvc.releaseLease(input.lease.id, "failed");
|
||||
return await environmentsSvc.releaseLease(input.lease.id, "failed", {
|
||||
failureReason: input.failureReason ?? "lease_destroyed",
|
||||
});
|
||||
},
|
||||
|
||||
async realizeWorkspace(input) {
|
||||
|
|
@ -1213,27 +1497,7 @@ export function environmentRuntimeService(
|
|||
const environment = await environmentsSvc.getById(leaseRow.environmentId);
|
||||
if (!environment) continue;
|
||||
|
||||
const leaseSnapshot: EnvironmentLease = {
|
||||
id: leaseRow.id,
|
||||
companyId: leaseRow.companyId,
|
||||
environmentId: leaseRow.environmentId,
|
||||
executionWorkspaceId: leaseRow.executionWorkspaceId ?? null,
|
||||
issueId: leaseRow.issueId ?? null,
|
||||
heartbeatRunId: leaseRow.heartbeatRunId ?? null,
|
||||
status: leaseRow.status as EnvironmentLease["status"],
|
||||
leasePolicy: leaseRow.leasePolicy as EnvironmentLease["leasePolicy"],
|
||||
provider: leaseRow.provider ?? null,
|
||||
providerLeaseId: leaseRow.providerLeaseId ?? null,
|
||||
acquiredAt: leaseRow.acquiredAt,
|
||||
lastUsedAt: leaseRow.lastUsedAt,
|
||||
expiresAt: leaseRow.expiresAt ?? null,
|
||||
releasedAt: leaseRow.releasedAt ?? null,
|
||||
failureReason: leaseRow.failureReason ?? null,
|
||||
cleanupStatus: leaseRow.cleanupStatus as EnvironmentLease["cleanupStatus"],
|
||||
metadata: (leaseRow.metadata as Record<string, unknown> | null) ?? null,
|
||||
createdAt: leaseRow.createdAt,
|
||||
updatedAt: leaseRow.updatedAt,
|
||||
};
|
||||
const leaseSnapshot = toEnvironmentLeaseSnapshot(leaseRow);
|
||||
const driver = getDriver(getLeaseDriverKey(leaseSnapshot, environment));
|
||||
const lease = driver
|
||||
? await driver.releaseRunLease({
|
||||
|
|
@ -1258,6 +1522,60 @@ export function environmentRuntimeService(
|
|||
return released;
|
||||
},
|
||||
|
||||
async destroyReusableSandboxLeases(input: {
|
||||
companyId: string;
|
||||
issueId?: string | null;
|
||||
executionWorkspaceId?: string | null;
|
||||
failureReason?: string;
|
||||
}): Promise<EnvironmentRuntimeLeaseRecord[]> {
|
||||
const scopeConditions = [
|
||||
input.issueId ? eq(environmentLeases.issueId, input.issueId) : undefined,
|
||||
input.executionWorkspaceId ? eq(environmentLeases.executionWorkspaceId, input.executionWorkspaceId) : undefined,
|
||||
].filter((condition): condition is NonNullable<typeof condition> => Boolean(condition));
|
||||
if (scopeConditions.length === 0) return [];
|
||||
|
||||
const leaseRows = await db
|
||||
.select()
|
||||
.from(environmentLeases)
|
||||
.where(
|
||||
and(
|
||||
eq(environmentLeases.companyId, input.companyId),
|
||||
eq(environmentLeases.leasePolicy, "reuse_by_environment"),
|
||||
inArray(environmentLeases.status, ["active", "released", "retained", "pending_cleanup"]),
|
||||
...scopeConditions,
|
||||
),
|
||||
);
|
||||
|
||||
const destroyed: EnvironmentRuntimeLeaseRecord[] = [];
|
||||
for (const leaseRow of leaseRows) {
|
||||
const environment = await environmentsSvc.getById(leaseRow.environmentId);
|
||||
if (!environment) continue;
|
||||
const leaseSnapshot = toEnvironmentLeaseSnapshot(leaseRow);
|
||||
const driver = getDriver(getLeaseDriverKey(leaseSnapshot, environment));
|
||||
const lease = driver?.destroyRunLease
|
||||
? await driver.destroyRunLease({
|
||||
environment,
|
||||
lease: leaseSnapshot,
|
||||
failureReason: input.failureReason ?? "reusable_lease_destroyed",
|
||||
})
|
||||
: await environmentsSvc.releaseLease(leaseSnapshot.id, "pending_cleanup", {
|
||||
failureReason: input.failureReason ?? "reusable_lease_destroyed",
|
||||
cleanupStatus: "failed",
|
||||
});
|
||||
if (!lease) continue;
|
||||
destroyed.push({
|
||||
environment,
|
||||
lease,
|
||||
leaseContext: {
|
||||
executionWorkspaceId: lease.executionWorkspaceId,
|
||||
executionWorkspaceMode:
|
||||
(lease.metadata?.executionWorkspaceMode as ExecutionWorkspace["mode"] | null | undefined) ?? null,
|
||||
},
|
||||
});
|
||||
}
|
||||
return destroyed;
|
||||
},
|
||||
|
||||
async resumeRunLease(input: EnvironmentDriverLeaseInput): Promise<PluginEnvironmentLease | EnvironmentLease | null> {
|
||||
const driver = requireDriverKey(getLeaseDriverKey(input.lease, input.environment));
|
||||
if (!driver.resumeRunLease) {
|
||||
|
|
|
|||
|
|
@ -499,7 +499,7 @@ export function environmentService(db: Db) {
|
|||
|
||||
releaseLease: async (
|
||||
id: string,
|
||||
status: Extract<EnvironmentLeaseStatus, "released" | "expired" | "failed" | "retained"> = "released",
|
||||
status: Extract<EnvironmentLeaseStatus, "released" | "expired" | "failed" | "retained" | "pending_cleanup"> = "released",
|
||||
options?: {
|
||||
failureReason?: string;
|
||||
cleanupStatus?: EnvironmentLeaseCleanupStatus;
|
||||
|
|
|
|||
|
|
@ -841,6 +841,7 @@ async function resolveRunScopedMentionedSkillKeys(input: {
|
|||
function leaseReleaseStatusForRunStatus(
|
||||
status: string | null | undefined,
|
||||
): Extract<EnvironmentLeaseStatus, "released" | "expired" | "failed"> {
|
||||
if (status === "cancelled") return "expired";
|
||||
return status === "failed" || status === "timed_out" ? "failed" : "released";
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -78,6 +78,7 @@ export interface SandboxExecuteResult {
|
|||
|
||||
export interface SandboxProvider {
|
||||
readonly provider: SandboxEnvironmentProvider;
|
||||
readonly supportsReusableLeases?: boolean;
|
||||
validateConfig(config: SandboxEnvironmentConfig): Promise<SandboxProviderValidationResult>;
|
||||
probe(config: SandboxEnvironmentConfig): Promise<EnvironmentProbeResult>;
|
||||
acquireLease(input: AcquireSandboxLeaseInput): Promise<SandboxLeaseHandle>;
|
||||
|
|
@ -117,6 +118,7 @@ function buildFakeSandboxProbe(config: FakeSandboxEnvironmentConfig): Environmen
|
|||
|
||||
class FakeSandboxProvider implements SandboxProvider {
|
||||
readonly provider = "fake" as const;
|
||||
readonly supportsReusableLeases = false;
|
||||
|
||||
async validateConfig(config: SandboxEnvironmentConfig): Promise<SandboxProviderValidationResult> {
|
||||
assertProviderConfig<FakeSandboxEnvironmentConfig>(this.provider, config);
|
||||
|
|
@ -138,7 +140,7 @@ class FakeSandboxProvider implements SandboxProvider {
|
|||
|
||||
async acquireLease(input: AcquireSandboxLeaseInput): Promise<SandboxLeaseHandle> {
|
||||
assertProviderConfig<FakeSandboxEnvironmentConfig>(this.provider, input.config);
|
||||
const providerLeaseId = input.config.reuseLease
|
||||
const providerLeaseId = input.config.reuseLease && this.supportsReusableLeases
|
||||
? `sandbox://fake/${input.environmentId}/${input.executionWorkspaceId ?? "workspace"}/${input.agentId ?? "agent"}`
|
||||
: `sandbox://fake/${input.heartbeatRunId}/${randomUUID()}`;
|
||||
|
||||
|
|
@ -336,7 +338,7 @@ export async function acquireSandboxProviderLease(input: {
|
|||
reusableProviderLeaseId?: string | null;
|
||||
}): Promise<SandboxLeaseHandle> {
|
||||
const provider = requireSandboxProvider(input.config.provider);
|
||||
if (input.config.reuseLease && input.reusableProviderLeaseId) {
|
||||
if (provider.supportsReusableLeases && input.config.reuseLease && input.reusableProviderLeaseId) {
|
||||
const resumedLease = await provider.resumeLease({
|
||||
config: input.config,
|
||||
providerLeaseId: input.reusableProviderLeaseId,
|
||||
|
|
|
|||
Loading…
Reference in New Issue