diff --git a/packages/adapter-utils/src/acpx-engine/execute.test.ts b/packages/adapter-utils/src/acpx-engine/execute.test.ts index 12ea70891d..6006e05e22 100644 --- a/packages/adapter-utils/src/acpx-engine/execute.test.ts +++ b/packages/adapter-utils/src/acpx-engine/execute.test.ts @@ -2116,6 +2116,80 @@ describe("ACPX engine remote sandbox staging seam (PR 1: workspace + cwd)", () = expect(payloadEnv.PAPERCLIP_API_KEY).not.toBe("real-run-jwt"); }); + it("publishes referenced-project workspace hints repointed at their staged sandbox directories", async () => { + const { root, stateDir, localCwd, executionTarget } = await setupRemoteSandbox(); + // A referenced project with a real host tree, so the sandbox transport stages it and returns a + // `project-` directory for it. + const referencedProjectDir = path.join(root, "referenced-project-a"); + await fs.mkdir(referencedProjectDir, { recursive: true }); + await fs.writeFile(path.join(referencedProjectDir, "note.txt"), "referenced", "utf8"); + + // Decode the process-session LAUNCH payload — the in-sandbox process env is carried there. + let launchPayload: Record | null = null; + (executionTarget as { runner: unknown }).runner = createLocalSandboxRunner((input) => { + if (input.env?.PAPERCLIP_SANDBOX_EXEC_CHANNEL === "bridge") { + const script = input.args?.[1] ?? ""; + const match = script.match(/PAPERCLIP_PROCESS_SESSION_COMMAND_B64='([^']+)'/); + if (match) { + launchPayload = JSON.parse(Buffer.from(match[1]!, "base64").toString("utf8")) as Record< + string, + unknown + >; + } + } + }); + + await runExecutor( + { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir, cwd: localCwd }, + { + authToken: "real-run-jwt", + executionTarget, + context: { + taskId: "issue-1", + wakeReason: "issue_assigned", + paperclipWorkspace: { + cwd: localCwd, + realization: { + additional: [ + { + path: referencedProjectDir, + projectId: "a", + projectWorkspaceId: "ws-a", + repoUrl: "https://example.test/a.git", + repoRef: "main", + }, + ], + }, + }, + // The plural workspace-hints channel the agent reads. The referenced hint points at the + // host path today; on a remote target the run must repoint it at the staged directory. + paperclipWorkspaces: [ + { + workspaceId: "ws-a", + cwd: referencedProjectDir, + repoUrl: "https://example.test/a.git", + repoRef: "main", + projectId: "a", + }, + ], + }, + }, + ); + + const payloadEnv = ((launchPayload as Record | null)?.env ?? {}) as Record< + string, + unknown + >; + const workspacesJson = payloadEnv.PAPERCLIP_WORKSPACES_JSON; + expect(typeof workspacesJson).toBe("string"); + const hints = JSON.parse(String(workspacesJson)) as Array>; + const referencedHint = hints.find((hint) => hint.projectId === "a"); + expect(referencedHint).toBeTruthy(); + // The referenced hint repoints from the host path to its staged in-sandbox directory. + expect(String(referencedHint!.cwd)).toContain("project-a"); + expect(referencedHint!.cwd).not.toBe(referencedProjectDir); + }); + it("stops the process-session bridge when the paperclip bridge fails under concurrency", async () => { const { stateDir, localCwd, executionTarget } = await setupRemoteSandbox(); // The paperclip bridge fails; the process-session bridge — started CONCURRENTLY diff --git a/packages/adapter-utils/src/acpx-engine/execute.ts b/packages/adapter-utils/src/acpx-engine/execute.ts index e3321c02ed..645c105834 100644 --- a/packages/adapter-utils/src/acpx-engine/execute.ts +++ b/packages/adapter-utils/src/acpx-engine/execute.ts @@ -1406,6 +1406,15 @@ async function buildRuntime(input: { contentSignature: await referencedSourceContentSignature(entry.localPath), })), ); + // Referenced-project workspace hints exposed to the agent through PAPERCLIP_WORKSPACES_JSON. The + // list joins the anchor project's alternative workspaces with the referenced (mentioned) projects. + // On the confined sandbox lane the run repoints each referenced hint at its staged directory after + // staging below. Empty unless run prep resolved referenced projects or alternative workspaces. + const workspaceHints = Array.isArray(context.paperclipWorkspaces) + ? context.paperclipWorkspaces.filter( + (value): value is Record => typeof value === "object" && value !== null, + ) + : []; const executionTarget = readAdapterExecutionTarget({ executionTarget: input.ctx.executionTarget, legacyRemoteExecution: input.ctx.executionTransport?.remoteExecution, @@ -1888,6 +1897,27 @@ async function buildRuntime(input: { remoteManagedHomeTeardown = staged.value.teardown; remoteStagingDispose = staged.value.dispose; remoteStagingEnvDelta = staged.value.envDelta; + // Publish the referenced-project workspace hints to the in-sandbox agent. The staged-directory + // map (`project-`) is known only after staging above, so this runs here rather than + // with the initial workspace shaping. Each referenced hint repoints at its staged directory; a + // referenced hint whose project did not stage loses its cwd, so the agent never receives an + // unstaged path. Only the confined sandbox lane stages referenced trees, so only it publishes + // the hints; the local and runner-less lanes keep their env untouched. The set `env` write wins + // over an inherited value in the merged launch env. + const stagedProjectDirs = stagedRuntime?.additionalSourceDirs ?? {}; + if (Object.keys(stagedProjectDirs).length > 0) { + const shapedHints = shapePaperclipWorkspaceEnvForExecution({ + workspaceCwd: effectiveWorkspaceCwd, + workspaceWorktreePath, + workspaceHints, + executionTargetIsRemote, + executionCwd: effectiveExecutionCwd, + stagedProjectDirs, + }).workspaceHints; + if (shapedHints.length > 0) { + env.PAPERCLIP_WORKSPACES_JSON = JSON.stringify(shapedHints); + } + } } // Both bridge starts run under one try so a failure at EITHER — including the // paperclip callback bridge — fires the same abandon-path cleanup. The @@ -2916,6 +2946,17 @@ export function createAcpxEngineExecutor(deps: AcpxEngineExecutorOptions = {}) { rootSpan.end(true); throw err; } + // Per-project staging outcomes for the referenced (mentioned) projects, surfaced back to the + // server on the run result. A referenced project that failed to stage into the sandbox is a + // first-class, counted failure in the requested-vs-synced observability, not only a warning. The + // list is empty on a local target, on a transport that does not stage referenced projects, or + // when every staged referenced project succeeded, so the spread adds the field only when there + // is a failure to report. + const referencedProjectStagingFailures = ( + prepared.stagedRuntime?.additionalSourceFailures ?? [] + ).map((failure) => ({ projectId: failure.projectId })); + const referencedProjectStagingFailuresField = + referencedProjectStagingFailures.length > 0 ? { referencedProjectStagingFailures } : {}; // State the effective wall-clock timeout and its source up front so a // later timeout is diagnosable from the run log alone. Goes to stderr: // the acpx stdout log stream carries JSON acpx.* event payloads and must @@ -3060,6 +3101,7 @@ export function createAcpxEngineExecutor(deps: AcpxEngineExecutorOptions = {}) { errorMessage: message, ...classified, ...billingFields, + ...referencedProjectStagingFailuresField, model: prepared.requestedModel || null, clearSession, resultJson: { phase: "ensure_session" }, @@ -3079,6 +3121,7 @@ export function createAcpxEngineExecutor(deps: AcpxEngineExecutorOptions = {}) { errorMessage: "ACPX did not return a runtime session handle.", errorCode: "acpx_runtime_error", ...billingFields, + ...referencedProjectStagingFailuresField, model: prepared.requestedModel || null, resultJson: { phase: "ensure_session" }, summary: "ACPX did not return a runtime session handle.", @@ -3122,6 +3165,7 @@ export function createAcpxEngineExecutor(deps: AcpxEngineExecutorOptions = {}) { errorMessage: message, ...classified, ...billingFields, + ...referencedProjectStagingFailuresField, model: prepared.requestedModel || null, clearSession, resultJson: { @@ -3322,6 +3366,7 @@ export function createAcpxEngineExecutor(deps: AcpxEngineExecutorOptions = {}) { sessionParams: buildSessionParams({ prepared, handle: sessionHandle }), sessionDisplayId: sessionHandle.agentSessionId ?? sessionHandle.backendSessionId ?? sessionHandle.runtimeSessionName, ...billingFields, + ...referencedProjectStagingFailuresField, model: prepared.requestedModel || null, ...(turnUsage.usage ? { usage: turnUsage.usage, usageBasis: "per_run" as const } : {}), costUsd: turnUsage.costUsd, @@ -3378,6 +3423,7 @@ export function createAcpxEngineExecutor(deps: AcpxEngineExecutorOptions = {}) { errorCode: timedOut ? "acpx_timeout" : classified.errorCode, errorMeta: classified.errorMeta, ...billingFields, + ...referencedProjectStagingFailuresField, model: prepared.requestedModel || null, clearSession: clearSession || timedOut, resultJson: { phase: "turn" }, diff --git a/packages/adapter-utils/src/execution-target.ts b/packages/adapter-utils/src/execution-target.ts index 9652fd2cb3..d1d878d7c2 100644 --- a/packages/adapter-utils/src/execution-target.ts +++ b/packages/adapter-utils/src/execution-target.ts @@ -14,8 +14,14 @@ import { prepareRemoteManagedRuntime, remoteExecutionSessionMatches, } from "./remote-managed-runtime.js"; -import type { SandboxAdditionalSource } from "./sandbox-managed-runtime.js"; -export type { SandboxAdditionalSource } from "./sandbox-managed-runtime.js"; +import type { + AdditionalSourceStagingFailure, + SandboxAdditionalSource, +} from "./sandbox-managed-runtime.js"; +export type { + AdditionalSourceStagingFailure, + SandboxAdditionalSource, +} from "./sandbox-managed-runtime.js"; import { createCommandManagedSandboxCallbackBridgeQueueClient, createSandboxCallbackBridgeAsset, @@ -122,6 +128,12 @@ export interface PreparedAdapterExecutionTargetRuntime { * additional sources were requested. */ additionalSourceDirs: Record; + /** + * Each additional (referenced) project whose staging failed, paired with the + * failure message. Empty for a local target, for a transport that does not + * stage referenced projects, or when every requested project staged. + */ + additionalSourceFailures: AdditionalSourceStagingFailure[]; restoreWorkspace(onProgress?: RuntimeProgressSink): Promise; } @@ -1135,6 +1147,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: { runtimeRootDir: null, assetDirs: {}, additionalSourceDirs: {}, + additionalSourceFailures: [], restoreWorkspace: async () => {}, }; } @@ -1157,6 +1170,9 @@ export async function prepareAdapterExecutionTargetRuntime(input: { runtimeRootDir: prepared.runtimeRootDir, assetDirs: prepared.assetDirs, additionalSourceDirs: prepared.additionalSourceDirs, + // The SSH transport does not stage referenced projects (it is out of scope), so it never + // reports a per-project staging failure. + additionalSourceFailures: [], restoreWorkspace: prepared.restoreWorkspace, }; } @@ -1192,6 +1208,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: { runtimeRootDir: prepared.runtimeRootDir, assetDirs: prepared.assetDirs, additionalSourceDirs: prepared.additionalSourceDirs, + additionalSourceFailures: prepared.additionalSourceFailures, restoreWorkspace: prepared.restoreWorkspace, }; } diff --git a/packages/adapter-utils/src/sandbox-managed-runtime.test.ts b/packages/adapter-utils/src/sandbox-managed-runtime.test.ts index db796761f2..4e82cb0d86 100644 --- a/packages/adapter-utils/src/sandbox-managed-runtime.test.ts +++ b/packages/adapter-utils/src/sandbox-managed-runtime.test.ts @@ -1904,6 +1904,14 @@ describe("sandbox managed runtime", () => { expect(prepared.additionalSourceDirs["proj-second"]).toBe(path.posix.join(runtimeRootDir, "project-proj-second")); expect(prepared.additionalSourceDirs["proj-missing"]).toBeUndefined(); + // The skipped project is a first-class per-project failure outcome, not only a warning, so the + // run can count it in the requested-vs-synced accounting. The two healthy projects do not + // appear as failures. + expect(prepared.additionalSourceFailures.map((failure) => failure.projectId)).toEqual([ + "proj-missing", + ]); + expect(prepared.additionalSourceFailures[0]!.error.length).toBeGreaterThan(0); + await expect(readFile(path.join(prepared.additionalSourceDirs["proj-first"], "docs", "guide.md"), "utf8")) .resolves.toBe("first guide\n"); await expect(readFile(path.join(prepared.additionalSourceDirs["proj-second"], "notes.md"), "utf8")) diff --git a/packages/adapter-utils/src/sandbox-managed-runtime.ts b/packages/adapter-utils/src/sandbox-managed-runtime.ts index 0cff256464..74b1f4520c 100644 --- a/packages/adapter-utils/src/sandbox-managed-runtime.ts +++ b/packages/adapter-utils/src/sandbox-managed-runtime.ts @@ -300,9 +300,23 @@ export interface PreparedSandboxManagedRuntime { * were requested. */ additionalSourceDirs: Record; + /** + * Each additional (referenced) project whose staging failed, paired with the + * failure message. Per-project failure isolation keeps one project's failure + * from aborting the run, so a failed project is absent from + * `additionalSourceDirs` and present here. Empty when every requested project + * staged, or when no additional sources were requested. + */ + additionalSourceFailures: AdditionalSourceStagingFailure[]; restoreWorkspace(onProgress?: RuntimeProgressSink): Promise; } +/** One additional (referenced) project that failed to stage into the sandbox. */ +export interface AdditionalSourceStagingFailure { + projectId: string; + error: string; +} + function asObject(value: unknown): Record { return value && typeof value === "object" && !Array.isArray(value) ? (value as Record) @@ -736,6 +750,10 @@ export async function prepareSandboxManagedRuntime(input: { // Remote directory of each additional (referenced) project that stages // successfully, keyed by projectId. A project that fails to stage is absent. const additionalSourceDirs: Record = {}; + // Each additional (referenced) project whose staging failed, paired with the + // failure message. Per-project failure isolation keeps the run and the other + // projects going; this list makes each failure a first-class, reported outcome. + const additionalSourceFailures: AdditionalSourceStagingFailure[] = []; // Additional projects stage as plain trees. Drop the heavy build/cache dirs a // reference tree does not need, and `.git` — additional sources never carry // git-history semantics (anchor-only). @@ -999,9 +1017,13 @@ export async function prepareSandboxManagedRuntime(input: { }); additionalSourceDirs[projectId] = remoteProjectDir; } catch (error) { + const message = error instanceof Error ? error.message : String(error); console.warn( - `[paperclip] Failed to stage referenced project ${projectId}; skipping it. ${String(error)}`, + `[paperclip] Failed to stage referenced project ${projectId}; skipping it. ${message}`, ); + // Record the failure as a first-class per-project outcome so the run can count it in the + // requested-vs-synced accounting instead of losing it to a warning line. + additionalSourceFailures.push({ projectId, error: message }); } } }); @@ -1017,6 +1039,7 @@ export async function prepareSandboxManagedRuntime(input: { runtimeRootDir, assetDirs, additionalSourceDirs, + additionalSourceFailures, restoreWorkspace: async (onProgress?: RuntimeProgressSink) => { const restoreSink = onProgress ?? input.onProgress; if (!syncWorkspace) { diff --git a/packages/adapter-utils/src/server-utils.test.ts b/packages/adapter-utils/src/server-utils.test.ts index 4d4c9d6523..74f032a968 100644 --- a/packages/adapter-utils/src/server-utils.test.ts +++ b/packages/adapter-utils/src/server-utils.test.ts @@ -2295,6 +2295,53 @@ describe("shapePaperclipWorkspaceEnvForExecution", () => { }); }); + it("repoints a referenced hint to its staged remote directory when the map has an entry", () => { + const shaped = shapePaperclipWorkspaceEnvForExecution({ + workspaceCwd: "/tmp/workspace", + workspaceWorktreePath: "/tmp/worktree", + workspaceHints: [ + // The anchor hint keeps its remote-cwd rewrite. + { workspaceId: "workspace-1", cwd: "/tmp/workspace" }, + // A referenced hint with a staged directory repoints at it. + { workspaceId: "workspace-2", cwd: "/tmp/referenced/project-a", projectId: "project-a" }, + // A referenced hint with no staged directory loses its cwd. + { workspaceId: "workspace-3", cwd: "/tmp/referenced/project-b", projectId: "project-b" }, + ], + executionTargetIsRemote: true, + executionCwd: "/remote/workspace", + stagedProjectDirs: { "project-a": "/remote/runtime/project-project-a" }, + }); + + expect(shaped).toEqual({ + workspaceCwd: "/remote/workspace", + workspaceWorktreePath: null, + workspaceHints: [ + { workspaceId: "workspace-1", cwd: "/remote/workspace" }, + { + workspaceId: "workspace-2", + cwd: "/remote/runtime/project-project-a", + projectId: "project-a", + }, + { workspaceId: "workspace-3", projectId: "project-b" }, + ], + }); + }); + + it("removes cwd from a referenced hint that has no staged directory", () => { + const shaped = shapePaperclipWorkspaceEnvForExecution({ + workspaceCwd: "/tmp/workspace", + workspaceHints: [ + { workspaceId: "workspace-2", cwd: "/tmp/referenced/project-a", projectId: "project-a" }, + ], + executionTargetIsRemote: true, + executionCwd: "/remote/workspace", + // The map is empty, so the referenced hint has no staged directory. + stagedProjectDirs: {}, + }); + + expect(shaped.workspaceHints).toEqual([{ workspaceId: "workspace-2", projectId: "project-a" }]); + }); + it("leaves local execution workspace paths unchanged", () => { const workspaceHints = [{ workspaceId: "workspace-1", cwd: "/tmp/workspace" }]; const shaped = shapePaperclipWorkspaceEnvForExecution({ diff --git a/packages/adapter-utils/src/server-utils.ts b/packages/adapter-utils/src/server-utils.ts index 5a3fc36cbb..41d96a2349 100644 --- a/packages/adapter-utils/src/server-utils.ts +++ b/packages/adapter-utils/src/server-utils.ts @@ -2026,6 +2026,15 @@ export function shapePaperclipWorkspaceEnvForExecution(input: { workspaceHints?: Array>; executionTargetIsRemote?: boolean; executionCwd?: string | null; + /** + * On a remote target, the map of referenced (mentioned) project id to the staged in-sandbox + * directory that received that project's tree (`project-`). A non-anchor hint whose + * `projectId` has an entry repoints its `cwd` to the staged directory. A non-anchor hint with no + * entry loses its `cwd`, so the agent never receives a path the transport did not stage. The map + * is empty on a local target and defaults to empty, so a caller that passes nothing keeps the + * previous behavior (every non-anchor hint loses its `cwd` on a remote target). + */ + stagedProjectDirs?: Record; }): { workspaceCwd: string | null; workspaceWorktreePath: string | null; @@ -2068,6 +2077,7 @@ export function shapePaperclipWorkspaceEnvForExecution(input: { } const realizedWorkspaceCwd = executionCwd; const localWorkspaceCwd = workspaceCwd ? path.resolve(workspaceCwd) : null; + const stagedProjectDirs = input.stagedProjectDirs ?? {}; const shapedWorkspaceHints = workspaceHints.map((hint) => { const nextHint = { ...hint }; const hintCwd = typeof nextHint.cwd === "string" ? nextHint.cwd.trim() : ""; @@ -2082,7 +2092,19 @@ export function shapePaperclipWorkspaceEnvForExecution(input: { return nextHint; } - delete nextHint.cwd; + // A referenced (mentioned) project hint carries its `projectId`. When the transport staged that + // project into the sandbox, repoint the hint at its staged `project-` directory so + // the agent reads it there. Without a staged directory the hint would point at a local path that + // the remote target cannot reach, so remove the `cwd` (fail loud — never expose an unstaged + // path). This also removes the `cwd` of a non-anchor hint that carries no `projectId`, such as an + // alternative anchor-project workspace, which keeps the previous behavior for those hints. + const hintProjectId = typeof nextHint.projectId === "string" ? nextHint.projectId : ""; + const stagedProjectDir = hintProjectId ? stagedProjectDirs[hintProjectId] : undefined; + if (stagedProjectDir && stagedProjectDir.trim().length > 0) { + nextHint.cwd = stagedProjectDir.trim(); + } else { + delete nextHint.cwd; + } return nextHint; }); @@ -2145,6 +2167,8 @@ export function refreshPaperclipWorkspaceEnvForExecution(input: { agentHome?: string | null; executionTargetIsRemote?: boolean; executionCwd?: string | null; + /** Referenced-project id to staged in-sandbox directory map; see {@link shapePaperclipWorkspaceEnvForExecution}. */ + stagedProjectDirs?: Record; }): { workspaceCwd: string | null; workspaceWorktreePath: string | null; @@ -2156,6 +2180,7 @@ export function refreshPaperclipWorkspaceEnvForExecution(input: { workspaceHints: input.workspaceHints, executionTargetIsRemote: input.executionTargetIsRemote, executionCwd: input.executionCwd, + stagedProjectDirs: input.stagedProjectDirs, }); delete input.env.PAPERCLIP_WORKSPACE_CWD; diff --git a/packages/adapter-utils/src/types.ts b/packages/adapter-utils/src/types.ts index 988e1318e7..d065722def 100644 --- a/packages/adapter-utils/src/types.ts +++ b/packages/adapter-utils/src/types.ts @@ -103,6 +103,14 @@ export interface AdapterExecutionResult { costUsd?: number | null; resultJson?: Record | null; runtimeServices?: AdapterRuntimeServiceReport[]; + /** + * Each referenced (mentioned) project that failed to stage into the remote sandbox for this run, + * by `projectId`. The run continues without a failed project (per-project failure isolation); this + * field carries the failure back so the server counts it in the requested-vs-synced observability + * instead of losing it to a warning line. Absent or empty on a local target, or when every staged + * referenced project succeeded. + */ + referencedProjectStagingFailures?: Array<{ projectId: string }>; summary?: string | null; clearSession?: boolean; question?: { diff --git a/packages/plugins/sdk/src/protocol.ts b/packages/plugins/sdk/src/protocol.ts index f85446a83c..91d1d58f66 100644 --- a/packages/plugins/sdk/src/protocol.ts +++ b/packages/plugins/sdk/src/protocol.ts @@ -638,6 +638,12 @@ export interface PluginEnvironmentRealizeWorkspaceParams extends PluginEnvironme }; } +/** + * A plugin `environmentRealizeWorkspace` handler returns only the realized cwd and provider + * metadata. The server, not the plugin, builds the full workspace-realization record from the run + * request and merges this cwd and metadata into it. Do not return a `workspaceRealization` record + * here; the server owns that record, so the referenced (mentioned) project sources reach the adapter. + */ export interface PluginEnvironmentRealizeWorkspaceResult { cwd: string; metadata?: Record; diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index cefe00fc8b..3c836883ab 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -699,6 +699,231 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentReleaseLease", expect.anything(), 31234); }); + it("builds the workspace-realization record with referenced sources for a plugin-backed sandbox realize", async () => { + // A provider plugin realize handler returns only its realized cwd and provider metadata; it does + // not build the workspace-realization record. The server must build that record from the run + // request, so the referenced (mentioned) project sources reach the adapter through + // `realization.additional`. Without the record the sandbox agent never receives the mentioned + // projects. This test drives the plugin-backed sandbox realize path and asserts the referenced + // source survives into the returned record. + const pluginId = randomUUID(); + const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); + const fakePluginConfig = { + provider: "fake-plugin", + image: "fake:test", + timeoutMs: 1234, + reuseLease: false, + }; + const environment = { + ...baseEnvironment, + name: "Fake Plugin Sandbox Realize", + driver: "sandbox", + config: fakePluginConfig, + }; + await environmentService(db).update(environment.id, { + driver: "sandbox", + name: environment.name, + config: fakePluginConfig, + }); + await db.insert(plugins).values({ + id: pluginId, + pluginKey: "paperclip.fake-plugin-sandbox-provider", + packageName: "@paperclipai/plugin-fake-sandbox", + version: "1.0.0", + apiVersion: 1, + categories: ["automation"], + manifestJson: { + id: "paperclip.fake-plugin-sandbox-provider", + apiVersion: 1, + version: "1.0.0", + displayName: "Fake Plugin Sandbox Provider", + description: "Test fake plugin provider", + author: "Paperclip", + categories: ["automation"], + capabilities: ["environment.drivers.register"], + entrypoints: { worker: "dist/worker.js" }, + environmentDrivers: [ + { + driverKey: "fake-plugin", + kind: "sandbox_provider", + displayName: "Fake Plugin", + configSchema: { type: "object" }, + }, + ], + }, + status: "ready", + installOrder: 1, + updatedAt: new Date(), + } as any); + const workerManager = { + isRunning: vi.fn((id: string) => id === pluginId), + call: vi.fn(async (_pluginId: string, method: string) => { + if (method === "environmentAcquireLease") { + return { + providerLeaseId: "sandbox-realize-1", + metadata: { + provider: "fake-plugin", + image: "fake:test", + timeoutMs: 1234, + reuseLease: false, + remoteCwd: "/workspace", + }, + }; + } + if (method === "environmentRealizeWorkspace") { + // Mimic a real provider (for example Daytona): return only the realized cwd and provider + // metadata, never a `workspaceRealization` record. + return { + cwd: "/workspace/project", + metadata: { + provider: "fake-plugin", + remoteCwd: "/workspace/project", + }, + }; + } + throw new Error(`Unexpected plugin method: ${method}`); + }), + } as unknown as PluginWorkerManager; + const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); + + const acquired = await runtimeWithPlugin.acquireRunLease({ + companyId, + environment, + issueId: null, + heartbeatRunId: runId, + persistedExecutionWorkspace: null, + }); + const workspaceRealizationRequest = { + version: 1, + adapterType: "codex_local", + companyId, + environmentId: environment.id, + executionWorkspaceId: null, + issueId: null, + heartbeatRunId: runId, + requestedMode: "ephemeral", + source: { + kind: "project_primary", + localPath: "/tmp/anchor", + projectId: "anchor-project", + projectWorkspaceId: "anchor-workspace", + repoUrl: null, + repoRef: null, + strategy: "project_primary", + branchName: null, + worktreePath: null, + }, + additionalSources: [ + { + localPath: "/tmp/referenced-project", + projectId: "referenced-project-1", + projectWorkspaceId: "referenced-workspace-1", + repoUrl: null, + repoRef: null, + }, + ], + }; + const realized = await runtimeWithPlugin.realizeWorkspace({ + environment, + lease: acquired.lease, + workspace: { + localPath: "/tmp/anchor", + mode: "ephemeral", + metadata: { workspaceRealizationRequest }, + }, + }); + + // The provider realized cwd and provider metadata survive. + expect(realized.cwd).toBe("/workspace/project"); + expect(realized.metadata?.provider).toBe("fake-plugin"); + // The server-built record carries the referenced source through `additional`, so the adapter can + // stage the mentioned project into the sandbox. + const realization = realized.metadata?.workspaceRealization as Record | undefined; + expect(realization).toBeDefined(); + expect(realization?.additional).toEqual([ + expect.objectContaining({ + path: "/tmp/referenced-project", + projectId: "referenced-project-1", + }), + ]); + }); + + it("builds the workspace-realization record with referenced sources for a built-in sandbox realize", async () => { + // The sandbox driver `realizeWorkspace` has two exits: a plugin-backed provider and a + // built-in provider. Both must build the same workspace-realization record, so the + // referenced (mentioned) project sources reach the adapter through `realization.additional`. + // The test above covers the plugin exit. This test covers the built-in exit (no provider + // plugin), so a regression on either exit fails a test. + const { companyId, environment, runId } = await seedEnvironment({ + driver: "sandbox", + name: "Fake Sandbox Realize", + config: { + provider: "fake", + image: "ubuntu:24.04", + reuseLease: true, + }, + }); + + const acquired = await runtime.acquireRunLease({ + companyId, + environment, + issueId: null, + heartbeatRunId: runId, + persistedExecutionWorkspace: null, + }); + + const workspaceRealizationRequest = { + version: 1, + adapterType: "codex_local", + companyId, + environmentId: environment.id, + executionWorkspaceId: null, + issueId: null, + heartbeatRunId: runId, + requestedMode: "ephemeral", + source: { + kind: "project_primary", + localPath: "/tmp/anchor", + projectId: "anchor-project", + projectWorkspaceId: "anchor-workspace", + repoUrl: null, + repoRef: null, + strategy: "project_primary", + branchName: null, + worktreePath: null, + }, + additionalSources: [ + { + localPath: "/tmp/referenced-project", + projectId: "referenced-project-1", + projectWorkspaceId: "referenced-workspace-1", + repoUrl: null, + repoRef: null, + }, + ], + }; + const realized = await runtime.realizeWorkspace({ + environment, + lease: acquired.lease, + workspace: { + localPath: "/tmp/anchor", + mode: "ephemeral", + metadata: { workspaceRealizationRequest }, + }, + }); + + // The built-in exit builds the record and carries the referenced source through `additional`, + // so the adapter can stage the mentioned project into the sandbox. + const realization = realized.metadata?.workspaceRealization as Record | undefined; + expect(realization).toBeDefined(); + expect(realization?.additional).toEqual([ + expect.objectContaining({ + path: "/tmp/referenced-project", + projectId: "referenced-project-1", + }), + ]); + }); + it("uses resolved secret-ref config for plugin-backed sandbox execute and release", async () => { const pluginId = randomUUID(); const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); diff --git a/server/src/__tests__/heartbeat-remote-referenced-projects.test.ts b/server/src/__tests__/heartbeat-remote-referenced-projects.test.ts new file mode 100644 index 0000000000..de2403f820 --- /dev/null +++ b/server/src/__tests__/heartbeat-remote-referenced-projects.test.ts @@ -0,0 +1,359 @@ +import { randomUUID } from "node:crypto"; +import { describe, expect, it } from "vitest"; +import { + isConfinedRemoteStagingDriver, + isMultiProjectWorkspaceSyncRemoteEnabled, + MAX_RUN_REFERENCED_ADDITIONAL_PROJECTS, + MAX_RUN_REFERENCED_CANDIDATE_EVALUATIONS, + MULTI_PROJECT_WORKSPACE_SYNC_REMOTE_ENV, + resolveAdditionalRunWorkspaces, + type ResolveAdditionalRunWorkspacesOptions, + type ResolvedAdditionalWorkspace, +} from "../services/heartbeat.ts"; +import type { AuthorizationActor, AuthorizationDecision } from "../services/authorization.ts"; + +// These tests exercise the remote gate in `resolveAdditionalRunWorkspaces` with fully injected +// dependencies (no database). The function reads mentions through `issues.findMentionedProjectIds`, +// hydrates candidates through `projects.listByIds`, authorizes each through `access.decide`, and +// resolves an admitted project to a workspace through `resolveProjectWorkspace`. Each dependency is +// a stub here, so a test asserts the exact gate behavior for a sandbox target, an SSH target, and +// the remote kill switch. + +const buildActor = (companyId: string): AuthorizationActor => ({ + type: "agent", + agentId: randomUUID(), + companyId, + source: "agent_key", +}); + +const decision = (allowed: boolean): AuthorizationDecision => ({ + allowed, + action: "project:read", + reason: allowed ? "allow_company_agent" : "deny_company_boundary", + explanation: "test decision", +}); + +// A `projects.listByIds` stub that returns one minimal record per requested id, in request order. +// The referenced-project record only needs an `id` here; the workspace resolution is stubbed, so no +// other field is read. The cast keeps the stub minimal without restating the full project row shape. +const listByIdsStub: ResolveAdditionalRunWorkspacesOptions["projects"] = { + listByIds: async (_companyId, ids) => + ids.map( + (id) => + ({ id, name: `Project ${id}`, status: "in_progress" }) as Awaited< + ReturnType + >[number], + ), +}; + +// A `findMentionedProjectIds` stub that returns a fixed mention set for any issue. +const mentionsStub = ( + mentionedProjectIds: string[], +): ResolveAdditionalRunWorkspacesOptions["issues"] => ({ + findMentionedProjectIds: async () => mentionedProjectIds, +}); + +// Records every `project:read` authorization call and answers via the supplied resolver. +const recordingAccess = ( + resolve: (projectId: string) => AuthorizationDecision, +): { decidedProjectIds: string[]; access: ResolveAdditionalRunWorkspacesOptions["access"] } => { + const decidedProjectIds: string[] = []; + const access: ResolveAdditionalRunWorkspacesOptions["access"] = { + decide: async (input) => { + const resource = input.resource; + const projectId = resource.type === "project" ? (resource.projectId ?? "") : ""; + decidedProjectIds.push(projectId); + return resolve(projectId); + }, + }; + return { decidedProjectIds, access }; +}; + +// Records every workspace that resolution stages and returns a read-only workspace stub for it. +const recordingResolveProjectWorkspace = (): { + stagedProjectIds: string[]; + resolveProjectWorkspace: ResolveAdditionalRunWorkspacesOptions["resolveProjectWorkspace"]; +} => { + const stagedProjectIds: string[] = []; + const resolveProjectWorkspace: ResolveAdditionalRunWorkspacesOptions["resolveProjectWorkspace"] = + async (project) => { + stagedProjectIds.push(project.projectId); + return { + cwd: `/tmp/referenced/${project.projectId}`, + projectId: project.projectId, + workspaceId: null, + repoUrl: null, + repoRef: null, + } satisfies ResolvedAdditionalWorkspace; + }; + return { stagedProjectIds, resolveProjectWorkspace }; +}; + +const baseOpts = ( + companyId: string, + overrides: Partial & + Pick, +): ResolveAdditionalRunWorkspacesOptions => ({ + enabled: true, + companyId, + actor: buildActor(companyId), + projects: listByIdsStub, + ...overrides, +}); + +describe("remote referenced-project kill switch", () => { + it("is ON by default and disabled only by an explicit false env value", () => { + // Default ON: an unset value resolves the remote path live (go-live default). + expect(isMultiProjectWorkspaceSyncRemoteEnabled({})).toBe(true); + // The targeted kill switch: an explicit false value (the rollback path) disables it. + expect( + isMultiProjectWorkspaceSyncRemoteEnabled({ [MULTI_PROJECT_WORKSPACE_SYNC_REMOTE_ENV]: "" }), + ).toBe(false); + expect( + isMultiProjectWorkspaceSyncRemoteEnabled({ + [MULTI_PROJECT_WORKSPACE_SYNC_REMOTE_ENV]: "false", + }), + ).toBe(false); + expect( + isMultiProjectWorkspaceSyncRemoteEnabled({ [MULTI_PROJECT_WORKSPACE_SYNC_REMOTE_ENV]: "0" }), + ).toBe(false); + expect( + isMultiProjectWorkspaceSyncRemoteEnabled({ [MULTI_PROJECT_WORKSPACE_SYNC_REMOTE_ENV]: "off" }), + ).toBe(false); + // Any other value keeps the remote path on. + expect( + isMultiProjectWorkspaceSyncRemoteEnabled({ [MULTI_PROJECT_WORKSPACE_SYNC_REMOTE_ENV]: "true" }), + ).toBe(true); + }); + + it("classifies only the sandbox driver as a confined remote staging transport", () => { + // Only the sandbox driver confines each staged referenced tree, so only it opens the gate. + expect(isConfinedRemoteStagingDriver("sandbox")).toBe(true); + // The SSH and plugin drivers keep dropping referenced projects. + expect(isConfinedRemoteStagingDriver("ssh")).toBe(false); + expect(isConfinedRemoteStagingDriver("plugin")).toBe(false); + expect(isConfinedRemoteStagingDriver("local")).toBe(false); + expect(isConfinedRemoteStagingDriver(null)).toBe(false); + expect(isConfinedRemoteStagingDriver(undefined)).toBe(false); + }); +}); + +describe("resolveAdditionalRunWorkspaces remote gate", () => { + it("resolves and authorizes referenced workspaces on a sandbox target with the remote flag on", async () => { + const companyId = randomUUID(); + const issueId = randomUUID(); + const anchorProjectId = randomUUID(); + const mentionedProjectId = randomUUID(); + const { decidedProjectIds, access } = recordingAccess(() => decision(true)); + const { stagedProjectIds, resolveProjectWorkspace } = recordingResolveProjectWorkspace(); + + const result = await resolveAdditionalRunWorkspaces( + issueId, + anchorProjectId, + baseOpts(companyId, { + executionTargetIsRemote: true, + targetStagesConfined: true, + remoteReferencedSyncEnabled: true, + issues: mentionsStub([mentionedProjectId]), + access, + resolveProjectWorkspace, + }), + ); + + // The run authorized the referenced project against the run actor and staged its workspace. + expect(decidedProjectIds).toEqual([mentionedProjectId]); + expect(stagedProjectIds).toEqual([mentionedProjectId]); + expect(result.additionalWorkspaces.map((workspace) => workspace.projectId)).toEqual([ + mentionedProjectId, + ]); + expect(result.failures).toEqual([]); + }); + + it("fails closed on a sandbox target when the remote flag is off (no authorization, no staging)", async () => { + const companyId = randomUUID(); + const issueId = randomUUID(); + const anchorProjectId = randomUUID(); + const mentionedProjectId = randomUUID(); + const { decidedProjectIds, access } = recordingAccess(() => decision(true)); + const { stagedProjectIds, resolveProjectWorkspace } = recordingResolveProjectWorkspace(); + + const result = await resolveAdditionalRunWorkspaces( + issueId, + anchorProjectId, + baseOpts(companyId, { + executionTargetIsRemote: true, + targetStagesConfined: true, + remoteReferencedSyncEnabled: false, + issues: mentionsStub([mentionedProjectId]), + access, + resolveProjectWorkspace, + }), + ); + + // Fail closed: no authorization decision and no workspace staging ran. + expect(decidedProjectIds).toEqual([]); + expect(stagedProjectIds).toEqual([]); + expect(result.additionalWorkspaces).toEqual([]); + // The whole referenced set is still counted as a staging-layer drop. + expect(result.failures).toEqual([{ projectId: mentionedProjectId, reason: "staging" }]); + }); + + it("drops referenced projects on an SSH target whether the remote flag is on or off", async () => { + const companyId = randomUUID(); + const issueId = randomUUID(); + const anchorProjectId = randomUUID(); + const mentionedProjectId = randomUUID(); + + for (const remoteReferencedSyncEnabled of [true, false]) { + const { decidedProjectIds, access } = recordingAccess(() => decision(true)); + const { stagedProjectIds, resolveProjectWorkspace } = recordingResolveProjectWorkspace(); + + const result = await resolveAdditionalRunWorkspaces( + issueId, + anchorProjectId, + baseOpts(companyId, { + executionTargetIsRemote: true, + // The SSH transport does not confine its staging path, so it stays out of scope. + targetStagesConfined: false, + remoteReferencedSyncEnabled, + issues: mentionsStub([mentionedProjectId]), + access, + resolveProjectWorkspace, + }), + ); + + expect(decidedProjectIds).toEqual([]); + expect(stagedProjectIds).toEqual([]); + expect(result.additionalWorkspaces).toEqual([]); + expect(result.failures).toEqual([{ projectId: mentionedProjectId, reason: "staging" }]); + } + }); + + it("drops a referenced project the run agent cannot read on a sandbox target", async () => { + const companyId = randomUUID(); + const issueId = randomUUID(); + const anchorProjectId = randomUUID(); + const readableProjectId = randomUUID(); + const unreadableProjectId = randomUUID(); + // Deny project:read for the unreadable project; allow it for the readable one. + const { decidedProjectIds, access } = recordingAccess((projectId) => + decision(projectId === readableProjectId), + ); + const { stagedProjectIds, resolveProjectWorkspace } = recordingResolveProjectWorkspace(); + + const result = await resolveAdditionalRunWorkspaces( + issueId, + anchorProjectId, + baseOpts(companyId, { + executionTargetIsRemote: true, + targetStagesConfined: true, + remoteReferencedSyncEnabled: true, + issues: mentionsStub([readableProjectId, unreadableProjectId]), + access, + resolveProjectWorkspace, + }), + ); + + // Both projects were authorized against the run actor, but only the readable one was staged. + expect(decidedProjectIds).toEqual([readableProjectId, unreadableProjectId]); + expect(stagedProjectIds).toEqual([readableProjectId]); + expect(result.additionalWorkspaces.map((workspace) => workspace.projectId)).toEqual([ + readableProjectId, + ]); + // The denied project is a first-class authorization failure. + expect(result.failures).toEqual([{ projectId: unreadableProjectId, reason: "authorization" }]); + }); + + it("never authorizes the anchor project on a sandbox target", async () => { + const companyId = randomUUID(); + const issueId = randomUUID(); + const anchorProjectId = randomUUID(); + const mentionedProjectId = randomUUID(); + const { decidedProjectIds, access } = recordingAccess(() => decision(true)); + const { resolveProjectWorkspace } = recordingResolveProjectWorkspace(); + + await resolveAdditionalRunWorkspaces( + issueId, + anchorProjectId, + baseOpts(companyId, { + executionTargetIsRemote: true, + targetStagesConfined: true, + remoteReferencedSyncEnabled: true, + // The mention set includes the anchor itself; the anchor must never be re-authorized. + issues: mentionsStub([anchorProjectId, mentionedProjectId]), + access, + resolveProjectWorkspace, + }), + ); + + // The anchor exemption drops the anchor from the mention set before the per-project check, so + // only the non-anchor referenced project is authorized. + expect(decidedProjectIds).toEqual([mentionedProjectId]); + expect(decidedProjectIds).not.toContain(anchorProjectId); + }); + + it("holds the admitted-project cap on a sandbox target", async () => { + const companyId = randomUUID(); + const issueId = randomUUID(); + const anchorProjectId = randomUUID(); + // One more mention than the admitted cap. + const mentionedProjectIds = Array.from( + { length: MAX_RUN_REFERENCED_ADDITIONAL_PROJECTS + 1 }, + () => randomUUID(), + ); + const { access } = recordingAccess(() => decision(true)); + const { stagedProjectIds, resolveProjectWorkspace } = recordingResolveProjectWorkspace(); + + const result = await resolveAdditionalRunWorkspaces( + issueId, + anchorProjectId, + baseOpts(companyId, { + executionTargetIsRemote: true, + targetStagesConfined: true, + remoteReferencedSyncEnabled: true, + issues: mentionsStub(mentionedProjectIds), + access, + resolveProjectWorkspace, + }), + ); + + // The cap bounds how many additional projects a sandbox run materializes. + expect(stagedProjectIds).toHaveLength(MAX_RUN_REFERENCED_ADDITIONAL_PROJECTS); + expect(result.additionalWorkspaces).toHaveLength(MAX_RUN_REFERENCED_ADDITIONAL_PROJECTS); + // The one project past the cap is a first-class resolution failure. + expect(result.failures).toEqual([ + { projectId: mentionedProjectIds[MAX_RUN_REFERENCED_ADDITIONAL_PROJECTS], reason: "resolution" }, + ]); + }); + + it("holds the candidate-evaluation cap on a sandbox target", async () => { + const companyId = randomUUID(); + const issueId = randomUUID(); + const anchorProjectId = randomUUID(); + // Flood the run with denied mentions past the evaluation cap. Every candidate is denied, so the + // admitted cap is never reached; the evaluation cap must still bound the authorization fan-out. + const mentionedProjectIds = Array.from( + { length: MAX_RUN_REFERENCED_CANDIDATE_EVALUATIONS + 5 }, + () => randomUUID(), + ); + const { decidedProjectIds, access } = recordingAccess(() => decision(false)); + const { resolveProjectWorkspace } = recordingResolveProjectWorkspace(); + + const result = await resolveAdditionalRunWorkspaces( + issueId, + anchorProjectId, + baseOpts(companyId, { + executionTargetIsRemote: true, + targetStagesConfined: true, + remoteReferencedSyncEnabled: true, + issues: mentionsStub(mentionedProjectIds), + access, + resolveProjectWorkspace, + }), + ); + + // The evaluation cap bounds authorization decisions even under a denied-mention flood. + expect(decidedProjectIds).toHaveLength(MAX_RUN_REFERENCED_CANDIDATE_EVALUATIONS); + expect(result.additionalWorkspaces).toEqual([]); + }); +}); diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index 7e7ab0cd67..7457718a50 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -1202,7 +1202,14 @@ function createSandboxEnvironmentDriver( }, async realizeWorkspace(input) { - // Plugin-backed sandbox providers: delegate workspace realization. + // Resolve the realized cwd and any provider metadata first, then build ONE + // workspace-realization record and wrap it the SAME way for every driver. A + // plugin-backed sandbox provider realizes the workspace remotely and returns its + // own cwd and metadata. A built-in driver has no plugin call; it uses the lease + // `remoteCwd`. Both paths must produce the record through the single build below, + // so the record can never drift between two exits. + let pluginRealizedCwd: string | null = null; + let providerMetadata: Record | null = null; if (input.lease.metadata?.sandboxProviderPlugin && pluginWorkerManager) { const pluginId = readString(input.lease.metadata?.pluginId); const providerKey = @@ -1216,7 +1223,7 @@ function createSandboxEnvironmentDriver( lease: input.lease, provider: providerKey, }); - return await pluginWorkerManager.call(pluginId, "environmentRealizeWorkspace", { + const pluginResult = await pluginWorkerManager.call(pluginId, "environmentRealizeWorkspace", { driverKey: providerKey, companyId: input.lease.companyId, environmentId: input.environment.id, @@ -1229,21 +1236,36 @@ function createSandboxEnvironmentDriver( }, workspace: input.workspace, }, resolvePluginSandboxRpcTimeoutMs(stripSandboxProviderEnvelope(config as SandboxEnvironmentConfig))); + pluginRealizedCwd = + typeof pluginResult.cwd === "string" && pluginResult.cwd.trim().length > 0 + ? pluginResult.cwd.trim() + : null; + providerMetadata = pluginResult.metadata ?? null; } } + // A plugin realize handler returns only its realized cwd and provider metadata; it + // does not build the full workspace-realization record. The server builds that record + // from the run request, so the referenced (mentioned) project sources reach the adapter + // through `realization.additional`. The adapter reads that field to stage each referenced + // tree into the sandbox; without the record the sandbox agent never receives the mentioned + // projects. The provider cwd and metadata still drive the remote path when a plugin realizes + // the workspace. const record = buildWorkspaceRealizationRecordFromDriverInput({ environment: input.environment, lease: input.lease, workspace: input.workspace, cwd: - typeof input.lease.metadata?.remoteCwd === "string" && input.lease.metadata.remoteCwd.trim().length > 0 + pluginRealizedCwd ?? + (typeof input.lease.metadata?.remoteCwd === "string" && input.lease.metadata.remoteCwd.trim().length > 0 ? input.lease.metadata.remoteCwd.trim() - : input.workspace.remotePath ?? input.workspace.localPath ?? null, + : input.workspace.remotePath ?? input.workspace.localPath ?? null), + providerMetadata, }); return { - cwd: record.remote.path ?? record.local.path, + cwd: pluginRealizedCwd ?? record.remote.path ?? record.local.path, metadata: { + ...(providerMetadata ?? {}), workspaceRealization: record, }, }; diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index bcc54e70c3..08d86a7672 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -2377,6 +2377,41 @@ export function isRemoteExecutionEnvironmentDriver(driver: string | null | undef return driver === "ssh" || driver === "sandbox" || driver === "plugin"; } +/** + * Environment flag (kill-switch, default ON) that gates whether a *remote* run stages the + * referenced (mentioned) project set into the sandbox. This is a targeted rollback lever: it + * disables only the remote referenced-project path and never regresses the working local path. + * The master flag {@link MULTI_PROJECT_WORKSPACE_SYNC_ENV} is the blunt switch that kills both + * local and remote. The remote path runs when both the master flag and this remote flag are ON — + * the default state. An unset value resolves ON; an operator disables it with an explicit false + * value (`"false"`, `"0"`, `"off"`, `"no"`, or `""`). The OFF state fails closed: a remote run + * runs no referenced-project authorization or staging and reverts to the remote drop path. + */ +export const MULTI_PROJECT_WORKSPACE_SYNC_REMOTE_ENV = + "PAPERCLIP_MULTI_PROJECT_WORKSPACE_SYNC_REMOTE"; + +export function isMultiProjectWorkspaceSyncRemoteEnabled( + env: Record = process.env, +): boolean { + // Default ON: an unset value is not false, so the remote path is live unless an operator sets + // an explicit false value as the targeted kill switch (rollback path). + return !isFalsyRuntimeEnvValue(env[MULTI_PROJECT_WORKSPACE_SYNC_REMOTE_ENV]); +} + +/** + * True when an environment driver stages a multi-source remote workspace through the confined + * sandbox/command runtime. Only the `sandbox` driver asserts per-project confinement on the + * staging path (`assertSyncOperationsConfined` in `sandbox-managed-runtime`). The `ssh` driver + * stages without that guard, and the `plugin` driver does not route through the confined command + * runtime in the workspace-realization step, so both keep dropping referenced projects. A `local` + * (or unknown) driver is not remote and never reaches this check. This gate is intentionally + * narrower than {@link isRemoteExecutionEnvironmentDriver}: it names the one transport that + * confines each staged referenced tree. + */ +export function isConfinedRemoteStagingDriver(driver: string | null | undefined): boolean { + return driver === "sandbox"; +} + /** * Upper bound on how many additional (mentioned) projects a single run may materialize * beyond the anchor. Bounds the fan-out of per-project authorization and workspace prep. @@ -2647,11 +2682,26 @@ export interface ResolveAdditionalRunWorkspacesOptions { maxCandidateEvaluations?: number; /** * True when the run executes on a non-local target (ssh, sandbox, or plugin). A referenced - * project realizes as a local directory only, and a remote target has no path yet to receive - * that tree, so a resolved cwd would not exist on the target. When true, the function skips - * referenced-project authorization and workspace work and returns no additional workspaces. + * project realizes as a local directory first. On a remote target that local tree reaches the + * agent only when a confined transport stages it into the sandbox and the remote flag is on + * (see `targetStagesConfined` and `remoteReferencedSyncEnabled`). Otherwise the run drops the + * whole referenced set and records it at the staging layer. */ executionTargetIsRemote?: boolean; + /** + * True when the remote target stages each referenced tree through the confined sandbox/command + * runtime (the `sandbox` driver; see {@link isConfinedRemoteStagingDriver}). The gate opens the + * referenced-project path on a remote target only when this is true. The SSH transport and any + * unconfined transport keep dropping referenced projects. Ignored on a local target. + */ + targetStagesConfined?: boolean; + /** + * The remote-only kill switch (default ON; see {@link isMultiProjectWorkspaceSyncRemoteEnabled}). + * When true, a confined remote target stages the referenced set. When false, a remote target + * fails closed: it runs no referenced-project authorization or staging and reverts to the remote + * drop path. Ignored on a local target. + */ + remoteReferencedSyncEnabled?: boolean; } /** @@ -2677,34 +2727,44 @@ export async function resolveAdditionalRunWorkspaces( return { additionalWorkspaces: [], warnings: [], failures: [] }; } - // A referenced project realizes as a local directory only. A remote execution target (ssh, - // sandbox, or plugin) has no path yet to receive the referenced tree, so a resolved cwd would - // not exist on the target and the anchor-only remote sync never carries it across. Skip the - // referenced-project authorization and clone work on a remote target, so the run neither does - // work it must discard nor exposes an inaccessible referenced path to the agent. Warn only when - // the issue actually mentions a project, so a remote run without any referenced mention stays - // silent. + // A referenced project realizes as a local directory first. On a remote target the run carries + // that tree to the agent only when a confined transport stages it into the sandbox and the + // remote flag is on. The confined sandbox transport asserts per-project confinement on each + // staged tree (`assertSyncOperationsConfined` in `sandbox-managed-runtime`). The SSH transport + // does not, so it stays out of scope. When the remote flag is off the run fails closed. In every + // one of those drop cases the run neither does authorization or clone work it must discard nor + // exposes an inaccessible referenced path to the agent. if (opts.executionTargetIsRemote) { - const mentionedIds = await opts.issues.findMentionedProjectIds(issueId, { - includeCommentBodies: true, - }); - // Each distinct non-anchor mention is a referenced project this remote run drops. A remote - // target has no path to receive the referenced tree, so the run drops the whole set at the - // staging layer. Record one failure per dropped project so the requested-vs-synced accounting - // counts the whole referenced set and the run still emits its structured sync log. - const droppedProjectIds = [ - ...new Set(mentionedIds.filter((projectId) => projectId !== anchorProjectId)), - ]; - return { - additionalWorkspaces: [], - warnings: - droppedProjectIds.length > 0 - ? [ - "Referenced-project workspaces are available only on a local execution target. This run uses a remote execution target, so no referenced-project workspace was attached.", - ] - : [], - failures: droppedProjectIds.map((projectId) => ({ projectId, reason: "staging" as const })), - }; + const remoteReferencedSyncOpen = + (opts.remoteReferencedSyncEnabled ?? false) && (opts.targetStagesConfined ?? false); + if (!remoteReferencedSyncOpen) { + const mentionedIds = await opts.issues.findMentionedProjectIds(issueId, { + includeCommentBodies: true, + }); + // Each distinct non-anchor mention is a referenced project this remote run drops. An SSH + // target (or the remote flag off) has no confined path to receive the referenced tree, so + // the run drops the whole set at the staging layer. Record one failure per dropped project + // so the requested-vs-synced accounting counts the whole referenced set and the run still + // emits its structured sync log. Warn only when the issue actually mentions a project, so a + // remote run without any referenced mention stays silent. + const droppedProjectIds = [ + ...new Set(mentionedIds.filter((projectId) => projectId !== anchorProjectId)), + ]; + return { + additionalWorkspaces: [], + warnings: + droppedProjectIds.length > 0 + ? [ + "Referenced-project workspaces are available only on a local execution target or a confined sandbox target. This run uses a different remote execution target, so no referenced-project workspace was attached.", + ] + : [], + failures: droppedProjectIds.map((projectId) => ({ projectId, reason: "staging" as const })), + }; + } + // Fall through: a confined sandbox target with the remote flag on resolves and authorizes the + // referenced set exactly like a local target. The resolver is driver-agnostic; the confined + // sandbox transport downstream stages each admitted tree into its own `project-` + // directory. The per-project `project:read` check below still runs against the run actor. } const referenced = await resolveRunReferencedProjects(issueId, anchorProjectId, { @@ -8192,20 +8252,27 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) agent: typeof agents.$inferSelect, context: Record, previousSessionParams: Record | null, - opts?: { useProjectWorkspace?: boolean | null; executionTargetIsRemote?: boolean }, + opts?: { useProjectWorkspace?: boolean | null; executionEnvironmentDriver?: string | null }, ): Promise { const anchor = await resolveAnchorWorkspaceForRun(agent, context, previousSessionParams, opts); if (!isMultiProjectWorkspaceSyncEnabled()) { return { ...anchor, additionalWorkspaces: [], referencedProjectFailures: [] }; } + // Derive the remote-transport facts from the selected environment driver. `executionTargetIsRemote` + // decides whether the referenced set needs the remote path at all; `targetStagesConfined` decides + // whether that remote target confines each staged tree (only the sandbox driver does). The remote + // flag is the targeted kill switch; with it off, a remote run fails closed. + const executionEnvironmentDriver = opts?.executionEnvironmentDriver ?? null; const issueId = readNonEmptyString(context.issueId) ?? readNonEmptyString(context.taskId); const { additionalWorkspaces, warnings, failures } = await resolveAdditionalRunWorkspaces( issueId, anchor.projectId, { enabled: true, - executionTargetIsRemote: opts?.executionTargetIsRemote ?? false, + executionTargetIsRemote: isRemoteExecutionEnvironmentDriver(executionEnvironmentDriver), + targetStagesConfined: isConfinedRemoteStagingDriver(executionEnvironmentDriver), + remoteReferencedSyncEnabled: isMultiProjectWorkspaceSyncRemoteEnabled(), companyId: agent.companyId, actor: { type: "agent", @@ -13259,12 +13326,11 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) previousSessionParams, { useProjectWorkspace: requestedExecutionWorkspaceMode !== "agent_default", - // Referenced-project workspaces attach on a local execution target only. Gate their - // resolution on the selected environment driver so a remote run never resolves a - // referenced path it cannot reach. This never changes the anchor workspace. - executionTargetIsRemote: isRemoteExecutionEnvironmentDriver( - selectedEnvironmentForConfig?.driver, - ), + // Thread the selected environment driver so run-workspace resolution can tell a local + // target from a remote one, and a confined sandbox target from an unconfined remote + // target. A remote run resolves referenced projects only for the confined sandbox + // transport with the remote flag on. This never changes the anchor workspace. + executionEnvironmentDriver: selectedEnvironmentForConfig?.driver ?? null, }, ), }); @@ -14518,6 +14584,37 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) ); } } + // Reconcile the referenced-project set against the real remote staging outcome. A referenced + // project can pass authorization and clone locally at run prep, then fail to stage into the + // sandbox during execution. The run-prep observability above counts such a project as synced, + // so emit a second, stage-time line that counts each staging failure as a first-class + // `staging` failure. The synced set is the resolved referenced projects minus the ones that + // failed to stage. A run with no staging failure stays silent, so the anchor-only and + // fully-synced paths add no noise. + const referencedProjectStagingFailures = adapterResult.referencedProjectStagingFailures ?? []; + if (referencedProjectStagingFailures.length > 0) { + const stagingFailedProjectIds = new Set( + referencedProjectStagingFailures.map((failure) => failure.projectId), + ); + const stagedProjectObservability = buildReferencedProjectRunObservability({ + syncedProjectIds: resolvedWorkspace.additionalWorkspaces + .map((additional) => additional.projectId) + .filter((projectId) => !stagingFailedProjectIds.has(projectId)), + failures: referencedProjectStagingFailures.map((failure) => ({ + projectId: failure.projectId, + reason: "staging" as const, + })), + }); + logger.info( + { + runId: run.id, + companyId: agent.companyId, + issueId: issueRef?.id ?? null, + ...stagedProjectObservability, + }, + "run referenced-project remote staging", + ); + } const adapterManagedRuntimeServices = adapterResult.runtimeServices ? await persistAdapterManagedRuntimeServices({ db, diff --git a/server/src/services/workspace-realization.ts b/server/src/services/workspace-realization.ts index 6049cfb36c..0c11f97d9d 100644 --- a/server/src/services/workspace-realization.ts +++ b/server/src/services/workspace-realization.ts @@ -298,6 +298,14 @@ export function buildWorkspaceRealizationRecord(input: { }; } +/** + * Build the workspace-realization record from the run request. The server owns the record; + * a driver realize handler (built-in or plugin) returns only a realized cwd and provider + * metadata. Every `realizeWorkspace` exit must route through this helper, so the record carries + * the referenced (mentioned) project sources in `additional`. The adapter reads `additional` to + * stage each referenced tree into the target; a realize exit that returns a raw provider result + * without this helper drops the mentioned projects. + */ export function buildWorkspaceRealizationRecordFromDriverInput(input: { environment: Environment; lease: EnvironmentLease;