From 53bcf3897f05bb279a33e6289a12d3f0cc2d16c3 Mon Sep 17 00:00:00 2001 From: Nicky Leach Date: Fri, 31 Jul 2026 13:31:17 -0700 Subject: [PATCH] feat: sync @-mentioned projects into remote sandboxes (confined sandbox transport, flag ON) (#10564) ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The run layer must move project context into the sandbox that executes the agent > - Local sandboxes already stage referenced projects for @-mentions > - Remote confined sandboxes dropped the whole referenced set, so the agent lost needed files and paths > - This pull request keeps the confined sandbox transport aligned with the local behavior for referenced projects > - It does this behind a remote-only flag that defaults on, while SSH keeps the old drop-only path > - The benefit is that remote runs can read the same referenced project context that local runs already provide ## Linked Issues or Issue Description **Subsystem affected** Cross-cutting. This change touches server orchestration, sandbox transport, and observability. **Problem or motivation** A run can @-mention another project. Local targets stage each referenced project and give the agent a path. Remote confined sandboxes dropped the full referenced set, so the agent could not read those project files or paths. **Proposed solution** Enable referenced-project sync for the confined sandbox transport behind `PAPERCLIP_MULTI_PROJECT_WORKSPACE_SYNC_REMOTE`, which defaults on. Keep the SSH transport out of scope and keep it dropping referenced projects. Repoint each referenced workspace hint at its staged `project-` sandbox directory. Publish `PAPERCLIP_WORKSPACES_JSON` on the confined sandbox lane. Count each per-project remote staging failure as a `staging` failure in the requested-vs-synced metrics. **Alternatives considered** Keep the remote path drop-only. That keeps the gap open. Move the change into SSH too. That expands scope beyond the target transport and adds risk. **Roadmap alignment** No matching item in `ROADMAP.md` showed up in this review. **Additional context** The change lands in three commits. The first commit opens the gate for the confined sandbox transport. The second commit repoints the workspace hints and publishes the workspace map. The third commit records per-project staging failure data. ## What Changed - Opened remote referenced-project sync for the confined sandbox transport behind `PAPERCLIP_MULTI_PROJECT_WORKSPACE_SYNC_REMOTE`. - Repointed referenced workspace hints to the staged `project-` sandbox directories and published `PAPERCLIP_WORKSPACES_JSON`. - Counted per-project remote staging failures as first-class `staging` failures in the requested-vs-synced observability. ## Verification - The pushed ref `refs/heads/feat/sync-referenced-projects-remote-sandbox` resolves to the authorized submit SHA. - `git log --oneline origin/master..origin/feat/sync-referenced-projects-remote-sandbox` shows exactly the three expected commits. - The handoff reports server typecheck clean, adapter-utils typecheck clean, and the listed unit tests passing. - The handoff also reports no open review comments and no Greptile score yet. ## Risks - The change touches authorization and sandbox path handling, so regressions could block remote runs or expose the wrong project context. - The new flag defaults on, so any bug in the remote path affects normal remote use. - SSH stays out of scope, so the two transport paths must remain distinct. ## Model Used OpenAI GPT-5 via Codex. Tool use enabled. Context window not reported in this run. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes: #` / `Refs: #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- .../src/acpx-engine/execute.test.ts | 74 ++++ .../adapter-utils/src/acpx-engine/execute.ts | 46 +++ .../adapter-utils/src/execution-target.ts | 21 +- .../src/sandbox-managed-runtime.test.ts | 8 + .../src/sandbox-managed-runtime.ts | 25 +- .../adapter-utils/src/server-utils.test.ts | 47 +++ packages/adapter-utils/src/server-utils.ts | 27 +- packages/adapter-utils/src/types.ts | 8 + packages/plugins/sdk/src/protocol.ts | 6 + .../src/__tests__/environment-runtime.test.ts | 225 +++++++++++ ...artbeat-remote-referenced-projects.test.ts | 359 ++++++++++++++++++ server/src/services/environment-runtime.ts | 32 +- server/src/services/heartbeat.ts | 173 +++++++-- server/src/services/workspace-realization.ts | 8 + 14 files changed, 1012 insertions(+), 47 deletions(-) create mode 100644 server/src/__tests__/heartbeat-remote-referenced-projects.test.ts diff --git a/packages/adapter-utils/src/acpx-engine/execute.test.ts b/packages/adapter-utils/src/acpx-engine/execute.test.ts index 12ea70891d..6006e05e22 100644 --- a/packages/adapter-utils/src/acpx-engine/execute.test.ts +++ b/packages/adapter-utils/src/acpx-engine/execute.test.ts @@ -2116,6 +2116,80 @@ describe("ACPX engine remote sandbox staging seam (PR 1: workspace + cwd)", () = expect(payloadEnv.PAPERCLIP_API_KEY).not.toBe("real-run-jwt"); }); + it("publishes referenced-project workspace hints repointed at their staged sandbox directories", async () => { + const { root, stateDir, localCwd, executionTarget } = await setupRemoteSandbox(); + // A referenced project with a real host tree, so the sandbox transport stages it and returns a + // `project-` directory for it. + const referencedProjectDir = path.join(root, "referenced-project-a"); + await fs.mkdir(referencedProjectDir, { recursive: true }); + await fs.writeFile(path.join(referencedProjectDir, "note.txt"), "referenced", "utf8"); + + // Decode the process-session LAUNCH payload — the in-sandbox process env is carried there. + let launchPayload: Record | null = null; + (executionTarget as { runner: unknown }).runner = createLocalSandboxRunner((input) => { + if (input.env?.PAPERCLIP_SANDBOX_EXEC_CHANNEL === "bridge") { + const script = input.args?.[1] ?? ""; + const match = script.match(/PAPERCLIP_PROCESS_SESSION_COMMAND_B64='([^']+)'/); + if (match) { + launchPayload = JSON.parse(Buffer.from(match[1]!, "base64").toString("utf8")) as Record< + string, + unknown + >; + } + } + }); + + await runExecutor( + { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir, cwd: localCwd }, + { + authToken: "real-run-jwt", + executionTarget, + context: { + taskId: "issue-1", + wakeReason: "issue_assigned", + paperclipWorkspace: { + cwd: localCwd, + realization: { + additional: [ + { + path: referencedProjectDir, + projectId: "a", + projectWorkspaceId: "ws-a", + repoUrl: "https://example.test/a.git", + repoRef: "main", + }, + ], + }, + }, + // The plural workspace-hints channel the agent reads. The referenced hint points at the + // host path today; on a remote target the run must repoint it at the staged directory. + paperclipWorkspaces: [ + { + workspaceId: "ws-a", + cwd: referencedProjectDir, + repoUrl: "https://example.test/a.git", + repoRef: "main", + projectId: "a", + }, + ], + }, + }, + ); + + const payloadEnv = ((launchPayload as Record | null)?.env ?? {}) as Record< + string, + unknown + >; + const workspacesJson = payloadEnv.PAPERCLIP_WORKSPACES_JSON; + expect(typeof workspacesJson).toBe("string"); + const hints = JSON.parse(String(workspacesJson)) as Array>; + const referencedHint = hints.find((hint) => hint.projectId === "a"); + expect(referencedHint).toBeTruthy(); + // The referenced hint repoints from the host path to its staged in-sandbox directory. + expect(String(referencedHint!.cwd)).toContain("project-a"); + expect(referencedHint!.cwd).not.toBe(referencedProjectDir); + }); + it("stops the process-session bridge when the paperclip bridge fails under concurrency", async () => { const { stateDir, localCwd, executionTarget } = await setupRemoteSandbox(); // The paperclip bridge fails; the process-session bridge — started CONCURRENTLY diff --git a/packages/adapter-utils/src/acpx-engine/execute.ts b/packages/adapter-utils/src/acpx-engine/execute.ts index e3321c02ed..645c105834 100644 --- a/packages/adapter-utils/src/acpx-engine/execute.ts +++ b/packages/adapter-utils/src/acpx-engine/execute.ts @@ -1406,6 +1406,15 @@ async function buildRuntime(input: { contentSignature: await referencedSourceContentSignature(entry.localPath), })), ); + // Referenced-project workspace hints exposed to the agent through PAPERCLIP_WORKSPACES_JSON. The + // list joins the anchor project's alternative workspaces with the referenced (mentioned) projects. + // On the confined sandbox lane the run repoints each referenced hint at its staged directory after + // staging below. Empty unless run prep resolved referenced projects or alternative workspaces. + const workspaceHints = Array.isArray(context.paperclipWorkspaces) + ? context.paperclipWorkspaces.filter( + (value): value is Record => typeof value === "object" && value !== null, + ) + : []; const executionTarget = readAdapterExecutionTarget({ executionTarget: input.ctx.executionTarget, legacyRemoteExecution: input.ctx.executionTransport?.remoteExecution, @@ -1888,6 +1897,27 @@ async function buildRuntime(input: { remoteManagedHomeTeardown = staged.value.teardown; remoteStagingDispose = staged.value.dispose; remoteStagingEnvDelta = staged.value.envDelta; + // Publish the referenced-project workspace hints to the in-sandbox agent. The staged-directory + // map (`project-`) is known only after staging above, so this runs here rather than + // with the initial workspace shaping. Each referenced hint repoints at its staged directory; a + // referenced hint whose project did not stage loses its cwd, so the agent never receives an + // unstaged path. Only the confined sandbox lane stages referenced trees, so only it publishes + // the hints; the local and runner-less lanes keep their env untouched. The set `env` write wins + // over an inherited value in the merged launch env. + const stagedProjectDirs = stagedRuntime?.additionalSourceDirs ?? {}; + if (Object.keys(stagedProjectDirs).length > 0) { + const shapedHints = shapePaperclipWorkspaceEnvForExecution({ + workspaceCwd: effectiveWorkspaceCwd, + workspaceWorktreePath, + workspaceHints, + executionTargetIsRemote, + executionCwd: effectiveExecutionCwd, + stagedProjectDirs, + }).workspaceHints; + if (shapedHints.length > 0) { + env.PAPERCLIP_WORKSPACES_JSON = JSON.stringify(shapedHints); + } + } } // Both bridge starts run under one try so a failure at EITHER — including the // paperclip callback bridge — fires the same abandon-path cleanup. The @@ -2916,6 +2946,17 @@ export function createAcpxEngineExecutor(deps: AcpxEngineExecutorOptions = {}) { rootSpan.end(true); throw err; } + // Per-project staging outcomes for the referenced (mentioned) projects, surfaced back to the + // server on the run result. A referenced project that failed to stage into the sandbox is a + // first-class, counted failure in the requested-vs-synced observability, not only a warning. The + // list is empty on a local target, on a transport that does not stage referenced projects, or + // when every staged referenced project succeeded, so the spread adds the field only when there + // is a failure to report. + const referencedProjectStagingFailures = ( + prepared.stagedRuntime?.additionalSourceFailures ?? [] + ).map((failure) => ({ projectId: failure.projectId })); + const referencedProjectStagingFailuresField = + referencedProjectStagingFailures.length > 0 ? { referencedProjectStagingFailures } : {}; // State the effective wall-clock timeout and its source up front so a // later timeout is diagnosable from the run log alone. Goes to stderr: // the acpx stdout log stream carries JSON acpx.* event payloads and must @@ -3060,6 +3101,7 @@ export function createAcpxEngineExecutor(deps: AcpxEngineExecutorOptions = {}) { errorMessage: message, ...classified, ...billingFields, + ...referencedProjectStagingFailuresField, model: prepared.requestedModel || null, clearSession, resultJson: { phase: "ensure_session" }, @@ -3079,6 +3121,7 @@ export function createAcpxEngineExecutor(deps: AcpxEngineExecutorOptions = {}) { errorMessage: "ACPX did not return a runtime session handle.", errorCode: "acpx_runtime_error", ...billingFields, + ...referencedProjectStagingFailuresField, model: prepared.requestedModel || null, resultJson: { phase: "ensure_session" }, summary: "ACPX did not return a runtime session handle.", @@ -3122,6 +3165,7 @@ export function createAcpxEngineExecutor(deps: AcpxEngineExecutorOptions = {}) { errorMessage: message, ...classified, ...billingFields, + ...referencedProjectStagingFailuresField, model: prepared.requestedModel || null, clearSession, resultJson: { @@ -3322,6 +3366,7 @@ export function createAcpxEngineExecutor(deps: AcpxEngineExecutorOptions = {}) { sessionParams: buildSessionParams({ prepared, handle: sessionHandle }), sessionDisplayId: sessionHandle.agentSessionId ?? sessionHandle.backendSessionId ?? sessionHandle.runtimeSessionName, ...billingFields, + ...referencedProjectStagingFailuresField, model: prepared.requestedModel || null, ...(turnUsage.usage ? { usage: turnUsage.usage, usageBasis: "per_run" as const } : {}), costUsd: turnUsage.costUsd, @@ -3378,6 +3423,7 @@ export function createAcpxEngineExecutor(deps: AcpxEngineExecutorOptions = {}) { errorCode: timedOut ? "acpx_timeout" : classified.errorCode, errorMeta: classified.errorMeta, ...billingFields, + ...referencedProjectStagingFailuresField, model: prepared.requestedModel || null, clearSession: clearSession || timedOut, resultJson: { phase: "turn" }, diff --git a/packages/adapter-utils/src/execution-target.ts b/packages/adapter-utils/src/execution-target.ts index 9652fd2cb3..d1d878d7c2 100644 --- a/packages/adapter-utils/src/execution-target.ts +++ b/packages/adapter-utils/src/execution-target.ts @@ -14,8 +14,14 @@ import { prepareRemoteManagedRuntime, remoteExecutionSessionMatches, } from "./remote-managed-runtime.js"; -import type { SandboxAdditionalSource } from "./sandbox-managed-runtime.js"; -export type { SandboxAdditionalSource } from "./sandbox-managed-runtime.js"; +import type { + AdditionalSourceStagingFailure, + SandboxAdditionalSource, +} from "./sandbox-managed-runtime.js"; +export type { + AdditionalSourceStagingFailure, + SandboxAdditionalSource, +} from "./sandbox-managed-runtime.js"; import { createCommandManagedSandboxCallbackBridgeQueueClient, createSandboxCallbackBridgeAsset, @@ -122,6 +128,12 @@ export interface PreparedAdapterExecutionTargetRuntime { * additional sources were requested. */ additionalSourceDirs: Record; + /** + * Each additional (referenced) project whose staging failed, paired with the + * failure message. Empty for a local target, for a transport that does not + * stage referenced projects, or when every requested project staged. + */ + additionalSourceFailures: AdditionalSourceStagingFailure[]; restoreWorkspace(onProgress?: RuntimeProgressSink): Promise; } @@ -1135,6 +1147,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: { runtimeRootDir: null, assetDirs: {}, additionalSourceDirs: {}, + additionalSourceFailures: [], restoreWorkspace: async () => {}, }; } @@ -1157,6 +1170,9 @@ export async function prepareAdapterExecutionTargetRuntime(input: { runtimeRootDir: prepared.runtimeRootDir, assetDirs: prepared.assetDirs, additionalSourceDirs: prepared.additionalSourceDirs, + // The SSH transport does not stage referenced projects (it is out of scope), so it never + // reports a per-project staging failure. + additionalSourceFailures: [], restoreWorkspace: prepared.restoreWorkspace, }; } @@ -1192,6 +1208,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: { runtimeRootDir: prepared.runtimeRootDir, assetDirs: prepared.assetDirs, additionalSourceDirs: prepared.additionalSourceDirs, + additionalSourceFailures: prepared.additionalSourceFailures, restoreWorkspace: prepared.restoreWorkspace, }; } diff --git a/packages/adapter-utils/src/sandbox-managed-runtime.test.ts b/packages/adapter-utils/src/sandbox-managed-runtime.test.ts index db796761f2..4e82cb0d86 100644 --- a/packages/adapter-utils/src/sandbox-managed-runtime.test.ts +++ b/packages/adapter-utils/src/sandbox-managed-runtime.test.ts @@ -1904,6 +1904,14 @@ describe("sandbox managed runtime", () => { expect(prepared.additionalSourceDirs["proj-second"]).toBe(path.posix.join(runtimeRootDir, "project-proj-second")); expect(prepared.additionalSourceDirs["proj-missing"]).toBeUndefined(); + // The skipped project is a first-class per-project failure outcome, not only a warning, so the + // run can count it in the requested-vs-synced accounting. The two healthy projects do not + // appear as failures. + expect(prepared.additionalSourceFailures.map((failure) => failure.projectId)).toEqual([ + "proj-missing", + ]); + expect(prepared.additionalSourceFailures[0]!.error.length).toBeGreaterThan(0); + await expect(readFile(path.join(prepared.additionalSourceDirs["proj-first"], "docs", "guide.md"), "utf8")) .resolves.toBe("first guide\n"); await expect(readFile(path.join(prepared.additionalSourceDirs["proj-second"], "notes.md"), "utf8")) diff --git a/packages/adapter-utils/src/sandbox-managed-runtime.ts b/packages/adapter-utils/src/sandbox-managed-runtime.ts index 0cff256464..74b1f4520c 100644 --- a/packages/adapter-utils/src/sandbox-managed-runtime.ts +++ b/packages/adapter-utils/src/sandbox-managed-runtime.ts @@ -300,9 +300,23 @@ export interface PreparedSandboxManagedRuntime { * were requested. */ additionalSourceDirs: Record; + /** + * Each additional (referenced) project whose staging failed, paired with the + * failure message. Per-project failure isolation keeps one project's failure + * from aborting the run, so a failed project is absent from + * `additionalSourceDirs` and present here. Empty when every requested project + * staged, or when no additional sources were requested. + */ + additionalSourceFailures: AdditionalSourceStagingFailure[]; restoreWorkspace(onProgress?: RuntimeProgressSink): Promise; } +/** One additional (referenced) project that failed to stage into the sandbox. */ +export interface AdditionalSourceStagingFailure { + projectId: string; + error: string; +} + function asObject(value: unknown): Record { return value && typeof value === "object" && !Array.isArray(value) ? (value as Record) @@ -736,6 +750,10 @@ export async function prepareSandboxManagedRuntime(input: { // Remote directory of each additional (referenced) project that stages // successfully, keyed by projectId. A project that fails to stage is absent. const additionalSourceDirs: Record = {}; + // Each additional (referenced) project whose staging failed, paired with the + // failure message. Per-project failure isolation keeps the run and the other + // projects going; this list makes each failure a first-class, reported outcome. + const additionalSourceFailures: AdditionalSourceStagingFailure[] = []; // Additional projects stage as plain trees. Drop the heavy build/cache dirs a // reference tree does not need, and `.git` — additional sources never carry // git-history semantics (anchor-only). @@ -999,9 +1017,13 @@ export async function prepareSandboxManagedRuntime(input: { }); additionalSourceDirs[projectId] = remoteProjectDir; } catch (error) { + const message = error instanceof Error ? error.message : String(error); console.warn( - `[paperclip] Failed to stage referenced project ${projectId}; skipping it. ${String(error)}`, + `[paperclip] Failed to stage referenced project ${projectId}; skipping it. ${message}`, ); + // Record the failure as a first-class per-project outcome so the run can count it in the + // requested-vs-synced accounting instead of losing it to a warning line. + additionalSourceFailures.push({ projectId, error: message }); } } }); @@ -1017,6 +1039,7 @@ export async function prepareSandboxManagedRuntime(input: { runtimeRootDir, assetDirs, additionalSourceDirs, + additionalSourceFailures, restoreWorkspace: async (onProgress?: RuntimeProgressSink) => { const restoreSink = onProgress ?? input.onProgress; if (!syncWorkspace) { diff --git a/packages/adapter-utils/src/server-utils.test.ts b/packages/adapter-utils/src/server-utils.test.ts index 4d4c9d6523..74f032a968 100644 --- a/packages/adapter-utils/src/server-utils.test.ts +++ b/packages/adapter-utils/src/server-utils.test.ts @@ -2295,6 +2295,53 @@ describe("shapePaperclipWorkspaceEnvForExecution", () => { }); }); + it("repoints a referenced hint to its staged remote directory when the map has an entry", () => { + const shaped = shapePaperclipWorkspaceEnvForExecution({ + workspaceCwd: "/tmp/workspace", + workspaceWorktreePath: "/tmp/worktree", + workspaceHints: [ + // The anchor hint keeps its remote-cwd rewrite. + { workspaceId: "workspace-1", cwd: "/tmp/workspace" }, + // A referenced hint with a staged directory repoints at it. + { workspaceId: "workspace-2", cwd: "/tmp/referenced/project-a", projectId: "project-a" }, + // A referenced hint with no staged directory loses its cwd. + { workspaceId: "workspace-3", cwd: "/tmp/referenced/project-b", projectId: "project-b" }, + ], + executionTargetIsRemote: true, + executionCwd: "/remote/workspace", + stagedProjectDirs: { "project-a": "/remote/runtime/project-project-a" }, + }); + + expect(shaped).toEqual({ + workspaceCwd: "/remote/workspace", + workspaceWorktreePath: null, + workspaceHints: [ + { workspaceId: "workspace-1", cwd: "/remote/workspace" }, + { + workspaceId: "workspace-2", + cwd: "/remote/runtime/project-project-a", + projectId: "project-a", + }, + { workspaceId: "workspace-3", projectId: "project-b" }, + ], + }); + }); + + it("removes cwd from a referenced hint that has no staged directory", () => { + const shaped = shapePaperclipWorkspaceEnvForExecution({ + workspaceCwd: "/tmp/workspace", + workspaceHints: [ + { workspaceId: "workspace-2", cwd: "/tmp/referenced/project-a", projectId: "project-a" }, + ], + executionTargetIsRemote: true, + executionCwd: "/remote/workspace", + // The map is empty, so the referenced hint has no staged directory. + stagedProjectDirs: {}, + }); + + expect(shaped.workspaceHints).toEqual([{ workspaceId: "workspace-2", projectId: "project-a" }]); + }); + it("leaves local execution workspace paths unchanged", () => { const workspaceHints = [{ workspaceId: "workspace-1", cwd: "/tmp/workspace" }]; const shaped = shapePaperclipWorkspaceEnvForExecution({ diff --git a/packages/adapter-utils/src/server-utils.ts b/packages/adapter-utils/src/server-utils.ts index 5a3fc36cbb..41d96a2349 100644 --- a/packages/adapter-utils/src/server-utils.ts +++ b/packages/adapter-utils/src/server-utils.ts @@ -2026,6 +2026,15 @@ export function shapePaperclipWorkspaceEnvForExecution(input: { workspaceHints?: Array>; executionTargetIsRemote?: boolean; executionCwd?: string | null; + /** + * On a remote target, the map of referenced (mentioned) project id to the staged in-sandbox + * directory that received that project's tree (`project-`). A non-anchor hint whose + * `projectId` has an entry repoints its `cwd` to the staged directory. A non-anchor hint with no + * entry loses its `cwd`, so the agent never receives a path the transport did not stage. The map + * is empty on a local target and defaults to empty, so a caller that passes nothing keeps the + * previous behavior (every non-anchor hint loses its `cwd` on a remote target). + */ + stagedProjectDirs?: Record; }): { workspaceCwd: string | null; workspaceWorktreePath: string | null; @@ -2068,6 +2077,7 @@ export function shapePaperclipWorkspaceEnvForExecution(input: { } const realizedWorkspaceCwd = executionCwd; const localWorkspaceCwd = workspaceCwd ? path.resolve(workspaceCwd) : null; + const stagedProjectDirs = input.stagedProjectDirs ?? {}; const shapedWorkspaceHints = workspaceHints.map((hint) => { const nextHint = { ...hint }; const hintCwd = typeof nextHint.cwd === "string" ? nextHint.cwd.trim() : ""; @@ -2082,7 +2092,19 @@ export function shapePaperclipWorkspaceEnvForExecution(input: { return nextHint; } - delete nextHint.cwd; + // A referenced (mentioned) project hint carries its `projectId`. When the transport staged that + // project into the sandbox, repoint the hint at its staged `project-` directory so + // the agent reads it there. Without a staged directory the hint would point at a local path that + // the remote target cannot reach, so remove the `cwd` (fail loud — never expose an unstaged + // path). This also removes the `cwd` of a non-anchor hint that carries no `projectId`, such as an + // alternative anchor-project workspace, which keeps the previous behavior for those hints. + const hintProjectId = typeof nextHint.projectId === "string" ? nextHint.projectId : ""; + const stagedProjectDir = hintProjectId ? stagedProjectDirs[hintProjectId] : undefined; + if (stagedProjectDir && stagedProjectDir.trim().length > 0) { + nextHint.cwd = stagedProjectDir.trim(); + } else { + delete nextHint.cwd; + } return nextHint; }); @@ -2145,6 +2167,8 @@ export function refreshPaperclipWorkspaceEnvForExecution(input: { agentHome?: string | null; executionTargetIsRemote?: boolean; executionCwd?: string | null; + /** Referenced-project id to staged in-sandbox directory map; see {@link shapePaperclipWorkspaceEnvForExecution}. */ + stagedProjectDirs?: Record; }): { workspaceCwd: string | null; workspaceWorktreePath: string | null; @@ -2156,6 +2180,7 @@ export function refreshPaperclipWorkspaceEnvForExecution(input: { workspaceHints: input.workspaceHints, executionTargetIsRemote: input.executionTargetIsRemote, executionCwd: input.executionCwd, + stagedProjectDirs: input.stagedProjectDirs, }); delete input.env.PAPERCLIP_WORKSPACE_CWD; diff --git a/packages/adapter-utils/src/types.ts b/packages/adapter-utils/src/types.ts index 988e1318e7..d065722def 100644 --- a/packages/adapter-utils/src/types.ts +++ b/packages/adapter-utils/src/types.ts @@ -103,6 +103,14 @@ export interface AdapterExecutionResult { costUsd?: number | null; resultJson?: Record | null; runtimeServices?: AdapterRuntimeServiceReport[]; + /** + * Each referenced (mentioned) project that failed to stage into the remote sandbox for this run, + * by `projectId`. The run continues without a failed project (per-project failure isolation); this + * field carries the failure back so the server counts it in the requested-vs-synced observability + * instead of losing it to a warning line. Absent or empty on a local target, or when every staged + * referenced project succeeded. + */ + referencedProjectStagingFailures?: Array<{ projectId: string }>; summary?: string | null; clearSession?: boolean; question?: { diff --git a/packages/plugins/sdk/src/protocol.ts b/packages/plugins/sdk/src/protocol.ts index f85446a83c..91d1d58f66 100644 --- a/packages/plugins/sdk/src/protocol.ts +++ b/packages/plugins/sdk/src/protocol.ts @@ -638,6 +638,12 @@ export interface PluginEnvironmentRealizeWorkspaceParams extends PluginEnvironme }; } +/** + * A plugin `environmentRealizeWorkspace` handler returns only the realized cwd and provider + * metadata. The server, not the plugin, builds the full workspace-realization record from the run + * request and merges this cwd and metadata into it. Do not return a `workspaceRealization` record + * here; the server owns that record, so the referenced (mentioned) project sources reach the adapter. + */ export interface PluginEnvironmentRealizeWorkspaceResult { cwd: string; metadata?: Record; diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index cefe00fc8b..3c836883ab 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -699,6 +699,231 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentReleaseLease", expect.anything(), 31234); }); + it("builds the workspace-realization record with referenced sources for a plugin-backed sandbox realize", async () => { + // A provider plugin realize handler returns only its realized cwd and provider metadata; it does + // not build the workspace-realization record. The server must build that record from the run + // request, so the referenced (mentioned) project sources reach the adapter through + // `realization.additional`. Without the record the sandbox agent never receives the mentioned + // projects. This test drives the plugin-backed sandbox realize path and asserts the referenced + // source survives into the returned record. + const pluginId = randomUUID(); + const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); + const fakePluginConfig = { + provider: "fake-plugin", + image: "fake:test", + timeoutMs: 1234, + reuseLease: false, + }; + const environment = { + ...baseEnvironment, + name: "Fake Plugin Sandbox Realize", + driver: "sandbox", + config: fakePluginConfig, + }; + await environmentService(db).update(environment.id, { + driver: "sandbox", + name: environment.name, + config: fakePluginConfig, + }); + await db.insert(plugins).values({ + id: pluginId, + pluginKey: "paperclip.fake-plugin-sandbox-provider", + packageName: "@paperclipai/plugin-fake-sandbox", + version: "1.0.0", + apiVersion: 1, + categories: ["automation"], + manifestJson: { + id: "paperclip.fake-plugin-sandbox-provider", + apiVersion: 1, + version: "1.0.0", + displayName: "Fake Plugin Sandbox Provider", + description: "Test fake plugin provider", + author: "Paperclip", + categories: ["automation"], + capabilities: ["environment.drivers.register"], + entrypoints: { worker: "dist/worker.js" }, + environmentDrivers: [ + { + driverKey: "fake-plugin", + kind: "sandbox_provider", + displayName: "Fake Plugin", + configSchema: { type: "object" }, + }, + ], + }, + status: "ready", + installOrder: 1, + updatedAt: new Date(), + } as any); + const workerManager = { + isRunning: vi.fn((id: string) => id === pluginId), + call: vi.fn(async (_pluginId: string, method: string) => { + if (method === "environmentAcquireLease") { + return { + providerLeaseId: "sandbox-realize-1", + metadata: { + provider: "fake-plugin", + image: "fake:test", + timeoutMs: 1234, + reuseLease: false, + remoteCwd: "/workspace", + }, + }; + } + if (method === "environmentRealizeWorkspace") { + // Mimic a real provider (for example Daytona): return only the realized cwd and provider + // metadata, never a `workspaceRealization` record. + return { + cwd: "/workspace/project", + metadata: { + provider: "fake-plugin", + remoteCwd: "/workspace/project", + }, + }; + } + throw new Error(`Unexpected plugin method: ${method}`); + }), + } as unknown as PluginWorkerManager; + const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); + + const acquired = await runtimeWithPlugin.acquireRunLease({ + companyId, + environment, + issueId: null, + heartbeatRunId: runId, + persistedExecutionWorkspace: null, + }); + const workspaceRealizationRequest = { + version: 1, + adapterType: "codex_local", + companyId, + environmentId: environment.id, + executionWorkspaceId: null, + issueId: null, + heartbeatRunId: runId, + requestedMode: "ephemeral", + source: { + kind: "project_primary", + localPath: "/tmp/anchor", + projectId: "anchor-project", + projectWorkspaceId: "anchor-workspace", + repoUrl: null, + repoRef: null, + strategy: "project_primary", + branchName: null, + worktreePath: null, + }, + additionalSources: [ + { + localPath: "/tmp/referenced-project", + projectId: "referenced-project-1", + projectWorkspaceId: "referenced-workspace-1", + repoUrl: null, + repoRef: null, + }, + ], + }; + const realized = await runtimeWithPlugin.realizeWorkspace({ + environment, + lease: acquired.lease, + workspace: { + localPath: "/tmp/anchor", + mode: "ephemeral", + metadata: { workspaceRealizationRequest }, + }, + }); + + // The provider realized cwd and provider metadata survive. + expect(realized.cwd).toBe("/workspace/project"); + expect(realized.metadata?.provider).toBe("fake-plugin"); + // The server-built record carries the referenced source through `additional`, so the adapter can + // stage the mentioned project into the sandbox. + const realization = realized.metadata?.workspaceRealization as Record | undefined; + expect(realization).toBeDefined(); + expect(realization?.additional).toEqual([ + expect.objectContaining({ + path: "/tmp/referenced-project", + projectId: "referenced-project-1", + }), + ]); + }); + + it("builds the workspace-realization record with referenced sources for a built-in sandbox realize", async () => { + // The sandbox driver `realizeWorkspace` has two exits: a plugin-backed provider and a + // built-in provider. Both must build the same workspace-realization record, so the + // referenced (mentioned) project sources reach the adapter through `realization.additional`. + // The test above covers the plugin exit. This test covers the built-in exit (no provider + // plugin), so a regression on either exit fails a test. + const { companyId, environment, runId } = await seedEnvironment({ + driver: "sandbox", + name: "Fake Sandbox Realize", + config: { + provider: "fake", + image: "ubuntu:24.04", + reuseLease: true, + }, + }); + + const acquired = await runtime.acquireRunLease({ + companyId, + environment, + issueId: null, + heartbeatRunId: runId, + persistedExecutionWorkspace: null, + }); + + const workspaceRealizationRequest = { + version: 1, + adapterType: "codex_local", + companyId, + environmentId: environment.id, + executionWorkspaceId: null, + issueId: null, + heartbeatRunId: runId, + requestedMode: "ephemeral", + source: { + kind: "project_primary", + localPath: "/tmp/anchor", + projectId: "anchor-project", + projectWorkspaceId: "anchor-workspace", + repoUrl: null, + repoRef: null, + strategy: "project_primary", + branchName: null, + worktreePath: null, + }, + additionalSources: [ + { + localPath: "/tmp/referenced-project", + projectId: "referenced-project-1", + projectWorkspaceId: "referenced-workspace-1", + repoUrl: null, + repoRef: null, + }, + ], + }; + const realized = await runtime.realizeWorkspace({ + environment, + lease: acquired.lease, + workspace: { + localPath: "/tmp/anchor", + mode: "ephemeral", + metadata: { workspaceRealizationRequest }, + }, + }); + + // The built-in exit builds the record and carries the referenced source through `additional`, + // so the adapter can stage the mentioned project into the sandbox. + const realization = realized.metadata?.workspaceRealization as Record | undefined; + expect(realization).toBeDefined(); + expect(realization?.additional).toEqual([ + expect.objectContaining({ + path: "/tmp/referenced-project", + projectId: "referenced-project-1", + }), + ]); + }); + it("uses resolved secret-ref config for plugin-backed sandbox execute and release", async () => { const pluginId = randomUUID(); const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); diff --git a/server/src/__tests__/heartbeat-remote-referenced-projects.test.ts b/server/src/__tests__/heartbeat-remote-referenced-projects.test.ts new file mode 100644 index 0000000000..de2403f820 --- /dev/null +++ b/server/src/__tests__/heartbeat-remote-referenced-projects.test.ts @@ -0,0 +1,359 @@ +import { randomUUID } from "node:crypto"; +import { describe, expect, it } from "vitest"; +import { + isConfinedRemoteStagingDriver, + isMultiProjectWorkspaceSyncRemoteEnabled, + MAX_RUN_REFERENCED_ADDITIONAL_PROJECTS, + MAX_RUN_REFERENCED_CANDIDATE_EVALUATIONS, + MULTI_PROJECT_WORKSPACE_SYNC_REMOTE_ENV, + resolveAdditionalRunWorkspaces, + type ResolveAdditionalRunWorkspacesOptions, + type ResolvedAdditionalWorkspace, +} from "../services/heartbeat.ts"; +import type { AuthorizationActor, AuthorizationDecision } from "../services/authorization.ts"; + +// These tests exercise the remote gate in `resolveAdditionalRunWorkspaces` with fully injected +// dependencies (no database). The function reads mentions through `issues.findMentionedProjectIds`, +// hydrates candidates through `projects.listByIds`, authorizes each through `access.decide`, and +// resolves an admitted project to a workspace through `resolveProjectWorkspace`. Each dependency is +// a stub here, so a test asserts the exact gate behavior for a sandbox target, an SSH target, and +// the remote kill switch. + +const buildActor = (companyId: string): AuthorizationActor => ({ + type: "agent", + agentId: randomUUID(), + companyId, + source: "agent_key", +}); + +const decision = (allowed: boolean): AuthorizationDecision => ({ + allowed, + action: "project:read", + reason: allowed ? "allow_company_agent" : "deny_company_boundary", + explanation: "test decision", +}); + +// A `projects.listByIds` stub that returns one minimal record per requested id, in request order. +// The referenced-project record only needs an `id` here; the workspace resolution is stubbed, so no +// other field is read. The cast keeps the stub minimal without restating the full project row shape. +const listByIdsStub: ResolveAdditionalRunWorkspacesOptions["projects"] = { + listByIds: async (_companyId, ids) => + ids.map( + (id) => + ({ id, name: `Project ${id}`, status: "in_progress" }) as Awaited< + ReturnType + >[number], + ), +}; + +// A `findMentionedProjectIds` stub that returns a fixed mention set for any issue. +const mentionsStub = ( + mentionedProjectIds: string[], +): ResolveAdditionalRunWorkspacesOptions["issues"] => ({ + findMentionedProjectIds: async () => mentionedProjectIds, +}); + +// Records every `project:read` authorization call and answers via the supplied resolver. +const recordingAccess = ( + resolve: (projectId: string) => AuthorizationDecision, +): { decidedProjectIds: string[]; access: ResolveAdditionalRunWorkspacesOptions["access"] } => { + const decidedProjectIds: string[] = []; + const access: ResolveAdditionalRunWorkspacesOptions["access"] = { + decide: async (input) => { + const resource = input.resource; + const projectId = resource.type === "project" ? (resource.projectId ?? "") : ""; + decidedProjectIds.push(projectId); + return resolve(projectId); + }, + }; + return { decidedProjectIds, access }; +}; + +// Records every workspace that resolution stages and returns a read-only workspace stub for it. +const recordingResolveProjectWorkspace = (): { + stagedProjectIds: string[]; + resolveProjectWorkspace: ResolveAdditionalRunWorkspacesOptions["resolveProjectWorkspace"]; +} => { + const stagedProjectIds: string[] = []; + const resolveProjectWorkspace: ResolveAdditionalRunWorkspacesOptions["resolveProjectWorkspace"] = + async (project) => { + stagedProjectIds.push(project.projectId); + return { + cwd: `/tmp/referenced/${project.projectId}`, + projectId: project.projectId, + workspaceId: null, + repoUrl: null, + repoRef: null, + } satisfies ResolvedAdditionalWorkspace; + }; + return { stagedProjectIds, resolveProjectWorkspace }; +}; + +const baseOpts = ( + companyId: string, + overrides: Partial & + Pick, +): ResolveAdditionalRunWorkspacesOptions => ({ + enabled: true, + companyId, + actor: buildActor(companyId), + projects: listByIdsStub, + ...overrides, +}); + +describe("remote referenced-project kill switch", () => { + it("is ON by default and disabled only by an explicit false env value", () => { + // Default ON: an unset value resolves the remote path live (go-live default). + expect(isMultiProjectWorkspaceSyncRemoteEnabled({})).toBe(true); + // The targeted kill switch: an explicit false value (the rollback path) disables it. + expect( + isMultiProjectWorkspaceSyncRemoteEnabled({ [MULTI_PROJECT_WORKSPACE_SYNC_REMOTE_ENV]: "" }), + ).toBe(false); + expect( + isMultiProjectWorkspaceSyncRemoteEnabled({ + [MULTI_PROJECT_WORKSPACE_SYNC_REMOTE_ENV]: "false", + }), + ).toBe(false); + expect( + isMultiProjectWorkspaceSyncRemoteEnabled({ [MULTI_PROJECT_WORKSPACE_SYNC_REMOTE_ENV]: "0" }), + ).toBe(false); + expect( + isMultiProjectWorkspaceSyncRemoteEnabled({ [MULTI_PROJECT_WORKSPACE_SYNC_REMOTE_ENV]: "off" }), + ).toBe(false); + // Any other value keeps the remote path on. + expect( + isMultiProjectWorkspaceSyncRemoteEnabled({ [MULTI_PROJECT_WORKSPACE_SYNC_REMOTE_ENV]: "true" }), + ).toBe(true); + }); + + it("classifies only the sandbox driver as a confined remote staging transport", () => { + // Only the sandbox driver confines each staged referenced tree, so only it opens the gate. + expect(isConfinedRemoteStagingDriver("sandbox")).toBe(true); + // The SSH and plugin drivers keep dropping referenced projects. + expect(isConfinedRemoteStagingDriver("ssh")).toBe(false); + expect(isConfinedRemoteStagingDriver("plugin")).toBe(false); + expect(isConfinedRemoteStagingDriver("local")).toBe(false); + expect(isConfinedRemoteStagingDriver(null)).toBe(false); + expect(isConfinedRemoteStagingDriver(undefined)).toBe(false); + }); +}); + +describe("resolveAdditionalRunWorkspaces remote gate", () => { + it("resolves and authorizes referenced workspaces on a sandbox target with the remote flag on", async () => { + const companyId = randomUUID(); + const issueId = randomUUID(); + const anchorProjectId = randomUUID(); + const mentionedProjectId = randomUUID(); + const { decidedProjectIds, access } = recordingAccess(() => decision(true)); + const { stagedProjectIds, resolveProjectWorkspace } = recordingResolveProjectWorkspace(); + + const result = await resolveAdditionalRunWorkspaces( + issueId, + anchorProjectId, + baseOpts(companyId, { + executionTargetIsRemote: true, + targetStagesConfined: true, + remoteReferencedSyncEnabled: true, + issues: mentionsStub([mentionedProjectId]), + access, + resolveProjectWorkspace, + }), + ); + + // The run authorized the referenced project against the run actor and staged its workspace. + expect(decidedProjectIds).toEqual([mentionedProjectId]); + expect(stagedProjectIds).toEqual([mentionedProjectId]); + expect(result.additionalWorkspaces.map((workspace) => workspace.projectId)).toEqual([ + mentionedProjectId, + ]); + expect(result.failures).toEqual([]); + }); + + it("fails closed on a sandbox target when the remote flag is off (no authorization, no staging)", async () => { + const companyId = randomUUID(); + const issueId = randomUUID(); + const anchorProjectId = randomUUID(); + const mentionedProjectId = randomUUID(); + const { decidedProjectIds, access } = recordingAccess(() => decision(true)); + const { stagedProjectIds, resolveProjectWorkspace } = recordingResolveProjectWorkspace(); + + const result = await resolveAdditionalRunWorkspaces( + issueId, + anchorProjectId, + baseOpts(companyId, { + executionTargetIsRemote: true, + targetStagesConfined: true, + remoteReferencedSyncEnabled: false, + issues: mentionsStub([mentionedProjectId]), + access, + resolveProjectWorkspace, + }), + ); + + // Fail closed: no authorization decision and no workspace staging ran. + expect(decidedProjectIds).toEqual([]); + expect(stagedProjectIds).toEqual([]); + expect(result.additionalWorkspaces).toEqual([]); + // The whole referenced set is still counted as a staging-layer drop. + expect(result.failures).toEqual([{ projectId: mentionedProjectId, reason: "staging" }]); + }); + + it("drops referenced projects on an SSH target whether the remote flag is on or off", async () => { + const companyId = randomUUID(); + const issueId = randomUUID(); + const anchorProjectId = randomUUID(); + const mentionedProjectId = randomUUID(); + + for (const remoteReferencedSyncEnabled of [true, false]) { + const { decidedProjectIds, access } = recordingAccess(() => decision(true)); + const { stagedProjectIds, resolveProjectWorkspace } = recordingResolveProjectWorkspace(); + + const result = await resolveAdditionalRunWorkspaces( + issueId, + anchorProjectId, + baseOpts(companyId, { + executionTargetIsRemote: true, + // The SSH transport does not confine its staging path, so it stays out of scope. + targetStagesConfined: false, + remoteReferencedSyncEnabled, + issues: mentionsStub([mentionedProjectId]), + access, + resolveProjectWorkspace, + }), + ); + + expect(decidedProjectIds).toEqual([]); + expect(stagedProjectIds).toEqual([]); + expect(result.additionalWorkspaces).toEqual([]); + expect(result.failures).toEqual([{ projectId: mentionedProjectId, reason: "staging" }]); + } + }); + + it("drops a referenced project the run agent cannot read on a sandbox target", async () => { + const companyId = randomUUID(); + const issueId = randomUUID(); + const anchorProjectId = randomUUID(); + const readableProjectId = randomUUID(); + const unreadableProjectId = randomUUID(); + // Deny project:read for the unreadable project; allow it for the readable one. + const { decidedProjectIds, access } = recordingAccess((projectId) => + decision(projectId === readableProjectId), + ); + const { stagedProjectIds, resolveProjectWorkspace } = recordingResolveProjectWorkspace(); + + const result = await resolveAdditionalRunWorkspaces( + issueId, + anchorProjectId, + baseOpts(companyId, { + executionTargetIsRemote: true, + targetStagesConfined: true, + remoteReferencedSyncEnabled: true, + issues: mentionsStub([readableProjectId, unreadableProjectId]), + access, + resolveProjectWorkspace, + }), + ); + + // Both projects were authorized against the run actor, but only the readable one was staged. + expect(decidedProjectIds).toEqual([readableProjectId, unreadableProjectId]); + expect(stagedProjectIds).toEqual([readableProjectId]); + expect(result.additionalWorkspaces.map((workspace) => workspace.projectId)).toEqual([ + readableProjectId, + ]); + // The denied project is a first-class authorization failure. + expect(result.failures).toEqual([{ projectId: unreadableProjectId, reason: "authorization" }]); + }); + + it("never authorizes the anchor project on a sandbox target", async () => { + const companyId = randomUUID(); + const issueId = randomUUID(); + const anchorProjectId = randomUUID(); + const mentionedProjectId = randomUUID(); + const { decidedProjectIds, access } = recordingAccess(() => decision(true)); + const { resolveProjectWorkspace } = recordingResolveProjectWorkspace(); + + await resolveAdditionalRunWorkspaces( + issueId, + anchorProjectId, + baseOpts(companyId, { + executionTargetIsRemote: true, + targetStagesConfined: true, + remoteReferencedSyncEnabled: true, + // The mention set includes the anchor itself; the anchor must never be re-authorized. + issues: mentionsStub([anchorProjectId, mentionedProjectId]), + access, + resolveProjectWorkspace, + }), + ); + + // The anchor exemption drops the anchor from the mention set before the per-project check, so + // only the non-anchor referenced project is authorized. + expect(decidedProjectIds).toEqual([mentionedProjectId]); + expect(decidedProjectIds).not.toContain(anchorProjectId); + }); + + it("holds the admitted-project cap on a sandbox target", async () => { + const companyId = randomUUID(); + const issueId = randomUUID(); + const anchorProjectId = randomUUID(); + // One more mention than the admitted cap. + const mentionedProjectIds = Array.from( + { length: MAX_RUN_REFERENCED_ADDITIONAL_PROJECTS + 1 }, + () => randomUUID(), + ); + const { access } = recordingAccess(() => decision(true)); + const { stagedProjectIds, resolveProjectWorkspace } = recordingResolveProjectWorkspace(); + + const result = await resolveAdditionalRunWorkspaces( + issueId, + anchorProjectId, + baseOpts(companyId, { + executionTargetIsRemote: true, + targetStagesConfined: true, + remoteReferencedSyncEnabled: true, + issues: mentionsStub(mentionedProjectIds), + access, + resolveProjectWorkspace, + }), + ); + + // The cap bounds how many additional projects a sandbox run materializes. + expect(stagedProjectIds).toHaveLength(MAX_RUN_REFERENCED_ADDITIONAL_PROJECTS); + expect(result.additionalWorkspaces).toHaveLength(MAX_RUN_REFERENCED_ADDITIONAL_PROJECTS); + // The one project past the cap is a first-class resolution failure. + expect(result.failures).toEqual([ + { projectId: mentionedProjectIds[MAX_RUN_REFERENCED_ADDITIONAL_PROJECTS], reason: "resolution" }, + ]); + }); + + it("holds the candidate-evaluation cap on a sandbox target", async () => { + const companyId = randomUUID(); + const issueId = randomUUID(); + const anchorProjectId = randomUUID(); + // Flood the run with denied mentions past the evaluation cap. Every candidate is denied, so the + // admitted cap is never reached; the evaluation cap must still bound the authorization fan-out. + const mentionedProjectIds = Array.from( + { length: MAX_RUN_REFERENCED_CANDIDATE_EVALUATIONS + 5 }, + () => randomUUID(), + ); + const { decidedProjectIds, access } = recordingAccess(() => decision(false)); + const { resolveProjectWorkspace } = recordingResolveProjectWorkspace(); + + const result = await resolveAdditionalRunWorkspaces( + issueId, + anchorProjectId, + baseOpts(companyId, { + executionTargetIsRemote: true, + targetStagesConfined: true, + remoteReferencedSyncEnabled: true, + issues: mentionsStub(mentionedProjectIds), + access, + resolveProjectWorkspace, + }), + ); + + // The evaluation cap bounds authorization decisions even under a denied-mention flood. + expect(decidedProjectIds).toHaveLength(MAX_RUN_REFERENCED_CANDIDATE_EVALUATIONS); + expect(result.additionalWorkspaces).toEqual([]); + }); +}); diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index 7e7ab0cd67..7457718a50 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -1202,7 +1202,14 @@ function createSandboxEnvironmentDriver( }, async realizeWorkspace(input) { - // Plugin-backed sandbox providers: delegate workspace realization. + // Resolve the realized cwd and any provider metadata first, then build ONE + // workspace-realization record and wrap it the SAME way for every driver. A + // plugin-backed sandbox provider realizes the workspace remotely and returns its + // own cwd and metadata. A built-in driver has no plugin call; it uses the lease + // `remoteCwd`. Both paths must produce the record through the single build below, + // so the record can never drift between two exits. + let pluginRealizedCwd: string | null = null; + let providerMetadata: Record | null = null; if (input.lease.metadata?.sandboxProviderPlugin && pluginWorkerManager) { const pluginId = readString(input.lease.metadata?.pluginId); const providerKey = @@ -1216,7 +1223,7 @@ function createSandboxEnvironmentDriver( lease: input.lease, provider: providerKey, }); - return await pluginWorkerManager.call(pluginId, "environmentRealizeWorkspace", { + const pluginResult = await pluginWorkerManager.call(pluginId, "environmentRealizeWorkspace", { driverKey: providerKey, companyId: input.lease.companyId, environmentId: input.environment.id, @@ -1229,21 +1236,36 @@ function createSandboxEnvironmentDriver( }, workspace: input.workspace, }, resolvePluginSandboxRpcTimeoutMs(stripSandboxProviderEnvelope(config as SandboxEnvironmentConfig))); + pluginRealizedCwd = + typeof pluginResult.cwd === "string" && pluginResult.cwd.trim().length > 0 + ? pluginResult.cwd.trim() + : null; + providerMetadata = pluginResult.metadata ?? null; } } + // A plugin realize handler returns only its realized cwd and provider metadata; it + // does not build the full workspace-realization record. The server builds that record + // from the run request, so the referenced (mentioned) project sources reach the adapter + // through `realization.additional`. The adapter reads that field to stage each referenced + // tree into the sandbox; without the record the sandbox agent never receives the mentioned + // projects. The provider cwd and metadata still drive the remote path when a plugin realizes + // the workspace. const record = buildWorkspaceRealizationRecordFromDriverInput({ environment: input.environment, lease: input.lease, workspace: input.workspace, cwd: - typeof input.lease.metadata?.remoteCwd === "string" && input.lease.metadata.remoteCwd.trim().length > 0 + pluginRealizedCwd ?? + (typeof input.lease.metadata?.remoteCwd === "string" && input.lease.metadata.remoteCwd.trim().length > 0 ? input.lease.metadata.remoteCwd.trim() - : input.workspace.remotePath ?? input.workspace.localPath ?? null, + : input.workspace.remotePath ?? input.workspace.localPath ?? null), + providerMetadata, }); return { - cwd: record.remote.path ?? record.local.path, + cwd: pluginRealizedCwd ?? record.remote.path ?? record.local.path, metadata: { + ...(providerMetadata ?? {}), workspaceRealization: record, }, }; diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index bcc54e70c3..08d86a7672 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -2377,6 +2377,41 @@ export function isRemoteExecutionEnvironmentDriver(driver: string | null | undef return driver === "ssh" || driver === "sandbox" || driver === "plugin"; } +/** + * Environment flag (kill-switch, default ON) that gates whether a *remote* run stages the + * referenced (mentioned) project set into the sandbox. This is a targeted rollback lever: it + * disables only the remote referenced-project path and never regresses the working local path. + * The master flag {@link MULTI_PROJECT_WORKSPACE_SYNC_ENV} is the blunt switch that kills both + * local and remote. The remote path runs when both the master flag and this remote flag are ON — + * the default state. An unset value resolves ON; an operator disables it with an explicit false + * value (`"false"`, `"0"`, `"off"`, `"no"`, or `""`). The OFF state fails closed: a remote run + * runs no referenced-project authorization or staging and reverts to the remote drop path. + */ +export const MULTI_PROJECT_WORKSPACE_SYNC_REMOTE_ENV = + "PAPERCLIP_MULTI_PROJECT_WORKSPACE_SYNC_REMOTE"; + +export function isMultiProjectWorkspaceSyncRemoteEnabled( + env: Record = process.env, +): boolean { + // Default ON: an unset value is not false, so the remote path is live unless an operator sets + // an explicit false value as the targeted kill switch (rollback path). + return !isFalsyRuntimeEnvValue(env[MULTI_PROJECT_WORKSPACE_SYNC_REMOTE_ENV]); +} + +/** + * True when an environment driver stages a multi-source remote workspace through the confined + * sandbox/command runtime. Only the `sandbox` driver asserts per-project confinement on the + * staging path (`assertSyncOperationsConfined` in `sandbox-managed-runtime`). The `ssh` driver + * stages without that guard, and the `plugin` driver does not route through the confined command + * runtime in the workspace-realization step, so both keep dropping referenced projects. A `local` + * (or unknown) driver is not remote and never reaches this check. This gate is intentionally + * narrower than {@link isRemoteExecutionEnvironmentDriver}: it names the one transport that + * confines each staged referenced tree. + */ +export function isConfinedRemoteStagingDriver(driver: string | null | undefined): boolean { + return driver === "sandbox"; +} + /** * Upper bound on how many additional (mentioned) projects a single run may materialize * beyond the anchor. Bounds the fan-out of per-project authorization and workspace prep. @@ -2647,11 +2682,26 @@ export interface ResolveAdditionalRunWorkspacesOptions { maxCandidateEvaluations?: number; /** * True when the run executes on a non-local target (ssh, sandbox, or plugin). A referenced - * project realizes as a local directory only, and a remote target has no path yet to receive - * that tree, so a resolved cwd would not exist on the target. When true, the function skips - * referenced-project authorization and workspace work and returns no additional workspaces. + * project realizes as a local directory first. On a remote target that local tree reaches the + * agent only when a confined transport stages it into the sandbox and the remote flag is on + * (see `targetStagesConfined` and `remoteReferencedSyncEnabled`). Otherwise the run drops the + * whole referenced set and records it at the staging layer. */ executionTargetIsRemote?: boolean; + /** + * True when the remote target stages each referenced tree through the confined sandbox/command + * runtime (the `sandbox` driver; see {@link isConfinedRemoteStagingDriver}). The gate opens the + * referenced-project path on a remote target only when this is true. The SSH transport and any + * unconfined transport keep dropping referenced projects. Ignored on a local target. + */ + targetStagesConfined?: boolean; + /** + * The remote-only kill switch (default ON; see {@link isMultiProjectWorkspaceSyncRemoteEnabled}). + * When true, a confined remote target stages the referenced set. When false, a remote target + * fails closed: it runs no referenced-project authorization or staging and reverts to the remote + * drop path. Ignored on a local target. + */ + remoteReferencedSyncEnabled?: boolean; } /** @@ -2677,34 +2727,44 @@ export async function resolveAdditionalRunWorkspaces( return { additionalWorkspaces: [], warnings: [], failures: [] }; } - // A referenced project realizes as a local directory only. A remote execution target (ssh, - // sandbox, or plugin) has no path yet to receive the referenced tree, so a resolved cwd would - // not exist on the target and the anchor-only remote sync never carries it across. Skip the - // referenced-project authorization and clone work on a remote target, so the run neither does - // work it must discard nor exposes an inaccessible referenced path to the agent. Warn only when - // the issue actually mentions a project, so a remote run without any referenced mention stays - // silent. + // A referenced project realizes as a local directory first. On a remote target the run carries + // that tree to the agent only when a confined transport stages it into the sandbox and the + // remote flag is on. The confined sandbox transport asserts per-project confinement on each + // staged tree (`assertSyncOperationsConfined` in `sandbox-managed-runtime`). The SSH transport + // does not, so it stays out of scope. When the remote flag is off the run fails closed. In every + // one of those drop cases the run neither does authorization or clone work it must discard nor + // exposes an inaccessible referenced path to the agent. if (opts.executionTargetIsRemote) { - const mentionedIds = await opts.issues.findMentionedProjectIds(issueId, { - includeCommentBodies: true, - }); - // Each distinct non-anchor mention is a referenced project this remote run drops. A remote - // target has no path to receive the referenced tree, so the run drops the whole set at the - // staging layer. Record one failure per dropped project so the requested-vs-synced accounting - // counts the whole referenced set and the run still emits its structured sync log. - const droppedProjectIds = [ - ...new Set(mentionedIds.filter((projectId) => projectId !== anchorProjectId)), - ]; - return { - additionalWorkspaces: [], - warnings: - droppedProjectIds.length > 0 - ? [ - "Referenced-project workspaces are available only on a local execution target. This run uses a remote execution target, so no referenced-project workspace was attached.", - ] - : [], - failures: droppedProjectIds.map((projectId) => ({ projectId, reason: "staging" as const })), - }; + const remoteReferencedSyncOpen = + (opts.remoteReferencedSyncEnabled ?? false) && (opts.targetStagesConfined ?? false); + if (!remoteReferencedSyncOpen) { + const mentionedIds = await opts.issues.findMentionedProjectIds(issueId, { + includeCommentBodies: true, + }); + // Each distinct non-anchor mention is a referenced project this remote run drops. An SSH + // target (or the remote flag off) has no confined path to receive the referenced tree, so + // the run drops the whole set at the staging layer. Record one failure per dropped project + // so the requested-vs-synced accounting counts the whole referenced set and the run still + // emits its structured sync log. Warn only when the issue actually mentions a project, so a + // remote run without any referenced mention stays silent. + const droppedProjectIds = [ + ...new Set(mentionedIds.filter((projectId) => projectId !== anchorProjectId)), + ]; + return { + additionalWorkspaces: [], + warnings: + droppedProjectIds.length > 0 + ? [ + "Referenced-project workspaces are available only on a local execution target or a confined sandbox target. This run uses a different remote execution target, so no referenced-project workspace was attached.", + ] + : [], + failures: droppedProjectIds.map((projectId) => ({ projectId, reason: "staging" as const })), + }; + } + // Fall through: a confined sandbox target with the remote flag on resolves and authorizes the + // referenced set exactly like a local target. The resolver is driver-agnostic; the confined + // sandbox transport downstream stages each admitted tree into its own `project-` + // directory. The per-project `project:read` check below still runs against the run actor. } const referenced = await resolveRunReferencedProjects(issueId, anchorProjectId, { @@ -8192,20 +8252,27 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) agent: typeof agents.$inferSelect, context: Record, previousSessionParams: Record | null, - opts?: { useProjectWorkspace?: boolean | null; executionTargetIsRemote?: boolean }, + opts?: { useProjectWorkspace?: boolean | null; executionEnvironmentDriver?: string | null }, ): Promise { const anchor = await resolveAnchorWorkspaceForRun(agent, context, previousSessionParams, opts); if (!isMultiProjectWorkspaceSyncEnabled()) { return { ...anchor, additionalWorkspaces: [], referencedProjectFailures: [] }; } + // Derive the remote-transport facts from the selected environment driver. `executionTargetIsRemote` + // decides whether the referenced set needs the remote path at all; `targetStagesConfined` decides + // whether that remote target confines each staged tree (only the sandbox driver does). The remote + // flag is the targeted kill switch; with it off, a remote run fails closed. + const executionEnvironmentDriver = opts?.executionEnvironmentDriver ?? null; const issueId = readNonEmptyString(context.issueId) ?? readNonEmptyString(context.taskId); const { additionalWorkspaces, warnings, failures } = await resolveAdditionalRunWorkspaces( issueId, anchor.projectId, { enabled: true, - executionTargetIsRemote: opts?.executionTargetIsRemote ?? false, + executionTargetIsRemote: isRemoteExecutionEnvironmentDriver(executionEnvironmentDriver), + targetStagesConfined: isConfinedRemoteStagingDriver(executionEnvironmentDriver), + remoteReferencedSyncEnabled: isMultiProjectWorkspaceSyncRemoteEnabled(), companyId: agent.companyId, actor: { type: "agent", @@ -13259,12 +13326,11 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) previousSessionParams, { useProjectWorkspace: requestedExecutionWorkspaceMode !== "agent_default", - // Referenced-project workspaces attach on a local execution target only. Gate their - // resolution on the selected environment driver so a remote run never resolves a - // referenced path it cannot reach. This never changes the anchor workspace. - executionTargetIsRemote: isRemoteExecutionEnvironmentDriver( - selectedEnvironmentForConfig?.driver, - ), + // Thread the selected environment driver so run-workspace resolution can tell a local + // target from a remote one, and a confined sandbox target from an unconfined remote + // target. A remote run resolves referenced projects only for the confined sandbox + // transport with the remote flag on. This never changes the anchor workspace. + executionEnvironmentDriver: selectedEnvironmentForConfig?.driver ?? null, }, ), }); @@ -14518,6 +14584,37 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) ); } } + // Reconcile the referenced-project set against the real remote staging outcome. A referenced + // project can pass authorization and clone locally at run prep, then fail to stage into the + // sandbox during execution. The run-prep observability above counts such a project as synced, + // so emit a second, stage-time line that counts each staging failure as a first-class + // `staging` failure. The synced set is the resolved referenced projects minus the ones that + // failed to stage. A run with no staging failure stays silent, so the anchor-only and + // fully-synced paths add no noise. + const referencedProjectStagingFailures = adapterResult.referencedProjectStagingFailures ?? []; + if (referencedProjectStagingFailures.length > 0) { + const stagingFailedProjectIds = new Set( + referencedProjectStagingFailures.map((failure) => failure.projectId), + ); + const stagedProjectObservability = buildReferencedProjectRunObservability({ + syncedProjectIds: resolvedWorkspace.additionalWorkspaces + .map((additional) => additional.projectId) + .filter((projectId) => !stagingFailedProjectIds.has(projectId)), + failures: referencedProjectStagingFailures.map((failure) => ({ + projectId: failure.projectId, + reason: "staging" as const, + })), + }); + logger.info( + { + runId: run.id, + companyId: agent.companyId, + issueId: issueRef?.id ?? null, + ...stagedProjectObservability, + }, + "run referenced-project remote staging", + ); + } const adapterManagedRuntimeServices = adapterResult.runtimeServices ? await persistAdapterManagedRuntimeServices({ db, diff --git a/server/src/services/workspace-realization.ts b/server/src/services/workspace-realization.ts index 6049cfb36c..0c11f97d9d 100644 --- a/server/src/services/workspace-realization.ts +++ b/server/src/services/workspace-realization.ts @@ -298,6 +298,14 @@ export function buildWorkspaceRealizationRecord(input: { }; } +/** + * Build the workspace-realization record from the run request. The server owns the record; + * a driver realize handler (built-in or plugin) returns only a realized cwd and provider + * metadata. Every `realizeWorkspace` exit must route through this helper, so the record carries + * the referenced (mentioned) project sources in `additional`. The adapter reads `additional` to + * stage each referenced tree into the target; a realize exit that returns a raw provider result + * without this helper drops the mentioned projects. + */ export function buildWorkspaceRealizationRecordFromDriverInput(input: { environment: Environment; lease: EnvironmentLease;