From b247bf7150484e6a0a0da82079617901f789d2a5 Mon Sep 17 00:00:00 2001 From: Nicky Leach Date: Wed, 22 Jul 2026 10:08:03 -0700 Subject: [PATCH] feat(runtime): opt-in sandbox file-sync lifecycle hooks (API + provider docs) (#10013) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is an open-source AI-agent management platform; agents run tasks inside sandboxed environments (Daytona, Kubernetes, E2B, etc.) > - The control-plane ↔ sandbox file-transfer path flows through the `environmentExecute` seam in `protocol.ts` — the only verb available to plugins — which forces a base64-over-exec chunked loop for every file move: workspace files, assets, Codex home sync > - This transport is correct and safe, but it bypasses provider-native bulk/streaming APIs (Daytona `uploadFiles`, K8s `FastUploadInterceptor` / volume mounts), leaving significant throughput on the table for large workspaces > - The right fix is an opt-in seam extension: providers with faster native transfer declare two optional verbs; providers that do not opt in stay on the existing fallback with zero code or behavior change required > - This PR adds the first layer of that extension — two optional verbs (`environmentSyncIn` / `environmentSyncOut`) in the plugin SDK, the runtime plumbing to prefer the native path for the two clean destroy-then-replace cases, and a doc for the contract > - The core correctness invariant is byte-identical fallback: if no provider opts in, execution is exactly what ships today; `assertSyncOperationsConfined` enforces host-side path confinement for providers that do opt in > - No provider advertises the verbs yet → zero production behavior change; future PRs wire up Daytona and K8s providers against this contract ## Linked Issues or Issue Description No public GitHub issue exists for this feature. Description follows the `feature_request` issue template: **Subsystem affected:** packages/plugins — plugin system; packages/adapter-utils — adapter runtime; server/ — EnvironmentRuntimeService **Problem or motivation:** Sandbox file transfers currently always use a base64-over-exec chunked loop regardless of what the underlying provider supports. For workspaces larger than a few MB this becomes the dominant wall-clock cost of every sandbox run, and it bypasses bulk/stream APIs that providers like Daytona already expose natively. **Proposed solution:** Add two optional, opt-in plugin hooks — `onEnvironmentSyncIn` / `onEnvironmentSyncOut` — to the plugin SDK. When a provider defines both hooks and both are advertised via the existing `supportedMethods` negotiation, the runtime prefers the native path for the two clean destroy-then-replace transfer cases; all other cases fall back to the existing byte-identical base64 transport. **Alternatives considered:** An unconditional verb would require every provider to implement or stub the verb. The opt-in / `METHOD_NOT_IMPLEMENTED` pattern (already used by `environmentExecute`) preserves backward compatibility with zero provider changes required. **Roadmap alignment:** Consistent with the ✅ "Cloud / Sandbox agents" and ✅ "Plugin system" milestones; extends the plugin seam rather than adding control-plane-level logic. **Additional context:** Searched open pull requests and issues for duplicate sandbox file-sync / native-transfer work; none found. ## What Changed - **`packages/plugins/sdk`** - `protocol.ts`: two new optional `HostToWorkerMethods` — `environmentSyncIn` / `environmentSyncOut` — plus generic `SyncOperation`, `SyncFileMapping`, and `SyncOutcome` types - `define-plugin.ts`: optional `onEnvironmentSyncIn` / `onEnvironmentSyncOut` fields on `PluginDefinition`; worker advertises each verb only when its hook is defined (else `METHOD_NOT_IMPLEMENTED`, mirroring `environmentExecute`) - `worker-rpc-host.ts`: route new verbs to plugin hooks - `index.ts`: re-export new public types - **`packages/adapter-utils`** - `command-managed-runtime.ts`: expose optional `syncIn` / `syncOut` on `CommandManagedRuntimeRunner` (available only when both verbs are advertised); add `assertSyncOperationsConfined` host-side path-confinement guard - `sandbox-managed-runtime.ts`: `SandboxManagedRuntimeClient` gains optional `syncIn` / `syncOut`; orchestrator prefers native path for default-provision asset inbound and workspace-download-into-fresh-dir outbound; all other paths keep the existing base64 fallback - `sandbox-file-sync.test.ts` (new): 234-line characterization suite — native-opt-in branch, fallback branch, `assertSyncOperationsConfined` escape-path rejection, `followSymlinks` → tar `-h` - `command-managed-runtime.test.ts`: negotiation + native-sync + confinement tests - **`server/src/services/environment-runtime.ts`**: `EnvironmentRuntimeService` delegates to `syncIn` / `syncOut`, gated on advertised support - **`server/src/services/environment-execution-target.ts`**: minor typing fix alongside the new verbs - **`doc/plugins/SANDBOX_FILE_SYNC_HOOKS.md`** (new): documents the full contract — opt-in / no-op guarantee, operation ordering, provider-may-tar, atomicity, `followSymlinks`, secret modes (0600, no window), path confinement, `operationId` opacity, resource bounds, shell-quoting ## Verification ```bash # SDK suite pnpm --filter packages/plugins/sdk test # Adapter-utils suite (includes new sandbox-file-sync characterization tests) pnpm --filter packages/adapter-utils test # Expected: 255 pass / 4 skip # Type-check across affected packages pnpm --filter packages/plugins/sdk typecheck pnpm --filter packages/adapter-utils typecheck # Server changed-file spot check: cd server && npx tsc --noEmit --skipLibCheck 2>&1 | grep -E "environment-(runtime|execution-target)" | head -20 ``` Key behavioral invariant to spot-check: with no provider opting in (the current state), run any sandbox task and confirm file-transfer behavior is byte-for-byte identical to what the pre-PR code produces. The characterization tests assert this at the unit level. ## Risks - **Zero production risk today**: no provider advertises `environmentSyncIn` / `environmentSyncOut`, so the new code paths are unreachable in production; all real traffic stays on the existing base64 fallback - **Path confinement**: `assertSyncOperationsConfined` rejects any `targetPath` that escapes the declared root — this is the primary security boundary for future providers. The test suite covers escape-path rejection - **Atomicity**: the contract delegates atomicity to providers; the doc explicitly calls out that directory-level ops are not guaranteed atomic - **Secret transport**: credential assets (e.g., Codex `auth.json`, directory mappings) continue to use the existing tar path — they do not go through the new verbs in any current provider > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used Provider: Anthropic Model: `claude-sonnet-4-6` (Claude Sonnet 4.6) Context window: 200 K tokens Capabilities: extended tool use, multi-file code generation, agentic reasoning via the Paperclip agent framework ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes: #` / `Refs: #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Harold Kim Co-authored-by: Paperclip --- doc/plugins/SANDBOX_FILE_SYNC_HOOKS.md | 205 +++++++++++++++ .../src/command-managed-runtime.test.ts | 54 ++++ .../src/command-managed-runtime.ts | 25 +- .../src/sandbox-file-sync.test.ts | 234 ++++++++++++++++++ .../src/sandbox-managed-runtime.ts | 208 ++++++++++++++-- packages/plugins/sdk/src/define-plugin.ts | 24 ++ packages/plugins/sdk/src/index.ts | 5 + packages/plugins/sdk/src/protocol.ts | 70 ++++++ packages/plugins/sdk/src/worker-rpc-host.ts | 24 ++ .../environment-sync-negotiation.test.ts | 191 ++++++++++++++ .../services/environment-execution-target.ts | 22 ++ server/src/services/environment-runtime.ts | 86 +++++++ 12 files changed, 1120 insertions(+), 28 deletions(-) create mode 100644 doc/plugins/SANDBOX_FILE_SYNC_HOOKS.md create mode 100644 packages/adapter-utils/src/sandbox-file-sync.test.ts create mode 100644 packages/plugins/sdk/tests/environment-sync-negotiation.test.ts diff --git a/doc/plugins/SANDBOX_FILE_SYNC_HOOKS.md b/doc/plugins/SANDBOX_FILE_SYNC_HOOKS.md new file mode 100644 index 0000000000..951bf02009 --- /dev/null +++ b/doc/plugins/SANDBOX_FILE_SYNC_HOOKS.md @@ -0,0 +1,205 @@ +# Sandbox file-sync lifecycle hooks + +A sandbox environment provider moves workspace and asset files between the host +and the sandbox around every run. By default the runtime synthesizes that +transfer over the single `environmentExecute` verb: it base64-encodes bytes and +pipes them through `base64 -d` shell commands, one bounded round-trip per chunk. +That works everywhere but is slow for large workspaces because it cannot use a +provider's native bulk file transport. + +The two **optional, opt-in** hooks documented here let a provider replace that +base64-over-exec transfer with its own native mechanism: + +- **`onEnvironmentSyncIn`** — before execution: place a set of host + files/directories at target sandbox paths. +- **`onEnvironmentSyncOut`** — after execution: copy a set of sandbox + files/directories back to target host paths. + +They are entirely opt-in. A provider that does not define them keeps the exact +base64 fallback, byte-for-byte — there is **zero behavior change** for existing +providers. + +## Opt-in / no-op semantics + +A hook is opted into exactly like `onEnvironmentExecute`: defining it on your +`PluginDefinition` makes the worker advertise the matching verb in +`InitializeResult.supportedMethods`; leaving it undefined omits the verb and the +guarded handler throws `METHOD_NOT_IMPLEMENTED` if it is ever called. + +**Both hooks are advertised and consumed as a pair.** The host runtime uses the +native path only when the worker advertises **both** `environmentSyncIn` and +`environmentSyncOut`; if a provider advertises only one, the orchestrator keeps +the base64 fallback for both directions. Define both or neither. + +```ts +export default definePlugin({ + async setup() { /* ... */ }, + async onEnvironmentSyncIn(params) { + return { operations: await transferInbound(params) }; + }, + async onEnvironmentSyncOut(params) { + return { operations: await transferOutbound(params) }; + }, +}); +``` + +## The operation / file-mapping contract + +Each hook receives an ordered list of **operations**. Each operation carries an +opaque id and a list of source→target **file mappings**: + +```ts +interface PluginSyncOperation { + operationId: string; // opaque, non-sensitive; do NOT interpret it + files: PluginSyncFileMapping[]; +} + +interface PluginSyncFileMapping { + sourcePath: string; // absolute + targetPath: string; // absolute + kind: "file" | "directory"; + mode?: number; // POSIX mode to apply at the target + exclude?: string[]; // glob excludes for a directory mapping + followSymlinks?: boolean; // directory symlink handling; see below +} + +interface PluginEnvironmentSyncResult { + operations: { operationId: string; filesTransferred: number; bytesTransferred: number }[]; +} +``` + +For `onEnvironmentSyncIn`, `sourcePath` is a **host** path and `targetPath` a +**sandbox** path. For `onEnvironmentSyncOut` the direction is reversed. All +sandbox paths are POSIX. Return per-operation `filesTransferred` / +`bytesTransferred` for observability. + +### Ordering + +Operations are applied strictly in array order, and the orchestrator invokes the +hooks in a fixed lifecycle order (inbound before execution, outbound after). The +orchestrator owns *what* and *when*; a provider only executes the opaque +transfers it is handed and must not reorder them. + +### A provider may tar internally + +The contract only describes the observable source→target result. How you move +the bytes is yours: bulk upload API, an internal `tar` stream, per-file +enumeration — all are fine. Whatever you do, the materialized target must be +observationally identical to the mapping (same files, same contents, same modes, +same symlink treatment) so the native and fallback paths are interchangeable. + +### `operationId` is opaque + +`operationId` is an opaque, non-sensitive token authored by the orchestrator. It +is **not** derived from any secret or user data, it is safe to log and safe to +expose to the sandbox, and a provider **must not** parse or depend on its value. +Do not echo it into a path or a place where it could collide with real data. + +## Symlink contract (`followSymlinks`) + +`followSymlinks` applies to `kind: "directory"` mappings and has exactly the +meaning of `tar`'s `-h` flag: + +- **falsy (default)** — archive and recreate symlinks **as links** (preserve). +- **`true`** — **dereference** each symlink to its target bytes. + +A provider honoring a directory mapping MUST reproduce this: preserve links when +falsy, dereference to bytes when `true`. The orchestrator passes the same value +it passes to its own tar create step, so native and fallback are observationally +identical. There is no separate extract-side symlink flag and no execution-time +special case — symlink handling lives entirely in this one flag. + +## Atomicity contract + +The required guarantee level is deliberately equal to the base64 fallback's +floor, so opting in never weakens integrity and never over-promises. + +- **Single-file mappings (`kind: "file"`) MUST be atomic-replace (REQUIRED).** + Stage the bytes to a provider-chosen temporary path, then atomically rename + onto `targetPath`, so an interrupted transfer never leaves a truncated file at + `targetPath`. This mirrors the fallback, which stages to + `.paperclip-upload` and then `mv -f`. + - The temp file MUST live in the **same directory (same filesystem)** as + `targetPath`. A cross-device rename degrades to copy-then-unlink and + reintroduces the truncation window it is meant to close. + - Reserve the `.paperclip-upload*` scratch names: a provider-chosen temp must + not collide with the fallback scratch name or with a real target. + +- **Directory mappings and the sync as a whole are NOT atomic / NOT + transactional.** A directory transfer is destroy-then-replace: a crash + mid-transfer can leave a partial tree, and the runtime does not roll back + already-moved bytes across operations. This matches today's behavior; do not + assume a directory operation is atomic. Where an individual file must be + integrity-protected, deliver it as its own `kind: "file"` mapping so it inherits + the single-file atomic-replace guarantee. + +- **Every operation is fail-loud.** An operation either completes or raises to + the orchestrator; never report partial success silently. The orchestrator may + then retry or fall back. + +## Secret material and file modes + +This seam can carry credential-bearing files (for example an auth directory). +Treat `mode` as mandatory for such mappings: + +- Apply the requested `mode` (e.g. `0o600`) with **no world-readable window** — + create the target with the mode, or `chmod` **before** writing any bytes, never + after. +- `mode` MUST be honored for files **inside a directory mapping** too, not only + for `kind: "file"` mappings. If you tar internally, preserve permissions; if + you enumerate, set the mode as each file lands. A credential that rides a + directory mapping otherwise silently loses its `0o600` guarantee. +- A directory mapping is not atomic (see above). If a directory carries an + individually-sensitive secret whose integrity matters, prefer delivering that + secret as a `kind: "file"` mapping so it gets atomic-replace, or protect its + integrity out of band. + +## Host-side path confinement (required of the orchestrator) + +The sandbox is untrusted relative to the host, so **the orchestrator — not the +provider — owns and confines every path**. Before an operation is handed to a +provider, the runtime canonicalizes each mapping's `sourcePath`/`targetPath` and +confines it to an orchestrator-owned root (the workspace directory or a specific +asset directory), rejecting absolute escapes and `..` traversal fail-closed. A +provider receives only already-confined, orchestrator-authored paths and MUST +NOT widen them (for example by following a sandbox-planted symlink out of the +intended root on an outbound write). Confinement is a host-side complete-mediation +guard and is never delegated below the trust boundary. + +## Resource bounds + +The base64 fallback enforces transfer caps so a runaway payload cannot exhaust +memory. A native provider MUST keep an equivalent bound — stream or chunk large +transfers rather than buffering unboundedly, and fail closed on an oversized +inline payload rather than silently removing the cap. + +## Shell safety (native providers that shell out) + +If your native transfer builds shell command strings (for example a pod-exec +`tar`/`base64`/`mv` pipeline), single-quote **every** interpolated path so a path +containing shell metacharacters is transferred literally, never interpreted. +Providers whose transport is a non-shell API (a typed bulk-upload call) do not +need this, but any shell interpolation must quote. + +## Reference: minimal shape + +```ts +async onEnvironmentSyncIn({ operations }) { + const results = []; + for (const op of operations) { // apply in order + let filesTransferred = 0; + let bytesTransferred = 0; + for (const f of op.files) { + if (f.kind === "file") { + // stage to a same-dir temp, then atomic rename onto f.targetPath, + // applying f.mode with no world-readable window + } else { + // materialize f.sourcePath at f.targetPath (destroy-then-replace), + // honoring f.exclude and f.followSymlinks, applying per-file modes + } + } + results.push({ operationId: op.operationId, filesTransferred, bytesTransferred }); + } + return { operations: results }; +} +``` diff --git a/packages/adapter-utils/src/command-managed-runtime.test.ts b/packages/adapter-utils/src/command-managed-runtime.test.ts index d9eb811780..2cd4d853a2 100644 --- a/packages/adapter-utils/src/command-managed-runtime.test.ts +++ b/packages/adapter-utils/src/command-managed-runtime.test.ts @@ -304,6 +304,60 @@ describe("command managed runtime", () => { expect(progress.at(-1)).toEqual({ done: payload.length, total: payload.length }); }); + it("stages a single-file write to .paperclip-upload then atomically renames it (single-stream path)", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-command-atomic-")); + cleanupDirs.push(rootDir); + const remotePath = path.join(rootDir, "nested", "payload.bin"); + + const { runner, calls } = makeSpawnRunner({ supportsSingleStreamStdinProgress: true }); + const client = createCommandManagedRuntimeClient({ runner, commandCwd: "/", timeoutMs: 30_000 }); + await client.writeFile(remotePath, toArrayBuffer(Buffer.from("hello atomic\n"))); + + // Characterization guardrail: the legacy single-file transport must keep its + // stage-then-atomic-rename shape (temp .paperclip-upload + `mv -f`). + const script = (calls[0].args ?? []).join(" "); + expect(script).toContain(`${remotePath}.paperclip-upload`); + expect(script).toContain(`mv -f`); + expect(script.indexOf(".paperclip-upload")).toBeLessThan(script.indexOf("mv -f")); + expect(await readFile(remotePath, "utf8")).toBe("hello atomic\n"); + }); + + it("stages a single-file write to a temp then renames it on the chunked fallback path too", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-command-atomic-fallback-")); + cleanupDirs.push(rootDir); + const remotePath = path.join(rootDir, "nested", "payload.bin"); + + const payload = Buffer.alloc(10 * 1024 * 1024); + for (let i = 0; i < payload.length; i++) payload[i] = i % 256; + const { runner, calls } = makeSpawnRunner({ supportsSingleStreamStdinProgress: false }); + const client = createCommandManagedRuntimeClient({ runner, commandCwd: "/", timeoutMs: 30_000 }); + await client.writeFile(remotePath, toArrayBuffer(payload)); + + const scripts = calls.map((call) => (call.args ?? []).join(" ")); + expect(scripts.some((script) => script.includes(`${remotePath}.paperclip-upload`))).toBe(true); + expect(scripts.some((script) => script.includes(`mv -f`))).toBe(true); + expect((await readFile(remotePath)).equals(payload)).toBe(true); + }); + + it("leaves the client without syncIn/syncOut unless the runner supports both (fallback preserved)", () => { + const base = makeSpawnRunner().runner; + expect(createCommandManagedRuntimeClient({ runner: base, commandCwd: "/", timeoutMs: 1 }).syncIn).toBeUndefined(); + + const onlyIn: CommandManagedRuntimeRunner = { ...base, syncIn: async () => ({ operations: [] }) }; + const partial = createCommandManagedRuntimeClient({ runner: onlyIn, commandCwd: "/", timeoutMs: 1 }); + expect(partial.syncIn).toBeUndefined(); + expect(partial.syncOut).toBeUndefined(); + + const both: CommandManagedRuntimeRunner = { + ...base, + syncIn: async () => ({ operations: [] }), + syncOut: async () => ({ operations: [] }), + }; + const native = createCommandManagedRuntimeClient({ runner: both, commandCwd: "/", timeoutMs: 1 }); + expect(native.syncIn).toBeTypeOf("function"); + expect(native.syncOut).toBeTypeOf("function"); + }); + it("falls back to chunked upload progress when the runner cannot report mid-stream stdin progress", async () => { const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-command-write-fallback-")); cleanupDirs.push(rootDir); diff --git a/packages/adapter-utils/src/command-managed-runtime.ts b/packages/adapter-utils/src/command-managed-runtime.ts index 77acad78cb..b8b3043cca 100644 --- a/packages/adapter-utils/src/command-managed-runtime.ts +++ b/packages/adapter-utils/src/command-managed-runtime.ts @@ -5,6 +5,8 @@ import { type SandboxManagedRuntimeAsset, type SandboxManagedRuntimeClient, type SandboxRemoteExecutionSpec, + type SandboxSyncOperation, + type SandboxSyncResult, } from "./sandbox-managed-runtime.js"; import { preferredShellForSandbox, shellCommandArgs } from "./sandbox-shell.js"; import type { RunProcessResult } from "./server-utils.js"; @@ -28,6 +30,16 @@ export interface CommandManagedRuntimeRunner { onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise; onSpawn?: (meta: { pid: number; startedAt: string }) => Promise; }): Promise; + /** + * Optional native inbound file transfer. Present only when the sandbox + * provider advertises both `environmentSyncIn` and `environmentSyncOut`; the + * client exposes `syncIn`/`syncOut` only when BOTH are present, so the + * orchestrator either uses the native path for both directions or falls back + * to the base64 transport for both. + */ + syncIn?(operations: SandboxSyncOperation[]): Promise; + /** Optional native outbound file transfer. See {@link syncIn}. */ + syncOut?(operations: SandboxSyncOperation[]): Promise; } export interface CommandManagedRuntimeSpec { @@ -118,7 +130,7 @@ export function createCommandManagedRuntimeClient(input: { return result; }; - return { + const client: SandboxManagedRuntimeClient = { makeDir: async (remotePath) => { await runShell(`mkdir -p ${shellQuote(remotePath)}`); }, @@ -240,6 +252,17 @@ export function createCommandManagedRuntimeClient(input: { requireSuccessfulResult(result, command); }, }; + + // Expose the native sync capability to the orchestrator only when the runner + // supports BOTH directions; a provider that advertises just one verb (or + // neither) keeps the byte-identical base64 fallback for both. + const { syncIn, syncOut } = input.runner; + if (syncIn && syncOut) { + client.syncIn = (operations) => syncIn(operations); + client.syncOut = (operations) => syncOut(operations); + } + + return client; } export async function prepareCommandManagedRuntime(input: { diff --git a/packages/adapter-utils/src/sandbox-file-sync.test.ts b/packages/adapter-utils/src/sandbox-file-sync.test.ts new file mode 100644 index 0000000000..206d053122 --- /dev/null +++ b/packages/adapter-utils/src/sandbox-file-sync.test.ts @@ -0,0 +1,234 @@ +import { lstat, mkdir, mkdtemp, readFile, readdir, rm, symlink, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { execFile as execFileCallback } from "node:child_process"; +import { promisify } from "node:util"; +import { afterEach, describe, expect, it } from "vitest"; + +import { + assertSyncOperationsConfined, + prepareSandboxManagedRuntime, + type SandboxManagedRuntimeClient, + type SandboxSyncOperation, +} from "./sandbox-managed-runtime.js"; + +const execFile = promisify(execFileCallback); + +interface RecordingClient { + client: SandboxManagedRuntimeClient; + syncInOps: SandboxSyncOperation[][]; + syncOutOps: SandboxSyncOperation[][]; +} + +// A filesystem-backed client that additionally exposes native syncIn/syncOut, +// mirroring a provider that opted into the sync verbs. The native transfer is a +// faithful destroy-then-replace directory copy honoring followSymlinks. +function makeNativeClient(): RecordingClient { + const syncInOps: SandboxSyncOperation[][] = []; + const syncOutOps: SandboxSyncOperation[][] = []; + + const transferDirectory = async ( + sourcePath: string, + targetPath: string, + followSymlinks: boolean | undefined, + ): Promise => { + await rm(targetPath, { recursive: true, force: true }); + await mkdir(targetPath, { recursive: true }); + // followSymlinks true dereferences to bytes (like tar -h); falsy preserves links. + const copyArgs = followSymlinks ? ["-RL"] : ["-a"]; + await execFile("cp", [...copyArgs, `${sourcePath}/.`, targetPath]); + const entries = await readdir(targetPath, { withFileTypes: true }).catch(() => []); + return entries.length; + }; + + const applyOperations = async (operations: SandboxSyncOperation[]) => ({ + operations: await Promise.all(operations.map(async (operation) => { + let filesTransferred = 0; + for (const mapping of operation.files) { + if (mapping.kind === "directory") { + filesTransferred += await transferDirectory(mapping.sourcePath, mapping.targetPath, mapping.followSymlinks); + } else { + await mkdir(path.dirname(mapping.targetPath), { recursive: true }); + await writeFile(mapping.targetPath, await readFile(mapping.sourcePath)); + filesTransferred += 1; + } + } + return { operationId: operation.operationId, filesTransferred, bytesTransferred: 0 }; + })), + }); + + const client: SandboxManagedRuntimeClient = { + makeDir: async (remotePath) => { await mkdir(remotePath, { recursive: true }); }, + writeFile: async (remotePath, bytes) => { + await mkdir(path.dirname(remotePath), { recursive: true }); + await writeFile(remotePath, Buffer.from(bytes)); + }, + readFile: async (remotePath) => await readFile(remotePath), + listFiles: async (remotePath) => { + const entries = await readdir(remotePath, { withFileTypes: true }).catch(() => []); + return entries.filter((e) => e.isFile()).map((e) => e.name).sort(); + }, + remove: async (remotePath) => { await rm(remotePath, { recursive: true, force: true }); }, + run: async (command) => { await execFile("sh", ["-c", command], { maxBuffer: 32 * 1024 * 1024 }); }, + syncIn: async (operations) => { syncInOps.push(operations); return applyOperations(operations); }, + syncOut: async (operations) => { syncOutOps.push(operations); return applyOperations(operations); }, + }; + + return { client, syncInOps, syncOutOps }; +} + +describe("sandbox native file sync", () => { + const cleanupDirs: string[] = []; + afterEach(async () => { + while (cleanupDirs.length > 0) { + const dir = cleanupDirs.pop(); + if (dir) await rm(dir, { recursive: true, force: true }).catch(() => undefined); + } + }); + + it("prefers the native path for default-provision asset inbound and workspace outbound", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-native-sync-")); + cleanupDirs.push(rootDir); + const localWorkspaceDir = path.join(rootDir, "local-workspace"); + const remoteWorkspaceDir = path.join(rootDir, "remote-workspace"); + const localAssetsDir = path.join(rootDir, "local-assets"); + await mkdir(localWorkspaceDir, { recursive: true }); + await mkdir(localAssetsDir, { recursive: true }); + await writeFile(path.join(localWorkspaceDir, "README.md"), "local workspace\n", "utf8"); + await writeFile(path.join(localAssetsDir, "skill.md"), "skill body\n", "utf8"); + + const { client, syncInOps, syncOutOps } = makeNativeClient(); + const prepared = await prepareSandboxManagedRuntime({ + spec: { transport: "sandbox", provider: "test", sandboxId: "s1", remoteCwd: remoteWorkspaceDir, timeoutMs: 30_000, apiKey: null }, + adapterKey: "test-adapter", + client, + workspaceLocalDir: localWorkspaceDir, + assets: [{ key: "skills", localDir: localAssetsDir }], + }); + + // The default-provision asset was transferred through syncIn as a single + // directory mapping with an opaque operationId; the file landed in place. + const inboundOps = syncInOps.flat(); + expect(inboundOps.length).toBe(1); + const assetOp = inboundOps[0]; + expect(assetOp.operationId).toMatch(/^sync-op-\d+$/); + expect(assetOp.operationId).not.toContain("skills"); + expect(assetOp.files).toEqual([ + { sourcePath: localAssetsDir, targetPath: prepared.assetDirs.skills, kind: "directory", exclude: undefined, followSymlinks: undefined }, + ]); + expect(await readFile(path.join(prepared.assetDirs.skills, "skill.md"), "utf8")).toBe("skill body\n"); + + // Mutate the sandbox workspace, then restore through the native outbound path. + await writeFile(path.join(remoteWorkspaceDir, "README.md"), "remote workspace\n", "utf8"); + await writeFile(path.join(remoteWorkspaceDir, "new.txt"), "added\n", "utf8"); + await prepared.restoreWorkspace(); + + const outboundOps = syncOutOps.flat(); + expect(outboundOps.length).toBe(1); + expect(outboundOps[0].files[0]).toMatchObject({ sourcePath: remoteWorkspaceDir, kind: "directory" }); + expect(await readFile(path.join(localWorkspaceDir, "README.md"), "utf8")).toBe("remote workspace\n"); + expect(await readFile(path.join(localWorkspaceDir, "new.txt"), "utf8")).toBe("added\n"); + }); + + it("keeps a custom-provision asset on the tar fallback even when native sync is available", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-native-custom-")); + cleanupDirs.push(rootDir); + const localWorkspaceDir = path.join(rootDir, "local-workspace"); + const remoteWorkspaceDir = path.join(rootDir, "remote-workspace"); + const localAssetsDir = path.join(rootDir, "local-assets"); + await mkdir(localWorkspaceDir, { recursive: true }); + await mkdir(localAssetsDir, { recursive: true }); + await writeFile(path.join(localWorkspaceDir, "README.md"), "ws\n", "utf8"); + await writeFile(path.join(localAssetsDir, "cred.txt"), "secret\n", "utf8"); + + const { client, syncInOps } = makeNativeClient(); + const prepared = await prepareSandboxManagedRuntime({ + spec: { transport: "sandbox", provider: "test", sandboxId: "s1", remoteCwd: remoteWorkspaceDir, timeoutMs: 30_000, apiKey: null }, + adapterKey: "test-adapter", + client, + workspaceLocalDir: localWorkspaceDir, + assets: [{ + key: "creds", + localDir: localAssetsDir, + // A bespoke extract command (e.g. a credential merge) cannot be a generic + // file mapping, so the orchestrator keeps it on the tar path. + provision: { extractCommand: ({ assetTarPath, assetDir }) => + `rm -rf ${assetDir} && mkdir -p ${assetDir} && tar -xf ${assetTarPath} -C ${assetDir} && rm -f ${assetTarPath}` }, + }], + }); + + // No syncIn operation for the custom asset; it still materializes via tar. + expect(syncInOps.flat().length).toBe(0); + expect(await readFile(path.join(prepared.assetDirs.creds, "cred.txt"), "utf8")).toBe("secret\n"); + }); + + it("dereferences symlinks only when followSymlinks is true (native honors the flag)", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-native-symlink-")); + cleanupDirs.push(rootDir); + const localWorkspaceDir = path.join(rootDir, "local-workspace"); + const remoteWorkspaceDir = path.join(rootDir, "remote-workspace"); + const assetsPreserve = path.join(rootDir, "assets-preserve"); + const assetsDeref = path.join(rootDir, "assets-deref"); + const target = path.join(rootDir, "target.md"); + await mkdir(localWorkspaceDir, { recursive: true }); + await mkdir(assetsPreserve, { recursive: true }); + await mkdir(assetsDeref, { recursive: true }); + await writeFile(path.join(localWorkspaceDir, "README.md"), "ws\n", "utf8"); + await writeFile(target, "link body\n", "utf8"); + await symlink(target, path.join(assetsPreserve, "link.md")); + await symlink(target, path.join(assetsDeref, "link.md")); + + const { client } = makeNativeClient(); + const prepared = await prepareSandboxManagedRuntime({ + spec: { transport: "sandbox", provider: "test", sandboxId: "s1", remoteCwd: remoteWorkspaceDir, timeoutMs: 30_000, apiKey: null }, + adapterKey: "test-adapter", + client, + workspaceLocalDir: localWorkspaceDir, + assets: [ + { key: "preserve", localDir: assetsPreserve, followSymlinks: false }, + { key: "deref", localDir: assetsDeref, followSymlinks: true }, + ], + }); + + expect((await lstat(path.join(prepared.assetDirs.preserve, "link.md"))).isSymbolicLink()).toBe(true); + const dereffed = await lstat(path.join(prepared.assetDirs.deref, "link.md")); + expect(dereffed.isSymbolicLink()).toBe(false); + expect(await readFile(path.join(prepared.assetDirs.deref, "link.md"), "utf8")).toBe("link body\n"); + }); +}); + +describe("assertSyncOperationsConfined", () => { + const op = (targetPath: string, sourcePath = "/host/src"): SandboxSyncOperation[] => [ + { operationId: "sync-op-1", files: [{ sourcePath, targetPath, kind: "directory" }] }, + ]; + + it("accepts targets within an allowed root", () => { + expect(() => assertSyncOperationsConfined(op("/remote/ws/sub"), { + sourceRoots: ["/host/src"], targetRoots: ["/remote/ws"], + })).not.toThrow(); + }); + + it("rejects a relative target", () => { + expect(() => assertSyncOperationsConfined(op("relative/path"), { + sourceRoots: ["/host/src"], targetRoots: ["/remote/ws"], + })).toThrow(/confined absolute path/); + }); + + it("rejects a parent-traversal escape", () => { + expect(() => assertSyncOperationsConfined(op("/remote/ws/../etc/passwd"), { + sourceRoots: ["/host/src"], targetRoots: ["/remote/ws"], + })).toThrow(/confined absolute path|escapes its confinement root/); + }); + + it("rejects an absolute target outside every root", () => { + expect(() => assertSyncOperationsConfined(op("/etc/passwd"), { + sourceRoots: ["/host/src"], targetRoots: ["/remote/ws"], + })).toThrow(/escapes its confinement root/); + }); + + it("rejects a source outside every source root", () => { + expect(() => assertSyncOperationsConfined(op("/remote/ws/ok", "/etc/shadow"), { + sourceRoots: ["/host/src"], targetRoots: ["/remote/ws"], + })).toThrow(/escapes its confinement root/); + }); +}); diff --git a/packages/adapter-utils/src/sandbox-managed-runtime.ts b/packages/adapter-utils/src/sandbox-managed-runtime.ts index ccce1d1312..62bf2cf7d7 100644 --- a/packages/adapter-utils/src/sandbox-managed-runtime.ts +++ b/packages/adapter-utils/src/sandbox-managed-runtime.ts @@ -121,6 +121,37 @@ export interface SandboxTransferProgressOptions { onProgress?: (transferredBytes: number, totalBytes: number | null) => void | Promise; } +/** + * A single source→target file or directory transfer within a sync operation. + * Mirrors the plugin SDK `PluginSyncFileMapping`; kept as a local structural + * type so `adapter-utils` does not depend on the plugin SDK. For `syncIn`, + * `sourcePath` is a host path and `targetPath` a sandbox path; for `syncOut` the + * direction is reversed. Sandbox paths are POSIX. + */ +export interface SandboxSyncFileMapping { + sourcePath: string; + targetPath: string; + kind: "file" | "directory"; + mode?: number; + exclude?: string[]; + followSymlinks?: boolean; +} + +/** + * An ordered, opaque unit of work handed to the native sync transport. The + * `operationId` is an opaque, non-sensitive token authored by the orchestrator + * (never a caller/asset identifier that could leak intent); a provider MUST NOT + * interpret it. + */ +export interface SandboxSyncOperation { + operationId: string; + files: SandboxSyncFileMapping[]; +} + +export interface SandboxSyncResult { + operations: { operationId: string; filesTransferred: number; bytesTransferred: number }[]; +} + export interface SandboxManagedRuntimeClient { makeDir(remotePath: string): Promise; writeFile(remotePath: string, bytes: ArrayBuffer, options?: SandboxTransferProgressOptions): Promise; @@ -131,6 +162,49 @@ export interface SandboxManagedRuntimeClient { listFiles(remotePath: string): Promise; remove(remotePath: string): Promise; run(command: string, options: { timeoutMs: number }): Promise; + /** + * Optional native inbound transfer. Present only when the sandbox provider + * advertises both `environmentSyncIn` and `environmentSyncOut`; otherwise the + * orchestrator falls back to the tar + base64 `writeFile`/`run` path so + * behavior is byte-identical to a provider that never opted in. + */ + syncIn?(operations: SandboxSyncOperation[]): Promise; + /** Optional native outbound transfer. See {@link syncIn}. */ + syncOut?(operations: SandboxSyncOperation[]): Promise; +} + +/** + * Host-side complete-mediation guard for native sync operations. The orchestrator + * authors every `targetPath`, but the native transport crosses the host↔sandbox + * trust boundary, so we canonicalize and confine each mapping's source and target + * to an orchestrator-owned root before handing the operation to a provider. + * Absolute escapes and `..` traversal are rejected fail-closed. Sandbox and host + * paths on the server are POSIX. + */ +export function assertSyncOperationsConfined( + operations: SandboxSyncOperation[], + roots: { sourceRoots: string[]; targetRoots: string[] }, +): void { + const confine = (candidate: string, allowed: string[], label: string): void => { + const normalized = path.posix.normalize(candidate); + if (!path.posix.isAbsolute(normalized) || normalized === ".." || normalized.includes("/../") || normalized.endsWith("/..")) { + throw new Error(`sync operation ${label} path is not a confined absolute path: ${candidate}`); + } + const within = allowed.some((root) => { + const normalizedRoot = path.posix.normalize(root); + const prefix = normalizedRoot.endsWith("/") ? normalizedRoot : `${normalizedRoot}/`; + return normalized === normalizedRoot || normalized.startsWith(prefix); + }); + if (!within) { + throw new Error(`sync operation ${label} path escapes its confinement root: ${candidate}`); + } + }; + for (const operation of operations) { + for (const mapping of operation.files) { + confine(mapping.sourcePath, roots.sourceRoots, "source"); + confine(mapping.targetPath, roots.targetRoots, "target"); + } + } } export interface PreparedSandboxManagedRuntime { @@ -517,6 +591,15 @@ export async function prepareSandboxManagedRuntime(input: { exclude: restoreExclude, }); + // Prefer the provider's native file transport when it advertised the sync + // verbs; otherwise every branch below falls back to the byte-identical tar + + // base64 `writeFile`/`run` path. `nextSyncOperationId` emits opaque, ordered, + // non-sensitive tokens — never a caller/asset identifier. + const nativeSyncIn = typeof input.client.syncIn === "function"; + const nativeSyncOut = typeof input.client.syncOut === "function"; + let syncOperationSeq = 0; + const nextSyncOperationId = () => `sync-op-${++syncOperationSeq}`; + await withTempDir("paperclip-sandbox-sync-", async (tempDir) => { const preservedNames = new Set([ ".paperclip-runtime", @@ -529,6 +612,12 @@ export async function prepareSandboxManagedRuntime(input: { localDir: input.workspaceLocalDir, snapshot: gitSnapshot, }, async (cloneDir) => { + // The git-workspace and workspace transfers preserve `.paperclip-runtime` + // on the target (and the git overlay merges on top rather than replacing), + // which the generic destroy-then-replace file mapping cannot express, so + // they always take the tar path. Native transfer is used for the clean + // destroy-then-replace cases (default-provision assets inbound; the + // workspace download into a fresh host dir outbound). const gitTarPath = path.join(tempDir, "git-workspace.tar"); await createTarballFromDirectory({ localDir: cloneDir, @@ -614,6 +703,41 @@ export async function prepareSandboxManagedRuntime(input: { for (const asset of input.assets ?? []) { await emitRuntimeStatus(input.onRuntimeProgress, "config_sync", "Syncing runtime assets to sandbox"); + const remoteAssetDir = path.posix.join(runtimeRootDir, asset.key); + // Assets with custom provisioning (staged helper files or a bespoke extract + // command such as a credential merge) do more than a plain directory + // replacement, so they cannot be expressed as a generic file mapping and + // always take the tar path. A default-provisioned asset is a clean + // destroy-then-replace of its own directory, which the native transport + // reproduces exactly. + const usesCustomProvision = + Boolean(asset.provision?.extractCommand) || (asset.provision?.stageFiles?.length ?? 0) > 0; + if (nativeSyncIn && !usesCustomProvision) { + const operations: SandboxSyncOperation[] = [{ + operationId: nextSyncOperationId(), + files: [{ + sourcePath: asset.localDir, + targetPath: remoteAssetDir, + kind: "directory", + exclude: asset.exclude, + followSymlinks: asset.followSymlinks, + }], + }]; + assertSyncOperationsConfined(operations, { + sourceRoots: [asset.localDir], + targetRoots: [runtimeRootDir], + }); + const assetUpload = makeTransferProgress( + input.onProgress, + "Syncing", + "to", + asset.key, + { sink: input.onRuntimeProgress, phase: "config_sync" }, + ); + await input.client.syncIn!(operations); + await assetUpload.finish(0, 0); + continue; + } const assetTarPath = path.join(tempDir, `${asset.key}.tar`); await createTarballFromDirectory({ localDir: asset.localDir, @@ -622,7 +746,6 @@ export async function prepareSandboxManagedRuntime(input: { exclude: asset.exclude, }); const assetTarBytes = await fs.readFile(assetTarPath); - const remoteAssetDir = path.posix.join(runtimeRootDir, asset.key); const remoteAssetTar = path.posix.join(runtimeRootDir, `${asset.key}-upload.tar`); const assetUpload = makeTransferProgress( input.onProgress, @@ -718,34 +841,65 @@ export async function prepareSandboxManagedRuntime(input: { }); } - const remoteWorkspaceTar = path.posix.join(runtimeRootDir, "workspace-download.tar"); await emitRuntimeStatus(input.onRuntimeProgress, "restore", "Restoring workspace from sandbox"); - await input.client.run( - `sh -c ${shellQuote(createRemoteTarballFromDirectoryCommand({ - remoteDir: workspaceRemoteDir, - archivePath: remoteWorkspaceTar, - exclude: restoreExclude, - }))}`, - { timeoutMs: input.spec.timeoutMs }, - ); - const workspaceRestore = makeTransferProgress( - restoreSink, - "Restoring", - "from", - "workspace", - { sink: input.onRuntimeProgress, phase: "restore" }, - ); - const archiveBytes = await input.client.readFile(remoteWorkspaceTar, workspaceRestore.options); - const archiveBuffer = toBuffer(archiveBytes); - await workspaceRestore.finish(archiveBuffer.byteLength, archiveBuffer.byteLength); - await input.client.remove(remoteWorkspaceTar).catch(() => undefined); - const localArchivePath = path.join(tempDir, "workspace.tar"); const extractedDir = path.join(tempDir, "workspace"); - await fs.writeFile(localArchivePath, archiveBuffer); - await extractTarballToDirectory({ - archivePath: localArchivePath, - localDir: extractedDir, - }); + if (nativeSyncOut) { + // Native outbound: the provider materializes the sandbox workspace into + // a fresh host directory. It is a clean destroy-then-replace into a + // temp dir the orchestrator just created, so it maps exactly to a + // generic directory file mapping; the host-side baseline merge below is + // unchanged. + const operations: SandboxSyncOperation[] = [{ + operationId: nextSyncOperationId(), + files: [{ + sourcePath: workspaceRemoteDir, + targetPath: extractedDir, + kind: "directory", + exclude: restoreExclude, + }], + }]; + assertSyncOperationsConfined(operations, { + sourceRoots: [workspaceRemoteDir], + targetRoots: [extractedDir], + }); + await fs.mkdir(extractedDir, { recursive: true }); + const workspaceRestore = makeTransferProgress( + restoreSink, + "Restoring", + "from", + "workspace", + { sink: input.onRuntimeProgress, phase: "restore" }, + ); + await input.client.syncOut!(operations); + await workspaceRestore.finish(0, 0); + } else { + const remoteWorkspaceTar = path.posix.join(runtimeRootDir, "workspace-download.tar"); + await input.client.run( + `sh -c ${shellQuote(createRemoteTarballFromDirectoryCommand({ + remoteDir: workspaceRemoteDir, + archivePath: remoteWorkspaceTar, + exclude: restoreExclude, + }))}`, + { timeoutMs: input.spec.timeoutMs }, + ); + const workspaceRestore = makeTransferProgress( + restoreSink, + "Restoring", + "from", + "workspace", + { sink: input.onRuntimeProgress, phase: "restore" }, + ); + const archiveBytes = await input.client.readFile(remoteWorkspaceTar, workspaceRestore.options); + const archiveBuffer = toBuffer(archiveBytes); + await workspaceRestore.finish(archiveBuffer.byteLength, archiveBuffer.byteLength); + await input.client.remove(remoteWorkspaceTar).catch(() => undefined); + const localArchivePath = path.join(tempDir, "workspace.tar"); + await fs.writeFile(localArchivePath, archiveBuffer); + await extractTarballToDirectory({ + archivePath: localArchivePath, + localDir: extractedDir, + }); + } const gitHeadToIntegrate = importedHead; await mergeDirectoryWithBaseline({ baseline: baselineSnapshot, diff --git a/packages/plugins/sdk/src/define-plugin.ts b/packages/plugins/sdk/src/define-plugin.ts index 668551633b..3c894672e5 100644 --- a/packages/plugins/sdk/src/define-plugin.ts +++ b/packages/plugins/sdk/src/define-plugin.ts @@ -55,6 +55,9 @@ import type { PluginEnvironmentDestroyLeaseParams, PluginEnvironmentExecuteParams, PluginEnvironmentExecuteResult, + PluginEnvironmentSyncInParams, + PluginEnvironmentSyncOutParams, + PluginEnvironmentSyncResult, PluginEnvironmentStartInteractiveSetupParams, PluginEnvironmentInteractiveSetupSession, PluginEnvironmentGetInteractiveSetupParams, @@ -336,6 +339,27 @@ export interface PluginDefinition { params: PluginEnvironmentExecuteParams, ): Promise; + /** + * Optional, opt-in: called before execution to place host files/directories at + * target sandbox paths using a provider-native transport instead of the default + * base64-over-exec fallback. Defining this hook (together with + * `onEnvironmentSyncOut`) advertises `environmentSyncIn`; leaving it undefined + * keeps the byte-identical fallback. See `doc/plugins/SANDBOX_FILE_SYNC_HOOKS.md`. + */ + onEnvironmentSyncIn?( + params: PluginEnvironmentSyncInParams, + ): Promise; + + /** + * Optional, opt-in: called after execution to copy sandbox files/directories + * back to target host paths using a provider-native transport. Defining this + * hook (together with `onEnvironmentSyncIn`) advertises `environmentSyncOut`. + * See `doc/plugins/SANDBOX_FILE_SYNC_HOOKS.md`. + */ + onEnvironmentSyncOut?( + params: PluginEnvironmentSyncOutParams, + ): Promise; + /** Called to start an interactive setup sandbox and return redacted connection metadata. */ onEnvironmentStartInteractiveSetup?( params: PluginEnvironmentStartInteractiveSetupParams, diff --git a/packages/plugins/sdk/src/index.ts b/packages/plugins/sdk/src/index.ts index d32b7820b8..1e6d3260e0 100644 --- a/packages/plugins/sdk/src/index.ts +++ b/packages/plugins/sdk/src/index.ts @@ -181,6 +181,11 @@ export type { PluginEnvironmentRealizeWorkspaceResult, PluginEnvironmentExecuteParams, PluginEnvironmentExecuteResult, + PluginSyncFileMapping, + PluginSyncOperation, + PluginEnvironmentSyncInParams, + PluginEnvironmentSyncOutParams, + PluginEnvironmentSyncResult, PluginEnvironmentInteractiveSetupStatus, PluginEnvironmentInteractiveSetupConnectionType, PluginEnvironmentTemplateRefKind, diff --git a/packages/plugins/sdk/src/protocol.ts b/packages/plugins/sdk/src/protocol.ts index ae3cbd86b5..8bdb2914b3 100644 --- a/packages/plugins/sdk/src/protocol.ts +++ b/packages/plugins/sdk/src/protocol.ts @@ -650,6 +650,66 @@ export interface PluginEnvironmentExecuteResult { metadata?: Record; } +/** + * A single source→target file or directory transfer within a sync operation. + * + * For `environmentSyncIn`, `sourcePath` is a host path and `targetPath` is a + * sandbox path; for `environmentSyncOut` the direction is reversed. All sandbox + * paths are POSIX. The contract is provider-agnostic: a provider may transfer a + * directory by whatever native mechanism it prefers (bulk upload, internal tar, + * per-file enumeration) as long as the observable result matches this mapping. + */ +export interface PluginSyncFileMapping { + /** Absolute path of the transfer source (host for syncIn, sandbox for syncOut). */ + sourcePath: string; + /** Absolute path of the transfer target (sandbox for syncIn, host for syncOut). */ + targetPath: string; + /** Whether the mapping transfers a single regular file or a directory tree. */ + kind: "file" | "directory"; + /** + * POSIX file mode to apply at the target (e.g. `0o600` for secret material). + * When set, providers MUST create the target with this mode with no + * world-readable window (create-with-mode or chmod-before-bytes, never after). + */ + mode?: number; + /** Glob patterns to exclude when `kind` is `"directory"`. */ + exclude?: string[]; + /** + * Symlink handling for `kind: "directory"` transfers. Falsy preserves symlinks + * as links; `true` dereferences them to their target bytes. Mirrors tar's `-h`. + */ + followSymlinks?: boolean; +} + +/** + * An ordered, opaque unit of work handed to a sync hook. The `operationId` is an + * opaque, non-sensitive token authored by the orchestrator; a provider MUST NOT + * interpret it. Operations are applied in array order. + */ +export interface PluginSyncOperation { + operationId: string; + files: PluginSyncFileMapping[]; +} + +export interface PluginEnvironmentSyncInParams extends PluginEnvironmentDriverBaseParams { + lease: PluginEnvironmentLease; + operations: PluginSyncOperation[]; +} + +export interface PluginEnvironmentSyncOutParams extends PluginEnvironmentDriverBaseParams { + lease: PluginEnvironmentLease; + operations: PluginSyncOperation[]; +} + +/** Per-operation transfer accounting returned by a sync hook, for observability. */ +export interface PluginEnvironmentSyncResult { + operations: { + operationId: string; + filesTransferred: number; + bytesTransferred: number; + }[]; +} + export type PluginEnvironmentInteractiveSetupStatus = | "starting" | "waiting_for_user" @@ -864,6 +924,14 @@ export interface HostToWorkerMethods { params: PluginEnvironmentExecuteParams, result: PluginEnvironmentExecuteResult, ]; + environmentSyncIn: [ + params: PluginEnvironmentSyncInParams, + result: PluginEnvironmentSyncResult, + ]; + environmentSyncOut: [ + params: PluginEnvironmentSyncOutParams, + result: PluginEnvironmentSyncResult, + ]; environmentStartInteractiveSetup: [ params: PluginEnvironmentStartInteractiveSetupParams, result: PluginEnvironmentInteractiveSetupSession, @@ -918,6 +986,8 @@ export const HOST_TO_WORKER_OPTIONAL_METHODS: readonly HostToWorkerMethodName[] "environmentDestroyLease", "environmentRealizeWorkspace", "environmentExecute", + "environmentSyncIn", + "environmentSyncOut", "environmentStartInteractiveSetup", "environmentGetInteractiveSetup", "environmentCaptureTemplate", diff --git a/packages/plugins/sdk/src/worker-rpc-host.ts b/packages/plugins/sdk/src/worker-rpc-host.ts index 730fbb4742..4c53eb9226 100644 --- a/packages/plugins/sdk/src/worker-rpc-host.ts +++ b/packages/plugins/sdk/src/worker-rpc-host.ts @@ -88,6 +88,8 @@ import type { PluginEnvironmentAcquireLeaseParams, PluginEnvironmentDestroyLeaseParams, PluginEnvironmentExecuteParams, + PluginEnvironmentSyncInParams, + PluginEnvironmentSyncOutParams, PluginEnvironmentRealizeWorkspaceParams, PluginEnvironmentReleaseLeaseParams, PluginEnvironmentResumeLeaseParams, @@ -1394,6 +1396,12 @@ export function startWorkerRpcHost(options: WorkerRpcHostOptions): WorkerRpcHost case "environmentExecute": return handleEnvironmentExecute(params as PluginEnvironmentExecuteParams); + case "environmentSyncIn": + return handleEnvironmentSyncIn(params as PluginEnvironmentSyncInParams); + + case "environmentSyncOut": + return handleEnvironmentSyncOut(params as PluginEnvironmentSyncOutParams); + case "environmentStartInteractiveSetup": return handleEnvironmentStartInteractiveSetup(params as PluginEnvironmentStartInteractiveSetupParams); @@ -1453,6 +1461,8 @@ export function startWorkerRpcHost(options: WorkerRpcHostOptions): WorkerRpcHost if (plugin.definition.onEnvironmentDestroyLease) supportedMethods.push("environmentDestroyLease"); if (plugin.definition.onEnvironmentRealizeWorkspace) supportedMethods.push("environmentRealizeWorkspace"); if (plugin.definition.onEnvironmentExecute) supportedMethods.push("environmentExecute"); + if (plugin.definition.onEnvironmentSyncIn) supportedMethods.push("environmentSyncIn"); + if (plugin.definition.onEnvironmentSyncOut) supportedMethods.push("environmentSyncOut"); if (plugin.definition.onEnvironmentStartInteractiveSetup) supportedMethods.push("environmentStartInteractiveSetup"); if (plugin.definition.onEnvironmentGetInteractiveSetup) supportedMethods.push("environmentGetInteractiveSetup"); if (plugin.definition.onEnvironmentCaptureTemplate) supportedMethods.push("environmentCaptureTemplate"); @@ -1715,6 +1725,20 @@ export function startWorkerRpcHost(options: WorkerRpcHostOptions): WorkerRpcHost return plugin.definition.onEnvironmentExecute(params); } + async function handleEnvironmentSyncIn(params: PluginEnvironmentSyncInParams) { + if (!plugin.definition.onEnvironmentSyncIn) { + throw methodNotImplemented("environmentSyncIn"); + } + return plugin.definition.onEnvironmentSyncIn(params); + } + + async function handleEnvironmentSyncOut(params: PluginEnvironmentSyncOutParams) { + if (!plugin.definition.onEnvironmentSyncOut) { + throw methodNotImplemented("environmentSyncOut"); + } + return plugin.definition.onEnvironmentSyncOut(params); + } + async function handleEnvironmentStartInteractiveSetup(params: PluginEnvironmentStartInteractiveSetupParams) { if (!plugin.definition.onEnvironmentStartInteractiveSetup) { throw methodNotImplemented("environmentStartInteractiveSetup"); diff --git a/packages/plugins/sdk/tests/environment-sync-negotiation.test.ts b/packages/plugins/sdk/tests/environment-sync-negotiation.test.ts new file mode 100644 index 0000000000..512f60cfd4 --- /dev/null +++ b/packages/plugins/sdk/tests/environment-sync-negotiation.test.ts @@ -0,0 +1,191 @@ +import { createInterface } from "node:readline"; +import { PassThrough } from "node:stream"; + +import { describe, expect, it } from "vitest"; + +import { definePlugin } from "../src/define-plugin.js"; +import { + createRequest, + isJsonRpcResponse, + parseMessage, + PLUGIN_RPC_ERROR_CODES, + serializeMessage, + type JsonRpcResponse, + type PluginEnvironmentSyncInParams, + type PluginEnvironmentSyncOutParams, + type PluginEnvironmentSyncResult, +} from "../src/protocol.js"; +import { startWorkerRpcHost } from "../src/worker-rpc-host.js"; + +const MANIFEST = { + id: "paperclip.sync-negotiation-test", + apiVersion: 1, + version: "1.0.0", + displayName: "Sync Negotiation Test", + description: "Test plugin", + author: "Paperclip", + categories: ["automation"], + capabilities: [], + entrypoints: {}, +} as const; + +function startTestWorker(plugin: ReturnType) { + const hostToWorker = new PassThrough(); + const workerToHost = new PassThrough(); + const hostReadline = createInterface({ input: workerToHost }); + const pending = new Map void>(); + let nextRequestId = 1; + + hostReadline.on("line", (line) => { + const message = parseMessage(line); + if (!isJsonRpcResponse(message)) return; + pending.get(String(message.id))?.(message); + pending.delete(String(message.id)); + }); + + const worker = startWorkerRpcHost({ plugin, stdin: hostToWorker, stdout: workerToHost }); + + function callWorker(method: string, params: unknown): Promise { + const id = `host-${nextRequestId++}`; + const result = new Promise((resolve, reject) => { + pending.set(id, (response) => { + if ("error" in response && response.error) { + reject(Object.assign(new Error(response.error.message), { code: response.error.code })); + return; + } + resolve((response as { result?: T }).result as T); + }); + }); + hostToWorker.write(serializeMessage(createRequest(method, params, id))); + return result; + } + + function stop() { + worker.stop(); + hostReadline.close(); + hostToWorker.destroy(); + workerToHost.destroy(); + } + + return { callWorker, stop }; +} + +describe("environment sync verb negotiation", () => { + it("advertises environmentSyncIn/environmentSyncOut only when the hooks are defined", async () => { + const withHooks = startTestWorker( + definePlugin({ + async setup() {}, + async onEnvironmentSyncIn(): Promise { + return { operations: [] }; + }, + async onEnvironmentSyncOut(): Promise { + return { operations: [] }; + }, + }), + ); + try { + const result = await withHooks.callWorker<{ ok: boolean; supportedMethods: string[] }>( + "initialize", + { manifest: MANIFEST, config: {}, databaseNamespace: null }, + ); + expect(result.supportedMethods).toContain("environmentSyncIn"); + expect(result.supportedMethods).toContain("environmentSyncOut"); + } finally { + withHooks.stop(); + } + + const withoutHooks = startTestWorker(definePlugin({ async setup() {} })); + try { + const result = await withoutHooks.callWorker<{ ok: boolean; supportedMethods: string[] }>( + "initialize", + { manifest: MANIFEST, config: {}, databaseNamespace: null }, + ); + expect(result.supportedMethods).not.toContain("environmentSyncIn"); + expect(result.supportedMethods).not.toContain("environmentSyncOut"); + } finally { + withoutHooks.stop(); + } + }); + + it("routes environmentSyncIn/environmentSyncOut to the hooks when defined", async () => { + const seen: string[] = []; + const worker = startTestWorker( + definePlugin({ + async setup() {}, + async onEnvironmentSyncIn(params): Promise { + seen.push("in"); + return { + operations: params.operations.map((op) => ({ + operationId: op.operationId, + filesTransferred: op.files.length, + bytesTransferred: 0, + })), + }; + }, + async onEnvironmentSyncOut(params): Promise { + seen.push("out"); + return { + operations: params.operations.map((op) => ({ + operationId: op.operationId, + filesTransferred: op.files.length, + bytesTransferred: 0, + })), + }; + }, + }), + ); + try { + await worker.callWorker("initialize", { manifest: MANIFEST, config: {}, databaseNamespace: null }); + const baseParams = { + driverKey: "sandbox", + companyId: "company", + environmentId: "env", + config: {}, + lease: { providerLeaseId: "lease-1" }, + }; + const inParams: PluginEnvironmentSyncInParams = { + ...baseParams, + operations: [ + { operationId: "op-a", files: [{ sourcePath: "/host/a", targetPath: "/remote/a", kind: "file" }] }, + ], + }; + const inResult = await worker.callWorker("environmentSyncIn", inParams); + expect(inResult.operations[0]).toMatchObject({ operationId: "op-a", filesTransferred: 1 }); + + const outParams: PluginEnvironmentSyncOutParams = { + ...baseParams, + operations: [ + { operationId: "op-b", files: [{ sourcePath: "/remote/b", targetPath: "/host/b", kind: "directory" }] }, + ], + }; + const outResult = await worker.callWorker("environmentSyncOut", outParams); + expect(outResult.operations[0]).toMatchObject({ operationId: "op-b", filesTransferred: 1 }); + expect(seen).toEqual(["in", "out"]); + } finally { + worker.stop(); + } + }); + + it("throws METHOD_NOT_IMPLEMENTED when the sync hooks are absent", async () => { + const worker = startTestWorker(definePlugin({ async setup() {} })); + try { + await worker.callWorker("initialize", { manifest: MANIFEST, config: {}, databaseNamespace: null }); + const params = { + driverKey: "sandbox", + companyId: "company", + environmentId: "env", + config: {}, + lease: { providerLeaseId: "lease-1" }, + operations: [], + }; + await expect(worker.callWorker("environmentSyncIn", params)).rejects.toMatchObject({ + code: PLUGIN_RPC_ERROR_CODES.METHOD_NOT_IMPLEMENTED, + }); + await expect(worker.callWorker("environmentSyncOut", params)).rejects.toMatchObject({ + code: PLUGIN_RPC_ERROR_CODES.METHOD_NOT_IMPLEMENTED, + }); + } finally { + worker.stop(); + } + }); +}); diff --git a/server/src/services/environment-execution-target.ts b/server/src/services/environment-execution-target.ts index 079b3563b7..bbd86d8ce5 100644 --- a/server/src/services/environment-execution-target.ts +++ b/server/src/services/environment-execution-target.ts @@ -103,6 +103,28 @@ export async function resolveEnvironmentExecutionTarget(input: { startedAt, }; }, + // Expose the native file-sync capability only when the provider's + // worker advertises BOTH sync verbs; otherwise leave syncIn/syncOut + // undefined so the orchestrator keeps the byte-identical base64 path. + ...(input.environmentRuntime.supportsSync({ + environment: input.environment as Environment, + lease: input.lease, + }) + ? { + syncIn: (operations) => + input.environmentRuntime!.syncIn({ + environment: input.environment as Environment, + lease: input.lease!, + operations, + }), + syncOut: (operations) => + input.environmentRuntime!.syncOut({ + environment: input.environment as Environment, + lease: input.lease!, + operations, + }), + } + : {}), } : undefined, }; diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index 151b181a7c..765c354fc9 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -14,6 +14,8 @@ import type { PluginEnvironmentExecuteResult, PluginEnvironmentLease, PluginEnvironmentRealizeWorkspaceResult, + PluginEnvironmentSyncResult, + PluginSyncOperation, } from "@paperclipai/plugin-sdk"; import { ensureSshWorkspaceReady } from "@paperclipai/adapter-utils/ssh"; import { environmentService } from "./environments.js"; @@ -185,6 +187,10 @@ export interface EnvironmentDriverExecuteInput extends EnvironmentDriverLeaseInp timeoutMs?: number; } +export interface EnvironmentDriverSyncInput extends EnvironmentDriverLeaseInput { + operations: PluginSyncOperation[]; +} + export interface EnvironmentRuntimeDriver { readonly driver: string; acquireRunLease(input: EnvironmentDriverAcquireInput): Promise; @@ -193,6 +199,16 @@ export interface EnvironmentRuntimeDriver { destroyRunLease?(input: EnvironmentDriverLeaseInput): Promise; realizeWorkspace?(input: EnvironmentDriverRealizeWorkspaceInput): Promise; execute?(input: EnvironmentDriverExecuteInput): Promise; + /** + * Optional native inbound/outbound file transfer, delegated to the plugin + * worker's `environmentSyncIn`/`environmentSyncOut` verbs. Only present for + * plugin-backed sandbox drivers whose worker advertises both verbs; callers + * gate on {@link EnvironmentRuntimeDriver.supportsSync}. + */ + syncIn?(input: EnvironmentDriverSyncInput): Promise; + syncOut?(input: EnvironmentDriverSyncInput): Promise; + /** True when the lease's plugin worker advertises both sync verbs. */ + supportsSync?(input: EnvironmentDriverLeaseInput): boolean; } export interface EnvironmentRuntimeLeaseRecord { @@ -722,6 +738,39 @@ function createSandboxEnvironmentDriver( } } + async function callPluginEnvironmentSync( + method: "environmentSyncIn" | "environmentSyncOut", + input: EnvironmentDriverSyncInput, + ): Promise { + if (!input.lease.metadata?.sandboxProviderPlugin || !pluginWorkerManager) { + throw new Error("Sandbox driver does not support native file sync for this lease."); + } + const pluginId = readString(input.lease.metadata?.pluginId); + const providerKey = readString(input.lease.metadata?.provider); + if (!pluginId || !providerKey) { + throw new Error("Sandbox lease is missing plugin/provider metadata for native file sync."); + } + const config = await resolvePluginSandboxRuntimeConfig({ + environment: input.environment, + lease: input.lease, + provider: providerKey, + }); + const sanitizedConfig = stripSandboxProviderEnvelope(config as SandboxEnvironmentConfig); + return await pluginWorkerManager.call(pluginId, method, { + driverKey: providerKey, + companyId: input.lease.companyId, + environmentId: input.environment.id, + issueId: input.lease.issueId, + config: sanitizedConfig, + lease: { + providerLeaseId: input.lease.providerLeaseId, + metadata: input.lease.metadata ?? undefined, + expiresAt: input.lease.expiresAt?.toISOString() ?? null, + }, + operations: input.operations, + }, resolvePluginSandboxRpcTimeoutMs(sanitizedConfig)); + } + return { driver: "sandbox", @@ -1235,6 +1284,22 @@ function createSandboxEnvironmentDriver( throw new Error("Sandbox driver does not support direct command execution for built-in providers."); }, + supportsSync(input) { + if (!input.lease.metadata?.sandboxProviderPlugin || !pluginWorkerManager) return false; + const pluginId = readString(input.lease.metadata?.pluginId); + if (!pluginId) return false; + const advertised = pluginWorkerManager.getWorker(pluginId)?.supportedMethods ?? []; + return advertised.includes("environmentSyncIn") && advertised.includes("environmentSyncOut"); + }, + + async syncIn(input) { + return await callPluginEnvironmentSync("environmentSyncIn", input); + }, + + async syncOut(input) { + return await callPluginEnvironmentSync("environmentSyncOut", input); + }, + async destroyRunLease(input) { return await destroyReusableSandboxLease({ environment: input.environment, @@ -1890,6 +1955,27 @@ export function environmentRuntimeService( } return await driver.execute(input); }, + + supportsSync(input: EnvironmentDriverLeaseInput): boolean { + const driver = getDriver(getLeaseDriverKey(input.lease, input.environment)); + return driver?.supportsSync?.(input) ?? false; + }, + + async syncIn(input: EnvironmentDriverSyncInput): Promise { + const driver = requireDriverKey(getLeaseDriverKey(input.lease, input.environment)); + if (!driver.syncIn) { + throw new Error(`Environment driver "${driver.driver}" does not support native file sync.`); + } + return await driver.syncIn(input); + }, + + async syncOut(input: EnvironmentDriverSyncInput): Promise { + const driver = requireDriverKey(getLeaseDriverKey(input.lease, input.environment)); + if (!driver.syncOut) { + throw new Error(`Environment driver "${driver.driver}" does not support native file sync.`); + } + return await driver.syncOut(input); + }, }; }