From 7f766526a67c5564d8e029986ede23aed488eba8 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 24 Jul 2026 09:58:58 -0500 Subject: [PATCH] feat(sandbox): add task-scoped egress grants (#10155) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Confinement providers protect agent runs with default-deny network policies > - Kubernetes environments currently apply only provider-level, namespace-wide egress allowances > - Tasks that legitimately need GitHub or package registries therefore cannot request narrow access, while network failures do not explain the governing policy or how to request a grant > - This pull request adds issue-scoped egress grants that become workload-owned, run-label-selected policies and carries the effective grant through lease audit metadata > - The benefit is that internet-dependent work can run without enabling broad egress for every concurrent task, and denied requests point operators to the exact grant path ## Linked Issues or Issue Description No public issue exists. Related but distinct: Refs #9944, which adds a provider-wide open-internet posture; this PR keeps provider defaults narrow and adds per-task grants. **Problem / motivation** Kubernetes sandbox egress is configured at the provider/tenant level. A task that needs to clone from GitHub or install from PyPI cannot request those destinations without changing the policy for every run in the tenant namespace. DNS/connectivity failures also surface as generic tool errors with no policy name or remediation path. **Proposed solution** Accept `executionWorkspaceSettings.networkEgress.allowFqdns` and `allowCidrs`, forward the setting through heartbeat environment acquisition, and create a workload-owned NetworkPolicy or CiliumNetworkPolicy selected by `paperclip.io/run-id`. Record the effective grant in lease activity/metadata, expose policy context through `PAPERCLIP_NETWORK_EGRESS_*`, and append the grant path to likely policy-related stderr failures. **Alternatives considered** A provider-wide open-internet switch is broader than required and is already covered by #9944. Mutating the existing namespace policy would leak each task's destinations to other concurrent runs. Standard Kubernetes NetworkPolicy cannot enforce FQDNs exactly, so standard mode uses the existing hardened public-IPv4 TCP 80/443 fallback only for the selected run; Cilium mode remains exact. **Roadmap alignment** This extends the existing cloud/sandbox agent roadmap capability with task-level control-plane policy and does not duplicate a planned roadmap item. ## What Changed - Added validated `networkEgress` grants to issue execution workspace settings and forwarded them through environment lease acquisition. - Added workload-owned, run-label-scoped NetworkPolicy/CiliumNetworkPolicy resources for task FQDN/CIDR grants. - Added lease audit metadata, sandbox policy environment variables, and actionable network-denial stderr guidance. - Added focused parser, manifest, policy creation, and denial-message tests plus Kubernetes provider documentation. ## Verification - `pnpm -C packages/shared exec vitest run src/validators/issue.test.ts` — 27 passed. - `pnpm -C packages/plugins/sandbox-providers/kubernetes test -- --run test/unit/network-policy.test.ts test/unit/cilium-network-policy.test.ts test/unit/scoped-network-egress.test.ts` — 21 passed. - `pnpm -C server exec vitest run src/__tests__/execution-workspace-policy.test.ts` — 15 passed. - `pnpm exec vitest run server/src/__tests__/heartbeat-plugin-environment.test.ts server/src/__tests__/environment-runtime.test.ts` — 26 passed. - `pnpm --dir packages/db build && pnpm --dir packages/shared build && pnpm --dir packages/plugins/sdk build` — passed, including migration safety checks. - `pnpm --dir packages/plugins/sandbox-providers/kubernetes typecheck && pnpm --dir server typecheck` — passed after refreshing the worktree's frozen offline dependencies. - End-to-end cluster validation of the `build-cython-ext` benchmark remains for CI/maintainer Kubernetes infrastructure; the focused tests assert `github.com` and `pypi.org` produce a policy selected only by the granted run. ## Risks - Standard NetworkPolicy cannot express FQDNs, so an FQDN grant allows hardened public IPv4 TCP 80/443 for that run; use Cilium mode for exact hostname enforcement. - The new field is additive and absent by default, so existing runs keep the current provider-level policy. - Workload owner references garbage-collect scoped policies with the Job/Sandbox; a cluster/controller that ignores owner references could temporarily strand a policy that still selects no future run ID. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used OpenAI Codex, exact model ID `gpt-5.6-sol`, high reasoning mode, tool use and code execution. The runtime did not expose a context-window size. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- .../sandbox-providers/kubernetes/README.md | 17 +++ .../kubernetes/src/cilium-network-policy.ts | 13 ++- .../kubernetes/src/network-policy.ts | 20 +++- .../kubernetes/src/plugin.ts | 53 ++++++++- .../kubernetes/src/scoped-network-egress.ts | 106 ++++++++++++++++++ .../sandbox-providers/kubernetes/src/types.ts | 5 + .../test/unit/cilium-network-policy.test.ts | 16 +++ .../test/unit/network-policy.test.ts | 17 +++ .../test/unit/scoped-network-egress.test.ts | 86 ++++++++++++++ packages/plugins/sdk/src/protocol.ts | 1 + .../shared/src/types/workspace-runtime.ts | 4 + packages/shared/src/validators/issue.test.ts | 28 +++++ packages/shared/src/validators/issue.ts | 44 ++++++++ .../src/__tests__/environment-runtime.test.ts | 4 + .../execution-workspace-policy.test.ts | 33 ++++++ .../heartbeat-plugin-environment.test.ts | 2 + .../services/environment-run-orchestrator.ts | 5 + server/src/services/environment-runtime.ts | 8 +- .../services/execution-workspace-policy.ts | 25 +++++ server/src/services/heartbeat.ts | 11 +- 20 files changed, 482 insertions(+), 16 deletions(-) create mode 100644 packages/plugins/sandbox-providers/kubernetes/src/scoped-network-egress.ts create mode 100644 packages/plugins/sandbox-providers/kubernetes/test/unit/scoped-network-egress.test.ts diff --git a/packages/plugins/sandbox-providers/kubernetes/README.md b/packages/plugins/sandbox-providers/kubernetes/README.md index 6f37a525b6..a51c6ecee6 100644 --- a/packages/plugins/sandbox-providers/kubernetes/README.md +++ b/packages/plugins/sandbox-providers/kubernetes/README.md @@ -77,6 +77,23 @@ Common optional fields: Full JSON Schema in `src/manifest.ts`. +### Task-scoped egress grants + +Keep provider-level egress defaults narrow, then grant only the destinations a task needs through its execution workspace settings: + +```json +{ + "executionWorkspaceSettings": { + "networkEgress": { + "allowFqdns": ["github.com", "pypi.org"], + "allowCidrs": [] + } + } +} +``` + +The provider creates a workload-owned policy selected by the task run label, so the additional destinations do not become reachable from other concurrent agent pods. Cilium mode enforces FQDNs directly. Standard NetworkPolicy mode cannot express FQDNs, so an FQDN grant permits public IPv4 TCP 80/443 for that run while excluding private, loopback, link-local, CGNAT, and multicast ranges. Network failures that look policy-related include the grant path in stderr, and the sandbox exposes the effective policy through `PAPERCLIP_NETWORK_EGRESS_*` environment variables. + ## What gets created in your cluster For each company that runs agents (created lazily on first dispatch): diff --git a/packages/plugins/sandbox-providers/kubernetes/src/cilium-network-policy.ts b/packages/plugins/sandbox-providers/kubernetes/src/cilium-network-policy.ts index 5dedcf73e9..68273834ff 100644 --- a/packages/plugins/sandbox-providers/kubernetes/src/cilium-network-policy.ts +++ b/packages/plugins/sandbox-providers/kubernetes/src/cilium-network-policy.ts @@ -3,6 +3,10 @@ export interface BuildCiliumNetworkPolicyInput { paperclipServerNamespace: string; egressAllowFqdns: string[]; egressAllowCidrs: string[]; + name?: string; + endpointSelector?: Record; + includeBaseRules?: boolean; + ownerReferences?: Record[]; } // Design note: no ingress rules are defined here. Paperclip-server does NOT @@ -12,7 +16,7 @@ export interface BuildCiliumNetworkPolicyInput { export function buildCiliumNetworkPolicyManifest(input: BuildCiliumNetworkPolicyInput): Record { const egress: Record[] = []; - egress.push({ + if (input.includeBaseRules !== false) egress.push({ toEndpoints: [ { matchLabels: { "k8s:io.kubernetes.pod.namespace": "kube-system", "k8s-app": "kube-dns" } }, ], @@ -34,7 +38,7 @@ export function buildCiliumNetworkPolicyManifest(input: BuildCiliumNetworkPolicy }); } - egress.push({ + if (input.includeBaseRules !== false) egress.push({ toEndpoints: [ { matchLabels: { @@ -56,12 +60,13 @@ export function buildCiliumNetworkPolicyManifest(input: BuildCiliumNetworkPolicy apiVersion: "cilium.io/v2", kind: "CiliumNetworkPolicy", metadata: { - name: "paperclip-egress-fqdn", + name: input.name ?? "paperclip-egress-fqdn", namespace: input.namespace, labels: { "paperclip.io/managed-by": "paperclip-k8s-plugin" }, + ...(input.ownerReferences ? { ownerReferences: input.ownerReferences } : {}), }, spec: { - endpointSelector: { matchLabels: { "paperclip.io/role": "agent" } }, + endpointSelector: { matchLabels: input.endpointSelector ?? { "paperclip.io/role": "agent" } }, egress, }, }; diff --git a/packages/plugins/sandbox-providers/kubernetes/src/network-policy.ts b/packages/plugins/sandbox-providers/kubernetes/src/network-policy.ts index 4878a3b73d..18d6c5759d 100644 --- a/packages/plugins/sandbox-providers/kubernetes/src/network-policy.ts +++ b/packages/plugins/sandbox-providers/kubernetes/src/network-policy.ts @@ -13,6 +13,10 @@ export interface BuildNetworkPolicyInput { * "cilium"` for exact FQDN allow-listing in production. */ egressAllowFqdns?: string[]; + name?: string; + podSelector?: Record; + includeBaseRules?: boolean; + ownerReferences?: Record[]; } /** @@ -59,15 +63,14 @@ export function buildNetworkPolicyManifests(input: BuildNetworkPolicyInput): Rec apiVersion: "networking.k8s.io/v1", kind: "NetworkPolicy", metadata: { - name: "paperclip-egress-allow", namespace: input.namespace, labels: { "paperclip.io/managed-by": "paperclip-k8s-plugin" }, }, spec: { - podSelector: { matchLabels: { "paperclip.io/role": "agent" } }, + podSelector: { matchLabels: input.podSelector ?? { "paperclip.io/role": "agent" } }, policyTypes: ["Egress"], egress: [ - { + ...(input.includeBaseRules === false ? [] : [{ to: [ { namespaceSelector: { matchLabels: { "kubernetes.io/metadata.name": "kube-system" } }, @@ -78,8 +81,8 @@ export function buildNetworkPolicyManifests(input: BuildNetworkPolicyInput): Rec { protocol: "UDP", port: 53 }, { protocol: "TCP", port: 53 }, ], - }, - { + }]), + ...(input.includeBaseRules === false ? [] : [{ to: [ { namespaceSelector: { matchLabels: { "kubernetes.io/metadata.name": input.paperclipServerNamespace } }, @@ -87,7 +90,7 @@ export function buildNetworkPolicyManifests(input: BuildNetworkPolicyInput): Rec }, ], ports: [{ protocol: "TCP", port: 3100 }], - }, + }]), // NOTE: operator-supplied CIDRs are intentionally NOT port-scoped — // operators may need them for non-HTTP services (e.g. private VCS // mirrors, S3 endpoints, internal artifact registries). Operators @@ -128,5 +131,10 @@ export function buildNetworkPolicyManifests(input: BuildNetworkPolicyInput): Rec }, }; + (egressAllow.metadata as Record).name = input.name ?? "paperclip-egress-allow"; + if (input.ownerReferences) { + (egressAllow.metadata as Record).ownerReferences = input.ownerReferences; + } + return [denyAll, egressAllow]; } diff --git a/packages/plugins/sandbox-providers/kubernetes/src/plugin.ts b/packages/plugins/sandbox-providers/kubernetes/src/plugin.ts index 133c6e7023..8ae47de8ec 100644 --- a/packages/plugins/sandbox-providers/kubernetes/src/plugin.ts +++ b/packages/plugins/sandbox-providers/kubernetes/src/plugin.ts @@ -39,6 +39,12 @@ import { import { execInPod, execInPodStreaming, wrapCommandWithEnv } from "./pod-exec.js"; import { performSyncIn, performSyncOut, type PodStreamExec } from "./file-sync.js"; import { checkLeaseResumable, destroyLeaseResources } from "./lease-lifecycle.js"; +import { + appendNetworkEgressDenyHint, + createScopedNetworkEgressPolicyOrReleaseWorkload, + NETWORK_EGRESS_GRANT_PATH, + parseScopedNetworkEgressGrant, +} from "./scoped-network-egress.js"; import { deriveCompanySlug, deriveNamespaceName, @@ -288,7 +294,10 @@ const plugin = definePlugin({ // SDK lease params grow that field (companion server-integration PR). The // plugin works without it: absent means "use the environment's configured // default adapter", so it stays compatible with the current SDK. - params: PluginEnvironmentAcquireLeaseParams & { adapterType?: string }, + params: PluginEnvironmentAcquireLeaseParams & { + adapterType?: string; + executionWorkspaceSettings?: Record | null; + }, ): Promise { const config = kubernetesProviderConfigSchema.parse(params.config); const namespace = deriveTenantNamespace(config, params.companyId); @@ -389,10 +398,34 @@ const plugin = definePlugin({ }); const { uid: ownerUid } = await orchestrator.claim(clients, namespace, manifest); + const scopedNetworkEgress = parseScopedNetworkEgressGrant(params.executionWorkspaceSettings); + const scopedNetworkPolicyName = await createScopedNetworkEgressPolicyOrReleaseWorkload( + { + clients, + namespace, + mode: config.egressMode, + runId: params.runId, + workloadName: jobName, + ownerReference: { + apiVersion: isSandboxCrBackend ? "agents.x-k8s.io/v1alpha1" : "batch/v1", + kind: isSandboxCrBackend ? "Sandbox" : "Job", + name: jobName, + uid: ownerUid, + controller: false, + blockOwnerDeletion: false, + }, + grant: scopedNetworkEgress, + }, + () => orchestrator.release(clients, namespace, jobName), + ); // defaultEnv (non-secret base, e.g. the inference base URL) is layered first; // the process-env secrets named by envKeys override it. const adapterEnv = buildAdapterEnv(adapterDefaults); + adapterEnv.PAPERCLIP_NETWORK_EGRESS_POLICY = "kubernetes-default-deny"; + adapterEnv.PAPERCLIP_NETWORK_EGRESS_GRANT_PATH = NETWORK_EGRESS_GRANT_PATH; + adapterEnv.PAPERCLIP_NETWORK_EGRESS_ALLOW_FQDNS = scopedNetworkEgress.allowFqdns.join(","); + adapterEnv.PAPERCLIP_NETWORK_EGRESS_ALLOW_CIDRS = scopedNetworkEgress.allowCidrs.join(","); const bootstrapToken = generateBootstrapToken(); // Secret ownerRef: for job backend, the Job owns the Secret (cascade delete). @@ -421,6 +454,8 @@ const plugin = definePlugin({ secretName, phase: "Pending", backend: config.backend, + scopedNetworkPolicyName, + scopedNetworkEgress, // Native file sync streams over a pod exec; only the sandbox-cr backend // exposes one. Flag the job backend so the server keeps the base64 fallback // rather than routing its sync to a hook that would reject immediately. @@ -494,6 +529,13 @@ const plugin = definePlugin({ secretName, phase: check.phase, backend: leaseBackend, + scopedNetworkPolicyName: + typeof params.leaseMetadata?.scopedNetworkPolicyName === "string" + ? params.leaseMetadata.scopedNetworkPolicyName + : null, + scopedNetworkEgress: parseScopedNetworkEgressGrant({ + networkEgress: params.leaseMetadata?.scopedNetworkEgress, + }), // See acquireLease: only the sandbox-cr backend has a pod-exec channel for // native sync, so a resumed job lease must keep the base64 fallback. nativeFileSyncUnsupported: leaseBackend !== "sandbox-cr", @@ -626,6 +668,9 @@ const plugin = definePlugin({ } const config = kubernetesProviderConfigSchema.parse(params.config); + const scopedNetworkEgress = parseScopedNetworkEgressGrant({ + networkEgress: lease.metadata?.scopedNetworkEgress, + }); const namespace = typeof lease.metadata?.namespace === "string" ? lease.metadata.namespace @@ -861,7 +906,7 @@ const plugin = definePlugin({ exitCode: null, timedOut: true, stdout: "", - stderr: err instanceof Error ? err.message : String(err), + stderr: appendNetworkEgressDenyHint(err instanceof Error ? err.message : String(err), scopedNetworkEgress), metadata: { provider: "kubernetes", backend: "sandbox-cr", @@ -876,7 +921,7 @@ const plugin = definePlugin({ exitCode: execResult.exitCode, timedOut: false, stdout: execResult.stdout, - stderr: execResult.stderr, + stderr: appendNetworkEgressDenyHint(execResult.stderr, scopedNetworkEgress), metadata: { provider: "kubernetes", backend: "sandbox-cr", @@ -940,7 +985,7 @@ const plugin = definePlugin({ exitCode: timedOut ? null : status?.phase === "Succeeded" ? 0 : 1, timedOut, stdout: stdoutChunks.join(""), - stderr: stderrChunks.join(""), + stderr: appendNetworkEgressDenyHint(stderrChunks.join(""), scopedNetworkEgress), metadata: { provider: "kubernetes", backend: "job", diff --git a/packages/plugins/sandbox-providers/kubernetes/src/scoped-network-egress.ts b/packages/plugins/sandbox-providers/kubernetes/src/scoped-network-egress.ts new file mode 100644 index 0000000000..d0b5c2fc30 --- /dev/null +++ b/packages/plugins/sandbox-providers/kubernetes/src/scoped-network-egress.ts @@ -0,0 +1,106 @@ +import type { KubeClients } from "./kube-client.js"; +import { buildNetworkPolicyManifests } from "./network-policy.js"; +import { buildCiliumNetworkPolicyManifest } from "./cilium-network-policy.js"; + +export const NETWORK_EGRESS_GRANT_PATH = "executionWorkspaceSettings.networkEgress"; + +export interface ScopedNetworkEgressGrant { + allowFqdns: string[]; + allowCidrs: string[]; +} + +export function parseScopedNetworkEgressGrant(settings: unknown): ScopedNetworkEgressGrant { + if (!settings || typeof settings !== "object" || Array.isArray(settings)) { + return { allowFqdns: [], allowCidrs: [] }; + } + const networkEgress = (settings as Record).networkEgress; + if (!networkEgress || typeof networkEgress !== "object" || Array.isArray(networkEgress)) { + return { allowFqdns: [], allowCidrs: [] }; + } + const record = networkEgress as Record; + const strings = (value: unknown) => Array.isArray(value) + ? [...new Set(value.filter((item): item is string => typeof item === "string").map((item) => item.trim()).filter(Boolean))] + : []; + return { + allowFqdns: strings(record.allowFqdns).map((fqdn) => fqdn.toLowerCase()), + allowCidrs: strings(record.allowCidrs), + }; +} + +export async function createScopedNetworkEgressPolicy(input: { + clients: KubeClients; + namespace: string; + mode: "standard" | "cilium"; + runId: string; + workloadName: string; + ownerReference: Record; + grant: ScopedNetworkEgressGrant; +}): Promise { + if (input.grant.allowFqdns.length === 0 && input.grant.allowCidrs.length === 0) return null; + const suffix = "-egress"; + const maxWorkloadLength = 253 - suffix.length; + const workloadName = input.workloadName.length <= maxWorkloadLength + ? input.workloadName + : `${input.workloadName.slice(0, maxWorkloadLength - 26)}-${input.workloadName.slice(-25)}`; + const name = `${workloadName}${suffix}`; + if (input.mode === "cilium") { + const manifest = buildCiliumNetworkPolicyManifest({ + namespace: input.namespace, + paperclipServerNamespace: "", + egressAllowFqdns: input.grant.allowFqdns, + egressAllowCidrs: input.grant.allowCidrs, + name, + endpointSelector: { "paperclip.io/run-id": input.runId }, + includeBaseRules: false, + ownerReferences: [input.ownerReference], + }); + await input.clients.custom.createNamespacedCustomObject({ + group: "cilium.io", + version: "v2", + namespace: input.namespace, + plural: "ciliumnetworkpolicies", + body: manifest, + }); + } else { + const [, manifest] = buildNetworkPolicyManifests({ + namespace: input.namespace, + paperclipServerNamespace: "", + egressAllowFqdns: input.grant.allowFqdns, + egressAllowCidrs: input.grant.allowCidrs, + name, + podSelector: { "paperclip.io/run-id": input.runId }, + includeBaseRules: false, + ownerReferences: [input.ownerReference], + }); + await input.clients.networking.createNamespacedNetworkPolicy({ namespace: input.namespace, body: manifest as never }); + } + return name; +} + +export async function createScopedNetworkEgressPolicyOrReleaseWorkload( + input: Parameters[0], + releaseWorkload: () => Promise, +): Promise { + try { + return await createScopedNetworkEgressPolicy(input); + } catch (policyError) { + try { + await releaseWorkload(); + } catch (releaseError) { + throw new AggregateError( + [policyError, releaseError], + "Failed to create scoped network egress policy and release its workload", + ); + } + throw policyError; + } +} + +export function appendNetworkEgressDenyHint(stderr: string, grant: ScopedNetworkEgressGrant): string { + if (!/(could not resolve host|network is unreachable|connection timed out|failed to connect|temporary failure in name resolution)/i.test(stderr)) { + return stderr; + } + const allowed = [...grant.allowFqdns, ...grant.allowCidrs]; + const detail = allowed.length > 0 ? ` Current task grant: ${allowed.join(", ")}.` : " No task-scoped destinations are granted."; + return `${stderr.trimEnd()}\nPaperclip network policy denied or could not route this request.${detail} Request access through ${NETWORK_EGRESS_GRANT_PATH}.\n`; +} diff --git a/packages/plugins/sandbox-providers/kubernetes/src/types.ts b/packages/plugins/sandbox-providers/kubernetes/src/types.ts index 1a6de44fee..5626daa938 100644 --- a/packages/plugins/sandbox-providers/kubernetes/src/types.ts +++ b/packages/plugins/sandbox-providers/kubernetes/src/types.ts @@ -90,6 +90,11 @@ export interface KubernetesLeaseMetadata { phase: "Pending" | "Running" | "Succeeded" | "Failed"; /** Which backend provisioned this lease. */ backend: "sandbox-cr" | "job"; + scopedNetworkPolicyName: string | null; + scopedNetworkEgress: { + allowFqdns: string[]; + allowCidrs: string[]; + }; /** * True when this lease's backend has NO data channel for the native file-sync * transport. Native sync streams over a pod exec, which only the `sandbox-cr` diff --git a/packages/plugins/sandbox-providers/kubernetes/test/unit/cilium-network-policy.test.ts b/packages/plugins/sandbox-providers/kubernetes/test/unit/cilium-network-policy.test.ts index 0e6503638a..419f4b1b33 100644 --- a/packages/plugins/sandbox-providers/kubernetes/test/unit/cilium-network-policy.test.ts +++ b/packages/plugins/sandbox-providers/kubernetes/test/unit/cilium-network-policy.test.ts @@ -57,4 +57,20 @@ describe("buildCiliumNetworkPolicyManifest", () => { const cidrRule = cnp.spec.egress.find((e: { toCIDRSet?: { cidr: string }[] }) => e.toCIDRSet); expect(cidrRule.toCIDRSet[0].cidr).toBe("10.0.0.0/8"); }); + + it("targets only the granted run when building a scoped policy", () => { + const cnp = buildCiliumNetworkPolicyManifest({ + ...baseInput, + name: "pc-run-egress", + endpointSelector: { "paperclip.io/run-id": "run-123" }, + includeBaseRules: false, + ownerReferences: [{ apiVersion: "batch/v1", kind: "Job", name: "pc-run", uid: "uid-1" }], + egressAllowFqdns: ["github.com", "pypi.org"], + }); + + expect(cnp.metadata.name).toBe("pc-run-egress"); + expect(cnp.metadata.ownerReferences).toHaveLength(1); + expect(cnp.spec.endpointSelector.matchLabels).toEqual({ "paperclip.io/run-id": "run-123" }); + expect(cnp.spec.egress).toHaveLength(1); + }); }); diff --git a/packages/plugins/sandbox-providers/kubernetes/test/unit/network-policy.test.ts b/packages/plugins/sandbox-providers/kubernetes/test/unit/network-policy.test.ts index 72df869e43..80338e7012 100644 --- a/packages/plugins/sandbox-providers/kubernetes/test/unit/network-policy.test.ts +++ b/packages/plugins/sandbox-providers/kubernetes/test/unit/network-policy.test.ts @@ -92,4 +92,21 @@ describe("buildNetworkPolicyManifests", () => { ); expect(fallback).toBeUndefined(); }); + + it("builds a task-scoped allow policy without namespace-wide base rules", () => { + const [, egress] = buildNetworkPolicyManifests({ + ...baseInput, + name: "pc-run-egress", + podSelector: { "paperclip.io/run-id": "run-123" }, + includeBaseRules: false, + egressAllowFqdns: ["github.com", "pypi.org"], + ownerReferences: [{ apiVersion: "batch/v1", kind: "Job", name: "pc-run", uid: "uid-1" }], + }); + + expect(egress.metadata.name).toBe("pc-run-egress"); + expect(egress.metadata.ownerReferences).toHaveLength(1); + expect(egress.spec.podSelector.matchLabels).toEqual({ "paperclip.io/run-id": "run-123" }); + expect(egress.spec.egress).toHaveLength(1); + expect(egress.spec.egress[0].to[0].ipBlock.cidr).toBe("0.0.0.0/0"); + }); }); diff --git a/packages/plugins/sandbox-providers/kubernetes/test/unit/scoped-network-egress.test.ts b/packages/plugins/sandbox-providers/kubernetes/test/unit/scoped-network-egress.test.ts new file mode 100644 index 0000000000..231c8875d4 --- /dev/null +++ b/packages/plugins/sandbox-providers/kubernetes/test/unit/scoped-network-egress.test.ts @@ -0,0 +1,86 @@ +import { describe, expect, it, vi } from "vitest"; +import { + appendNetworkEgressDenyHint, + createScopedNetworkEgressPolicy, + createScopedNetworkEgressPolicyOrReleaseWorkload, + parseScopedNetworkEgressGrant, +} from "../../src/scoped-network-egress.js"; + +describe("scoped network egress", () => { + it("normalizes task grants", () => { + expect(parseScopedNetworkEgressGrant({ + networkEgress: { + allowFqdns: ["GitHub.com", "pypi.org"], + allowCidrs: ["203.0.113.0/24"], + }, + })).toEqual({ + allowFqdns: ["github.com", "pypi.org"], + allowCidrs: ["203.0.113.0/24"], + }); + }); + + it("creates a standard policy scoped to the run label", async () => { + const createNamespacedNetworkPolicy = vi.fn().mockResolvedValue({}); + await createScopedNetworkEgressPolicy({ + clients: { networking: { createNamespacedNetworkPolicy } } as never, + namespace: "paperclip-acme", + mode: "standard", + runId: "run-123", + workloadName: "pc-workload", + ownerReference: { apiVersion: "batch/v1", kind: "Job", name: "pc-workload", uid: "uid-1" }, + grant: { allowFqdns: ["github.com", "pypi.org"], allowCidrs: [] }, + }); + expect(createNamespacedNetworkPolicy).toHaveBeenCalledWith(expect.objectContaining({ + namespace: "paperclip-acme", + body: expect.objectContaining({ + metadata: expect.objectContaining({ name: "pc-workload-egress" }), + spec: expect.objectContaining({ podSelector: { matchLabels: { "paperclip.io/run-id": "run-123" } } }), + }), + })); + }); + + it("caps scoped policy names while preserving the workload tail", async () => { + const createNamespacedNetworkPolicy = vi.fn().mockResolvedValue({}); + const workloadName = `pc-${"a".repeat(260)}-unique-tail`; + + const name = await createScopedNetworkEgressPolicy({ + clients: { networking: { createNamespacedNetworkPolicy } } as never, + namespace: "paperclip-acme", + mode: "standard", + runId: "run-123", + workloadName, + ownerReference: { apiVersion: "batch/v1", kind: "Job", name: workloadName, uid: "uid-1" }, + grant: { allowFqdns: ["github.com"], allowCidrs: [] }, + }); + + expect(name).toHaveLength(253); + expect(name).toMatch(/unique-tail-egress$/); + }); + + it("adds the policy and grant path to likely network denials", () => { + expect(appendNetworkEgressDenyHint("curl: Could not resolve host: example.com", { + allowFqdns: ["github.com"], + allowCidrs: [], + })).toContain("executionWorkspaceSettings.networkEgress"); + }); + + it("releases the workload when scoped policy creation fails", async () => { + const policyError = new Error("policy denied"); + const releaseWorkload = vi.fn().mockResolvedValue(undefined); + + await expect(createScopedNetworkEgressPolicyOrReleaseWorkload({ + clients: { + networking: { + createNamespacedNetworkPolicy: vi.fn().mockRejectedValue(policyError), + }, + } as never, + namespace: "paperclip-acme", + mode: "standard", + runId: "run-123", + workloadName: "pc-workload", + ownerReference: { apiVersion: "batch/v1", kind: "Job", name: "pc-workload", uid: "uid-1" }, + grant: { allowFqdns: ["github.com"], allowCidrs: [] }, + }, releaseWorkload)).rejects.toBe(policyError); + expect(releaseWorkload).toHaveBeenCalledOnce(); + }); +}); diff --git a/packages/plugins/sdk/src/protocol.ts b/packages/plugins/sdk/src/protocol.ts index ad11522d93..631ccd0a01 100644 --- a/packages/plugins/sdk/src/protocol.ts +++ b/packages/plugins/sdk/src/protocol.ts @@ -613,6 +613,7 @@ export interface PluginEnvironmentAcquireLeaseParams extends PluginEnvironmentDr * per-run sandbox should use this to select the runtime image and per-run env. */ adapterType?: string; + executionWorkspaceSettings?: Record | null; } export interface PluginEnvironmentResumeLeaseParams extends PluginEnvironmentDriverBaseParams { diff --git a/packages/shared/src/types/workspace-runtime.ts b/packages/shared/src/types/workspace-runtime.ts index 4e7fdcc2d2..c53be28db1 100644 --- a/packages/shared/src/types/workspace-runtime.ts +++ b/packages/shared/src/types/workspace-runtime.ts @@ -165,6 +165,10 @@ export interface IssueExecutionWorkspaceSettings { environmentId?: string | null; workspaceStrategy?: ExecutionWorkspaceStrategy | null; workspaceRuntime?: Record | null; + networkEgress?: { + allowFqdns?: string[]; + allowCidrs?: string[]; + } | null; } export interface ExecutionWorkspaceSummary { diff --git a/packages/shared/src/validators/issue.test.ts b/packages/shared/src/validators/issue.test.ts index 264152c62f..5db3925761 100644 --- a/packages/shared/src/validators/issue.test.ts +++ b/packages/shared/src/validators/issue.test.ts @@ -71,6 +71,34 @@ describe("issue validators", () => { }).success).toBe(false); }); + it("rejects invalid task-scoped network egress CIDRs", () => { + expect(updateIssueSchema.safeParse({ + executionWorkspaceSettings: { + networkEgress: { allowCidrs: ["203.0.113.0/24"] }, + }, + }).success).toBe(true); + expect(updateIssueSchema.safeParse({ + executionWorkspaceSettings: { + networkEgress: { allowCidrs: ["999.0.0.0/8"] }, + }, + }).success).toBe(false); + expect(updateIssueSchema.safeParse({ + executionWorkspaceSettings: { + networkEgress: { allowCidrs: ["1.2.3.4/33"] }, + }, + }).success).toBe(false); + expect(updateIssueSchema.safeParse({ + executionWorkspaceSettings: { + networkEgress: { allowCidrs: ["10.0.0.0/8"] }, + }, + }).success).toBe(false); + expect(updateIssueSchema.safeParse({ + executionWorkspaceSettings: { + networkEgress: { allowCidrs: ["0.0.0.0/0"] }, + }, + }).success).toBe(false); + }); + it("keeps issue attribution fields create-only", () => { const created = createIssueSchema.parse({ title: "Preserve attribution input for route checks", diff --git a/packages/shared/src/validators/issue.ts b/packages/shared/src/validators/issue.ts index e25c34bc54..af6b7efcff 100644 --- a/packages/shared/src/validators/issue.ts +++ b/packages/shared/src/validators/issue.ts @@ -116,12 +116,56 @@ const executionWorkspaceStrategySchema = z }) .strict(); +const ipv4CidrPattern = /^(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\/(?:3[0-2]|[12]?\d)$/; +const protectedTaskEgressCidrs = [ + "0.0.0.0/8", + "10.0.0.0/8", + "100.64.0.0/10", + "127.0.0.0/8", + "169.254.0.0/16", + "172.16.0.0/12", + "192.168.0.0/16", + "224.0.0.0/4", +] as const; + +function ipv4CidrRange(cidr: string): [number, number] | null { + if (!ipv4CidrPattern.test(cidr)) return null; + const [address, prefixText] = cidr.split("/"); + const addressValue = address.split(".").reduce((value, octet) => value * 256 + Number(octet), 0); + const prefix = Number(prefixText); + const blockSize = 2 ** (32 - prefix); + const start = Math.floor(addressValue / blockSize) * blockSize; + return [start, start + blockSize - 1]; +} + +function isAllowedTaskEgressCidr(cidr: string): boolean { + const range = ipv4CidrRange(cidr); + if (!range) return false; + return protectedTaskEgressCidrs.every((protectedCidr) => { + const protectedRange = ipv4CidrRange(protectedCidr); + return protectedRange !== null && (range[1] < protectedRange[0] || range[0] > protectedRange[1]); + }); +} + export const issueExecutionWorkspaceSettingsSchema = z .object({ mode: z.enum(ISSUE_EXECUTION_WORKSPACE_PREFERENCES).optional(), environmentId: z.string().uuid().optional().nullable(), workspaceStrategy: executionWorkspaceStrategySchema.optional().nullable(), workspaceRuntime: z.record(z.string(), z.unknown()).optional().nullable(), + networkEgress: z.object({ + allowFqdns: z.array(z.string().trim().toLowerCase().regex( + /^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)*[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/, + "Network egress FQDNs must be hostnames without a URL scheme or path", + ).max(253)).max(100).optional(), + allowCidrs: z.array(z.string().trim().regex( + ipv4CidrPattern, + "Invalid IPv4 CIDR (must use octets 0-255 and prefix 0-32)", + ).max(64).refine( + isAllowedTaskEgressCidr, + "Task-scoped network egress CIDRs cannot overlap private, loopback, link-local, CGNAT, or multicast ranges", + )).max(100).optional(), + }).strict().optional().nullable(), }) .strict(); diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index ee7ce887f0..d1557f65f2 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -2106,8 +2106,12 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { driverKey: "fake-plugin", companyId, environmentId: environment.id, + executionWorkspaceId: undefined, + executionWorkspaceSettings: null, issueId: null, config: { template: "base" }, + agentId: undefined, + adapterType: undefined, runId, workspaceMode: undefined, }); diff --git a/server/src/__tests__/execution-workspace-policy.test.ts b/server/src/__tests__/execution-workspace-policy.test.ts index c4c37e65be..e487181f90 100644 --- a/server/src/__tests__/execution-workspace-policy.test.ts +++ b/server/src/__tests__/execution-workspace-policy.test.ts @@ -10,6 +10,7 @@ import { resolveExecutionWorkspaceEnvironmentId, resolvePinnedIssueWorkspaceStrategyType, resolveExecutionWorkspaceMode, + selectEnvironmentExecutionWorkspaceSettings, } from "../services/execution-workspace-policy.ts"; describe("execution workspace policy helpers", () => { @@ -291,6 +292,38 @@ describe("execution workspace policy helpers", () => { mode: "shared_workspace", environmentId: "11111111-1111-4111-8111-111111111111", }); + expect( + parseIssueExecutionWorkspaceSettings({ + mode: "isolated_workspace", + networkEgress: { + allowFqdns: ["github.com", "pypi.org"], + allowCidrs: ["203.0.113.0/24"], + }, + }), + ).toEqual({ + mode: "isolated_workspace", + networkEgress: { + allowFqdns: ["github.com", "pypi.org"], + allowCidrs: ["203.0.113.0/24"], + }, + }); + }); + + it("keeps egress grants independent from isolated workspace mode", () => { + const parsedSettings = { + mode: "isolated_workspace" as const, + workspaceRuntime: { image: "example/image" }, + networkEgress: { + allowFqdns: ["github.com"], + allowCidrs: ["203.0.113.0/24"], + }, + }; + + expect(selectEnvironmentExecutionWorkspaceSettings(parsedSettings, false)).toEqual({ + networkEgress: parsedSettings.networkEgress, + }); + expect(selectEnvironmentExecutionWorkspaceSettings(parsedSettings, true)).toEqual(parsedSettings); + expect(selectEnvironmentExecutionWorkspaceSettings({ mode: "isolated_workspace" }, false)).toBeNull(); }); it("prefers the agent default environment", () => { diff --git a/server/src/__tests__/heartbeat-plugin-environment.test.ts b/server/src/__tests__/heartbeat-plugin-environment.test.ts index 13f3d76c0b..4750b686fb 100644 --- a/server/src/__tests__/heartbeat-plugin-environment.test.ts +++ b/server/src/__tests__/heartbeat-plugin-environment.test.ts @@ -213,6 +213,7 @@ describeEmbeddedPostgres("heartbeat plugin environments", () => { companyId, environmentId, executionWorkspaceId: expect.any(String), + executionWorkspaceSettings: null, issueId: null, config: { template: "base" }, agentId, @@ -674,6 +675,7 @@ describeEmbeddedPostgres("heartbeat plugin environments", () => { companyId, environmentId: newEnvironmentId, executionWorkspaceId: expect.any(String), + executionWorkspaceSettings: { mode: "shared_workspace" }, issueId, config: { template: "new" }, agentId, diff --git a/server/src/services/environment-run-orchestrator.ts b/server/src/services/environment-run-orchestrator.ts index e69d9ef669..c1b3656974 100644 --- a/server/src/services/environment-run-orchestrator.ts +++ b/server/src/services/environment-run-orchestrator.ts @@ -23,6 +23,7 @@ import type { EnvironmentLeaseStatus, ExecutionWorkspace, ExecutionWorkspaceConfig, + IssueExecutionWorkspaceSettings, } from "@paperclipai/shared"; import { environmentService } from "./environments.js"; import { @@ -202,6 +203,7 @@ export function environmentRunOrchestrator( agentId: string; heartbeatRunId: string; persistedExecutionWorkspace: Pick | null; + executionWorkspaceSettings: IssueExecutionWorkspaceSettings | null; adapterType: string | null; }): Promise { try { @@ -262,6 +264,7 @@ export function environmentRunOrchestrator( heartbeatRunId: string; agentId: string; persistedExecutionWorkspace: Pick | null; + executionWorkspaceSettings: IssueExecutionWorkspaceSettings | null; }): Promise { // Step 1: Resolve environment const environment = await resolveEnvironment({ @@ -278,6 +281,7 @@ export function environmentRunOrchestrator( agentId: input.agentId, heartbeatRunId: input.heartbeatRunId, persistedExecutionWorkspace: input.persistedExecutionWorkspace, + executionWorkspaceSettings: input.executionWorkspaceSettings, adapterType: input.adapterType ?? null, }); @@ -299,6 +303,7 @@ export function environmentRunOrchestrator( provider: leaseRecord.lease.provider, executionWorkspaceId: leaseRecord.leaseContext.executionWorkspaceId, issueId: input.issueId, + networkEgress: input.executionWorkspaceSettings?.networkEgress ?? null, }, }); diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index 79410df061..7e7ab0cd67 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -7,10 +7,12 @@ import type { EnvironmentLease, EnvironmentLeaseStatus, ExecutionWorkspace, + IssueExecutionWorkspaceSettings, PluginEnvironmentConfig, SandboxEnvironmentConfig, } from "@paperclipai/shared"; import type { + PluginEnvironmentAcquireLeaseParams, PluginEnvironmentExecuteResult, PluginEnvironmentLease, PluginEnvironmentRealizeWorkspaceResult, @@ -123,6 +125,7 @@ export interface EnvironmentDriverAcquireInput { heartbeatRunId: string | null; executionWorkspaceId: string | null; executionWorkspaceMode: ExecutionWorkspace["mode"] | null; + executionWorkspaceSettings: IssueExecutionWorkspaceSettings | null; /** * The harness/adapter type for this run (the agent's adapter). Drivers that * materialize a per-run sandbox use it to select the runtime image so a single @@ -1585,7 +1588,8 @@ function createPluginEnvironmentDriver( agentId: input.agentId ?? undefined, executionWorkspaceId: input.executionWorkspaceId ?? undefined, adapterType: input.adapterType ?? undefined, - }); + executionWorkspaceSettings: input.executionWorkspaceSettings, + } as PluginEnvironmentAcquireLeaseParams); return await environmentsSvc.acquireLease({ companyId: input.companyId, @@ -1804,6 +1808,7 @@ export function environmentRuntimeService( /** Null for ad-hoc invocations (e.g. operator-initiated `Test` probes). */ heartbeatRunId: string | null; persistedExecutionWorkspace: Pick | null; + executionWorkspaceSettings?: IssueExecutionWorkspaceSettings | null; /** The agent's adapter type for this run (mixed-harness environments). */ adapterType?: string | null; /** @@ -1829,6 +1834,7 @@ export function environmentRuntimeService( heartbeatRunId: input.heartbeatRunId, executionWorkspaceId: leaseContext.executionWorkspaceId, executionWorkspaceMode: leaseContext.executionWorkspaceMode, + executionWorkspaceSettings: input.executionWorkspaceSettings ?? null, adapterType: input.adapterType ?? null, applyCustomImageTemplate: input.applyCustomImageTemplate ?? false, }); diff --git a/server/src/services/execution-workspace-policy.ts b/server/src/services/execution-workspace-policy.ts index f9221a3488..6c61872e7a 100644 --- a/server/src/services/execution-workspace-policy.ts +++ b/server/src/services/execution-workspace-policy.ts @@ -182,6 +182,17 @@ export function parseIssueExecutionWorkspaceSettings( if (mode === "isolated") return "isolated_workspace"; return ""; })(); + const networkEgress = parseObject(parsed.networkEgress); + const allowFqdns = Array.isArray(networkEgress.allowFqdns) + ? networkEgress.allowFqdns + .filter((value): value is string => typeof value === "string" && value.trim().length > 0) + .map((value) => value.trim().toLowerCase()) + : []; + const allowCidrs = Array.isArray(networkEgress.allowCidrs) + ? networkEgress.allowCidrs + .filter((value): value is string => typeof value === "string" && value.trim().length > 0) + .map((value) => value.trim()) + : []; return { ...(normalizedMode ? { mode: normalizedMode as IssueExecutionWorkspaceSettings["mode"] } @@ -193,9 +204,23 @@ export function parseIssueExecutionWorkspaceSettings( ...(parsed.workspaceRuntime && typeof parsed.workspaceRuntime === "object" && !Array.isArray(parsed.workspaceRuntime) ? { workspaceRuntime: { ...(parsed.workspaceRuntime as Record) } } : {}), + ...(allowFqdns.length > 0 || allowCidrs.length > 0 + ? { networkEgress: { allowFqdns, allowCidrs } } + : {}), }; } +export function selectEnvironmentExecutionWorkspaceSettings( + parsedSettings: IssueExecutionWorkspaceSettings | null, + isolatedWorkspacesEnabled: boolean, +): IssueExecutionWorkspaceSettings | null { + if (!parsedSettings) return null; + if (isolatedWorkspacesEnabled) return parsedSettings; + return parsedSettings.networkEgress + ? { networkEgress: parsedSettings.networkEgress } + : null; +} + export type ExecutionWorkspaceEnvironmentSource = | "agent" | "instance" diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 58de99abb8..8b91e802bc 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -174,6 +174,7 @@ import { resolveEffectiveWorkspaceStrategyType, resolveExecutionWorkspaceEnvironmentId, resolveExecutionWorkspaceMode, + selectEnvironmentExecutionWorkspaceSettings, WORKSPACE_WORKTREE_REQUIRES_PROJECT_CODE, WORKSPACE_WORKTREE_REQUIRES_PROJECT_MESSAGE, WORKSPACE_WORKTREE_REQUIRES_PROJECT_REMEDIATION, @@ -11928,9 +11929,16 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) ) : null; const isolatedWorkspacesEnabled = (await instanceSettings.getExperimental()).enableIsolatedWorkspaces; + const parsedIssueExecutionWorkspaceSettings = parseIssueExecutionWorkspaceSettings( + issueContext?.executionWorkspaceSettings, + ); const issueExecutionWorkspaceSettings = isolatedWorkspacesEnabled - ? parseIssueExecutionWorkspaceSettings(issueContext?.executionWorkspaceSettings) + ? parsedIssueExecutionWorkspaceSettings : null; + const environmentExecutionWorkspaceSettings = selectEnvironmentExecutionWorkspaceSettings( + parsedIssueExecutionWorkspaceSettings, + isolatedWorkspacesEnabled, + ); const contextProjectId = readNonEmptyString(context.projectId); const executionProjectId = issueContext?.projectId ?? contextProjectId; const projectContext = executionProjectId @@ -12810,6 +12818,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) heartbeatRunId: run.id, agentId: agent.id, persistedExecutionWorkspace, + executionWorkspaceSettings: environmentExecutionWorkspaceSettings, }); const selectedEnvironment = acquiredEnvironment.environment; // Defense-in-depth: re-check the actually-acquired environment against the