diff --git a/server/src/__tests__/cloud-control-task-drain.test.ts b/server/src/__tests__/cloud-control-task-drain.test.ts new file mode 100644 index 0000000000..0e500157bb --- /dev/null +++ b/server/src/__tests__/cloud-control-task-drain.test.ts @@ -0,0 +1,326 @@ +import { generateKeyPairSync, sign } from "node:crypto"; +import express from "express"; +import request from "supertest"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { + CLOUD_CONTROL_AUDIENCE, + CLOUD_CONTROL_HEADER, + CLOUD_CONTROL_JWS_TYPE, + CLOUD_RUNTIME_IDENTITY_AUDIENCE, + CLOUD_RUNTIME_IDENTITY_ISSUER, + CLOUD_RUNTIME_IDENTITY_JWS_TYPE, + resetCloudControlReplayFenceForTests, + verifyCloudControlAssertion, + type CloudControlAction, +} from "../services/cloud-runtime-identity.js"; +import { cloudControlMiddleware } from "../middleware/cloud-control.js"; +import { boardMutationGuard } from "../middleware/board-mutation-guard.js"; + +const STACK_ID = "stack-drain-test"; +const NOW = new Date("2099-01-01T00:00:00.000Z"); + +const pair = generateKeyPairSync("ed25519"); +const otherPair = generateKeyPairSync("ed25519"); +const publicJwk = { + ...pair.publicKey.export({ format: "jwk" }), + kid: "cloud-control-test-key", + use: "sig", + alg: "EdDSA", +}; + +const ENV = { + PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: JSON.stringify({ keys: [publicJwk] }), + PAPERCLIP_CLOUD_STACK_ID: STACK_ID, +} as NodeJS.ProcessEnv; + +function encodeJson(value: Record) { + return Buffer.from(JSON.stringify(value)).toString("base64url"); +} + +let requestIdCounter = 0; + +function controlAssertion(input: { + claims?: Record; + header?: Record; + signingKey?: typeof pair.privateKey; + action?: CloudControlAction; +} = {}) { + const iat = Math.floor(NOW.getTime() / 1000); + const header = encodeJson({ + alg: "EdDSA", + typ: CLOUD_CONTROL_JWS_TYPE, + kid: publicJwk.kid, + ...input.header, + }); + const payload = encodeJson({ + v: 1, + iss: CLOUD_RUNTIME_IDENTITY_ISSUER, + aud: CLOUD_CONTROL_AUDIENCE, + sub: STACK_ID, + action: input.action ?? "task-drain:start", + // Unique per assertion: request ids are single-use by design. + requestId: `drain-req-${(requestIdCounter += 1)}`, + iat, + exp: iat + 60, + ...input.claims, + }); + const signature = sign( + null, + Buffer.from(`${header}.${payload}`, "ascii"), + input.signingKey ?? pair.privateKey, + ).toString("base64url"); + return `${header}.${payload}.${signature}`; +} + +describe("verifyCloudControlAssertion", () => { + beforeEach(() => { + resetCloudControlReplayFenceForTests(); + }); + + const verify = (jws: string, expectedAction: CloudControlAction = "task-drain:start") => + verifyCloudControlAssertion({ compactJws: jws, expectedAction, env: ENV, now: NOW }); + + it("accepts a well-formed assertion bound to the expected action", () => { + const claims = verify(controlAssertion()); + expect(claims.sub).toBe(STACK_ID); + expect(claims.action).toBe("task-drain:start"); + expect(claims.requestId).toMatch(/^drain-req-\d+$/); + }); + + it("rejects a replay: each assertion's request id is single-use", () => { + const jws = controlAssertion(); + verify(jws); + expect(() => verify(jws)).toThrow(/already been used/); + // A distinct assertion (fresh request id) still verifies. + verify(controlAssertion()); + }); + + it("a rejected assertion does not burn its request id", () => { + // The consume runs last: replaying a mangled copy first must not + // deny the legitimate call. + const jws = controlAssertion(); + expect(() => verify(jws, "task-drain:stop")).toThrow(/does not authorize/); + verify(jws); + }); + + it("rejects an assertion for a different action — read cannot start a drain", () => { + expect(() => verify(controlAssertion({ action: "task-drain:read" }))).toThrow( + /does not authorize this action/, + ); + }); + + it("rejects an unknown action even when it matches the expectation", () => { + expect(() => + verifyCloudControlAssertion({ + compactJws: controlAssertion({ claims: { action: "instance:shutdown" } }), + expectedAction: "instance:shutdown" as CloudControlAction, + env: ENV, + now: NOW, + }), + ).toThrow(/does not authorize this action/); + }); + + it("rejects a runtime identity assertion replayed as a control assertion", () => { + // Same key, disjoint typ/aud/claims — the one-time bootstrap claim can + // never double as a management credential. + const iat = Math.floor(NOW.getTime() / 1000); + const header = encodeJson({ alg: "EdDSA", typ: CLOUD_RUNTIME_IDENTITY_JWS_TYPE, kid: publicJwk.kid }); + const payload = encodeJson({ + v: 1, + iss: CLOUD_RUNTIME_IDENTITY_ISSUER, + aud: CLOUD_RUNTIME_IDENTITY_AUDIENCE, + sub: STACK_ID, + claimId: "claim-1", + previousOrigin: "https://pool-1.staging.paperclip.app", + canonicalOrigin: "https://gonzo.staging.paperclip.app", + stackSlug: "gonzo", + iat, + exp: iat + 60, + }); + const signature = sign(null, Buffer.from(`${header}.${payload}`, "ascii"), pair.privateKey).toString("base64url"); + expect(() => verify(`${header}.${payload}.${signature}`)).toThrow(/protected header is invalid/); + }); + + it("rejects a control assertion whose audience is the runtime identity audience", () => { + expect(() => verify(controlAssertion({ claims: { aud: CLOUD_RUNTIME_IDENTITY_AUDIENCE } }))).toThrow( + /claims are incomplete|is invalid/, + ); + }); + + it("rejects an assertion for another stack, and any assertion when the instance is self-hosted", () => { + expect(() => verify(controlAssertion({ claims: { sub: "stack-other" } }))).toThrow( + /does not match this instance/, + ); + expect(() => + verifyCloudControlAssertion({ + compactJws: controlAssertion(), + expectedAction: "task-drain:start", + env: { PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: ENV.PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS } as NodeJS.ProcessEnv, + now: NOW, + }), + ).toThrow(/does not match this instance/); + }); + + it("rejects expired assertions and oversized lifetimes", () => { + const iat = Math.floor(NOW.getTime() / 1000); + expect(() => verify(controlAssertion({ claims: { iat: iat - 600, exp: iat - 300 } }))).toThrow( + /expired or has an invalid lifetime/, + ); + expect(() => verify(controlAssertion({ claims: { exp: iat + 3600 } }))).toThrow( + /expired or has an invalid lifetime/, + ); + }); + + it("rejects a signature from an unknown key", () => { + expect(() => verify(controlAssertion({ signingKey: otherPair.privateKey }))).toThrow( + /signature is invalid/, + ); + }); + + it("rejects a blank or padded request id", () => { + expect(() => verify(controlAssertion({ claims: { requestId: "" } }))).toThrow(/claims are incomplete|request id/); + expect(() => verify(controlAssertion({ claims: { requestId: " padded " } }))).toThrow(/request id is invalid/); + }); +}); + +describe("cloudControlMiddleware", () => { + const savedEnv: Record = {}; + + beforeEach(() => { + resetCloudControlReplayFenceForTests(); + savedEnv.PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS = process.env.PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS; + savedEnv.PAPERCLIP_CLOUD_STACK_ID = process.env.PAPERCLIP_CLOUD_STACK_ID; + process.env.PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS = ENV.PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS; + process.env.PAPERCLIP_CLOUD_STACK_ID = STACK_ID; + }); + + afterEach(() => { + for (const [key, value] of Object.entries(savedEnv)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + }); + + function createApp() { + const app = express(); + app.use((req, _res, next) => { + req.actor = { type: "none", source: "none" }; + next(); + }); + app.use(cloudControlMiddleware()); + app.all("/api/instance/task-drain", (req, res) => { + res.json({ actor: req.actor }); + }); + app.get("/api/instance/settings", (req, res) => { + res.json({ actor: req.actor }); + }); + return app; + } + + function freshAssertion(action: CloudControlAction) { + // The middleware verifies against the real clock; sign a live token. + const iat = Math.floor(Date.now() / 1000); + const header = encodeJson({ alg: "EdDSA", typ: CLOUD_CONTROL_JWS_TYPE, kid: publicJwk.kid }); + const payload = encodeJson({ + v: 1, + iss: CLOUD_RUNTIME_IDENTITY_ISSUER, + aud: CLOUD_CONTROL_AUDIENCE, + sub: STACK_ID, + action, + requestId: `drain-req-live-${(requestIdCounter += 1)}`, + iat, + exp: iat + 60, + }); + const signature = sign(null, Buffer.from(`${header}.${payload}`, "ascii"), pair.privateKey).toString("base64url"); + return `${header}.${payload}.${signature}`; + } + + it("installs a synthetic instance-admin board actor for a valid assertion, per method", async () => { + const app = createApp(); + for (const [method, action] of [ + ["get", "task-drain:read"], + ["post", "task-drain:start"], + ["delete", "task-drain:stop"], + ] as const) { + const res = await (request(app) as any)[method]("/api/instance/task-drain") + .set(CLOUD_CONTROL_HEADER, freshAssertion(action)); + expect(res.status).toBe(200); + expect(res.body.actor).toMatchObject({ + type: "board", + userId: "paperclip-cloud", + isInstanceAdmin: true, + source: "cloud_control", + }); + } + }); + + it("rejects an assertion bound to a different method's action", async () => { + const app = createApp(); + const res = await request(app) + .post("/api/instance/task-drain") + .set(CLOUD_CONTROL_HEADER, freshAssertion("task-drain:read")); + expect(res.status).toBe(401); + expect(res.body.error).toBe("invalid_cloud_control_assertion"); + }); + + it("accepts the conventional trailing-slash form of the endpoint", async () => { + const app = createApp(); + const res = await request(app) + .get("/api/instance/task-drain/") + .set(CLOUD_CONTROL_HEADER, freshAssertion("task-drain:read")); + expect(res.status).toBe(200); + expect(res.body.actor).toMatchObject({ source: "cloud_control" }); + }); + + it("rejects a replayed assertion at the middleware", async () => { + const app = createApp(); + const jws = freshAssertion("task-drain:read"); + const first = await request(app).get("/api/instance/task-drain").set(CLOUD_CONTROL_HEADER, jws); + expect(first.status).toBe(200); + const replay = await request(app).get("/api/instance/task-drain").set(CLOUD_CONTROL_HEADER, jws); + expect(replay.status).toBe(401); + }); + + it("rejects the header anywhere but the task-drain endpoint", async () => { + const app = createApp(); + const res = await request(app) + .get("/api/instance/settings") + .set(CLOUD_CONTROL_HEADER, freshAssertion("task-drain:read")); + expect(res.status).toBe(400); + expect(res.body.error).toBe("cloud_control_wrong_endpoint"); + }); + + it("rejects methods with no bound action even on the right endpoint", async () => { + const app = createApp(); + const res = await request(app) + .patch("/api/instance/task-drain") + .set(CLOUD_CONTROL_HEADER, freshAssertion("task-drain:start")); + expect(res.status).toBe(400); + }); + + it("passes requests without the header through untouched", async () => { + const app = createApp(); + const res = await request(app).get("/api/instance/task-drain"); + expect(res.status).toBe(200); + expect(res.body.actor).toMatchObject({ type: "none" }); + }); + + it("the board mutation guard exempts cloud_control mutations from the browser-origin check", async () => { + const app = express(); + app.use((req, _res, next) => { + req.actor = { + type: "board", + userId: "paperclip-cloud", + isInstanceAdmin: true, + source: "cloud_control", + }; + next(); + }); + app.use(boardMutationGuard()); + app.post("/api/instance/task-drain", (_req, res) => { + res.json({ ok: true }); + }); + const res = await request(app).post("/api/instance/task-drain"); + expect(res.status).toBe(200); + }); +}); diff --git a/server/src/app.ts b/server/src/app.ts index be2d067ab5..0cf096b17d 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -34,6 +34,7 @@ import { import { companyTransferRunService } from "./services/company-transfer-runs.js"; import { healthRoutes } from "./routes/health.js"; import { cloudRuntimeIdentityMiddleware } from "./middleware/cloud-runtime-identity.js"; +import { cloudControlMiddleware } from "./middleware/cloud-control.js"; import { cloudRoutes } from "./routes/cloud.js"; import { companyRoutes } from "./routes/companies.js"; import { companySkillRoutes } from "./routes/company-skills.js"; @@ -553,6 +554,10 @@ export async function createApp( resolveSession: opts.resolveSession, }), ); + // After the actor middleware on purpose: a valid Cloud control assertion + // REPLACES whatever actor the request otherwise resolved to, and only on + // the one endpoint it authorizes (see the middleware for the contract). + app.use(cloudControlMiddleware()); app.use("/api/auth", authRoutes(db)); if (opts.betterAuthHandler) { app.all("/api/auth/{*authPath}", opts.betterAuthHandler); diff --git a/server/src/middleware/board-mutation-guard.ts b/server/src/middleware/board-mutation-guard.ts index 6913ac5787..eee3d9dc25 100644 --- a/server/src/middleware/board-mutation-guard.ts +++ b/server/src/middleware/board-mutation-guard.ts @@ -85,13 +85,14 @@ export function boardMutationGuard(): RequestHandler { return; } - // Local-trusted mode, board bearer keys, and trusted Cloud tenant calls are - // not browser-session requests. + // Local-trusted mode, board bearer keys, trusted Cloud tenant calls, and + // signed Cloud control assertions are not browser-session requests. // In these modes, origin/referer headers can be absent; do not block those mutations. if ( req.actor.source === "local_implicit" || req.actor.source === "board_key" || req.actor.source === "cloud_tenant" + || req.actor.source === "cloud_control" ) { next(); return; diff --git a/server/src/middleware/cloud-control.ts b/server/src/middleware/cloud-control.ts new file mode 100644 index 0000000000..e4e490dfb8 --- /dev/null +++ b/server/src/middleware/cloud-control.ts @@ -0,0 +1,60 @@ +import type { RequestHandler } from "express"; +import { logger } from "./logger.js"; +import { + CLOUD_CONTROL_HEADER, + verifyCloudControlAssertion, + type CloudControlAction, +} from "../services/cloud-runtime-identity.js"; + +/** Method → the one action a control assertion must name to take it. */ +const ACTION_BY_METHOD: Record = { + GET: "task-drain:read", + POST: "task-drain:start", + DELETE: "task-drain:stop", +}; + +/** + * Accepts Cloud's signed control assertion only on the task-drain endpoint, + * so the Cloud control plane can hold new agent work and wait for quiescence + * before restarting the container for a deploy. The JWS is the entire + * authorization: a valid assertion installs a synthetic instance-admin board + * actor (replacing whatever weaker actor the request carried), each assertion + * is bound to exactly one method's action, and the header is rejected loudly + * anywhere else so it can never become an ambient credential. Instances + * without a Cloud stack identity reject every assertion. The browser-facing + * Cloud proxy strips this header, and possession of the shared tenant-session + * token cannot mint it. + */ +export function cloudControlMiddleware(): RequestHandler { + return (req, res, next) => { + const assertion = req.get(CLOUD_CONTROL_HEADER)?.trim(); + if (!assertion) { + next(); + return; + } + const expectedAction = ACTION_BY_METHOD[req.method]; + // Express's non-strict routing treats a trailing slash as the same + // route; the endpoint check must agree with it. + const normalizedPath = req.path.length > 1 && req.path.endsWith("/") ? req.path.slice(0, -1) : req.path; + if (normalizedPath !== "/api/instance/task-drain" || !expectedAction) { + res.status(400).json({ error: "cloud_control_wrong_endpoint" }); + return; + } + try { + verifyCloudControlAssertion({ compactJws: assertion, expectedAction }); + } catch (error) { + logger.warn({ err: error }, "Rejected Cloud control assertion"); + res.status(401).json({ error: "invalid_cloud_control_assertion" }); + return; + } + req.actor = { + type: "board", + userId: "paperclip-cloud", + userName: "Paperclip Cloud", + userEmail: null, + isInstanceAdmin: true, + source: "cloud_control", + }; + next(); + }; +} diff --git a/server/src/services/authorization.ts b/server/src/services/authorization.ts index c6cad46937..a63b1a407c 100644 --- a/server/src/services/authorization.ts +++ b/server/src/services/authorization.ts @@ -54,6 +54,7 @@ export type AuthorizationActor = | "agent_key" | "agent_jwt" | "cloud_tenant" + | "cloud_control" | "none"; }; diff --git a/server/src/services/cloud-runtime-identity.ts b/server/src/services/cloud-runtime-identity.ts index 5e1b8b0dae..db8f5b77bd 100644 --- a/server/src/services/cloud-runtime-identity.ts +++ b/server/src/services/cloud-runtime-identity.ts @@ -427,3 +427,153 @@ export function resetCloudRuntimeIdentityForTests() { startupOrigin = null; currentIdentity = null; } + +// --------------------------------------------------------------------------- +// Cloud control assertions +// +// A second, deliberately separate use of the same Cloud signing key: where the +// runtime identity assertion above is a ONE-TIME origin claim applied at +// bootstrap, a control assertion authorizes a single management call from the +// Cloud control plane — today, only the task-drain admission hold, so Cloud +// can stop new agent work and wait for quiescence before it restarts the +// container for a deploy. The audience, JWS type, and claim shape are +// disjoint from the runtime identity assertion, so neither token can ever be +// replayed as the other, and the verifier binds each assertion to one exact +// action so a "read status" token cannot start or stop a drain. +// --------------------------------------------------------------------------- + +export const CLOUD_CONTROL_HEADER = "x-paperclip-cloud-control"; +export const CLOUD_CONTROL_AUDIENCE = "paperclip-cloud-control/v1"; +export const CLOUD_CONTROL_JWS_TYPE = "paperclip-cloud-control+jwt"; +export const CLOUD_CONTROL_ACTIONS = [ + "task-drain:read", + "task-drain:start", + "task-drain:stop", +] as const; +export type CloudControlAction = (typeof CLOUD_CONTROL_ACTIONS)[number]; + +/** Control calls are immediate; a stolen assertion should age out fast. */ +const CLOUD_CONTROL_MAX_LIFETIME_SECONDS = 5 * 60; + +/** + * Single-use fence: a verified assertion's requestId is consumed atomically + * (module state; JS execution is single-threaded per process) and a replay + * of the same id is rejected for as long as the original could still be + * alive. Process-local on purpose — the drain state this protects is + * itself process-local, so a restart clears both together. Entries prune + * lazily at their expiry. + */ +const consumedControlRequestIds = new Map(); + +function consumeControlRequestId(requestId: string, expSeconds: number, nowMs: number): boolean { + for (const [id, expiresAtMs] of consumedControlRequestIds) { + if (expiresAtMs <= nowMs) consumedControlRequestIds.delete(id); + } + if (consumedControlRequestIds.has(requestId)) { + return false; + } + consumedControlRequestIds.set(requestId, expSeconds * 1000); + return true; +} + +/** Test seam: modules sharing a process must be able to reset the fence. */ +export function resetCloudControlReplayFenceForTests() { + consumedControlRequestIds.clear(); +} + +export type CloudControlClaims = { + v: 1; + iss: typeof CLOUD_RUNTIME_IDENTITY_ISSUER; + aud: typeof CLOUD_CONTROL_AUDIENCE; + sub: string; + action: CloudControlAction; + requestId: string; + iat: number; + exp: number; +}; + +/** + * Verify a Cloud control assertion for one exact action on this instance. + * Signed with the same key set as the runtime identity assertion + * (PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS) but under its own JWS type and + * audience. Instances without a Cloud stack identity reject every assertion — + * the feature is inert when self-hosted. + */ +export function verifyCloudControlAssertion(input: { + compactJws: string; + expectedAction: CloudControlAction; + env?: NodeJS.ProcessEnv; + now?: Date; +}): CloudControlClaims { + const env = input.env ?? process.env; + const now = input.now ?? new Date(); + const parts = input.compactJws.split("."); + if (parts.length !== 3 || parts.some((part) => part.length === 0)) { + throw new Error("Cloud control assertion is not a compact JWS"); + } + const [encodedHeader, encodedPayload, encodedSignature] = parts; + const header = decodeJsonPart(encodedHeader, "protected header"); + if ( + header.alg !== "EdDSA" + || header.typ !== CLOUD_CONTROL_JWS_TYPE + || typeof header.kid !== "string" + || !header.kid + ) { + throw new Error("Cloud control protected header is invalid"); + } + const key = publicKeyForKid(env, header.kid); + const signature = Buffer.from(encodedSignature, "base64url"); + const signingInput = Buffer.from(`${encodedHeader}.${encodedPayload}`, "ascii"); + if (!verify(null, signingInput, key, signature)) { + throw new Error("Cloud control signature is invalid"); + } + + const payload = decodeJsonPart(encodedPayload, "payload"); + const configuredStackId = nonEmpty(env.PAPERCLIP_CLOUD_STACK_ID); + const nowSeconds = Math.floor(now.getTime() / 1000); + if ( + payload.v !== 1 + || payload.iss !== CLOUD_RUNTIME_IDENTITY_ISSUER + || payload.aud !== CLOUD_CONTROL_AUDIENCE + || typeof payload.sub !== "string" + || typeof payload.action !== "string" + || typeof payload.requestId !== "string" + || typeof payload.iat !== "number" + || !Number.isInteger(payload.iat) + || typeof payload.exp !== "number" + || !Number.isInteger(payload.exp) + ) { + throw new Error("Cloud control claims are incomplete"); + } + if (!configuredStackId || payload.sub !== configuredStackId) { + throw new Error("Cloud control assertion stack does not match this instance"); + } + if ( + !(CLOUD_CONTROL_ACTIONS as readonly string[]).includes(payload.action) + || payload.action !== input.expectedAction + ) { + throw new Error("Cloud control assertion does not authorize this action"); + } + if ( + !payload.requestId + || payload.requestId.trim() !== payload.requestId + || payload.requestId.length > 256 + ) { + throw new Error("Cloud control assertion request id is invalid"); + } + if ( + payload.exp <= nowSeconds + || payload.iat > nowSeconds + MAX_CLOCK_SKEW_SECONDS + || payload.exp <= payload.iat + || payload.exp - payload.iat > CLOUD_CONTROL_MAX_LIFETIME_SECONDS + ) { + throw new Error("Cloud control assertion is expired or has an invalid lifetime"); + } + // Consumed LAST, only after every other check passed: a rejected + // assertion must not burn its request id, or an attacker could deny a + // legitimate call by replaying a mangled copy of it first. + if (!consumeControlRequestId(payload.requestId, payload.exp + MAX_CLOCK_SKEW_SECONDS, now.getTime())) { + throw new Error("Cloud control assertion has already been used"); + } + return payload as CloudControlClaims; +} diff --git a/server/src/types/express.d.ts b/server/src/types/express.d.ts index cd4eb5d80c..33c09c1604 100644 --- a/server/src/types/express.d.ts +++ b/server/src/types/express.d.ts @@ -30,7 +30,7 @@ declare global { runId?: string; onBehalfOfUserId?: string | null; identityContextId?: string | null; - source?: "local_implicit" | "session" | "board_key" | "agent_key" | "agent_jwt" | "cloud_tenant" | "none"; + source?: "local_implicit" | "session" | "board_key" | "agent_key" | "agent_jwt" | "cloud_tenant" | "cloud_control" | "none"; }; } }