diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 793568d0b5..714dad0c0e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -13,6 +13,11 @@ on: # the new tag ref instead. The tag mapping below keys off github.ref either # way. workflow_dispatch: + inputs: + staging_artifact_base_url: + description: Optional HTTPS object-storage/CDN prefix for staging migrator artifacts (build only; no releases) + type: string + default: "" permissions: contents: read @@ -26,6 +31,39 @@ concurrency: cancel-in-progress: false jobs: + staging-migrator: + if: github.event_name == 'workflow_dispatch' && inputs.staging_artifact_base_url != '' + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + steps: + - name: Require a non-default branch + env: + REF: ${{ github.ref }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: | + set -euo pipefail + [[ "$REF" == refs/heads/* && "$REF" != "refs/heads/$DEFAULT_BRANCH" ]] + - uses: actions/checkout@v7 + - uses: pnpm/action-setup@v6 + with: + version: 9.15.4 + - uses: actions/setup-node@v7 + with: + node-version: 24 + - run: pnpm install --frozen-lockfile + - name: Build matching migrator artifacts + env: + ARTIFACT_BASE_URL: ${{ inputs.staging_artifact_base_url }} + run: node scripts/build-preview-migrator.mjs "$RUNNER_TEMP/staging-migrator" "$ARTIFACT_BASE_URL" + - uses: actions/upload-artifact@v7 + with: + name: staging-migrator-${{ github.sha }} + path: ${{ runner.temp }}/staging-migrator/ + if-no-files-found: error + retention-days: 30 + build-and-push: runs-on: ubuntu-latest timeout-minutes: 60 diff --git a/doc/sandbox-work-folders.md b/doc/sandbox-work-folders.md index 333ca444b8..759af6a5c4 100644 --- a/doc/sandbox-work-folders.md +++ b/doc/sandbox-work-folders.md @@ -137,10 +137,17 @@ interrupted saves, and recovery without the original sandbox or app volume. Passing acceptance does not authorize a merge or mainline release. Both require the user's explicit sign-off. -Staging migrator artifacts must use a staging-only distribution path; do not -create GitHub releases to transport them. The initial prerelease publication -job has been removed. Its replacement remains pending, so the existing -release-backed preview resolver is not an approved deployment path. The -replacement must preserve commit identity, artifact integrity, migration -coverage and dependency-lockfile checks, and restrict deployment to explicitly -selected pinned staging stacks without changing the fleet default. +Staging migrator artifacts use an immutable object-storage prefix. The Docker +workflow's optional `staging_artifact_base_url` input builds DB/shared tarballs +and an integrity manifest as a GitHub Actions artifact; it has no release-write +permission. Transfer those artifacts to the staging bucket using conditional +creates, publishing the manifest last. Do not create GitHub releases or publish +npm packages for this flow. + +Cloud enables this lane only in staging through +`CLOUD_HARNESS_STAGING_ARTIFACT_BASE_URL`. Resolve `preview:` through +its authenticated deployment API, then target the dedicated pinned stack. +The configured origin, commit identity, artifact integrity, migration coverage, +dependency lockfile and tenant readiness are checked before acceptance. Preview +artifacts cannot become the fleet default. Record both the build workflow and +the resulting object identities with the acceptance evidence. diff --git a/packages/paperclip-runner/src/drivers/codex/codex-boundaries.test.ts b/packages/paperclip-runner/src/drivers/codex/codex-boundaries.test.ts index 8c7cbc881d..9270f7d308 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-boundaries.test.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-boundaries.test.ts @@ -161,6 +161,21 @@ describe("Codex value and workspace boundaries", () => { ).toThrow("filesystem root"); }); + it("accepts only the host-bound scoped home in external sandboxes", () => { + const home = "/home/daytona"; + const env = { HOME: home, PAPERCLIP_RUNNER_EXTERNAL_SANDBOX: "1", + PAPERCLIP_WORKSPACE_CWD: `${home}/repos/main`, PAPERCLIP_PRIMARY_REPO: `${home}/repos/main`, + PAPERCLIP_TASK_DIR: `${home}/task`, PAPERCLIP_AGENT_DIR: `${home}/agent`, + PAPERCLIP_USER_DIR: `${home}/user`, PAPERCLIP_PROJECT_DIR: `${home}/project`, PAPERCLIP_REPOS_DIR: `${home}/repos` }; + expect(validateCodexWorkingDirectory(home, env, "remote_runner")).toBe(home); + for (const cwd of ["/home", `${home}/.codex`, `${home}/repos/other`]) { + expect(() => validateCodexWorkingDirectory(cwd, env, "remote_runner")).toThrow("does not match"); + } + expect(() => validateCodexWorkingDirectory(home, { ...env, PAPERCLIP_RUNNER_EXTERNAL_SANDBOX: undefined }, "remote_runner")).toThrow("does not match"); + expect(() => validateCodexWorkingDirectory(home, { ...env, PAPERCLIP_PRIMARY_REPO: `${home}/repos/other` }, "remote_runner")).toThrow("does not match"); + expect(() => validateCodexWorkingDirectory(home, { ...env, PAPERCLIP_USER_DIR: "/etc" }, "remote_runner")).toThrow("does not match its home"); + }); + it("bounds retained values and redacts protected diagnostics", () => { const bounded = boundedCodexPayload({ short: "ok", diff --git a/packages/paperclip-runner/src/drivers/codex/codex-boundaries.ts b/packages/paperclip-runner/src/drivers/codex/codex-boundaries.ts index 16be810799..ae25639107 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-boundaries.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-boundaries.ts @@ -1,3 +1,4 @@ +import { externalWorkFolderEnvironment } from "../../work-folder-environment.js"; import { realpathSync, statSync } from "node:fs"; import { basename, @@ -150,7 +151,10 @@ function validateRemoteRunnerWorkingDirectory( // The controller cannot inspect a provider-owned filesystem. Pin the facade // to the exact remote workspace while runnerd validates existence, type, and // canonical identity inside the authoritative filesystem before launch. - if (workingDirectory !== configuredRoot) { + const scoped = externalWorkFolderEnvironment(environment); + const scopedHome = scoped.HOME; + const hostBoundHome = scopedHome && scoped.PAPERCLIP_PRIMARY_REPO === configuredRoot; + if (workingDirectory !== configuredRoot && !(hostBoundHome && workingDirectory === scopedHome)) { throw new Error( "Remote Codex working directory does not match the assigned workspace", ); diff --git a/scripts/build-preview-migrator.mjs b/scripts/build-preview-migrator.mjs index c8eae1a0b9..3d920c5419 100644 --- a/scripts/build-preview-migrator.mjs +++ b/scripts/build-preview-migrator.mjs @@ -7,21 +7,23 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import { materializePublishManifest, prepareBundledPackage } from "./prepare-bundled-package.mjs"; -export function previewIdentity(sha, date, repository = "paperclipai/paperclip") { +export function previewIdentity(sha, date, artifactBaseUrl) { if (!/^[a-f0-9]{40}$/.test(sha) || Number.isNaN(date.getTime())) throw new Error("Invalid preview commit"); + const base = new URL(artifactBaseUrl); + if (base.protocol !== "https:" || base.username || base.password || base.search || base.hash) throw new Error("Invalid staging artifact base URL"); const day = `${date.getUTCMonth() + 1}${String(date.getUTCDate()).padStart(2, "0")}`; const second = date.getUTCHours() * 3600 + date.getUTCMinutes() * 60 + date.getUTCSeconds() + 1; return { tag: `preview/${sha}`, version: `${date.getUTCFullYear()}.${day}.${second}-preview.sha${sha}`, - baseUrl: `https://github.com/${repository}/releases/download/${encodeURIComponent(`preview/${sha}`)}` }; + baseUrl: `${base.href.replace(/\/$/, "")}/${sha}` }; } -export function buildPreviewMigrator(outputDirectory) { +export function buildPreviewMigrator(outputDirectory, artifactBaseUrl) { const repo = path.resolve(fileURLToPath(new URL("..", import.meta.url))); const git = (...args) => execFileSync("git", args, { cwd: repo, encoding: "utf8" }).trim(); const sha = git("rev-parse", "HEAD"); if (process.env.GITHUB_SHA && process.env.GITHUB_SHA !== sha) throw new Error("Preview checkout differs from the workflow commit"); git("diff", "--quiet", "HEAD"); - const identity = previewIdentity(sha, new Date(git("show", "-s", "--format=%cI", "HEAD"))); + const identity = previewIdentity(sha, new Date(git("show", "-s", "--format=%cI", "HEAD")), artifactBaseUrl); execFileSync("pnpm", ["--filter", "@paperclipai/db...", "build"], { cwd: repo, stdio: "inherit" }); const output = path.resolve(outputDirectory); mkdirSync(output, { recursive: true }); @@ -58,6 +60,6 @@ export function buildPreviewMigrator(outputDirectory) { } if (process.argv[1] === fileURLToPath(import.meta.url)) { - if (!process.argv[2]) throw new Error("Usage: build-preview-migrator.mjs "); - console.log(JSON.stringify(buildPreviewMigrator(process.argv[2]))); + if (!process.argv[2] || !process.argv[3]) throw new Error("Usage: build-preview-migrator.mjs "); + console.log(JSON.stringify(buildPreviewMigrator(process.argv[2], process.argv[3]))); } diff --git a/scripts/build-preview-migrator.test.mjs b/scripts/build-preview-migrator.test.mjs index d779e89bfe..ac3a0cccbd 100644 --- a/scripts/build-preview-migrator.test.mjs +++ b/scripts/build-preview-migrator.test.mjs @@ -4,10 +4,16 @@ import { previewIdentity } from "./build-preview-migrator.mjs"; test("preview artifact identity is immutable, namespaced, and ordered by commit time", () => { const sha = "2f42a4968d5761fd62172e35ecf8188195b8d431"; - const identity = previewIdentity(sha, new Date("2026-07-19T09:30:00.000Z")); + const identity = previewIdentity(sha, new Date("2026-07-19T09:30:00.000Z"), "https://staging.example/migrators"); assert.equal(identity.version, `2026.719.34201-preview.sha${sha}`); assert.equal(identity.tag, `preview/${sha}`); - assert.equal(identity.baseUrl, `https://github.com/paperclipai/paperclip/releases/download/preview%2F${sha}`); + assert.equal(identity.baseUrl, `https://staging.example/migrators/${sha}`); assert.throws(() => previewIdentity("master", new Date()), /Invalid preview/); assert.throws(() => previewIdentity(sha, new Date("invalid")), /Invalid preview/); }); + +test("staging artifact URLs cannot carry credentials or use plaintext", () => { + for (const url of ["http://staging.example", "https://user:secret@staging.example", "https://staging.example?token=secret"]) { + assert.throws(() => previewIdentity("a".repeat(40), new Date(), url), /Invalid staging/); + } +}); diff --git a/server/src/__tests__/openapi-routes.test.ts b/server/src/__tests__/openapi-routes.test.ts index c623323b21..13b4697c21 100644 --- a/server/src/__tests__/openapi-routes.test.ts +++ b/server/src/__tests__/openapi-routes.test.ts @@ -405,3 +405,16 @@ describe("openapi routes", () => { expect(codes).toEqual(["200", "401", "403", "404"]); }); }); + +it("declares work-folder retry and executable headers for generated clients", () => { + const spec = buildOpenApiSpec(); + const root = "/api/companies/{companyId}/work-folders/{scope}/{ownerId}"; + expect(spec.paths[`${root}/content`].put.parameters).toEqual(expect.arrayContaining([ + expect.objectContaining({ name: "Idempotency-Key", in: "header", required: false }), + expect.objectContaining({ name: "X-File-Executable", in: "header", schema: { type: "string", enum: ["true", "false"] } }), + expect.objectContaining({ name: "X-File-Content-Type", in: "header" }), + ])); + expect(spec.paths[`${root}/operations`].post.parameters).toEqual(expect.arrayContaining([ + expect.objectContaining({ name: "Idempotency-Key", in: "header" }), + ])); +}); diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.ts index 37e352ca97..955d1cef41 100644 --- a/server/src/routes/openapi.ts +++ b/server/src/routes/openapi.ts @@ -255,6 +255,7 @@ type OpenApiPathRegistration = { request?: { params?: z.ZodTypeAny; query?: z.ZodTypeAny; + headers?: z.ZodTypeAny; body?: { content: Record; required?: boolean; @@ -487,7 +488,7 @@ function normalizeResponses(responses: Record = {}) { ); } -function parametersFromSchema(schema: z.ZodTypeAny, location: "path" | "query") { +function parametersFromSchema(schema: z.ZodTypeAny, location: "path" | "query" | "header") { const objectSchema = unwrapSchema(schema); if (zodTypeName(objectSchema) !== "object") return []; const shape = zodDef(objectSchema).shape as Record; @@ -528,6 +529,12 @@ class OpenAPIRegistry { ...parametersFromSchema(request.query, "query"), ]; } + if (request?.headers) { + normalizedOperation.parameters = [ + ...((normalizedOperation.parameters as unknown[]) ?? []), + ...parametersFromSchema(request.headers, "header"), + ]; + } if (request?.body) { normalizedOperation.requestBody = { ...request.body, @@ -5277,12 +5284,15 @@ registry.registerPath({ method: "get", path: `${workFolderPath}/content`, tags: }); registry.registerPath({ method: "put", path: `${workFolderPath}/content`, tags: ["work-folders"], summary: "Upload or replace a scoped file", description: "Send raw bytes as application/octet-stream, including an empty body for an empty file. Idempotency-Key identifies a retry. X-File-Executable: true preserves executable permission. X-File-Content-Type specifies the stored media type.", - request: { params: workFolderParams, query: z.object({ path: z.string() }), body: { required: true, content: { "application/octet-stream": { schema: { type: "string", format: "binary" } } } } }, + request: { params: workFolderParams, query: z.object({ path: z.string() }), headers: z.object({ + "Idempotency-Key": z.string().optional(), "X-File-Executable": z.enum(["true", "false"]).optional(), + "X-File-Content-Type": z.string().optional(), + }), body: { required: true, content: { "application/octet-stream": { schema: { type: "string", format: "binary" } } } } }, responses: { ...workFolderErrors, 200: r.ok() }, }); registry.registerPath({ method: "post", path: `${workFolderPath}/operations`, tags: ["work-folders"], summary: "Create a directory, delete, restore, or permanently purge files", description: "Deletion retains a recoverable copy. Restore rejects occupied paths. Purge removes the deleted copy permanently. Idempotency-Key identifies retries.", - request: { params: workFolderParams, body: jsonBody(z.discriminatedUnion("action", [ + request: { params: workFolderParams, headers: z.object({ "Idempotency-Key": z.string().optional() }), body: jsonBody(z.discriminatedUnion("action", [ z.object({ action: z.literal("mkdir"), path: z.string() }), z.object({ action: z.literal("delete"), path: z.string() }), z.object({ action: z.literal("restore"), fileId: z.uuid() }), z.object({ action: z.literal("purge"), fileId: z.uuid() }), ])) }, responses: { ...workFolderErrors, 200: r.ok(z.object({ applied: z.boolean() })) }, diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index df9910fc49..8a60439a30 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -30,6 +30,7 @@ import { import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js"; type BackendFactoryOptions = { + environment?: NodeJS.ProcessEnv; runnerInstanceId?: string; acpxRuntimeDirectory?: string; workingDirectoryAuthority?: "local_filesystem" | "remote_runner"; @@ -6011,6 +6012,88 @@ describe("runnerd provider runtime wiring", () => { ); }); + it("starts sandbox sessions in the scoped home while preserving the primary workspace", async () => { + const remoteCwd = "/home/daytona/repos/main"; + const home = "/home/daytona"; + const remoteExecution = { + ...execution, + binding: { ...execution.binding, runId: "run-scoped-home-test" }, + task: { + identifier: "DOT-REMOTE", + title: "Remote workspace test", + description: null, + prompt: "Verify the remote workspace.", + workMode: "standard", + }, + workspace: { + cwd: "/host/paperclip-workspace", + repoUrl: null, + repoRef: null, + branchName: null, + }, + session: { + normalizedSessionId: "scoped-home-session", + driverKind: "codex_app_server", + protocolVersion: 2, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, + }, + provider: { + kind: "codex", + model: null, + approvalPolicy: "never", + }, + executionMode: "default", + planningContext: null, + interactionResponses: [], + credentialBindings: [], + } as unknown as NativeExecutionInputV1; + state.createBackend.mockClear(); + state.execute.mockReset().mockResolvedValue({ + result: { summary: "completed" }, + terminal: { runTerminalState: "succeeded" }, + turnId: "turn", + normalizedSessionId: "session", + providerSessionId: null, + driverKind: "test", + driverVersion: "1", + nativeEventCount: 1, + highestContiguousSourceSeq: 1, + }); + + await executePaperclipNativeSession({ + db: leaseDb(remoteExecution), + execution: remoteExecution, + runnerInstanceId: "runner", + useRunnerd: true, + runnerExecutionTarget: { + kind: "remote", transport: "sandbox", remoteCwd, workFolderHome: home, + environmentId: "environment", leaseId: "lease", providerKey: "daytona", + runner: { execute: vi.fn(), syncIn: vi.fn() }, + } as never, + runnerPublicUrl: "wss://paperclip.example.test", + }); + + expect(state.createBackend).toHaveBeenCalledWith( + expect.objectContaining({ + workspace: expect.objectContaining({ cwd: home }), + }), + expect.objectContaining({ + workingDirectoryAuthority: "remote_runner", + }), + ); + expect(state.execute).toHaveBeenCalledWith( + expect.objectContaining({ + input: expect.objectContaining({ + workspace: expect.objectContaining({ cwd: home }), + }), + }), + ); + expect(state.createBackend.mock.calls[0]![1].environment).toEqual(expect.objectContaining({ + HOME: home, CODEX_HOME: `${home}/.codex`, PAPERCLIP_WORKSPACE_CWD: remoteCwd, + PAPERCLIP_RUNNER_EXTERNAL_SANDBOX: "1", + })); + }); + it("uses the image's shared Codex without uploading or installing artifacts", async () => { const syncIn = vi.fn(async () => undefined); const remoteExecute = vi.fn( diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 553ccadbfe..4222d68c91 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -4344,7 +4344,9 @@ async function executePaperclipNativeSessionWithinScope( ...input.execution, workspace: { ...input.execution.workspace, - cwd: input.runnerExecutionTarget.remoteCwd, + cwd: input.runnerExecutionTarget.transport === "sandbox" + ? input.runnerExecutionTarget.workFolderHome ?? input.runnerExecutionTarget.remoteCwd + : input.runnerExecutionTarget.remoteCwd, }, } : input.execution; @@ -7613,7 +7615,7 @@ async function createRunnerdBackendWithinSessionClaim( // host-home deny rules cannot shadow the assigned workspace. HOME: remoteTarget!.transport === "sandbox" && remoteTarget!.workFolderHome ? remoteTarget!.workFolderHome : posix.join(remoteRunnerFilesystemRoot!, "codex-home"), CODEX_HOME: remoteTarget!.transport === "sandbox" && remoteTarget!.workFolderHome ? posix.join(remoteTarget!.workFolderHome, ".codex") : posix.join(remoteRunnerFilesystemRoot!, "codex-home"), - PAPERCLIP_WORKSPACE_CWD: runnerExecution.workspace.cwd, + PAPERCLIP_WORKSPACE_CWD: remoteTarget!.remoteCwd, ...(remoteTarget!.transport === "sandbox" ? { PAPERCLIP_RUNNER_EXTERNAL_SANDBOX: "1" } : {}), diff --git a/tests/runner-e2e/deployed-stack.ts b/tests/runner-e2e/deployed-stack.ts index 36573c546a..1a3ca295cf 100644 --- a/tests/runner-e2e/deployed-stack.ts +++ b/tests/runner-e2e/deployed-stack.ts @@ -12,6 +12,7 @@ export function isStagingOrigin(value: string) { export interface DeployedStack { baseURL: string; stackId: string; commit: string; appImage: string; migratorVersion: string; sandboxImage: string; companyId: string; taskId: string; agentId: string; projectId: string; userId: string; + excludedAdapters?: Array<{ adapterType: string; reason: string }>; profiles: Array<{ id: string; adapterType: string; engine: string; model: string; qualification: string; agentId: string }>; } @@ -28,6 +29,12 @@ export function loadDeployedStack(): DeployedStack { assert(/@sha256:[a-f0-9]{64}$/.test(manifest.sandboxImage), "Expected immutable sandbox image"); const uuid = /^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i; for (const key of ["companyId", "taskId", "agentId", "projectId"] as const) assert(uuid.test(manifest[key]), `Invalid ${key}`); + if (manifest.excludedAdapters !== undefined) { + assert(Array.isArray(manifest.excludedAdapters), "Invalid adapter exclusions"); + for (const exclusion of manifest.excludedAdapters) { + assert(typeof exclusion.adapterType === "string" && typeof exclusion.reason === "string" && exclusion.reason.trim().length > 0, "Exclusions require an explicit reason"); + } + } assert(Array.isArray(manifest.profiles) && manifest.profiles.length >= 7, "The seven baseline profiles are required"); for (const profile of manifest.profiles) { for (const key of ["id", "adapterType", "engine", "model", "qualification", "agentId"] as const) { diff --git a/tests/runner-e2e/deployed-work-folders.spec.ts b/tests/runner-e2e/deployed-work-folders.spec.ts index 0608a3fb90..5cd752d069 100644 --- a/tests/runner-e2e/deployed-work-folders.spec.ts +++ b/tests/runner-e2e/deployed-work-folders.spec.ts @@ -18,7 +18,8 @@ test("deployed candidate and complete supported adapter inventory", async ({}, i expect(capabilities.sandboxProviders.daytona?.supportsRunExecution).toBe(true); const adapters = await api.json>("/api/adapters"); const required: string[] = []; - for (const adapter of adapters.filter((entry) => !entry.disabled)) { + const excluded = new Set(stack.excludedAdapters?.map((entry) => entry.adapterType)); + for (const adapter of adapters.filter((entry) => !entry.disabled && !excluded.has(entry.type))) { if (capabilities.adapters.find((entry) => entry.adapterType === adapter.type)?.drivers.sandbox !== "supported") continue; if (adapter.type === "paperclip_runner") { required.push("paperclip_runner:codex", "paperclip_runner:opencode",