From dba1a2d4f54ef7b0c21d69d4cb538e467463c789 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sun, 30 Aug 2026 17:51:33 -0500 Subject: [PATCH] feat(runner): stage managed Codex credentials (#12398) ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Codex can authenticate through an API key or an `auth.json` document. > - Forwarding managed credentials in process arguments, durable session options, or general environment state would leak them across the runner boundary. > - The ACPX sandbox therefore needs a private Codex home, bounded staging, crash-safe cleanup intent, and cross-process ownership that fails closed without relying on writable marker files. > - This pull request establishes that credential boundary before later PRs connect the Codex ACP process. ## Linked Issues or Issue Description **Agent or provider** The qualified Codex ACPX profile. **Why this adapter is useful** Managed installations may already have a Codex login document, while service deployments may provide an API key or inline JSON. The runner needs all three modes without copying ambient home state, accepting ambiguous sources, persisting secret values in ordinary runner state, following links, or allowing concurrent processes to mutate the same credential home. **How the agent is invoked** A later runtime-host pull request acquires this credential lease before opening Codex and closes it during shutdown and failed-start paths. API keys remain launch-only environment values. JSON credentials are atomically staged as `auth.json` inside the isolated Codex home. This pull request does not spawn Codex, add a dependency, register an adapter, or change execution selection. **Additional context** This is the lowest unmerged PR in the runner series and targets `master` directly. It deliberately requires one explicit credential source and never reads the Paperclip server user's ambient home by default. ## What Changed - Add one-use managed Codex credential leases for API-key, inline-JSON, and explicit managed-file modes. - Reject missing and ambiguous source combinations. - Require absolute external managed-file paths, private ownership and permissions on POSIX, bounded documents, no-follow opens, and stable file identity during reads. - Stage JSON through a deterministic private no-follow staging path, scrub stale staging state on admission, sync the file, rename atomically, and sync the parent directory. - Persist a bounded cleanup intent and remove staged or provider-generated authentication files without following links. - Fence each canonical credential home with a markerless 2-of-3 exclusive loopback quorum derived from the current UID and canonical home. One unrelated occupied candidate is tolerated; two occupied candidates fail closed before credential mutation. - Bound stuck parent filesystem operations and cleanup helpers process-wide so an unresponsive filesystem cannot create unbounded work. - Zero temporary credential buffers and make cleanup retryable without allowing a stale cleanup generation to affect a successor. - Add focused tests for credential modes, permissions, malformed and ambiguous sources, symlink rejection, deterministic staging, quorum contention and crash release, stuck-operation bounds, and idempotent cleanup. ## Verification - Prettier and `git diff --check` passed for the exact pushed head. - Greptile reviewed the exact head at 5/5 with no blocking finding; independent static audits found no remaining correctness, security, test-contract, or resource-bound blocker. - GitHub Actions passed on the exact head `29e75afb7b8529b9d070070e0c1462e795d115bb`, including Build, Typecheck, the complete test matrix, e2e, verify, and security checks. One unrelated hosted-runner `npm ENOENT` shard passed on its GitHub rerun. - The diff contains 2 files and does not change `pnpm-lock.yaml`, workflows, dependencies, public package exports, server selection, or UI behavior. ## Risks The primary risks are credential disclosure, concurrent ownership, and cleanup that outlives its authority. Secret values never enter returned identity or durable runner data. Source and destination operations are bounded, no-follow, privately permissioned, and crash-durable. The markerless kernel quorum prevents concurrent home mutation without trusting a replaceable marker, while process-global operation/helper caps prevent unresponsive filesystem work from growing without bound. Cleanup is generation-scoped so a stale retry cannot remove a successor's credential. ## Model Used OpenAI Codex with GPT-5 and repository tool use. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either linked an existing public item or described the issue in this PR - [x] I have not referenced internal or instance-local Paperclip issues or links - [x] My branch name describes the change and contains no internal task identifier - [ ] I have run the affected tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have documented the credential and cleanup boundary - [x] All applicable GitHub Actions are green - [x] Greptile is 5/5 with every actionable comment resolved - [x] I will address all review findings before requesting merge --- .../drivers/acpx/codex-credentials.test.ts | 1696 +++++++++++++++++ .../src/drivers/acpx/codex-credentials.ts | 1231 ++++++++++++ 2 files changed, 2927 insertions(+) create mode 100644 packages/paperclip-runner/src/drivers/acpx/codex-credentials.test.ts create mode 100644 packages/paperclip-runner/src/drivers/acpx/codex-credentials.ts diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-credentials.test.ts b/packages/paperclip-runner/src/drivers/acpx/codex-credentials.test.ts new file mode 100644 index 0000000000..9b2a5d4eb4 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/acpx/codex-credentials.test.ts @@ -0,0 +1,1696 @@ +import { fork, type ChildProcess } from "node:child_process"; +import { createHash } from "node:crypto"; +import { EventEmitter } from "node:events"; +import { + chmod, + mkdir, + mkdtemp, + open, + readdir, + readFile, + realpath, + rm, + stat, + symlink, + writeFile, + type FileHandle, +} from "node:fs/promises"; +import { createRequire } from "node:module"; +import { createServer, type Server, type Socket } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { stageManagedCodexCredential } from "./codex-credentials.js"; + +const temporaryDirectories: string[] = []; + +afterEach(async () => { + vi.useRealTimers(); + vi.doUnmock("node:child_process"); + vi.doUnmock("node:fs/promises"); + vi.resetModules(); + ( + globalThis as typeof globalThis & { + __paperclipDirectorySyncHelperRegistryV1?: { + activeParentOperations: Set; + }; + } + ).__paperclipDirectorySyncHelperRegistryV1?.activeParentOperations.clear(); + await Promise.all( + temporaryDirectories + .splice(0) + .map((directory) => rm(directory, { force: true, recursive: true })), + ); +}); + +describe("managed Codex credentials", () => { + it("derives deterministic quorum candidates from distinct user scopes", () => { + expect(credentialLeasePortsForScope("1000", "/canonical/home")).toEqual( + credentialLeasePortsForScope("1000", "/canonical/home"), + ); + expect(credentialLeasePortsForScope("1000", "/canonical/home")).not.toEqual( + credentialLeasePortsForScope("1001", "/canonical/home"), + ); + expect(credentialLeasePortsForScope("1000", "/canonical/home")).not.toEqual( + credentialLeasePortsForScope("1000", "/canonical/other-home"), + ); + }); + + it("stages inline JSON privately and removes it idempotently", async () => { + const fixture = await credentialFixture(); + const lease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: JSON.stringify({ + tokens: { access_token: "inline-canary" }, + }), + }, + }); + + expect(lease.mode).toBe("inline_json"); + const cleanupIntent = join( + fixture.home, + ".paperclip-auth-cleanup-required", + ); + await expect(readFile(lease.path, "utf8")).resolves.toContain( + "inline-canary", + ); + await expect(readFile(cleanupIntent, "utf8")).resolves.toBe( + "paperclip-managed-codex-cleanup-v1\n", + ); + if (process.platform !== "win32") { + expect((await stat(lease.path)).mode & 0o777).toBe(0o600); + } + await lease.close(); + await lease.close(); + await expect(readFile(lease.path)).rejects.toMatchObject({ + code: "ENOENT", + }); + await expect(readFile(cleanupIntent)).rejects.toMatchObject({ + code: "ENOENT", + }); + }); + + it("fences overlapping leases for the same isolated home", async () => { + const fixture = await credentialFixture(); + const firstLease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"first"}', + }, + }); + + await expect( + stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"second"}', + }, + }), + ).rejects.toThrow("already has an active lease"); + await expect(readFile(firstLease.path, "utf8")).resolves.toBe( + '{"owner":"first"}', + ); + + await firstLease.close(); + const secondLease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"second"}', + }, + }); + await expect(readFile(secondLease.path, "utf8")).resolves.toBe( + '{"owner":"second"}', + ); + await secondLease.close(); + }); + + it("fences a contender loaded through a fresh module instance", async () => { + const fixture = await credentialFixture(); + const firstLease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"first"}', + }, + }); + + vi.resetModules(); + const freshCredentials = await import("./codex-credentials.js"); + await expect( + freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"contender"}', + }, + }), + ).rejects.toThrow("already has an active lease"); + + await firstLease.close(); + }); + + it.each([ + ["primary", 0], + ["non-primary", 1], + ] as const)( + "tolerates one unrelated silent %s quorum listener", + async (_label, occupiedIndex) => { + const fixture = await credentialFixture(); + const ports = credentialLeasePorts(await realpath(fixture.home)); + const occupied = await listenSilently(ports[occupiedIndex]); + try { + const lease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"paperclip"}', + }, + }); + await expect(readFile(lease.path, "utf8")).resolves.toBe( + '{"owner":"paperclip"}', + ); + await lease.close(); + } finally { + await occupied.close(); + } + }, + ); + + it("fails before auth mutation when two quorum candidates are occupied", async () => { + const fixture = await credentialFixture(); + const destination = join(fixture.home, "auth.json"); + await writeFile(destination, '{"sentinel":true}', { mode: 0o600 }); + const ports = credentialLeasePorts(await realpath(fixture.home)); + const first = await listenSilently(ports[0]); + const second = await listenSilently(ports[1]); + try { + await expect( + stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"contender"}', + }, + }), + ).rejects.toThrow("already has an active lease"); + await expect(readFile(destination, "utf8")).resolves.toBe( + '{"sentinel":true}', + ); + } finally { + await Promise.all([first.close(), second.close()]); + } + }); + + it("admits only one of two concurrent fresh-module contenders", async () => { + const fixture = await credentialFixture(); + vi.resetModules(); + const firstCredentials = await import("./codex-credentials.js"); + vi.resetModules(); + const secondCredentials = await import("./codex-credentials.js"); + + const results = await Promise.allSettled([ + firstCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"first"}', + }, + }), + secondCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"second"}', + }, + }), + ]); + const winners = results.filter( + ( + result, + ): result is PromiseFulfilledResult< + Awaited> + > => result.status === "fulfilled", + ); + expect(winners).toHaveLength(1); + expect( + results.filter((result) => result.status === "rejected"), + ).toHaveLength(1); + await winners[0].value.close(); + }); + + it("fences another process and recovers only after its kernel lease dies", async () => { + const fixture = await credentialFixture(); + const destination = join(fixture.home, "auth.json"); + const childScript = join(fixture.root, "credential-owner.mjs"); + const credentialModule = new URL("./codex-credentials.ts", import.meta.url) + .href; + // paperclip-runner intentionally does not ship a TS runtime dependency. + // Resolve the existing monorepo dev loader from a workspace that declares + // it, instead of asking the child to resolve an undeclared bare package. + const tsxLoader = createRequire( + new URL("../../../../../server/package.json", import.meta.url), + ).resolve("tsx"); + await writeFile( + childScript, + [ + `const { stageManagedCodexCredential } = await import(${JSON.stringify(credentialModule)});`, + "const lease = await stageManagedCodexCredential({", + " agentHomeDirectory: process.argv[2],", + ' environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: \'{"owner":"first"}\' },', + "});", + 'process.send?.({ type: "ready", path: lease.path });', + "process.on('message', async (message) => {", + " if (message?.type !== 'close') return;", + " await lease.close();", + " process.exit(0);", + "});", + ].join("\n"), + ); + const owner = fork(childScript, [fixture.home], { + execArgv: ["--import", tsxLoader], + stdio: ["ignore", "pipe", "pipe", "ipc"], + }); + try { + await waitForChildMessage(owner, "ready"); + await expect(readFile(destination, "utf8")).resolves.toBe( + '{"owner":"first"}', + ); + + if (process.platform !== "win32") { + owner.kill("SIGSTOP"); + await new Promise((resolveSignal) => + setTimeout(resolveSignal, 50), + ); + } + await expect( + stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"second"}', + }, + }), + ).rejects.toThrow("already has an active lease"); + await expect(readFile(destination, "utf8")).resolves.toBe( + '{"owner":"first"}', + ); + + if (process.platform !== "win32") owner.kill("SIGCONT"); + owner.kill(process.platform === "win32" ? undefined : "SIGKILL"); + await waitForChildExit(owner); + + const successor = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"second"}', + }, + }); + await expect(readFile(destination, "utf8")).resolves.toBe( + '{"owner":"second"}', + ); + await successor.close(); + expect( + (await readdir(fixture.home)).filter( + (name) => + name.includes("paperclip-auth-lease") || + name.includes("paperclip-auth-home-claim"), + ), + ).toEqual([]); + } finally { + if (owner.exitCode === null && owner.signalCode === null) { + if (process.platform !== "win32") owner.kill("SIGCONT"); + owner.kill(process.platform === "win32" ? undefined : "SIGKILL"); + await waitForChildExit(owner).catch(() => undefined); + } + } + }); + it.runIf(process.platform !== "win32")( + "holds kernel ownership until credential cleanup is durable", + async () => { + const fixture = await credentialFixture(); + const lease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" }, + }); + const probe = await open(fixture.home, "r"); + const prototype = Object.getPrototypeOf(probe) as { + sync(this: FileHandle): Promise; + }; + await probe.close(); + const originalSync = prototype.sync; + let releaseCleanup!: () => void; + const cleanupGate = new Promise((resolveCleanup) => { + releaseCleanup = resolveCleanup; + }); + let signalCleanupStarted!: () => void; + const cleanupStarted = new Promise((resolveStarted) => { + signalCleanupStarted = resolveStarted; + }); + let heldCleanup = false; + const syncSpy = vi + .spyOn(prototype, "sync") + .mockImplementation(async function (this: FileHandle): Promise { + if (!heldCleanup && (await this.stat()).isDirectory()) { + heldCleanup = true; + signalCleanupStarted(); + await cleanupGate; + } + await originalSync.call(this); + }); + try { + const closing = lease.close(); + await cleanupStarted; + vi.resetModules(); + const freshCredentials = await import("./codex-credentials.js"); + await expect( + freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" }, + }), + ).rejects.toThrow("already has an active lease"); + + releaseCleanup(); + await expect(closing).resolves.toBeUndefined(); + const successor = await freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" }, + }); + await successor.close(); + } finally { + releaseCleanup(); + syncSpy.mockRestore(); + } + }, + ); + + it("does not let an older failed close remove a successor credential", async () => { + const fixture = await credentialFixture(); + const firstLease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"first"}', + }, + }); + await rm(firstLease.path, { force: true }); + await mkdir(firstLease.path); + + await expect(firstLease.close()).rejects.toThrow( + "credential destination is a directory", + ); + await rm(firstLease.path, { force: true, recursive: true }); + + const secondLease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"second"}', + }, + }); + await expect(firstLease.close()).resolves.toBeUndefined(); + await expect(readFile(secondLease.path, "utf8")).resolves.toBe( + '{"owner":"second"}', + ); + await secondLease.close(); + }); + + it("leaves no ownership artifacts in the credential home", async () => { + const fixture = await credentialFixture(); + const lease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + await lease.close(); + + expect( + (await readdir(fixture.home)).filter( + (name) => + name.includes("paperclip-auth-lease") || + name.includes("paperclip-auth-home-claim") || + name.includes("paperclip-auth-lock"), + ), + ).toEqual([]); + }); + it("recovers a persisted cleanup intent before admitting another provider", async () => { + const fixture = await credentialFixture(); + const destination = join(fixture.home, "auth.json"); + const cleanupIntent = join( + fixture.home, + ".paperclip-auth-cleanup-required", + ); + await writeFile(destination, '{"crash_stale":true}', { mode: 0o600 }); + await writeFile(cleanupIntent, "paperclip-managed-codex-cleanup-v1\n", { + mode: 0o600, + }); + + const lease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + await expect(readFile(destination)).rejects.toMatchObject({ + code: "ENOENT", + }); + await expect(readFile(cleanupIntent, "utf8")).resolves.toBe( + "paperclip-managed-codex-cleanup-v1\n", + ); + await lease.close(); + await expect(readFile(cleanupIntent)).rejects.toMatchObject({ + code: "ENOENT", + }); + }); + + it("scrubs a crash-left credential staging file before admission", async () => { + const fixture = await credentialFixture(); + const stagingPath = join(fixture.home, ".paperclip-auth-staging-v1"); + const cleanupIntent = join( + fixture.home, + ".paperclip-auth-cleanup-required", + ); + await writeFile(stagingPath, '{"crash_secret":"must-not-survive"}', { + mode: 0o600, + }); + await writeFile(cleanupIntent, "paperclip-managed-codex-cleanup-v1\n", { + mode: 0o600, + }); + + vi.resetModules(); + const freshCredentials = await import("./codex-credentials.js"); + const lease = await freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"successor"}', + }, + }); + await expect(readFile(stagingPath)).rejects.toMatchObject({ + code: "ENOENT", + }); + await expect(readFile(lease.path, "utf8")).resolves.toBe( + '{"owner":"successor"}', + ); + await lease.close(); + }); + + it.runIf(process.platform !== "win32")( + "unlinks a crash-left staging symlink without touching its target", + async () => { + const fixture = await credentialFixture(); + const target = join(fixture.root, "external-secret.json"); + const stagingPath = join(fixture.home, ".paperclip-auth-staging-v1"); + await writeFile(target, '{"external":"unchanged"}', { mode: 0o600 }); + await symlink(target, stagingPath); + + const lease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + await expect(readFile(stagingPath)).rejects.toMatchObject({ + code: "ENOENT", + }); + await expect(readFile(target, "utf8")).resolves.toBe( + '{"external":"unchanged"}', + ); + await lease.close(); + }, + ); + + it("fails closed instead of recursively removing a staging directory", async () => { + const fixture = await credentialFixture(); + const stagingPath = join(fixture.home, ".paperclip-auth-staging-v1"); + await mkdir(stagingPath, { mode: 0o700 }); + + await expect( + stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }), + ).rejects.toThrow("credential destination is a directory"); + expect((await stat(stagingPath)).isDirectory()).toBe(true); + await expect( + readFile(join(fixture.home, "auth.json")), + ).rejects.toMatchObject({ + code: "ENOENT", + }); + }); + + it.runIf(process.platform !== "win32")( + "retries the directory sync after unlink already succeeded", + async () => { + const fixture = await credentialFixture(); + const lease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" }, + }); + const probe = await open(fixture.home, "r"); + const prototype = Object.getPrototypeOf(probe) as { + sync(this: FileHandle): Promise; + }; + await probe.close(); + const originalSync = prototype.sync; + let syncAttempts = 0; + const syncSpy = vi + .spyOn(prototype, "sync") + .mockImplementation(async function (this: FileHandle): Promise { + if ((await this.stat()).isDirectory()) { + syncAttempts += 1; + if (syncAttempts === 1) { + throw new Error("injected directory sync failure"); + } + } + await originalSync.call(this); + }); + try { + await expect(lease.close()).resolves.toBeUndefined(); + await expect(readFile(lease.path)).rejects.toMatchObject({ + code: "ENOENT", + }); + expect(syncAttempts).toBe(3); + } finally { + syncSpy.mockRestore(); + } + }, + ); + + it.runIf(process.platform !== "win32")( + "retries preflight, installation, and removal until each is durable", + async () => { + const fixture = await credentialFixture(); + const probe = await open(fixture.home, "r"); + const prototype = Object.getPrototypeOf(probe) as { + sync(this: FileHandle): Promise; + }; + await probe.close(); + const originalSync = prototype.sync; + let directorySyncAttempts = 0; + const syncSpy = vi + .spyOn(prototype, "sync") + .mockImplementation(async function (this: FileHandle): Promise { + if ((await this.stat()).isDirectory()) { + directorySyncAttempts += 1; + } + // The first attempt at each namespace boundary fails; the durable + // helper must retry before staging or cleanup reports success. + if ([1, 3, 5].includes(directorySyncAttempts)) { + throw new Error("injected directory sync failure"); + } + await originalSync.call(this); + }); + try { + const lease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" }, + }); + await expect(readFile(lease.path, "utf8")).resolves.toBe("{}"); + await expect(lease.close()).resolves.toBeUndefined(); + await expect(readFile(lease.path)).rejects.toMatchObject({ + code: "ENOENT", + }); + expect(directorySyncAttempts).toBe(8); + } finally { + syncSpy.mockRestore(); + } + }, + ); + + it("copies an explicit private source without changing the source", async () => { + const fixture = await credentialFixture(); + const source = join(fixture.root, "managed-auth.json"); + await writeFile( + source, + JSON.stringify({ tokens: { access_token: "managed-canary" } }), + { mode: 0o600 }, + ); + await chmod(source, 0o600); + + const lease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + sourcePath: source, + }); + expect(lease.mode).toBe("managed_file"); + await expect(readFile(lease.path, "utf8")).resolves.toContain( + "managed-canary", + ); + await lease.close(); + await expect(readFile(source, "utf8")).resolves.toContain("managed-canary"); + }); + + it("replaces a stale regular auth destination in JSON modes", async () => { + const fixture = await credentialFixture(); + const destination = join(fixture.home, "auth.json"); + await writeFile(destination, '{"stale":true}', { mode: 0o600 }); + + const lease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"fresh":true}', + }, + }); + await expect(readFile(destination, "utf8")).resolves.toBe('{"fresh":true}'); + await lease.close(); + }); + + it("cleans stale and provider-generated auth in API-key mode", async () => { + const fixture = await credentialFixture(); + const destination = join(fixture.home, "auth.json"); + await writeFile(destination, '{"stale":true}'); + + const lease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + expect(lease.mode).toBe("api_key"); + await expect(readFile(destination)).rejects.toMatchObject({ + code: "ENOENT", + }); + await writeFile(destination, '{"provider_generated":true}'); + await lease.close(); + await expect(readFile(destination)).rejects.toMatchObject({ + code: "ENOENT", + }); + }); + + it.runIf(process.platform !== "win32")( + "keeps API-key staging pending until stale removal is durable", + async () => { + const fixture = await credentialFixture(); + const destination = join(fixture.home, "auth.json"); + await writeFile(destination, '{"stale":true}', { mode: 0o600 }); + const probe = await open(fixture.home, "r"); + const prototype = Object.getPrototypeOf(probe) as { + sync(this: FileHandle): Promise; + }; + await probe.close(); + const originalSync = prototype.sync; + let syncAttempts = 0; + const syncSpy = vi + .spyOn(prototype, "sync") + .mockImplementation(async function (this: FileHandle): Promise { + if ((await this.stat()).isDirectory()) { + syncAttempts += 1; + if (syncAttempts === 1) { + throw new Error("injected directory sync failure"); + } + } + await originalSync.call(this); + }); + try { + const lease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + await expect(readFile(destination)).rejects.toMatchObject({ + code: "ENOENT", + }); + await expect(lease.close()).resolves.toBeUndefined(); + expect(syncAttempts).toBe(5); + } finally { + syncSpy.mockRestore(); + } + }, + ); + + it.runIf(process.platform !== "win32")( + "fails a non-durable admission within a bound and scrubs again on retry", + async () => { + const fixture = await credentialFixture(); + const probe = await open(fixture.home, "r"); + const prototype = Object.getPrototypeOf(probe) as { + sync(this: FileHandle): Promise; + }; + await probe.close(); + const originalSync = prototype.sync; + const syncSpy = vi + .spyOn(prototype, "sync") + .mockImplementation(async function (this: FileHandle): Promise { + if ((await this.stat()).isDirectory()) { + throw new Error("persistent directory sync failure"); + } + await originalSync.call(this); + }); + try { + const staging = stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + await expect(staging).rejects.toThrow( + "remained non-durable after 8 attempts", + ); + expect(syncSpy.mock.calls.length).toBeGreaterThanOrEqual(8); + syncSpy.mockRestore(); + const lease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + await expect(lease.close()).resolves.toBeUndefined(); + } finally { + syncSpy.mockRestore(); + } + }, + ); + + it.runIf(process.platform !== "win32")( + "shares the four-slot parent filesystem budget across fresh modules", + async () => { + const fixtures = await Promise.all( + Array.from({ length: 5 }, () => credentialFixture()), + ); + const retainedHomes = new Set( + fixtures.slice(0, 4).map((fixture) => fixture.home), + ); + const actualFs = + await vi.importActual( + "node:fs/promises", + ); + let directoryOpenAttempts = 0; + let fifthHomeOpenAttempts = 0; + let observeFourOpens!: () => void; + const fourOpens = new Promise((resolveOpens) => { + observeFourOpens = resolveOpens; + }); + vi.doMock("node:fs/promises", () => ({ + ...actualFs, + open: async ( + path: Parameters[0], + flags: Parameters[1], + mode?: Parameters[2], + ): Promise => { + const pathname = String(path); + if (retainedHomes.has(pathname)) { + directoryOpenAttempts += 1; + if (directoryOpenAttempts === 4) observeFourOpens(); + return await new Promise(() => undefined); + } + if (pathname === fixtures[4].home) fifthHomeOpenAttempts += 1; + return await actualFs.open(path, flags, mode); + }, + })); + const spawnMock = vi.fn(() => { + const child = Object.assign(new EventEmitter(), { + kill: vi.fn(() => true), + pid: 12345, + unref: vi.fn(), + }) as unknown as ChildProcess; + queueMicrotask(() => child.emit("exit", 0, null)); + return child; + }); + vi.doMock("node:child_process", () => ({ spawn: spawnMock })); + vi.resetModules(); + const firstCredentials = await import("./codex-credentials.js"); + vi.useFakeTimers(); + + const firstStaging = Promise.all( + fixtures.slice(0, 4).map((fixture) => + firstCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }), + ), + ); + let leases: Awaited>[] = + []; + try { + await fourOpens; + await vi.advanceTimersByTimeAsync(1_001); + leases = await firstStaging; + const registry = ( + globalThis as typeof globalThis & { + __paperclipDirectorySyncHelperRegistryV1?: { + activeParentOperations: Set; + }; + } + ).__paperclipDirectorySyncHelperRegistryV1; + expect(registry?.activeParentOperations.size).toBe(4); + + vi.resetModules(); + const freshCredentials = await import("./codex-credentials.js"); + const fifthLease = await freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixtures[4].home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + leases.push(fifthLease); + expect(directoryOpenAttempts).toBe(4); + expect(fifthHomeOpenAttempts).toBe(0); + expect(spawnMock).toHaveBeenCalled(); + } finally { + await Promise.allSettled(leases.map((lease) => lease.close())); + } + }, + ); + + it.runIf(process.platform !== "win32")( + "isolates retries after a directory open never settles", + async () => { + const fixture = await credentialFixture(); + const actualFs = + await vi.importActual( + "node:fs/promises", + ); + let directoryOpenAttempts = 0; + let observeFirstOpen!: () => void; + const firstOpen = new Promise((resolveOpen) => { + observeFirstOpen = resolveOpen; + }); + const retainedOpen = new Promise(() => undefined); + vi.doMock("node:fs/promises", () => ({ + ...actualFs, + open: async ( + path: Parameters[0], + flags: Parameters[1], + mode?: Parameters[2], + ): Promise => { + if (String(path) === fixture.home) { + directoryOpenAttempts += 1; + if (directoryOpenAttempts === 1) { + observeFirstOpen(); + return await retainedOpen; + } + } + return await actualFs.open(path, flags, mode); + }, + })); + vi.resetModules(); + const freshCredentials = await import("./codex-credentials.js"); + vi.useFakeTimers(); + + const staging = freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + await firstOpen; + await vi.advanceTimersByTimeAsync(1_001); + const lease = await staging; + expect(directoryOpenAttempts).toBe(1); + await lease.close(); + const successor = await freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + await successor.close(); + expect(directoryOpenAttempts).toBe(1); + }, + ); + + it.runIf(process.platform !== "win32")( + "isolates retries after a directory fsync never settles", + async () => { + const fixture = await credentialFixture(); + const actualFs = + await vi.importActual( + "node:fs/promises", + ); + let directorySyncAttempts = 0; + let observeFirstSync!: () => void; + const firstSync = new Promise((resolveSync) => { + observeFirstSync = resolveSync; + }); + const stalledDirectoryHandle = { + close: async (): Promise => undefined, + sync: async (): Promise => { + directorySyncAttempts += 1; + observeFirstSync(); + return await new Promise(() => undefined); + }, + } as FileHandle; + vi.doMock("node:fs/promises", () => ({ + ...actualFs, + open: async ( + path: Parameters[0], + flags: Parameters[1], + mode?: Parameters[2], + ): Promise => { + if (String(path) === fixture.home) return stalledDirectoryHandle; + return await actualFs.open(path, flags, mode); + }, + })); + vi.resetModules(); + const freshCredentials = await import("./codex-credentials.js"); + vi.useFakeTimers(); + + const staging = freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + await firstSync; + await vi.advanceTimersByTimeAsync(1_001); + const lease = await staging; + await lease.close(); + const successor = await freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + await successor.close(); + expect(directorySyncAttempts).toBe(1); + }, + ); + + it.runIf(process.platform !== "win32")( + "isolates later syncs after a directory close never settles", + async () => { + const fixture = await credentialFixture(); + const actualFs = + await vi.importActual( + "node:fs/promises", + ); + let directoryCloseAttempts = 0; + let observeFirstClose!: () => void; + const firstClose = new Promise((resolveClose) => { + observeFirstClose = resolveClose; + }); + const retainedClose = new Promise(() => undefined); + vi.doMock("node:fs/promises", () => ({ + ...actualFs, + open: async ( + path: Parameters[0], + flags: Parameters[1], + mode?: Parameters[2], + ): Promise => { + if (String(path) === fixture.home) { + return { + close: async (): Promise => { + directoryCloseAttempts += 1; + if (directoryCloseAttempts === 1) { + observeFirstClose(); + return await retainedClose; + } + }, + sync: async (): Promise => undefined, + } as FileHandle; + } + return await actualFs.open(path, flags, mode); + }, + })); + vi.resetModules(); + const freshCredentials = await import("./codex-credentials.js"); + vi.useFakeTimers(); + + const staging = freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + await firstClose; + await vi.advanceTimersByTimeAsync(1_001); + const lease = await staging; + expect(directoryCloseAttempts).toBe(1); + await lease.close(); + const successor = await freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + await successor.close(); + expect(directoryCloseAttempts).toBe(1); + }, + ); + + it.runIf(process.platform !== "win32")( + "permanently fails closed when a killed sync helper never exits", + async () => { + const fixture = await credentialFixture(); + const actualFs = + await vi.importActual( + "node:fs/promises", + ); + let observeDirectoryOpen!: () => void; + const directoryOpen = new Promise((resolveOpen) => { + observeDirectoryOpen = resolveOpen; + }); + let directoryOpenAttempts = 0; + vi.doMock("node:fs/promises", () => ({ + ...actualFs, + open: async ( + path: Parameters[0], + flags: Parameters[1], + mode?: Parameters[2], + ): Promise => { + if (String(path) === fixture.home) { + directoryOpenAttempts += 1; + observeDirectoryOpen(); + return await new Promise(() => undefined); + } + return await actualFs.open(path, flags, mode); + }, + })); + let spawnAttempts = 0; + const stuckChild = Object.assign(new EventEmitter(), { + kill: vi.fn(() => true), + // Signal-0 must observe a live process for this retained-helper fixture. + pid: process.pid, + unref: vi.fn(), + }) as unknown as ChildProcess; + vi.doMock("node:child_process", () => ({ + spawn: vi.fn(() => { + spawnAttempts += 1; + return stuckChild; + }), + })); + vi.resetModules(); + const freshCredentials = await import("./codex-credentials.js"); + vi.useFakeTimers(); + + const staging = freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + const rejection = expect(staging).rejects.toThrow( + /remained non-durable after 1 attempt/, + ); + await directoryOpen; + await vi.advanceTimersByTimeAsync(1_001); + await vi.advanceTimersByTimeAsync(1_001); + await vi.advanceTimersByTimeAsync(1_001); + await rejection; + expect(spawnAttempts).toBe(1); + expect(directoryOpenAttempts).toBe(1); + expect(stuckChild.unref).toHaveBeenCalledOnce(); + + await expect( + freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }), + ).rejects.toThrow(/remained non-durable after 1 attempt/); + expect(spawnAttempts).toBe(1); + expect(directoryOpenAttempts).toBe(1); + stuckChild.emit("exit", null, "SIGKILL"); + }, + ); + + it.runIf(process.platform !== "win32").each([ + ["returns false", (): boolean => false], + [ + "throws", + (): boolean => { + throw new Error("injected kill failure"); + }, + ], + ])( + "permanently fences a home when helper kill %s", + async (_label, killImplementation) => { + const fixture = await credentialFixture(); + const actualFs = + await vi.importActual( + "node:fs/promises", + ); + let observeDirectoryOpen!: () => void; + const directoryOpen = new Promise((resolveOpen) => { + observeDirectoryOpen = resolveOpen; + }); + vi.doMock("node:fs/promises", () => ({ + ...actualFs, + open: async ( + path: Parameters[0], + flags: Parameters[1], + mode?: Parameters[2], + ): Promise => { + if (String(path) === fixture.home) { + observeDirectoryOpen(); + return await new Promise(() => undefined); + } + return await actualFs.open(path, flags, mode); + }, + })); + const child = Object.assign(new EventEmitter(), { + kill: vi.fn(killImplementation), + // Signal-0 must observe a live process for this retained-helper fixture. + pid: process.pid, + unref: vi.fn(), + }) as unknown as ChildProcess; + const spawnMock = vi.fn(() => child); + vi.doMock("node:child_process", () => ({ spawn: spawnMock })); + vi.resetModules(); + const freshCredentials = await import("./codex-credentials.js"); + vi.useFakeTimers(); + + const staging = freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + const rejection = expect(staging).rejects.toThrow( + /remained non-durable after 1 attempt/, + ); + await directoryOpen; + await vi.advanceTimersByTimeAsync(1_001); + await vi.advanceTimersByTimeAsync(1_001); + await rejection; + await expect( + freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }), + ).rejects.toThrow(/remained non-durable after 1 attempt/); + expect(spawnMock).toHaveBeenCalledOnce(); + expect(child.unref).toHaveBeenCalledOnce(); + child.emit("exit", null, "SIGKILL"); + }, + ); + + it.runIf(process.platform !== "win32")( + "permanently fences a home after an asynchronous helper error", + async () => { + const fixture = await credentialFixture(); + const actualFs = + await vi.importActual( + "node:fs/promises", + ); + let observeDirectoryOpen!: () => void; + const directoryOpen = new Promise((resolveOpen) => { + observeDirectoryOpen = resolveOpen; + }); + vi.doMock("node:fs/promises", () => ({ + ...actualFs, + open: async ( + path: Parameters[0], + flags: Parameters[1], + mode?: Parameters[2], + ): Promise => { + if (String(path) === fixture.home) { + observeDirectoryOpen(); + return await new Promise(() => undefined); + } + return await actualFs.open(path, flags, mode); + }, + })); + const child = Object.assign(new EventEmitter(), { + kill: vi.fn(() => true), + // Signal-0 must observe a live process for this retained-helper fixture. + pid: process.pid, + unref: vi.fn(), + }) as unknown as ChildProcess; + const spawnMock = vi.fn(() => { + queueMicrotask(() => child.emit("error", new Error("spawn failed"))); + return child; + }); + vi.doMock("node:child_process", () => ({ spawn: spawnMock })); + vi.resetModules(); + const freshCredentials = await import("./codex-credentials.js"); + vi.useFakeTimers(); + + const staging = freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + const rejection = expect(staging).rejects.toThrow( + /remained non-durable after 1 attempt/, + ); + await directoryOpen; + await vi.advanceTimersByTimeAsync(1_001); + await rejection; + await expect( + freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }), + ).rejects.toThrow(/remained non-durable after 1 attempt/); + expect(spawnMock).toHaveBeenCalledOnce(); + child.emit("exit", null, null); + }, + ); + + it.runIf(process.platform !== "win32")( + "shares the helper process cap across fresh module instances", + async () => { + const fixture = await credentialFixture(); + const registry = ( + globalThis as typeof globalThis & { + __paperclipDirectorySyncHelperRegistryV1?: { + activeChildren: Set; + }; + } + ).__paperclipDirectorySyncHelperRegistryV1; + expect(registry).toBeDefined(); + const reservations = Array.from( + { length: 4 }, + () => new EventEmitter() as unknown as ChildProcess, + ); + for (const reservation of reservations) { + registry!.activeChildren.add(reservation); + } + const actualFs = + await vi.importActual( + "node:fs/promises", + ); + let observeDirectoryOpen!: () => void; + const directoryOpen = new Promise((resolveOpen) => { + observeDirectoryOpen = resolveOpen; + }); + vi.doMock("node:fs/promises", () => ({ + ...actualFs, + open: async ( + path: Parameters[0], + flags: Parameters[1], + mode?: Parameters[2], + ): Promise => { + if (String(path) === fixture.home) { + observeDirectoryOpen(); + return await new Promise(() => undefined); + } + return await actualFs.open(path, flags, mode); + }, + })); + const spawnMock = vi.fn(() => { + throw new Error("helper cap was bypassed"); + }); + vi.doMock("node:child_process", () => ({ spawn: spawnMock })); + vi.resetModules(); + const freshCredentials = await import("./codex-credentials.js"); + vi.useFakeTimers(); + try { + const staging = freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + const rejection = expect(staging).rejects.toThrow( + /remained non-durable after 8 attempts/, + ); + await directoryOpen; + await vi.advanceTimersByTimeAsync(3_000); + await rejection; + expect(spawnMock).not.toHaveBeenCalled(); + } finally { + for (const reservation of reservations) { + registry!.activeChildren.delete(reservation); + } + } + }, + ); + + it.runIf(process.platform !== "win32")( + "reclaims kernel-confirmed helper exits when child events are lost", + async () => { + const fixture = await credentialFixture(); + const registry = ( + globalThis as typeof globalThis & { + __paperclipDirectorySyncHelperRegistryV1?: { + activeChildren: Set; + childDirectories: Map; + activeParentOperations: Set; + failedHomes: Set; + stuckChildren: Map; + }; + } + ).__paperclipDirectorySyncHelperRegistryV1; + expect(registry).toBeDefined(); + + const parentReservations = Array.from({ length: 4 }, () => Symbol()); + for (const reservation of parentReservations) { + registry!.activeParentOperations.add(reservation); + } + const departedPids = new Set(); + const departedHelpers = Array.from({ length: 4 }, (_, index) => { + const pid = 41_000 + index; + departedPids.add(pid); + const child = Object.assign(new EventEmitter(), { + exitCode: null, + kill: vi.fn(() => true), + pid, + signalCode: null, + unref: vi.fn(), + }) as unknown as ChildProcess; + const directory = `/departed-credential-helper-${String(index)}`; + registry!.activeChildren.add(child); + registry!.childDirectories.set(child, directory); + if (index < 2) { + registry!.failedHomes.add(directory); + registry!.stuckChildren.set(directory, child); + } + return { child, directory }; + }); + const killSpy = vi + .spyOn(process, "kill") + .mockImplementation((pid, signal) => { + if (signal === 0 && departedPids.has(Number(pid))) { + throw Object.assign(new Error("process no longer exists"), { + code: "ESRCH", + }); + } + return true; + }); + const observedActiveChildren: number[] = []; + let nextPid = 42_000; + const spawnMock = vi.fn(() => { + observedActiveChildren.push(registry!.activeChildren.size); + const child = Object.assign(new EventEmitter(), { + exitCode: null, + kill: vi.fn(() => true), + pid: nextPid++, + signalCode: null, + unref: vi.fn(), + }) as unknown as ChildProcess; + queueMicrotask(() => { + child.emit("exit", 0, null); + child.emit("close", 0, null); + }); + return child; + }); + vi.doMock("node:child_process", () => ({ spawn: spawnMock })); + vi.resetModules(); + const freshCredentials = await import("./codex-credentials.js"); + + try { + const lease = await freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + await lease.close(); + + expect(spawnMock).toHaveBeenCalled(); + expect(observedActiveChildren.every((count) => count < 4)).toBe(true); + for (const { child, directory } of departedHelpers) { + expect(registry!.activeChildren).not.toContain(child); + expect(registry!.childDirectories.has(child)).toBe(false); + expect(registry!.failedHomes).not.toContain(directory); + expect(registry!.stuckChildren.has(directory)).toBe(false); + } + } finally { + killSpy.mockRestore(); + for (const reservation of parentReservations) { + registry!.activeParentOperations.delete(reservation); + } + for (const { child, directory } of departedHelpers) { + registry!.activeChildren.delete(child); + registry!.childDirectories.delete(child); + registry!.failedHomes.delete(directory); + registry!.stuckChildren.delete(directory); + } + } + }, + ); + + it.runIf(process.platform !== "win32")( + "keeps intent-publication failure before credential mutation and scrubs without process memory", + async () => { + const fixture = await credentialFixture(); + const destination = join(fixture.home, "auth.json"); + const cleanupIntent = join( + fixture.home, + ".paperclip-auth-cleanup-required", + ); + const probe = await open(fixture.home, "r"); + const prototype = Object.getPrototypeOf(probe) as { + sync(this: FileHandle): Promise; + }; + await probe.close(); + const originalSync = prototype.sync; + let directorySyncAttempts = 0; + const syncSpy = vi + .spyOn(prototype, "sync") + .mockImplementation(async function (this: FileHandle): Promise { + if ((await this.stat()).isDirectory()) { + directorySyncAttempts += 1; + if (directorySyncAttempts > 1) { + throw new Error("persistent intent sync failure"); + } + } + await originalSync.call(this); + }); + try { + await expect( + stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" }, + }), + ).rejects.toThrow("remained non-durable after 8 attempts"); + await expect(readFile(destination)).rejects.toMatchObject({ + code: "ENOENT", + }); + + // Model a crash losing the unsynced intent directory entry and the + // next process finding an unexpected auth file. A fresh module has no + // quarantine map from the failed process, so admission must rely on + // the isolated-home scrub rather than process memory. + syncSpy.mockRestore(); + await rm(cleanupIntent, { force: true }); + await writeFile(destination, '{"orphaned":true}', { mode: 0o600 }); + vi.resetModules(); + const freshCredentials = await import("./codex-credentials.js"); + const persistentSyncFailure = vi + .spyOn(prototype, "sync") + .mockImplementation(async function (this: FileHandle): Promise { + if ((await this.stat()).isDirectory()) { + throw new Error("persistent recovery sync failure"); + } + await originalSync.call(this); + }); + try { + await expect( + freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }), + ).rejects.toThrow("remained non-durable after 8 attempts"); + await expect(readFile(destination)).rejects.toMatchObject({ + code: "ENOENT", + }); + } finally { + persistentSyncFailure.mockRestore(); + } + } finally { + syncSpy.mockRestore(); + } + }, + ); + + it.runIf(process.platform !== "win32")( + "owns cleanup when post-rename directory durability fails", + async () => { + const fixture = await credentialFixture(); + const destination = join(fixture.home, "auth.json"); + const probe = await open(fixture.home, "r"); + const prototype = Object.getPrototypeOf(probe) as { + sync(this: FileHandle): Promise; + }; + await probe.close(); + const originalSync = prototype.sync; + let directorySyncAttempts = 0; + const syncSpy = vi + .spyOn(prototype, "sync") + .mockImplementation(async function (this: FileHandle): Promise { + if ((await this.stat()).isDirectory()) { + directorySyncAttempts += 1; + if (directorySyncAttempts > 2) { + throw new Error("persistent post-rename sync failure"); + } + } + await originalSync.call(this); + }); + try { + await expect( + stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" }, + }), + ).rejects.toThrow("remained non-durable after 8 attempts"); + await expect(readFile(destination, "utf8")).resolves.toBe("{}"); + + syncSpy.mockRestore(); + const lease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" }, + }); + await expect(readFile(destination, "utf8")).resolves.toBe("{}"); + await lease.close(); + } finally { + syncSpy.mockRestore(); + } + }, + ); + + it("rejects missing, ambiguous, malformed, and unsafe sources", async () => { + const fixture = await credentialFixture(); + await expect( + stageManagedCodexCredential({ agentHomeDirectory: fixture.home }), + ).rejects.toThrow(/credential missing/); + await expect( + stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + OPENAI_API_KEY: "key", + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}", + }, + }), + ).rejects.toThrow(/ambiguous/); + await expect( + stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "[]" }, + }), + ).rejects.toThrow(/malformed/); + + const source = join(fixture.root, "unsafe-auth.json"); + await writeFile(source, "{}", { mode: 0o644 }); + await chmod(source, 0o644); + if (process.platform !== "win32") { + await expect( + stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + sourcePath: source, + }), + ).rejects.toThrow(/permissions are unsafe/); + } + }); + + it.runIf(process.platform !== "win32")( + "rejects a credential home that is not private", + async () => { + const fixture = await credentialFixture(); + await chmod(fixture.home, 0o755); + + await expect( + stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }), + ).rejects.toThrow(/home permissions are unsafe/); + }, + ); + + it.runIf(process.platform !== "win32")( + "rejects a symbolic-link source", + async () => { + const fixture = await credentialFixture(); + const target = join(fixture.root, "auth-target.json"); + const source = join(fixture.root, "auth-link.json"); + await writeFile(target, "{}", { mode: 0o600 }); + await symlink(target, source); + + await expect( + stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + sourcePath: source, + }), + ).rejects.toThrow(/credential missing/); + }, + ); + + it.runIf(process.platform !== "win32")( + "replaces a stale destination link without touching its target", + async () => { + const fixture = await credentialFixture(); + const target = join(fixture.root, "outside.json"); + const destination = join(fixture.home, "auth.json"); + await writeFile(target, '{"outside":true}', { mode: 0o600 }); + await symlink(target, destination); + + const lease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" }, + }); + await expect(readFile(target, "utf8")).resolves.toBe('{"outside":true}'); + expect((await stat(lease.path)).isFile()).toBe(true); + await lease.close(); + }, + ); +}); + +async function credentialFixture(): Promise<{ root: string; home: string }> { + const root = await mkdtemp(join(tmpdir(), "paperclip-acpx-credential-")); + temporaryDirectories.push(root); + const home = join(root, "codex-home"); + await mkdir(home, { mode: 0o700 }); + await chmod(home, 0o700); + return { root, home }; +} + +function credentialLeasePorts(home: string): readonly number[] { + const userScope = + typeof process.getuid === "function" ? String(process.getuid()) : "win32"; + return credentialLeasePortsForScope(userScope, home); +} + +function credentialLeasePortsForScope( + userScope: string, + home: string, +): readonly number[] { + const digest = createHash("sha256") + .update("paperclip-managed-codex-lease-v2:") + .update(userScope) + .update("\0") + .update(home) + .digest(); + const start = digest.readUInt16BE(0) % 16_384; + const step = (digest.readUInt16BE(2) | 1) % 16_384; + return Array.from( + { length: 3 }, + (_, index) => 49_152 + ((start + index * step) % 16_384), + ); +} + +async function listenSilently( + port: number, +): Promise<{ close(): Promise }> { + const sockets = new Set(); + const server: Server = createServer((socket) => { + sockets.add(socket); + socket.once("close", () => sockets.delete(socket)); + socket.pause(); + }); + await new Promise((resolveListen, rejectListen) => { + server.once("error", rejectListen); + server.listen( + { + exclusive: true, + host: "127.0.0.1", + port, + }, + resolveListen, + ); + }); + return { + async close(): Promise { + for (const socket of sockets) socket.destroy(); + await new Promise((resolveClose, rejectClose) => { + server.close((error) => { + if (error) rejectClose(error); + else resolveClose(); + }); + }); + }, + }; +} + +async function waitForChildMessage( + child: ChildProcess, + type: string, +): Promise { + let diagnostic = ""; + child.stderr?.on("data", (chunk: Buffer) => { + diagnostic = `${diagnostic}${chunk.toString("utf8")}`.slice(-4_096); + }); + await new Promise((resolveMessage, rejectMessage) => { + const timeout = setTimeout(() => { + rejectMessage(new Error(`child message timed out: ${diagnostic}`)); + }, 10_000); + const finish = (error?: Error): void => { + clearTimeout(timeout); + child.off("message", onMessage); + child.off("exit", onExit); + if (error) rejectMessage(error); + else resolveMessage(); + }; + const onMessage = (message: unknown): void => { + if ( + typeof message === "object" && + message !== null && + "type" in message && + (message as { type?: unknown }).type === type + ) { + finish(); + } + }; + const onExit = (code: number | null): void => { + finish( + new Error( + `credential owner exited before ${type} (code ${String(code)}): ${diagnostic}`, + ), + ); + }; + child.on("message", onMessage); + child.once("exit", onExit); + }); +} + +async function waitForChildExit(child: ChildProcess): Promise { + if (child.exitCode !== null || child.signalCode !== null) return; + await new Promise((resolveExit, rejectExit) => { + const timeout = setTimeout( + () => rejectExit(new Error("credential owner exit timed out")), + 10_000, + ); + child.once("exit", () => { + clearTimeout(timeout); + resolveExit(); + }); + }); +} diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-credentials.ts b/packages/paperclip-runner/src/drivers/acpx/codex-credentials.ts new file mode 100644 index 0000000000..5abec9e887 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/acpx/codex-credentials.ts @@ -0,0 +1,1231 @@ +import { spawn, type ChildProcess } from "node:child_process"; +import { createHash } from "node:crypto"; +import { constants } from "node:fs"; +import { + lstat, + open, + realpath, + rename, + unlink, + type FileHandle, +} from "node:fs/promises"; +import { createServer, type Server } from "node:net"; +import { isAbsolute, join, resolve } from "node:path"; + +const MAX_CODEX_CREDENTIAL_BYTES = 256 * 1024; +const PRIVATE_FILE_MODE = 0o600; +const MAX_DIRECTORY_SYNC_ATTEMPTS = 8; +const DIRECTORY_SYNC_OPERATION_TIMEOUT_MS = 1_000; +const MAX_AUTONOMOUS_CREDENTIAL_CLEANUP_ATTEMPTS = 8; +const CREDENTIAL_CLEANUP_INTENT = ".paperclip-auth-cleanup-required"; +const CREDENTIAL_STAGING_FILE = ".paperclip-auth-staging-v1"; +const CREDENTIAL_LEASE_HOST = "127.0.0.1"; +const CREDENTIAL_LEASE_PORT_MIN = 49_152; +const CREDENTIAL_LEASE_PORT_COUNT = 16_384; +const CREDENTIAL_LEASE_CANDIDATES = 3; +const CREDENTIAL_LEASE_QUORUM = 2; +const DIRECTORY_SYNC_HELPER_KILL_ACK_TIMEOUT_MS = 1_000; +const MAX_DIRECTORY_SYNC_HELPERS = 4; +const MAX_PARENT_DIRECTORY_SYNC_OPERATIONS = 4; +const DIRECTORY_SYNC_HELPER_SOURCE = String.raw` +import { constants } from "node:fs"; +import { open } from "node:fs/promises"; + +const directory = process.argv[1]; +if (typeof directory !== "string") throw new Error("directory is required"); +const handle = await open( + directory, + constants.O_RDONLY | (constants.O_DIRECTORY ?? 0), +); +try { + await handle.sync(); +} finally { + await handle.close(); +} +`; + +interface CredentialHomeLock { + assertHeld(): void; + release(): Promise; +} + +interface QuarantinedCredentialCleanup { + path: string; + home: string; + intentPath: string; + ownerGeneration: CredentialLeaseGeneration; + lock: CredentialHomeLock; + recovery: Promise | null; +} + +type CredentialLeaseGeneration = number; + +interface DirectorySyncHelperRegistry { + activeChildren: Set; + childDirectories: Map; + activeParentOperations: Set; + attempts: Map>; + failedHomes: Set; + isolatedHomes: Set; + pendingCleanups: Map>; + stuckChildren: Map; +} + +const quarantinedCredentialCleanups = new Map< + string, + QuarantinedCredentialCleanup +>(); +const processDirectorySyncHelperState = globalThis as typeof globalThis & { + __paperclipDirectorySyncHelperRegistryV1?: DirectorySyncHelperRegistry; +}; +const directorySyncHelperRegistry = + processDirectorySyncHelperState.__paperclipDirectorySyncHelperRegistryV1 ?? { + activeChildren: new Set(), + childDirectories: new Map(), + activeParentOperations: new Set(), + attempts: new Map>(), + failedHomes: new Set(), + isolatedHomes: new Set(), + pendingCleanups: new Map>(), + stuckChildren: new Map(), + }; +processDirectorySyncHelperState.__paperclipDirectorySyncHelperRegistryV1 = + directorySyncHelperRegistry; +const pendingDirectorySyncCleanups = + directorySyncHelperRegistry.pendingCleanups; +const isolatedDirectorySyncHomes = directorySyncHelperRegistry.isolatedHomes; +const isolatedDirectorySyncAttempts = directorySyncHelperRegistry.attempts; +const failedIsolatedDirectorySyncHomes = + directorySyncHelperRegistry.failedHomes; +const stuckDirectorySyncHelpers = directorySyncHelperRegistry.stuckChildren; +const directorySyncHelperDirectories = + directorySyncHelperRegistry.childDirectories; +const activeCredentialLeaseGenerations = new Map< + string, + CredentialLeaseGeneration +>(); +let nextCredentialLeaseGeneration = 0; + +export type ManagedCodexCredentialMode = + "api_key" | "inline_json" | "managed_file"; + +export interface ManagedCodexCredentialLease { + readonly path: string; + readonly mode: ManagedCodexCredentialMode; + close(): Promise; +} + +/** Stage one explicit Codex authentication source in its isolated runtime home. */ +export async function stageManagedCodexCredential(input: { + agentHomeDirectory: string; + environment?: NodeJS.ProcessEnv; + sourcePath?: string; +}): Promise { + const home = await realpath(input.agentHomeDirectory); + const homeMetadata = await lstat(home); + if (!homeMetadata.isDirectory() || homeMetadata.isSymbolicLink()) { + throw new Error("Managed Codex credential home must be a real directory"); + } + if ( + process.platform !== "win32" && + ((homeMetadata.mode & 0o077) !== 0 || + (typeof process.getuid === "function" && + homeMetadata.uid !== process.getuid())) + ) { + throw new Error("Managed Codex credential home permissions are unsafe"); + } + // Join an older failed close before claiming the next generation. This + // keeps quarantine recovery authoritative over the shared paths without + // mistaking a waiting admission for an already-active successor. + await recoverQuarantinedCredentialCleanup(join(home, "auth.json"), home); + const ownerGeneration = claimCredentialLeaseGeneration(home); + let lock: CredentialHomeLock | null = null; + try { + lock = await acquireCredentialHomeLock(home); + return await stageClaimedManagedCodexCredential( + input, + home, + ownerGeneration, + lock, + ); + } catch (error) { + if ( + lock !== null && + quarantinedCredentialCleanups.get(home)?.lock !== lock + ) { + await lock.release().catch(() => undefined); + } + releaseCredentialLeaseGeneration(home, ownerGeneration); + throw error; + } +} + +async function stageClaimedManagedCodexCredential( + input: { + environment?: NodeJS.ProcessEnv; + sourcePath?: string; + }, + home: string, + ownerGeneration: CredentialLeaseGeneration, + lock: CredentialHomeLock, +): Promise { + const destination = join(home, "auth.json"); + const stagingPath = join(home, CREDENTIAL_STAGING_FILE); + const intentPath = join(home, CREDENTIAL_CLEANUP_INTENT); + lock.assertHeld(); + await recoverPersistedCredentialCleanup( + destination, + stagingPath, + home, + intentPath, + ); + // The isolated home is itself the durable recovery anchor. Scrub both the + // installed credential and the deterministic staging pathname before every + // admission, even when a prior cleanup-intent entry was lost with a runner + // crash. Only after their absence is durable may a new intent be created. + await removeCredentialArtifacts([destination, stagingPath], home); + const environment = input.environment ?? {}; + const hasApiKey = Boolean( + environment.CODEX_API_KEY || environment.OPENAI_API_KEY, + ); + const inlineJson = environment.PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET; + const hasInlineJson = typeof inlineJson === "string" && inlineJson.length > 0; + const hasManagedFile = + typeof input.sourcePath === "string" && input.sourcePath.length > 0; + const sourceCount = [hasApiKey, hasInlineJson, hasManagedFile].filter( + Boolean, + ).length; + if (sourceCount === 0) { + throw new Error( + "provider_initialize_protocol_error: provider=acpx stage=credential.stage managed Codex credential missing", + ); + } + if (sourceCount !== 1) { + throw new Error("Managed Codex credential source is ambiguous"); + } + + if ( + hasManagedFile && + (!isAbsolute(input.sourcePath!) || + resolve(input.sourcePath!) === destination) + ) { + throw new Error( + "Managed Codex credential source must be an external absolute path", + ); + } + + if (hasApiKey) { + // Codex will read the API key from the launch environment. Persist cleanup + // intent before admitting the provider and retain it for the lease + // lifetime, so a replacement runner removes provider-generated auth after + // a crash before it admits another provider. + // Failure while publishing the intent cannot strand a staged credential: + // the unconditional admission scrub above is already durable and this + // mode has not allowed the provider to create auth.json yet. + await createCredentialCleanupIntent(intentPath, home); + return credentialLease( + destination, + home, + intentPath, + "api_key", + ownerGeneration, + lock, + ); + } + + const credential = hasInlineJson + ? boundedInlineCredential(inlineJson!) + : await readManagedCredential(input.sourcePath!); + try { + validateCredentialDocument(credential); + // As with API-key mode, an intent-publication failure occurs before any + // credential mutation and therefore needs no process-only quarantine. + await createCredentialCleanupIntent(intentPath, home); + try { + await writeCredential(destination, stagingPath, home, credential); + } catch (error) { + // Rename may already have installed the credential before directory + // durability failed. Retain a bounded process owner and the persisted + // intent so later staging must recover both before admission. + quarantineCredentialCleanup( + destination, + home, + intentPath, + ownerGeneration, + lock, + ); + throw error; + } + } finally { + credential.fill(0); + } + return credentialLease( + destination, + home, + intentPath, + hasInlineJson ? "inline_json" : "managed_file", + ownerGeneration, + lock, + ); +} + +function claimCredentialLeaseGeneration( + home: string, +): CredentialLeaseGeneration { + if (activeCredentialLeaseGenerations.has(home)) { + throw new Error( + "Managed Codex credential home already has an active lease", + ); + } + if (nextCredentialLeaseGeneration >= Number.MAX_SAFE_INTEGER) { + throw new Error("Managed Codex credential lease generation exhausted"); + } + nextCredentialLeaseGeneration += 1; + activeCredentialLeaseGenerations.set(home, nextCredentialLeaseGeneration); + return nextCredentialLeaseGeneration; +} + +function releaseCredentialLeaseGeneration( + home: string, + ownerGeneration: CredentialLeaseGeneration, +): void { + if (activeCredentialLeaseGenerations.get(home) === ownerGeneration) { + activeCredentialLeaseGenerations.delete(home); + } +} + +async function acquireCredentialHomeLock( + home: string, +): Promise { + // This is deliberately markerless: authority is the live kernel ownership + // of any two candidates. Any two subsets of a three-port set intersect, so + // contenders cannot both reach quorum; one unrelated occupied listener is + // tolerated without probing or trusting the process behind it. + const servers: Server[] = []; + let invalid: Error | null = null; + let released = false; + try { + for (const port of credentialLeasePorts(home)) { + const server = createServer((socket) => socket.destroy()); + try { + await listenForCredentialLease(server, port); + } catch (error) { + await closeCredentialLeaseServer(server); + if (errorCode(error) === "EADDRINUSE") continue; + throw new Error( + "Managed Codex credential ownership could not be established", + { cause: error }, + ); + } + server.on("error", (error) => { + invalid ??= error; + }); + server.on("close", () => { + if (!released) { + invalid ??= new Error( + "Managed Codex credential ownership listener closed unexpectedly", + ); + } + }); + servers.push(server); + if (servers.length === CREDENTIAL_LEASE_QUORUM) break; + } + if (servers.length !== CREDENTIAL_LEASE_QUORUM) { + throw new Error( + "Managed Codex credential home already has an active lease", + ); + } + } catch (error) { + released = true; + await Promise.allSettled(servers.map(closeCredentialLeaseServer)); + throw error; + } + + return Object.freeze({ + assertHeld(): void { + if ( + released || + invalid !== null || + servers.filter((server) => server.listening).length < + CREDENTIAL_LEASE_QUORUM + ) { + throw new Error("Managed Codex credential ownership was lost"); + } + }, + async release(): Promise { + if (released) return; + const outcomes = await Promise.allSettled( + servers.map(closeCredentialLeaseServer), + ); + const listenersStillHeld = servers.filter( + (server) => server.listening, + ).length; + if (listenersStillHeld >= CREDENTIAL_LEASE_QUORUM) { + const failure = outcomes.find( + (outcome): outcome is PromiseRejectedResult => + outcome.status === "rejected", + ); + throw new Error( + "Managed Codex credential ownership could not be released", + { cause: failure?.reason }, + ); + } + // Once fewer than two listeners remain, quorum authority is gone. Never + // throw after that point: a stale quarantine must not touch a successor. + released = true; + }, + }); +} + +function credentialLeasePorts(home: string): readonly number[] { + const userScope = + typeof process.getuid === "function" ? String(process.getuid()) : "win32"; + const digest = createHash("sha256") + .update("paperclip-managed-codex-lease-v2:") + .update(userScope) + .update("\0") + .update(home) + .digest(); + const start = digest.readUInt16BE(0) % CREDENTIAL_LEASE_PORT_COUNT; + const step = (digest.readUInt16BE(2) | 1) % CREDENTIAL_LEASE_PORT_COUNT; + return Array.from( + { length: CREDENTIAL_LEASE_CANDIDATES }, + (_, index) => + CREDENTIAL_LEASE_PORT_MIN + + ((start + index * step) % CREDENTIAL_LEASE_PORT_COUNT), + ); +} + +async function listenForCredentialLease( + server: Server, + port: number, +): Promise { + await new Promise((resolveListen, rejectListen) => { + const onError = (error: Error): void => { + server.off("listening", onListening); + rejectListen(error); + }; + const onListening = (): void => { + server.off("error", onError); + resolveListen(); + }; + server.once("error", onError); + server.once("listening", onListening); + server.listen({ + exclusive: true, + host: CREDENTIAL_LEASE_HOST, + port, + }); + }); +} + +async function closeCredentialLeaseServer(server: Server): Promise { + if (!server.listening) return; + await new Promise((resolveClose, rejectClose) => { + server.close((error) => { + if (error) rejectClose(error); + else resolveClose(); + }); + }); +} +function quarantineCredentialCleanup( + path: string, + home: string, + intentPath: string, + ownerGeneration: CredentialLeaseGeneration, + lock: CredentialHomeLock, +): void { + const existing = quarantinedCredentialCleanups.get(home); + if (existing !== undefined) return; + const cleanup: QuarantinedCredentialCleanup = { + path, + home, + intentPath, + ownerGeneration, + lock, + recovery: null, + }; + quarantinedCredentialCleanups.set(home, cleanup); + startCredentialCleanupRecovery( + cleanup, + MAX_AUTONOMOUS_CREDENTIAL_CLEANUP_ATTEMPTS, + ); +} + +function startCredentialCleanupRecovery( + cleanup: QuarantinedCredentialCleanup, + maxAttempts: number, +): Promise { + if (cleanup.recovery) return cleanup.recovery; + const recovery = (async () => { + let retryDelayMs = 10; + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + try { + // Never let a stale cleanup callback mutate a successor's credential + // after kernel ownership has been lost. + assertCredentialCleanupAuthority(cleanup); + await removeReplaceableCredential(cleanup.path); + await removeReplaceableCredential( + join(cleanup.home, CREDENTIAL_STAGING_FILE), + ); + assertCredentialCleanupAuthority(cleanup); + await syncDirectory(cleanup.home); + assertCredentialCleanupAuthority(cleanup); + await removeCredentialCleanupIntent(cleanup.intentPath, cleanup.home); + assertCredentialCleanupAuthority(cleanup); + await cleanup.lock.release(); + quarantinedCredentialCleanups.delete(cleanup.home); + return; + } catch { + if (attempt === maxAttempts) return; + await new Promise((resolveRetry) => { + const timer = setTimeout(resolveRetry, retryDelayMs); + timer.unref?.(); + }); + retryDelayMs = Math.min(retryDelayMs * 2, 1_000); + } + } + })(); + cleanup.recovery = recovery; + void recovery + .finally(() => { + if (cleanup.recovery === recovery) cleanup.recovery = null; + }) + .catch(() => undefined); + return recovery; +} + +function assertCredentialCleanupAuthority( + cleanup: QuarantinedCredentialCleanup, +): void { + const activeGeneration = activeCredentialLeaseGenerations.get(cleanup.home); + if ( + activeGeneration !== undefined && + activeGeneration !== cleanup.ownerGeneration + ) { + throw new Error( + "Managed Codex credential cleanup was superseded by an active lease", + ); + } + cleanup.lock.assertHeld(); +} + +async function recoverQuarantinedCredentialCleanup( + path: string, + home: string, +): Promise { + const cleanup = quarantinedCredentialCleanups.get(home); + if (cleanup === undefined) return; + await (cleanup.recovery ?? startCredentialCleanupRecovery(cleanup, 1)); + if (!quarantinedCredentialCleanups.has(home)) return; + const admissionRecovery = startCredentialCleanupRecovery(cleanup, 1); + await admissionRecovery; + if (quarantinedCredentialCleanups.has(home)) { + throw new Error( + `Managed Codex credential cleanup remains non-durable for ${path}`, + ); + } +} + +async function recoverPersistedCredentialCleanup( + path: string, + stagingPath: string, + home: string, + intentPath: string, +): Promise { + if (!(await pathExists(intentPath))) return; + await removeCredentialArtifacts([path, stagingPath], home); + await removeCredentialCleanupIntent(intentPath, home); +} + +function credentialLease( + path: string, + home: string, + intentPath: string, + mode: ManagedCodexCredentialMode, + ownerGeneration: CredentialLeaseGeneration, + lock: CredentialHomeLock, +): ManagedCodexCredentialLease { + // Do not admit a provider if kernel ownership was lost while its credential + // was being staged. + lock.assertHeld(); + let closed = false; + let closeAttempt: Promise | null = null; + return Object.freeze({ + path, + mode, + async close(): Promise { + if (closed) return; + if (closeAttempt !== null) return await closeAttempt; + const attempt = (async () => { + const activeGeneration = activeCredentialLeaseGenerations.get(home); + if (activeGeneration !== ownerGeneration) { + // A failed close releases its generation only after publishing a + // quarantine owner. If no successor exists, synchronously join that + // cleanup before acknowledging the retry. If a successor does + // exist, this stale lease has no authority over its credential. + if (activeGeneration === undefined) { + await recoverQuarantinedCredentialCleanup(path, home); + await lock.release(); + } + closed = true; + return; + } + try { + lock.assertHeld(); + await removeCredential(path, home); + await removeCredentialCleanupIntent(intentPath, home); + await lock.release(); + closed = true; + } catch (error) { + quarantineCredentialCleanup( + path, + home, + intentPath, + ownerGeneration, + lock, + ); + throw error; + } finally { + releaseCredentialLeaseGeneration(home, ownerGeneration); + } + })(); + closeAttempt = attempt; + try { + await attempt; + } finally { + if (closeAttempt === attempt) closeAttempt = null; + } + }, + }); +} + +async function createCredentialCleanupIntent( + intentPath: string, + home: string, +): Promise { + let handle: FileHandle | undefined; + try { + handle = await open( + intentPath, + constants.O_WRONLY | + constants.O_CREAT | + constants.O_EXCL | + (constants.O_NOFOLLOW ?? 0), + PRIVATE_FILE_MODE, + ); + await handle.chmod(PRIVATE_FILE_MODE); + await handle.writeFile("paperclip-managed-codex-cleanup-v1\n", "utf8"); + await handle.sync(); + await handle.close(); + handle = undefined; + await syncDirectoryDurably(home); + } finally { + await handle?.close().catch(() => undefined); + } +} + +async function removeCredentialCleanupIntent( + intentPath: string, + home: string, +): Promise { + await removeReplaceableCredential(intentPath); + await syncDirectoryDurably(home); +} + +async function pathExists(path: string): Promise { + try { + await lstat(path); + return true; + } catch (error) { + if (errorCode(error) === "ENOENT") return false; + throw error; + } +} + +function boundedInlineCredential(value: string): Buffer { + const bytes = Buffer.from(value, "utf8"); + if (bytes.length < 1 || bytes.length > MAX_CODEX_CREDENTIAL_BYTES) { + bytes.fill(0); + throw new Error( + "Managed Codex credential document exceeds its bounded size", + ); + } + return bytes; +} + +async function readManagedCredential(sourcePath: string): Promise { + let handle: FileHandle; + try { + handle = await open( + sourcePath, + constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0), + ); + } catch { + throw new Error( + "provider_initialize_protocol_error: provider=acpx stage=credential.stage managed Codex credential missing", + ); + } + try { + const before = await handle.stat({ bigint: true }); + if ( + !before.isFile() || + before.size < 1n || + before.size > BigInt(MAX_CODEX_CREDENTIAL_BYTES) + ) { + throw new Error( + "Managed Codex credential source is not a bounded regular file", + ); + } + if (process.platform !== "win32" && (before.mode & 0o077n) !== 0n) { + throw new Error("Managed Codex credential source permissions are unsafe"); + } + if ( + process.platform !== "win32" && + typeof process.getuid === "function" && + before.uid !== BigInt(process.getuid()) + ) { + throw new Error("Managed Codex credential source ownership is unsafe"); + } + const bytes = await readHandle(handle, Number(before.size)); + const after = await handle.stat({ bigint: true }); + if ( + before.dev !== after.dev || + before.ino !== after.ino || + before.size !== after.size || + before.mtimeNs !== after.mtimeNs || + before.ctimeNs !== after.ctimeNs || + after.size !== BigInt(bytes.length) + ) { + bytes.fill(0); + throw new Error("Managed Codex credential source changed while read"); + } + return bytes; + } finally { + await handle.close(); + } +} + +async function readHandle(handle: FileHandle, size: number): Promise { + const bytes = Buffer.alloc(size); + let offset = 0; + while (offset < size) { + const result = await handle.read(bytes, offset, size - offset, offset); + if (result.bytesRead === 0) break; + offset += result.bytesRead; + } + if (offset !== size) { + bytes.fill(0); + throw new Error("Managed Codex credential source ended while read"); + } + return bytes; +} + +function validateCredentialDocument(bytes: Buffer): void { + let value: unknown; + try { + value = JSON.parse(bytes.toString("utf8")); + } catch { + throw new Error("Managed Codex credential source is malformed"); + } + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error("Managed Codex credential source is malformed"); + } +} + +async function writeCredential( + destination: string, + temporaryPath: string, + home: string, + bytes: Buffer, +): Promise { + let handle: FileHandle; + try { + handle = await open( + temporaryPath, + constants.O_WRONLY | + constants.O_CREAT | + constants.O_EXCL | + (constants.O_NOFOLLOW ?? 0), + PRIVATE_FILE_MODE, + ); + } catch { + throw new Error("Managed Codex credential destination could not be opened"); + } + try { + await handle.chmod(PRIVATE_FILE_MODE); + await handle.writeFile(bytes); + await handle.sync(); + await handle.close(); + try { + await rename(temporaryPath, destination); + } catch (error) { + if ( + process.platform !== "win32" || + !["EACCES", "EEXIST", "ENOTEMPTY", "EPERM"].includes( + errorCode(error) ?? "", + ) + ) { + throw error; + } + // Win32 rename does not replace an existing destination. Remove only + // the already-conflicting pathname (never a real directory), then move + // the fully synced private temporary file into place. + await removeReplaceableCredential(destination); + await rename(temporaryPath, destination); + } + } finally { + await handle.close().catch(() => undefined); + await unlink(temporaryPath).catch(() => undefined); + } + // Do not acknowledge the lease until the namespace update is durable. A + // directory-sync failure is a fail-closed admission condition: retry here so + // neither a returned lease nor a thrown pre-lease error can lose ownership + // of auth.json across a crash. + await syncDirectoryDurably(home); +} + +async function removeReplaceableCredential(path: string): Promise { + try { + const metadata = await lstat(path); + if (metadata.isDirectory() && !metadata.isSymbolicLink()) { + throw new Error("Managed Codex credential destination is a directory"); + } + await unlink(path); + } catch (error) { + if (errorCode(error) !== "ENOENT") throw error; + } +} + +async function removeCredential(path: string, home: string): Promise { + await removeCredentialArtifacts([path], home); +} + +async function removeCredentialArtifacts( + paths: readonly string[], + home: string, +): Promise { + for (const path of paths) { + await removeReplaceableCredential(path); + } + // Sync even after ENOENT: a previous unlink may have succeeded before its + // directory sync failed. Never report cleanup or finish preflight while the + // removal can still be rolled back by a crash. + await syncDirectoryDurably(home); +} + +async function syncDirectoryDurably(directory: string): Promise { + let retryDelayMs = 10; + let lastError: unknown; + let attempts = 0; + for (let attempt = 1; attempt <= MAX_DIRECTORY_SYNC_ATTEMPTS; attempt += 1) { + attempts = attempt; + try { + await syncDirectory(directory); + return; + } catch (error) { + lastError = error; + if ( + attempt === MAX_DIRECTORY_SYNC_ATTEMPTS || + error instanceof DirectorySyncOperationTimeoutError || + error instanceof UnconfirmedDirectorySyncHelperTerminationError + ) { + break; + } + // Keep admission closed during transient failures, while bounding total + // startup/shutdown latency for a persistently unhealthy filesystem. + await new Promise((resolveRetry) => { + setTimeout(resolveRetry, retryDelayMs); + }); + retryDelayMs = Math.min(retryDelayMs * 2, 1_000); + } + } + const attemptNoun = attempts === 1 ? "attempt" : "attempts"; + throw new Error( + `Managed Codex credential directory remained non-durable after ${attempts} ${attemptNoun}`, + { cause: lastError }, + ); +} + +async function syncDirectory(directory: string): Promise { + if (process.platform === "win32") return; + reclaimConfirmedExitedDirectorySyncHelpers(); + // Once an in-process filesystem request times out it cannot be cancelled. + // Keep that single request observed, and permanently use killable helper + // processes for this home so recovery remains available without accumulating + // additional parent-process handles or requests. + if (isolatedDirectorySyncHomes.has(directory)) { + if (failedIsolatedDirectorySyncHomes.has(directory)) { + throw new UnconfirmedDirectorySyncHelperTerminationError(directory); + } + await syncDirectoryInIsolatedProcess(directory); + return; + } + const parentOperation = reserveParentDirectorySyncOperation(); + if (parentOperation === null) { + // Never start a fifth uncancellable parent filesystem request. The helper + // pool has its own global bound and can be killed independently. + isolatedDirectorySyncHomes.add(directory); + await syncDirectoryInIsolatedProcess(directory); + return; + } + let retainParentOperation = false; + try { + const openAttempt = open( + directory, + constants.O_RDONLY | (constants.O_DIRECTORY ?? 0), + ); + let handle: FileHandle; + try { + handle = await waitForDirectorySyncOperation( + openAttempt, + "open", + directory, + ); + } catch (error) { + if (error instanceof DirectorySyncOperationTimeoutError) { + // open(2) cannot be cancelled. Retain this process-global reservation + // for the process lifetime and observe/close a late handle without + // ever starting another parent request for this home. + retainParentOperation = true; + const cleanupAttempt = openAttempt.then( + (lateHandle) => closeDirectoryHandle(lateHandle, directory), + () => undefined, + ); + retainDirectorySyncCleanup(directory, cleanupAttempt); + isolatedDirectorySyncHomes.add(directory); + await syncDirectoryInIsolatedProcess(directory); + return; + } + throw error; + } + + let syncAttempt: Promise; + try { + syncAttempt = handle.sync(); + } catch (error) { + retainParentOperation = await closeDirectoryHandle(handle, directory); + throw error; + } + let syncTimedOut = false; + try { + await waitForDirectorySyncOperation(syncAttempt, "fsync", directory); + } catch (error) { + syncTimedOut = error instanceof DirectorySyncOperationTimeoutError; + if (syncTimedOut) { + // FileHandle.close() waits for outstanding operations. Retain both the + // cleanup observer and the parent-operation slot for process lifetime. + retainParentOperation = true; + const cleanupAttempt = syncAttempt.then( + () => handle.close().catch(() => undefined), + () => handle.close().catch(() => undefined), + ); + retainDirectorySyncCleanup(directory, cleanupAttempt); + isolatedDirectorySyncHomes.add(directory); + await syncDirectoryInIsolatedProcess(directory); + return; + } + throw error; + } finally { + // A completed fsync is the durability boundary. Close is resource + // cleanup; a timed-out close retains the global parent-operation slot. + if (!syncTimedOut) { + retainParentOperation = + (await closeDirectoryHandle(handle, directory)) || + retainParentOperation; + } + } + } finally { + if (!retainParentOperation) { + directorySyncHelperRegistry.activeParentOperations.delete( + parentOperation, + ); + } + } +} + +async function closeDirectoryHandle( + handle: FileHandle, + directory: string, +): Promise { + try { + const closeAttempt = handle.close(); + try { + await waitForDirectorySyncOperation(closeAttempt, "close", directory); + } catch (error) { + // close(2) cannot be cancelled. Keep observing a late rejection without + // leaking additional directory handles. Admission remains closed until + // the real close settles, while a completed fsync stays successful. + if (error instanceof DirectorySyncOperationTimeoutError) { + retainDirectorySyncCleanup(directory, closeAttempt); + isolatedDirectorySyncHomes.add(directory); + return true; + } else { + void closeAttempt.catch(() => undefined); + } + } + } catch { + // Closing cannot invalidate an fsync that already completed, and callers + // with a failed fsync must retain that original durability error. + } + return false; +} + +function reserveParentDirectorySyncOperation(): symbol | null { + if ( + directorySyncHelperRegistry.activeParentOperations.size >= + MAX_PARENT_DIRECTORY_SYNC_OPERATIONS + ) { + return null; + } + const reservation = Symbol("parent-directory-sync"); + directorySyncHelperRegistry.activeParentOperations.add(reservation); + return reservation; +} + +async function syncDirectoryInIsolatedProcess( + directory: string, +): Promise { + reclaimConfirmedExitedDirectorySyncHelpers(); + if (failedIsolatedDirectorySyncHomes.has(directory)) { + throw new UnconfirmedDirectorySyncHelperTerminationError(directory); + } + const activeAttempt = isolatedDirectorySyncAttempts.get(directory); + if (activeAttempt !== undefined) return await activeAttempt; + const attempt = runDirectorySyncHelper(directory); + isolatedDirectorySyncAttempts.set(directory, attempt); + try { + await attempt; + } finally { + if (isolatedDirectorySyncAttempts.get(directory) === attempt) { + isolatedDirectorySyncAttempts.delete(directory); + } + } +} + +async function runDirectorySyncHelper(directory: string): Promise { + reclaimConfirmedExitedDirectorySyncHelpers(); + if ( + directorySyncHelperRegistry.activeChildren.size >= + MAX_DIRECTORY_SYNC_HELPERS + ) { + throw new Error( + `Managed Codex credential directory helper process limit of ${MAX_DIRECTORY_SYNC_HELPERS} was reached`, + ); + } + let child: ChildProcess; + try { + child = spawn( + process.execPath, + [ + "--input-type=module", + "--eval", + DIRECTORY_SYNC_HELPER_SOURCE, + directory, + ], + { + // The helper imports only Node built-ins. Do not inherit loader hooks or + // any credential-bearing process environment into the durability worker. + env: {}, + stdio: "ignore", + windowsHide: true, + }, + ); + directorySyncHelperRegistry.activeChildren.add(child); + directorySyncHelperDirectories.set(child, directory); + child.unref(); + } catch (error) { + throw new Error( + `Managed Codex credential directory helper could not start for ${directory}`, + { cause: error }, + ); + } + await new Promise((resolveHelper, rejectHelper) => { + let timeout: NodeJS.Timeout | undefined; + let killAcknowledgementTimeout: NodeJS.Timeout | undefined; + let timedOut = false; + let settled = false; + let reaped = false; + const settle = (error?: Error): void => { + if (settled) return; + settled = true; + if (timeout !== undefined) clearTimeout(timeout); + if (killAcknowledgementTimeout !== undefined) { + clearTimeout(killAcknowledgementTimeout); + } + if (error === undefined) resolveHelper(); + else rejectHelper(error); + }; + const reap = (): void => { + if (reaped) return; + reaped = true; + directorySyncHelperRegistry.activeChildren.delete(child); + directorySyncHelperDirectories.delete(child); + if (stuckDirectorySyncHelpers.get(directory) === child) { + stuckDirectorySyncHelpers.delete(directory); + failedIsolatedDirectorySyncHomes.delete(directory); + } + child.off("error", onError); + child.off("exit", onExit); + child.off("close", onClose); + }; + const fenceUntilExitConfirmed = (): void => { + failedIsolatedDirectorySyncHomes.add(directory); + stuckDirectorySyncHelpers.set(directory, child); + settle(new UnconfirmedDirectorySyncHelperTerminationError(directory)); + }; + const onError = (): void => { + fenceUntilExitConfirmed(); + // A missing pid proves spawn never created a process, so no retained + // reaper or global capacity slot is necessary. + if (child.pid === undefined) reap(); + }; + const onExit = ( + code: number | null, + signal: NodeJS.Signals | null, + ): void => { + reap(); + if (timedOut) { + settle(new DirectorySyncOperationTimeoutError("helper", directory)); + } else if (code === 0) { + settle(); + } else { + settle( + new Error( + `Managed Codex credential directory helper failed with ${ + signal === null ? `code ${String(code)}` : `signal ${signal}` + }`, + ), + ); + } + }; + const onClose = (): void => reap(); + child.on("error", onError); + child.once("exit", onExit); + child.once("close", onClose); + timeout = setTimeout(() => { + timedOut = true; + // Wait for the exit event after SIGKILL before permitting another helper; + // this is the resource-release acknowledgement the in-process API lacks. + try { + if (!child.kill("SIGKILL")) { + fenceUntilExitConfirmed(); + reclaimConfirmedExitedDirectorySyncHelpers(); + return; + } + killAcknowledgementTimeout = setTimeout(() => { + // A child that does not acknowledge SIGKILL is retained as the sole + // reaper for this home. Fail closed until an event or a kernel probe + // confirms exit, so no later attempt can accumulate another process. + fenceUntilExitConfirmed(); + reclaimConfirmedExitedDirectorySyncHelpers(); + }, DIRECTORY_SYNC_HELPER_KILL_ACK_TIMEOUT_MS); + killAcknowledgementTimeout.unref?.(); + } catch { + fenceUntilExitConfirmed(); + reclaimConfirmedExitedDirectorySyncHelpers(); + } + }, DIRECTORY_SYNC_OPERATION_TIMEOUT_MS); + timeout.unref?.(); + }); +} + +function reclaimConfirmedExitedDirectorySyncHelpers(): void { + for (const child of directorySyncHelperRegistry.activeChildren) { + let exited = + (child.exitCode !== null && child.exitCode !== undefined) || + (child.signalCode !== null && child.signalCode !== undefined); + if (!exited && child.pid !== undefined) { + try { + // ChildProcess events are advisory for capacity accounting: an exited + // helper can fail to deliver `exit`/`close` while its owner is under + // pressure. A signal-0 ESRCH result is the kernel confirmation that + // reclaiming this global slot cannot permit another live helper. + process.kill(child.pid, 0); + } catch (error) { + exited = errorCode(error) === "ESRCH"; + } + } + if (!exited) continue; + directorySyncHelperRegistry.activeChildren.delete(child); + const directory = + directorySyncHelperDirectories.get(child) ?? + [...stuckDirectorySyncHelpers].find(([, owner]) => owner === child)?.[0]; + directorySyncHelperDirectories.delete(child); + if (directory && stuckDirectorySyncHelpers.get(directory) === child) { + stuckDirectorySyncHelpers.delete(directory); + failedIsolatedDirectorySyncHomes.delete(directory); + } + } +} + +function retainDirectorySyncCleanup( + directory: string, + attempt: Promise, +): void { + const observed = attempt.then( + () => undefined, + () => undefined, + ); + const prior = pendingDirectorySyncCleanups.get(directory); + const barrier = + prior === undefined + ? observed + : Promise.allSettled([prior, observed]).then(() => undefined); + pendingDirectorySyncCleanups.set(directory, barrier); + void barrier + .finally(() => { + if (pendingDirectorySyncCleanups.get(directory) === barrier) { + pendingDirectorySyncCleanups.delete(directory); + } + }) + .catch(() => undefined); +} + +class DirectorySyncOperationTimeoutError extends Error { + constructor( + operation: "open" | "fsync" | "close" | "helper", + directory: string, + ) { + super( + `Managed Codex credential directory ${operation} timed out after ${DIRECTORY_SYNC_OPERATION_TIMEOUT_MS}ms for ${directory}`, + ); + this.name = "DirectorySyncOperationTimeoutError"; + } +} + +class UnconfirmedDirectorySyncHelperTerminationError extends Error { + constructor(directory: string) { + super( + `Managed Codex credential directory helper termination was not acknowledged for ${directory}`, + ); + this.name = "UnconfirmedDirectorySyncHelperTerminationError"; + } +} + +async function waitForDirectorySyncOperation( + operationAttempt: Promise, + operation: "open" | "fsync" | "close", + directory: string, +): Promise { + let timeout: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + operationAttempt, + new Promise((_resolve, reject) => { + timeout = setTimeout(() => { + reject(new DirectorySyncOperationTimeoutError(operation, directory)); + }, DIRECTORY_SYNC_OPERATION_TIMEOUT_MS); + timeout.unref?.(); + }), + ]); + } finally { + if (timeout !== undefined) clearTimeout(timeout); + } +} + +function errorCode(error: unknown): string | null { + return typeof error === "object" && error !== null && "code" in error + ? String((error as { code?: unknown }).code) + : null; +}