diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-credentials.test.ts b/packages/paperclip-runner/src/drivers/acpx/codex-credentials.test.ts new file mode 100644 index 0000000000..9b2a5d4eb4 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/acpx/codex-credentials.test.ts @@ -0,0 +1,1696 @@ +import { fork, type ChildProcess } from "node:child_process"; +import { createHash } from "node:crypto"; +import { EventEmitter } from "node:events"; +import { + chmod, + mkdir, + mkdtemp, + open, + readdir, + readFile, + realpath, + rm, + stat, + symlink, + writeFile, + type FileHandle, +} from "node:fs/promises"; +import { createRequire } from "node:module"; +import { createServer, type Server, type Socket } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { stageManagedCodexCredential } from "./codex-credentials.js"; + +const temporaryDirectories: string[] = []; + +afterEach(async () => { + vi.useRealTimers(); + vi.doUnmock("node:child_process"); + vi.doUnmock("node:fs/promises"); + vi.resetModules(); + ( + globalThis as typeof globalThis & { + __paperclipDirectorySyncHelperRegistryV1?: { + activeParentOperations: Set; + }; + } + ).__paperclipDirectorySyncHelperRegistryV1?.activeParentOperations.clear(); + await Promise.all( + temporaryDirectories + .splice(0) + .map((directory) => rm(directory, { force: true, recursive: true })), + ); +}); + +describe("managed Codex credentials", () => { + it("derives deterministic quorum candidates from distinct user scopes", () => { + expect(credentialLeasePortsForScope("1000", "/canonical/home")).toEqual( + credentialLeasePortsForScope("1000", "/canonical/home"), + ); + expect(credentialLeasePortsForScope("1000", "/canonical/home")).not.toEqual( + credentialLeasePortsForScope("1001", "/canonical/home"), + ); + expect(credentialLeasePortsForScope("1000", "/canonical/home")).not.toEqual( + credentialLeasePortsForScope("1000", "/canonical/other-home"), + ); + }); + + it("stages inline JSON privately and removes it idempotently", async () => { + const fixture = await credentialFixture(); + const lease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: JSON.stringify({ + tokens: { access_token: "inline-canary" }, + }), + }, + }); + + expect(lease.mode).toBe("inline_json"); + const cleanupIntent = join( + fixture.home, + ".paperclip-auth-cleanup-required", + ); + await expect(readFile(lease.path, "utf8")).resolves.toContain( + "inline-canary", + ); + await expect(readFile(cleanupIntent, "utf8")).resolves.toBe( + "paperclip-managed-codex-cleanup-v1\n", + ); + if (process.platform !== "win32") { + expect((await stat(lease.path)).mode & 0o777).toBe(0o600); + } + await lease.close(); + await lease.close(); + await expect(readFile(lease.path)).rejects.toMatchObject({ + code: "ENOENT", + }); + await expect(readFile(cleanupIntent)).rejects.toMatchObject({ + code: "ENOENT", + }); + }); + + it("fences overlapping leases for the same isolated home", async () => { + const fixture = await credentialFixture(); + const firstLease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"first"}', + }, + }); + + await expect( + stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"second"}', + }, + }), + ).rejects.toThrow("already has an active lease"); + await expect(readFile(firstLease.path, "utf8")).resolves.toBe( + '{"owner":"first"}', + ); + + await firstLease.close(); + const secondLease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"second"}', + }, + }); + await expect(readFile(secondLease.path, "utf8")).resolves.toBe( + '{"owner":"second"}', + ); + await secondLease.close(); + }); + + it("fences a contender loaded through a fresh module instance", async () => { + const fixture = await credentialFixture(); + const firstLease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"first"}', + }, + }); + + vi.resetModules(); + const freshCredentials = await import("./codex-credentials.js"); + await expect( + freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"contender"}', + }, + }), + ).rejects.toThrow("already has an active lease"); + + await firstLease.close(); + }); + + it.each([ + ["primary", 0], + ["non-primary", 1], + ] as const)( + "tolerates one unrelated silent %s quorum listener", + async (_label, occupiedIndex) => { + const fixture = await credentialFixture(); + const ports = credentialLeasePorts(await realpath(fixture.home)); + const occupied = await listenSilently(ports[occupiedIndex]); + try { + const lease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"paperclip"}', + }, + }); + await expect(readFile(lease.path, "utf8")).resolves.toBe( + '{"owner":"paperclip"}', + ); + await lease.close(); + } finally { + await occupied.close(); + } + }, + ); + + it("fails before auth mutation when two quorum candidates are occupied", async () => { + const fixture = await credentialFixture(); + const destination = join(fixture.home, "auth.json"); + await writeFile(destination, '{"sentinel":true}', { mode: 0o600 }); + const ports = credentialLeasePorts(await realpath(fixture.home)); + const first = await listenSilently(ports[0]); + const second = await listenSilently(ports[1]); + try { + await expect( + stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"contender"}', + }, + }), + ).rejects.toThrow("already has an active lease"); + await expect(readFile(destination, "utf8")).resolves.toBe( + '{"sentinel":true}', + ); + } finally { + await Promise.all([first.close(), second.close()]); + } + }); + + it("admits only one of two concurrent fresh-module contenders", async () => { + const fixture = await credentialFixture(); + vi.resetModules(); + const firstCredentials = await import("./codex-credentials.js"); + vi.resetModules(); + const secondCredentials = await import("./codex-credentials.js"); + + const results = await Promise.allSettled([ + firstCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"first"}', + }, + }), + secondCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"second"}', + }, + }), + ]); + const winners = results.filter( + ( + result, + ): result is PromiseFulfilledResult< + Awaited> + > => result.status === "fulfilled", + ); + expect(winners).toHaveLength(1); + expect( + results.filter((result) => result.status === "rejected"), + ).toHaveLength(1); + await winners[0].value.close(); + }); + + it("fences another process and recovers only after its kernel lease dies", async () => { + const fixture = await credentialFixture(); + const destination = join(fixture.home, "auth.json"); + const childScript = join(fixture.root, "credential-owner.mjs"); + const credentialModule = new URL("./codex-credentials.ts", import.meta.url) + .href; + // paperclip-runner intentionally does not ship a TS runtime dependency. + // Resolve the existing monorepo dev loader from a workspace that declares + // it, instead of asking the child to resolve an undeclared bare package. + const tsxLoader = createRequire( + new URL("../../../../../server/package.json", import.meta.url), + ).resolve("tsx"); + await writeFile( + childScript, + [ + `const { stageManagedCodexCredential } = await import(${JSON.stringify(credentialModule)});`, + "const lease = await stageManagedCodexCredential({", + " agentHomeDirectory: process.argv[2],", + ' environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: \'{"owner":"first"}\' },', + "});", + 'process.send?.({ type: "ready", path: lease.path });', + "process.on('message', async (message) => {", + " if (message?.type !== 'close') return;", + " await lease.close();", + " process.exit(0);", + "});", + ].join("\n"), + ); + const owner = fork(childScript, [fixture.home], { + execArgv: ["--import", tsxLoader], + stdio: ["ignore", "pipe", "pipe", "ipc"], + }); + try { + await waitForChildMessage(owner, "ready"); + await expect(readFile(destination, "utf8")).resolves.toBe( + '{"owner":"first"}', + ); + + if (process.platform !== "win32") { + owner.kill("SIGSTOP"); + await new Promise((resolveSignal) => + setTimeout(resolveSignal, 50), + ); + } + await expect( + stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"second"}', + }, + }), + ).rejects.toThrow("already has an active lease"); + await expect(readFile(destination, "utf8")).resolves.toBe( + '{"owner":"first"}', + ); + + if (process.platform !== "win32") owner.kill("SIGCONT"); + owner.kill(process.platform === "win32" ? undefined : "SIGKILL"); + await waitForChildExit(owner); + + const successor = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"second"}', + }, + }); + await expect(readFile(destination, "utf8")).resolves.toBe( + '{"owner":"second"}', + ); + await successor.close(); + expect( + (await readdir(fixture.home)).filter( + (name) => + name.includes("paperclip-auth-lease") || + name.includes("paperclip-auth-home-claim"), + ), + ).toEqual([]); + } finally { + if (owner.exitCode === null && owner.signalCode === null) { + if (process.platform !== "win32") owner.kill("SIGCONT"); + owner.kill(process.platform === "win32" ? undefined : "SIGKILL"); + await waitForChildExit(owner).catch(() => undefined); + } + } + }); + it.runIf(process.platform !== "win32")( + "holds kernel ownership until credential cleanup is durable", + async () => { + const fixture = await credentialFixture(); + const lease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" }, + }); + const probe = await open(fixture.home, "r"); + const prototype = Object.getPrototypeOf(probe) as { + sync(this: FileHandle): Promise; + }; + await probe.close(); + const originalSync = prototype.sync; + let releaseCleanup!: () => void; + const cleanupGate = new Promise((resolveCleanup) => { + releaseCleanup = resolveCleanup; + }); + let signalCleanupStarted!: () => void; + const cleanupStarted = new Promise((resolveStarted) => { + signalCleanupStarted = resolveStarted; + }); + let heldCleanup = false; + const syncSpy = vi + .spyOn(prototype, "sync") + .mockImplementation(async function (this: FileHandle): Promise { + if (!heldCleanup && (await this.stat()).isDirectory()) { + heldCleanup = true; + signalCleanupStarted(); + await cleanupGate; + } + await originalSync.call(this); + }); + try { + const closing = lease.close(); + await cleanupStarted; + vi.resetModules(); + const freshCredentials = await import("./codex-credentials.js"); + await expect( + freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" }, + }), + ).rejects.toThrow("already has an active lease"); + + releaseCleanup(); + await expect(closing).resolves.toBeUndefined(); + const successor = await freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" }, + }); + await successor.close(); + } finally { + releaseCleanup(); + syncSpy.mockRestore(); + } + }, + ); + + it("does not let an older failed close remove a successor credential", async () => { + const fixture = await credentialFixture(); + const firstLease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"first"}', + }, + }); + await rm(firstLease.path, { force: true }); + await mkdir(firstLease.path); + + await expect(firstLease.close()).rejects.toThrow( + "credential destination is a directory", + ); + await rm(firstLease.path, { force: true, recursive: true }); + + const secondLease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"second"}', + }, + }); + await expect(firstLease.close()).resolves.toBeUndefined(); + await expect(readFile(secondLease.path, "utf8")).resolves.toBe( + '{"owner":"second"}', + ); + await secondLease.close(); + }); + + it("leaves no ownership artifacts in the credential home", async () => { + const fixture = await credentialFixture(); + const lease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + await lease.close(); + + expect( + (await readdir(fixture.home)).filter( + (name) => + name.includes("paperclip-auth-lease") || + name.includes("paperclip-auth-home-claim") || + name.includes("paperclip-auth-lock"), + ), + ).toEqual([]); + }); + it("recovers a persisted cleanup intent before admitting another provider", async () => { + const fixture = await credentialFixture(); + const destination = join(fixture.home, "auth.json"); + const cleanupIntent = join( + fixture.home, + ".paperclip-auth-cleanup-required", + ); + await writeFile(destination, '{"crash_stale":true}', { mode: 0o600 }); + await writeFile(cleanupIntent, "paperclip-managed-codex-cleanup-v1\n", { + mode: 0o600, + }); + + const lease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + await expect(readFile(destination)).rejects.toMatchObject({ + code: "ENOENT", + }); + await expect(readFile(cleanupIntent, "utf8")).resolves.toBe( + "paperclip-managed-codex-cleanup-v1\n", + ); + await lease.close(); + await expect(readFile(cleanupIntent)).rejects.toMatchObject({ + code: "ENOENT", + }); + }); + + it("scrubs a crash-left credential staging file before admission", async () => { + const fixture = await credentialFixture(); + const stagingPath = join(fixture.home, ".paperclip-auth-staging-v1"); + const cleanupIntent = join( + fixture.home, + ".paperclip-auth-cleanup-required", + ); + await writeFile(stagingPath, '{"crash_secret":"must-not-survive"}', { + mode: 0o600, + }); + await writeFile(cleanupIntent, "paperclip-managed-codex-cleanup-v1\n", { + mode: 0o600, + }); + + vi.resetModules(); + const freshCredentials = await import("./codex-credentials.js"); + const lease = await freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"successor"}', + }, + }); + await expect(readFile(stagingPath)).rejects.toMatchObject({ + code: "ENOENT", + }); + await expect(readFile(lease.path, "utf8")).resolves.toBe( + '{"owner":"successor"}', + ); + await lease.close(); + }); + + it.runIf(process.platform !== "win32")( + "unlinks a crash-left staging symlink without touching its target", + async () => { + const fixture = await credentialFixture(); + const target = join(fixture.root, "external-secret.json"); + const stagingPath = join(fixture.home, ".paperclip-auth-staging-v1"); + await writeFile(target, '{"external":"unchanged"}', { mode: 0o600 }); + await symlink(target, stagingPath); + + const lease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + await expect(readFile(stagingPath)).rejects.toMatchObject({ + code: "ENOENT", + }); + await expect(readFile(target, "utf8")).resolves.toBe( + '{"external":"unchanged"}', + ); + await lease.close(); + }, + ); + + it("fails closed instead of recursively removing a staging directory", async () => { + const fixture = await credentialFixture(); + const stagingPath = join(fixture.home, ".paperclip-auth-staging-v1"); + await mkdir(stagingPath, { mode: 0o700 }); + + await expect( + stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }), + ).rejects.toThrow("credential destination is a directory"); + expect((await stat(stagingPath)).isDirectory()).toBe(true); + await expect( + readFile(join(fixture.home, "auth.json")), + ).rejects.toMatchObject({ + code: "ENOENT", + }); + }); + + it.runIf(process.platform !== "win32")( + "retries the directory sync after unlink already succeeded", + async () => { + const fixture = await credentialFixture(); + const lease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" }, + }); + const probe = await open(fixture.home, "r"); + const prototype = Object.getPrototypeOf(probe) as { + sync(this: FileHandle): Promise; + }; + await probe.close(); + const originalSync = prototype.sync; + let syncAttempts = 0; + const syncSpy = vi + .spyOn(prototype, "sync") + .mockImplementation(async function (this: FileHandle): Promise { + if ((await this.stat()).isDirectory()) { + syncAttempts += 1; + if (syncAttempts === 1) { + throw new Error("injected directory sync failure"); + } + } + await originalSync.call(this); + }); + try { + await expect(lease.close()).resolves.toBeUndefined(); + await expect(readFile(lease.path)).rejects.toMatchObject({ + code: "ENOENT", + }); + expect(syncAttempts).toBe(3); + } finally { + syncSpy.mockRestore(); + } + }, + ); + + it.runIf(process.platform !== "win32")( + "retries preflight, installation, and removal until each is durable", + async () => { + const fixture = await credentialFixture(); + const probe = await open(fixture.home, "r"); + const prototype = Object.getPrototypeOf(probe) as { + sync(this: FileHandle): Promise; + }; + await probe.close(); + const originalSync = prototype.sync; + let directorySyncAttempts = 0; + const syncSpy = vi + .spyOn(prototype, "sync") + .mockImplementation(async function (this: FileHandle): Promise { + if ((await this.stat()).isDirectory()) { + directorySyncAttempts += 1; + } + // The first attempt at each namespace boundary fails; the durable + // helper must retry before staging or cleanup reports success. + if ([1, 3, 5].includes(directorySyncAttempts)) { + throw new Error("injected directory sync failure"); + } + await originalSync.call(this); + }); + try { + const lease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" }, + }); + await expect(readFile(lease.path, "utf8")).resolves.toBe("{}"); + await expect(lease.close()).resolves.toBeUndefined(); + await expect(readFile(lease.path)).rejects.toMatchObject({ + code: "ENOENT", + }); + expect(directorySyncAttempts).toBe(8); + } finally { + syncSpy.mockRestore(); + } + }, + ); + + it("copies an explicit private source without changing the source", async () => { + const fixture = await credentialFixture(); + const source = join(fixture.root, "managed-auth.json"); + await writeFile( + source, + JSON.stringify({ tokens: { access_token: "managed-canary" } }), + { mode: 0o600 }, + ); + await chmod(source, 0o600); + + const lease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + sourcePath: source, + }); + expect(lease.mode).toBe("managed_file"); + await expect(readFile(lease.path, "utf8")).resolves.toContain( + "managed-canary", + ); + await lease.close(); + await expect(readFile(source, "utf8")).resolves.toContain("managed-canary"); + }); + + it("replaces a stale regular auth destination in JSON modes", async () => { + const fixture = await credentialFixture(); + const destination = join(fixture.home, "auth.json"); + await writeFile(destination, '{"stale":true}', { mode: 0o600 }); + + const lease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"fresh":true}', + }, + }); + await expect(readFile(destination, "utf8")).resolves.toBe('{"fresh":true}'); + await lease.close(); + }); + + it("cleans stale and provider-generated auth in API-key mode", async () => { + const fixture = await credentialFixture(); + const destination = join(fixture.home, "auth.json"); + await writeFile(destination, '{"stale":true}'); + + const lease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + expect(lease.mode).toBe("api_key"); + await expect(readFile(destination)).rejects.toMatchObject({ + code: "ENOENT", + }); + await writeFile(destination, '{"provider_generated":true}'); + await lease.close(); + await expect(readFile(destination)).rejects.toMatchObject({ + code: "ENOENT", + }); + }); + + it.runIf(process.platform !== "win32")( + "keeps API-key staging pending until stale removal is durable", + async () => { + const fixture = await credentialFixture(); + const destination = join(fixture.home, "auth.json"); + await writeFile(destination, '{"stale":true}', { mode: 0o600 }); + const probe = await open(fixture.home, "r"); + const prototype = Object.getPrototypeOf(probe) as { + sync(this: FileHandle): Promise; + }; + await probe.close(); + const originalSync = prototype.sync; + let syncAttempts = 0; + const syncSpy = vi + .spyOn(prototype, "sync") + .mockImplementation(async function (this: FileHandle): Promise { + if ((await this.stat()).isDirectory()) { + syncAttempts += 1; + if (syncAttempts === 1) { + throw new Error("injected directory sync failure"); + } + } + await originalSync.call(this); + }); + try { + const lease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + await expect(readFile(destination)).rejects.toMatchObject({ + code: "ENOENT", + }); + await expect(lease.close()).resolves.toBeUndefined(); + expect(syncAttempts).toBe(5); + } finally { + syncSpy.mockRestore(); + } + }, + ); + + it.runIf(process.platform !== "win32")( + "fails a non-durable admission within a bound and scrubs again on retry", + async () => { + const fixture = await credentialFixture(); + const probe = await open(fixture.home, "r"); + const prototype = Object.getPrototypeOf(probe) as { + sync(this: FileHandle): Promise; + }; + await probe.close(); + const originalSync = prototype.sync; + const syncSpy = vi + .spyOn(prototype, "sync") + .mockImplementation(async function (this: FileHandle): Promise { + if ((await this.stat()).isDirectory()) { + throw new Error("persistent directory sync failure"); + } + await originalSync.call(this); + }); + try { + const staging = stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + await expect(staging).rejects.toThrow( + "remained non-durable after 8 attempts", + ); + expect(syncSpy.mock.calls.length).toBeGreaterThanOrEqual(8); + syncSpy.mockRestore(); + const lease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + await expect(lease.close()).resolves.toBeUndefined(); + } finally { + syncSpy.mockRestore(); + } + }, + ); + + it.runIf(process.platform !== "win32")( + "shares the four-slot parent filesystem budget across fresh modules", + async () => { + const fixtures = await Promise.all( + Array.from({ length: 5 }, () => credentialFixture()), + ); + const retainedHomes = new Set( + fixtures.slice(0, 4).map((fixture) => fixture.home), + ); + const actualFs = + await vi.importActual( + "node:fs/promises", + ); + let directoryOpenAttempts = 0; + let fifthHomeOpenAttempts = 0; + let observeFourOpens!: () => void; + const fourOpens = new Promise((resolveOpens) => { + observeFourOpens = resolveOpens; + }); + vi.doMock("node:fs/promises", () => ({ + ...actualFs, + open: async ( + path: Parameters[0], + flags: Parameters[1], + mode?: Parameters[2], + ): Promise => { + const pathname = String(path); + if (retainedHomes.has(pathname)) { + directoryOpenAttempts += 1; + if (directoryOpenAttempts === 4) observeFourOpens(); + return await new Promise(() => undefined); + } + if (pathname === fixtures[4].home) fifthHomeOpenAttempts += 1; + return await actualFs.open(path, flags, mode); + }, + })); + const spawnMock = vi.fn(() => { + const child = Object.assign(new EventEmitter(), { + kill: vi.fn(() => true), + pid: 12345, + unref: vi.fn(), + }) as unknown as ChildProcess; + queueMicrotask(() => child.emit("exit", 0, null)); + return child; + }); + vi.doMock("node:child_process", () => ({ spawn: spawnMock })); + vi.resetModules(); + const firstCredentials = await import("./codex-credentials.js"); + vi.useFakeTimers(); + + const firstStaging = Promise.all( + fixtures.slice(0, 4).map((fixture) => + firstCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }), + ), + ); + let leases: Awaited>[] = + []; + try { + await fourOpens; + await vi.advanceTimersByTimeAsync(1_001); + leases = await firstStaging; + const registry = ( + globalThis as typeof globalThis & { + __paperclipDirectorySyncHelperRegistryV1?: { + activeParentOperations: Set; + }; + } + ).__paperclipDirectorySyncHelperRegistryV1; + expect(registry?.activeParentOperations.size).toBe(4); + + vi.resetModules(); + const freshCredentials = await import("./codex-credentials.js"); + const fifthLease = await freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixtures[4].home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + leases.push(fifthLease); + expect(directoryOpenAttempts).toBe(4); + expect(fifthHomeOpenAttempts).toBe(0); + expect(spawnMock).toHaveBeenCalled(); + } finally { + await Promise.allSettled(leases.map((lease) => lease.close())); + } + }, + ); + + it.runIf(process.platform !== "win32")( + "isolates retries after a directory open never settles", + async () => { + const fixture = await credentialFixture(); + const actualFs = + await vi.importActual( + "node:fs/promises", + ); + let directoryOpenAttempts = 0; + let observeFirstOpen!: () => void; + const firstOpen = new Promise((resolveOpen) => { + observeFirstOpen = resolveOpen; + }); + const retainedOpen = new Promise(() => undefined); + vi.doMock("node:fs/promises", () => ({ + ...actualFs, + open: async ( + path: Parameters[0], + flags: Parameters[1], + mode?: Parameters[2], + ): Promise => { + if (String(path) === fixture.home) { + directoryOpenAttempts += 1; + if (directoryOpenAttempts === 1) { + observeFirstOpen(); + return await retainedOpen; + } + } + return await actualFs.open(path, flags, mode); + }, + })); + vi.resetModules(); + const freshCredentials = await import("./codex-credentials.js"); + vi.useFakeTimers(); + + const staging = freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + await firstOpen; + await vi.advanceTimersByTimeAsync(1_001); + const lease = await staging; + expect(directoryOpenAttempts).toBe(1); + await lease.close(); + const successor = await freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + await successor.close(); + expect(directoryOpenAttempts).toBe(1); + }, + ); + + it.runIf(process.platform !== "win32")( + "isolates retries after a directory fsync never settles", + async () => { + const fixture = await credentialFixture(); + const actualFs = + await vi.importActual( + "node:fs/promises", + ); + let directorySyncAttempts = 0; + let observeFirstSync!: () => void; + const firstSync = new Promise((resolveSync) => { + observeFirstSync = resolveSync; + }); + const stalledDirectoryHandle = { + close: async (): Promise => undefined, + sync: async (): Promise => { + directorySyncAttempts += 1; + observeFirstSync(); + return await new Promise(() => undefined); + }, + } as FileHandle; + vi.doMock("node:fs/promises", () => ({ + ...actualFs, + open: async ( + path: Parameters[0], + flags: Parameters[1], + mode?: Parameters[2], + ): Promise => { + if (String(path) === fixture.home) return stalledDirectoryHandle; + return await actualFs.open(path, flags, mode); + }, + })); + vi.resetModules(); + const freshCredentials = await import("./codex-credentials.js"); + vi.useFakeTimers(); + + const staging = freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + await firstSync; + await vi.advanceTimersByTimeAsync(1_001); + const lease = await staging; + await lease.close(); + const successor = await freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + await successor.close(); + expect(directorySyncAttempts).toBe(1); + }, + ); + + it.runIf(process.platform !== "win32")( + "isolates later syncs after a directory close never settles", + async () => { + const fixture = await credentialFixture(); + const actualFs = + await vi.importActual( + "node:fs/promises", + ); + let directoryCloseAttempts = 0; + let observeFirstClose!: () => void; + const firstClose = new Promise((resolveClose) => { + observeFirstClose = resolveClose; + }); + const retainedClose = new Promise(() => undefined); + vi.doMock("node:fs/promises", () => ({ + ...actualFs, + open: async ( + path: Parameters[0], + flags: Parameters[1], + mode?: Parameters[2], + ): Promise => { + if (String(path) === fixture.home) { + return { + close: async (): Promise => { + directoryCloseAttempts += 1; + if (directoryCloseAttempts === 1) { + observeFirstClose(); + return await retainedClose; + } + }, + sync: async (): Promise => undefined, + } as FileHandle; + } + return await actualFs.open(path, flags, mode); + }, + })); + vi.resetModules(); + const freshCredentials = await import("./codex-credentials.js"); + vi.useFakeTimers(); + + const staging = freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + await firstClose; + await vi.advanceTimersByTimeAsync(1_001); + const lease = await staging; + expect(directoryCloseAttempts).toBe(1); + await lease.close(); + const successor = await freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + await successor.close(); + expect(directoryCloseAttempts).toBe(1); + }, + ); + + it.runIf(process.platform !== "win32")( + "permanently fails closed when a killed sync helper never exits", + async () => { + const fixture = await credentialFixture(); + const actualFs = + await vi.importActual( + "node:fs/promises", + ); + let observeDirectoryOpen!: () => void; + const directoryOpen = new Promise((resolveOpen) => { + observeDirectoryOpen = resolveOpen; + }); + let directoryOpenAttempts = 0; + vi.doMock("node:fs/promises", () => ({ + ...actualFs, + open: async ( + path: Parameters[0], + flags: Parameters[1], + mode?: Parameters[2], + ): Promise => { + if (String(path) === fixture.home) { + directoryOpenAttempts += 1; + observeDirectoryOpen(); + return await new Promise(() => undefined); + } + return await actualFs.open(path, flags, mode); + }, + })); + let spawnAttempts = 0; + const stuckChild = Object.assign(new EventEmitter(), { + kill: vi.fn(() => true), + // Signal-0 must observe a live process for this retained-helper fixture. + pid: process.pid, + unref: vi.fn(), + }) as unknown as ChildProcess; + vi.doMock("node:child_process", () => ({ + spawn: vi.fn(() => { + spawnAttempts += 1; + return stuckChild; + }), + })); + vi.resetModules(); + const freshCredentials = await import("./codex-credentials.js"); + vi.useFakeTimers(); + + const staging = freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + const rejection = expect(staging).rejects.toThrow( + /remained non-durable after 1 attempt/, + ); + await directoryOpen; + await vi.advanceTimersByTimeAsync(1_001); + await vi.advanceTimersByTimeAsync(1_001); + await vi.advanceTimersByTimeAsync(1_001); + await rejection; + expect(spawnAttempts).toBe(1); + expect(directoryOpenAttempts).toBe(1); + expect(stuckChild.unref).toHaveBeenCalledOnce(); + + await expect( + freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }), + ).rejects.toThrow(/remained non-durable after 1 attempt/); + expect(spawnAttempts).toBe(1); + expect(directoryOpenAttempts).toBe(1); + stuckChild.emit("exit", null, "SIGKILL"); + }, + ); + + it.runIf(process.platform !== "win32").each([ + ["returns false", (): boolean => false], + [ + "throws", + (): boolean => { + throw new Error("injected kill failure"); + }, + ], + ])( + "permanently fences a home when helper kill %s", + async (_label, killImplementation) => { + const fixture = await credentialFixture(); + const actualFs = + await vi.importActual( + "node:fs/promises", + ); + let observeDirectoryOpen!: () => void; + const directoryOpen = new Promise((resolveOpen) => { + observeDirectoryOpen = resolveOpen; + }); + vi.doMock("node:fs/promises", () => ({ + ...actualFs, + open: async ( + path: Parameters[0], + flags: Parameters[1], + mode?: Parameters[2], + ): Promise => { + if (String(path) === fixture.home) { + observeDirectoryOpen(); + return await new Promise(() => undefined); + } + return await actualFs.open(path, flags, mode); + }, + })); + const child = Object.assign(new EventEmitter(), { + kill: vi.fn(killImplementation), + // Signal-0 must observe a live process for this retained-helper fixture. + pid: process.pid, + unref: vi.fn(), + }) as unknown as ChildProcess; + const spawnMock = vi.fn(() => child); + vi.doMock("node:child_process", () => ({ spawn: spawnMock })); + vi.resetModules(); + const freshCredentials = await import("./codex-credentials.js"); + vi.useFakeTimers(); + + const staging = freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + const rejection = expect(staging).rejects.toThrow( + /remained non-durable after 1 attempt/, + ); + await directoryOpen; + await vi.advanceTimersByTimeAsync(1_001); + await vi.advanceTimersByTimeAsync(1_001); + await rejection; + await expect( + freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }), + ).rejects.toThrow(/remained non-durable after 1 attempt/); + expect(spawnMock).toHaveBeenCalledOnce(); + expect(child.unref).toHaveBeenCalledOnce(); + child.emit("exit", null, "SIGKILL"); + }, + ); + + it.runIf(process.platform !== "win32")( + "permanently fences a home after an asynchronous helper error", + async () => { + const fixture = await credentialFixture(); + const actualFs = + await vi.importActual( + "node:fs/promises", + ); + let observeDirectoryOpen!: () => void; + const directoryOpen = new Promise((resolveOpen) => { + observeDirectoryOpen = resolveOpen; + }); + vi.doMock("node:fs/promises", () => ({ + ...actualFs, + open: async ( + path: Parameters[0], + flags: Parameters[1], + mode?: Parameters[2], + ): Promise => { + if (String(path) === fixture.home) { + observeDirectoryOpen(); + return await new Promise(() => undefined); + } + return await actualFs.open(path, flags, mode); + }, + })); + const child = Object.assign(new EventEmitter(), { + kill: vi.fn(() => true), + // Signal-0 must observe a live process for this retained-helper fixture. + pid: process.pid, + unref: vi.fn(), + }) as unknown as ChildProcess; + const spawnMock = vi.fn(() => { + queueMicrotask(() => child.emit("error", new Error("spawn failed"))); + return child; + }); + vi.doMock("node:child_process", () => ({ spawn: spawnMock })); + vi.resetModules(); + const freshCredentials = await import("./codex-credentials.js"); + vi.useFakeTimers(); + + const staging = freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + const rejection = expect(staging).rejects.toThrow( + /remained non-durable after 1 attempt/, + ); + await directoryOpen; + await vi.advanceTimersByTimeAsync(1_001); + await rejection; + await expect( + freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }), + ).rejects.toThrow(/remained non-durable after 1 attempt/); + expect(spawnMock).toHaveBeenCalledOnce(); + child.emit("exit", null, null); + }, + ); + + it.runIf(process.platform !== "win32")( + "shares the helper process cap across fresh module instances", + async () => { + const fixture = await credentialFixture(); + const registry = ( + globalThis as typeof globalThis & { + __paperclipDirectorySyncHelperRegistryV1?: { + activeChildren: Set; + }; + } + ).__paperclipDirectorySyncHelperRegistryV1; + expect(registry).toBeDefined(); + const reservations = Array.from( + { length: 4 }, + () => new EventEmitter() as unknown as ChildProcess, + ); + for (const reservation of reservations) { + registry!.activeChildren.add(reservation); + } + const actualFs = + await vi.importActual( + "node:fs/promises", + ); + let observeDirectoryOpen!: () => void; + const directoryOpen = new Promise((resolveOpen) => { + observeDirectoryOpen = resolveOpen; + }); + vi.doMock("node:fs/promises", () => ({ + ...actualFs, + open: async ( + path: Parameters[0], + flags: Parameters[1], + mode?: Parameters[2], + ): Promise => { + if (String(path) === fixture.home) { + observeDirectoryOpen(); + return await new Promise(() => undefined); + } + return await actualFs.open(path, flags, mode); + }, + })); + const spawnMock = vi.fn(() => { + throw new Error("helper cap was bypassed"); + }); + vi.doMock("node:child_process", () => ({ spawn: spawnMock })); + vi.resetModules(); + const freshCredentials = await import("./codex-credentials.js"); + vi.useFakeTimers(); + try { + const staging = freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + const rejection = expect(staging).rejects.toThrow( + /remained non-durable after 8 attempts/, + ); + await directoryOpen; + await vi.advanceTimersByTimeAsync(3_000); + await rejection; + expect(spawnMock).not.toHaveBeenCalled(); + } finally { + for (const reservation of reservations) { + registry!.activeChildren.delete(reservation); + } + } + }, + ); + + it.runIf(process.platform !== "win32")( + "reclaims kernel-confirmed helper exits when child events are lost", + async () => { + const fixture = await credentialFixture(); + const registry = ( + globalThis as typeof globalThis & { + __paperclipDirectorySyncHelperRegistryV1?: { + activeChildren: Set; + childDirectories: Map; + activeParentOperations: Set; + failedHomes: Set; + stuckChildren: Map; + }; + } + ).__paperclipDirectorySyncHelperRegistryV1; + expect(registry).toBeDefined(); + + const parentReservations = Array.from({ length: 4 }, () => Symbol()); + for (const reservation of parentReservations) { + registry!.activeParentOperations.add(reservation); + } + const departedPids = new Set(); + const departedHelpers = Array.from({ length: 4 }, (_, index) => { + const pid = 41_000 + index; + departedPids.add(pid); + const child = Object.assign(new EventEmitter(), { + exitCode: null, + kill: vi.fn(() => true), + pid, + signalCode: null, + unref: vi.fn(), + }) as unknown as ChildProcess; + const directory = `/departed-credential-helper-${String(index)}`; + registry!.activeChildren.add(child); + registry!.childDirectories.set(child, directory); + if (index < 2) { + registry!.failedHomes.add(directory); + registry!.stuckChildren.set(directory, child); + } + return { child, directory }; + }); + const killSpy = vi + .spyOn(process, "kill") + .mockImplementation((pid, signal) => { + if (signal === 0 && departedPids.has(Number(pid))) { + throw Object.assign(new Error("process no longer exists"), { + code: "ESRCH", + }); + } + return true; + }); + const observedActiveChildren: number[] = []; + let nextPid = 42_000; + const spawnMock = vi.fn(() => { + observedActiveChildren.push(registry!.activeChildren.size); + const child = Object.assign(new EventEmitter(), { + exitCode: null, + kill: vi.fn(() => true), + pid: nextPid++, + signalCode: null, + unref: vi.fn(), + }) as unknown as ChildProcess; + queueMicrotask(() => { + child.emit("exit", 0, null); + child.emit("close", 0, null); + }); + return child; + }); + vi.doMock("node:child_process", () => ({ spawn: spawnMock })); + vi.resetModules(); + const freshCredentials = await import("./codex-credentials.js"); + + try { + const lease = await freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }); + await lease.close(); + + expect(spawnMock).toHaveBeenCalled(); + expect(observedActiveChildren.every((count) => count < 4)).toBe(true); + for (const { child, directory } of departedHelpers) { + expect(registry!.activeChildren).not.toContain(child); + expect(registry!.childDirectories.has(child)).toBe(false); + expect(registry!.failedHomes).not.toContain(directory); + expect(registry!.stuckChildren.has(directory)).toBe(false); + } + } finally { + killSpy.mockRestore(); + for (const reservation of parentReservations) { + registry!.activeParentOperations.delete(reservation); + } + for (const { child, directory } of departedHelpers) { + registry!.activeChildren.delete(child); + registry!.childDirectories.delete(child); + registry!.failedHomes.delete(directory); + registry!.stuckChildren.delete(directory); + } + } + }, + ); + + it.runIf(process.platform !== "win32")( + "keeps intent-publication failure before credential mutation and scrubs without process memory", + async () => { + const fixture = await credentialFixture(); + const destination = join(fixture.home, "auth.json"); + const cleanupIntent = join( + fixture.home, + ".paperclip-auth-cleanup-required", + ); + const probe = await open(fixture.home, "r"); + const prototype = Object.getPrototypeOf(probe) as { + sync(this: FileHandle): Promise; + }; + await probe.close(); + const originalSync = prototype.sync; + let directorySyncAttempts = 0; + const syncSpy = vi + .spyOn(prototype, "sync") + .mockImplementation(async function (this: FileHandle): Promise { + if ((await this.stat()).isDirectory()) { + directorySyncAttempts += 1; + if (directorySyncAttempts > 1) { + throw new Error("persistent intent sync failure"); + } + } + await originalSync.call(this); + }); + try { + await expect( + stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" }, + }), + ).rejects.toThrow("remained non-durable after 8 attempts"); + await expect(readFile(destination)).rejects.toMatchObject({ + code: "ENOENT", + }); + + // Model a crash losing the unsynced intent directory entry and the + // next process finding an unexpected auth file. A fresh module has no + // quarantine map from the failed process, so admission must rely on + // the isolated-home scrub rather than process memory. + syncSpy.mockRestore(); + await rm(cleanupIntent, { force: true }); + await writeFile(destination, '{"orphaned":true}', { mode: 0o600 }); + vi.resetModules(); + const freshCredentials = await import("./codex-credentials.js"); + const persistentSyncFailure = vi + .spyOn(prototype, "sync") + .mockImplementation(async function (this: FileHandle): Promise { + if ((await this.stat()).isDirectory()) { + throw new Error("persistent recovery sync failure"); + } + await originalSync.call(this); + }); + try { + await expect( + freshCredentials.stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }), + ).rejects.toThrow("remained non-durable after 8 attempts"); + await expect(readFile(destination)).rejects.toMatchObject({ + code: "ENOENT", + }); + } finally { + persistentSyncFailure.mockRestore(); + } + } finally { + syncSpy.mockRestore(); + } + }, + ); + + it.runIf(process.platform !== "win32")( + "owns cleanup when post-rename directory durability fails", + async () => { + const fixture = await credentialFixture(); + const destination = join(fixture.home, "auth.json"); + const probe = await open(fixture.home, "r"); + const prototype = Object.getPrototypeOf(probe) as { + sync(this: FileHandle): Promise; + }; + await probe.close(); + const originalSync = prototype.sync; + let directorySyncAttempts = 0; + const syncSpy = vi + .spyOn(prototype, "sync") + .mockImplementation(async function (this: FileHandle): Promise { + if ((await this.stat()).isDirectory()) { + directorySyncAttempts += 1; + if (directorySyncAttempts > 2) { + throw new Error("persistent post-rename sync failure"); + } + } + await originalSync.call(this); + }); + try { + await expect( + stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" }, + }), + ).rejects.toThrow("remained non-durable after 8 attempts"); + await expect(readFile(destination, "utf8")).resolves.toBe("{}"); + + syncSpy.mockRestore(); + const lease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" }, + }); + await expect(readFile(destination, "utf8")).resolves.toBe("{}"); + await lease.close(); + } finally { + syncSpy.mockRestore(); + } + }, + ); + + it("rejects missing, ambiguous, malformed, and unsafe sources", async () => { + const fixture = await credentialFixture(); + await expect( + stageManagedCodexCredential({ agentHomeDirectory: fixture.home }), + ).rejects.toThrow(/credential missing/); + await expect( + stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { + OPENAI_API_KEY: "key", + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}", + }, + }), + ).rejects.toThrow(/ambiguous/); + await expect( + stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "[]" }, + }), + ).rejects.toThrow(/malformed/); + + const source = join(fixture.root, "unsafe-auth.json"); + await writeFile(source, "{}", { mode: 0o644 }); + await chmod(source, 0o644); + if (process.platform !== "win32") { + await expect( + stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + sourcePath: source, + }), + ).rejects.toThrow(/permissions are unsafe/); + } + }); + + it.runIf(process.platform !== "win32")( + "rejects a credential home that is not private", + async () => { + const fixture = await credentialFixture(); + await chmod(fixture.home, 0o755); + + await expect( + stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { OPENAI_API_KEY: "launch-only-key" }, + }), + ).rejects.toThrow(/home permissions are unsafe/); + }, + ); + + it.runIf(process.platform !== "win32")( + "rejects a symbolic-link source", + async () => { + const fixture = await credentialFixture(); + const target = join(fixture.root, "auth-target.json"); + const source = join(fixture.root, "auth-link.json"); + await writeFile(target, "{}", { mode: 0o600 }); + await symlink(target, source); + + await expect( + stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + sourcePath: source, + }), + ).rejects.toThrow(/credential missing/); + }, + ); + + it.runIf(process.platform !== "win32")( + "replaces a stale destination link without touching its target", + async () => { + const fixture = await credentialFixture(); + const target = join(fixture.root, "outside.json"); + const destination = join(fixture.home, "auth.json"); + await writeFile(target, '{"outside":true}', { mode: 0o600 }); + await symlink(target, destination); + + const lease = await stageManagedCodexCredential({ + agentHomeDirectory: fixture.home, + environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" }, + }); + await expect(readFile(target, "utf8")).resolves.toBe('{"outside":true}'); + expect((await stat(lease.path)).isFile()).toBe(true); + await lease.close(); + }, + ); +}); + +async function credentialFixture(): Promise<{ root: string; home: string }> { + const root = await mkdtemp(join(tmpdir(), "paperclip-acpx-credential-")); + temporaryDirectories.push(root); + const home = join(root, "codex-home"); + await mkdir(home, { mode: 0o700 }); + await chmod(home, 0o700); + return { root, home }; +} + +function credentialLeasePorts(home: string): readonly number[] { + const userScope = + typeof process.getuid === "function" ? String(process.getuid()) : "win32"; + return credentialLeasePortsForScope(userScope, home); +} + +function credentialLeasePortsForScope( + userScope: string, + home: string, +): readonly number[] { + const digest = createHash("sha256") + .update("paperclip-managed-codex-lease-v2:") + .update(userScope) + .update("\0") + .update(home) + .digest(); + const start = digest.readUInt16BE(0) % 16_384; + const step = (digest.readUInt16BE(2) | 1) % 16_384; + return Array.from( + { length: 3 }, + (_, index) => 49_152 + ((start + index * step) % 16_384), + ); +} + +async function listenSilently( + port: number, +): Promise<{ close(): Promise }> { + const sockets = new Set(); + const server: Server = createServer((socket) => { + sockets.add(socket); + socket.once("close", () => sockets.delete(socket)); + socket.pause(); + }); + await new Promise((resolveListen, rejectListen) => { + server.once("error", rejectListen); + server.listen( + { + exclusive: true, + host: "127.0.0.1", + port, + }, + resolveListen, + ); + }); + return { + async close(): Promise { + for (const socket of sockets) socket.destroy(); + await new Promise((resolveClose, rejectClose) => { + server.close((error) => { + if (error) rejectClose(error); + else resolveClose(); + }); + }); + }, + }; +} + +async function waitForChildMessage( + child: ChildProcess, + type: string, +): Promise { + let diagnostic = ""; + child.stderr?.on("data", (chunk: Buffer) => { + diagnostic = `${diagnostic}${chunk.toString("utf8")}`.slice(-4_096); + }); + await new Promise((resolveMessage, rejectMessage) => { + const timeout = setTimeout(() => { + rejectMessage(new Error(`child message timed out: ${diagnostic}`)); + }, 10_000); + const finish = (error?: Error): void => { + clearTimeout(timeout); + child.off("message", onMessage); + child.off("exit", onExit); + if (error) rejectMessage(error); + else resolveMessage(); + }; + const onMessage = (message: unknown): void => { + if ( + typeof message === "object" && + message !== null && + "type" in message && + (message as { type?: unknown }).type === type + ) { + finish(); + } + }; + const onExit = (code: number | null): void => { + finish( + new Error( + `credential owner exited before ${type} (code ${String(code)}): ${diagnostic}`, + ), + ); + }; + child.on("message", onMessage); + child.once("exit", onExit); + }); +} + +async function waitForChildExit(child: ChildProcess): Promise { + if (child.exitCode !== null || child.signalCode !== null) return; + await new Promise((resolveExit, rejectExit) => { + const timeout = setTimeout( + () => rejectExit(new Error("credential owner exit timed out")), + 10_000, + ); + child.once("exit", () => { + clearTimeout(timeout); + resolveExit(); + }); + }); +} diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-credentials.ts b/packages/paperclip-runner/src/drivers/acpx/codex-credentials.ts new file mode 100644 index 0000000000..5abec9e887 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/acpx/codex-credentials.ts @@ -0,0 +1,1231 @@ +import { spawn, type ChildProcess } from "node:child_process"; +import { createHash } from "node:crypto"; +import { constants } from "node:fs"; +import { + lstat, + open, + realpath, + rename, + unlink, + type FileHandle, +} from "node:fs/promises"; +import { createServer, type Server } from "node:net"; +import { isAbsolute, join, resolve } from "node:path"; + +const MAX_CODEX_CREDENTIAL_BYTES = 256 * 1024; +const PRIVATE_FILE_MODE = 0o600; +const MAX_DIRECTORY_SYNC_ATTEMPTS = 8; +const DIRECTORY_SYNC_OPERATION_TIMEOUT_MS = 1_000; +const MAX_AUTONOMOUS_CREDENTIAL_CLEANUP_ATTEMPTS = 8; +const CREDENTIAL_CLEANUP_INTENT = ".paperclip-auth-cleanup-required"; +const CREDENTIAL_STAGING_FILE = ".paperclip-auth-staging-v1"; +const CREDENTIAL_LEASE_HOST = "127.0.0.1"; +const CREDENTIAL_LEASE_PORT_MIN = 49_152; +const CREDENTIAL_LEASE_PORT_COUNT = 16_384; +const CREDENTIAL_LEASE_CANDIDATES = 3; +const CREDENTIAL_LEASE_QUORUM = 2; +const DIRECTORY_SYNC_HELPER_KILL_ACK_TIMEOUT_MS = 1_000; +const MAX_DIRECTORY_SYNC_HELPERS = 4; +const MAX_PARENT_DIRECTORY_SYNC_OPERATIONS = 4; +const DIRECTORY_SYNC_HELPER_SOURCE = String.raw` +import { constants } from "node:fs"; +import { open } from "node:fs/promises"; + +const directory = process.argv[1]; +if (typeof directory !== "string") throw new Error("directory is required"); +const handle = await open( + directory, + constants.O_RDONLY | (constants.O_DIRECTORY ?? 0), +); +try { + await handle.sync(); +} finally { + await handle.close(); +} +`; + +interface CredentialHomeLock { + assertHeld(): void; + release(): Promise; +} + +interface QuarantinedCredentialCleanup { + path: string; + home: string; + intentPath: string; + ownerGeneration: CredentialLeaseGeneration; + lock: CredentialHomeLock; + recovery: Promise | null; +} + +type CredentialLeaseGeneration = number; + +interface DirectorySyncHelperRegistry { + activeChildren: Set; + childDirectories: Map; + activeParentOperations: Set; + attempts: Map>; + failedHomes: Set; + isolatedHomes: Set; + pendingCleanups: Map>; + stuckChildren: Map; +} + +const quarantinedCredentialCleanups = new Map< + string, + QuarantinedCredentialCleanup +>(); +const processDirectorySyncHelperState = globalThis as typeof globalThis & { + __paperclipDirectorySyncHelperRegistryV1?: DirectorySyncHelperRegistry; +}; +const directorySyncHelperRegistry = + processDirectorySyncHelperState.__paperclipDirectorySyncHelperRegistryV1 ?? { + activeChildren: new Set(), + childDirectories: new Map(), + activeParentOperations: new Set(), + attempts: new Map>(), + failedHomes: new Set(), + isolatedHomes: new Set(), + pendingCleanups: new Map>(), + stuckChildren: new Map(), + }; +processDirectorySyncHelperState.__paperclipDirectorySyncHelperRegistryV1 = + directorySyncHelperRegistry; +const pendingDirectorySyncCleanups = + directorySyncHelperRegistry.pendingCleanups; +const isolatedDirectorySyncHomes = directorySyncHelperRegistry.isolatedHomes; +const isolatedDirectorySyncAttempts = directorySyncHelperRegistry.attempts; +const failedIsolatedDirectorySyncHomes = + directorySyncHelperRegistry.failedHomes; +const stuckDirectorySyncHelpers = directorySyncHelperRegistry.stuckChildren; +const directorySyncHelperDirectories = + directorySyncHelperRegistry.childDirectories; +const activeCredentialLeaseGenerations = new Map< + string, + CredentialLeaseGeneration +>(); +let nextCredentialLeaseGeneration = 0; + +export type ManagedCodexCredentialMode = + "api_key" | "inline_json" | "managed_file"; + +export interface ManagedCodexCredentialLease { + readonly path: string; + readonly mode: ManagedCodexCredentialMode; + close(): Promise; +} + +/** Stage one explicit Codex authentication source in its isolated runtime home. */ +export async function stageManagedCodexCredential(input: { + agentHomeDirectory: string; + environment?: NodeJS.ProcessEnv; + sourcePath?: string; +}): Promise { + const home = await realpath(input.agentHomeDirectory); + const homeMetadata = await lstat(home); + if (!homeMetadata.isDirectory() || homeMetadata.isSymbolicLink()) { + throw new Error("Managed Codex credential home must be a real directory"); + } + if ( + process.platform !== "win32" && + ((homeMetadata.mode & 0o077) !== 0 || + (typeof process.getuid === "function" && + homeMetadata.uid !== process.getuid())) + ) { + throw new Error("Managed Codex credential home permissions are unsafe"); + } + // Join an older failed close before claiming the next generation. This + // keeps quarantine recovery authoritative over the shared paths without + // mistaking a waiting admission for an already-active successor. + await recoverQuarantinedCredentialCleanup(join(home, "auth.json"), home); + const ownerGeneration = claimCredentialLeaseGeneration(home); + let lock: CredentialHomeLock | null = null; + try { + lock = await acquireCredentialHomeLock(home); + return await stageClaimedManagedCodexCredential( + input, + home, + ownerGeneration, + lock, + ); + } catch (error) { + if ( + lock !== null && + quarantinedCredentialCleanups.get(home)?.lock !== lock + ) { + await lock.release().catch(() => undefined); + } + releaseCredentialLeaseGeneration(home, ownerGeneration); + throw error; + } +} + +async function stageClaimedManagedCodexCredential( + input: { + environment?: NodeJS.ProcessEnv; + sourcePath?: string; + }, + home: string, + ownerGeneration: CredentialLeaseGeneration, + lock: CredentialHomeLock, +): Promise { + const destination = join(home, "auth.json"); + const stagingPath = join(home, CREDENTIAL_STAGING_FILE); + const intentPath = join(home, CREDENTIAL_CLEANUP_INTENT); + lock.assertHeld(); + await recoverPersistedCredentialCleanup( + destination, + stagingPath, + home, + intentPath, + ); + // The isolated home is itself the durable recovery anchor. Scrub both the + // installed credential and the deterministic staging pathname before every + // admission, even when a prior cleanup-intent entry was lost with a runner + // crash. Only after their absence is durable may a new intent be created. + await removeCredentialArtifacts([destination, stagingPath], home); + const environment = input.environment ?? {}; + const hasApiKey = Boolean( + environment.CODEX_API_KEY || environment.OPENAI_API_KEY, + ); + const inlineJson = environment.PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET; + const hasInlineJson = typeof inlineJson === "string" && inlineJson.length > 0; + const hasManagedFile = + typeof input.sourcePath === "string" && input.sourcePath.length > 0; + const sourceCount = [hasApiKey, hasInlineJson, hasManagedFile].filter( + Boolean, + ).length; + if (sourceCount === 0) { + throw new Error( + "provider_initialize_protocol_error: provider=acpx stage=credential.stage managed Codex credential missing", + ); + } + if (sourceCount !== 1) { + throw new Error("Managed Codex credential source is ambiguous"); + } + + if ( + hasManagedFile && + (!isAbsolute(input.sourcePath!) || + resolve(input.sourcePath!) === destination) + ) { + throw new Error( + "Managed Codex credential source must be an external absolute path", + ); + } + + if (hasApiKey) { + // Codex will read the API key from the launch environment. Persist cleanup + // intent before admitting the provider and retain it for the lease + // lifetime, so a replacement runner removes provider-generated auth after + // a crash before it admits another provider. + // Failure while publishing the intent cannot strand a staged credential: + // the unconditional admission scrub above is already durable and this + // mode has not allowed the provider to create auth.json yet. + await createCredentialCleanupIntent(intentPath, home); + return credentialLease( + destination, + home, + intentPath, + "api_key", + ownerGeneration, + lock, + ); + } + + const credential = hasInlineJson + ? boundedInlineCredential(inlineJson!) + : await readManagedCredential(input.sourcePath!); + try { + validateCredentialDocument(credential); + // As with API-key mode, an intent-publication failure occurs before any + // credential mutation and therefore needs no process-only quarantine. + await createCredentialCleanupIntent(intentPath, home); + try { + await writeCredential(destination, stagingPath, home, credential); + } catch (error) { + // Rename may already have installed the credential before directory + // durability failed. Retain a bounded process owner and the persisted + // intent so later staging must recover both before admission. + quarantineCredentialCleanup( + destination, + home, + intentPath, + ownerGeneration, + lock, + ); + throw error; + } + } finally { + credential.fill(0); + } + return credentialLease( + destination, + home, + intentPath, + hasInlineJson ? "inline_json" : "managed_file", + ownerGeneration, + lock, + ); +} + +function claimCredentialLeaseGeneration( + home: string, +): CredentialLeaseGeneration { + if (activeCredentialLeaseGenerations.has(home)) { + throw new Error( + "Managed Codex credential home already has an active lease", + ); + } + if (nextCredentialLeaseGeneration >= Number.MAX_SAFE_INTEGER) { + throw new Error("Managed Codex credential lease generation exhausted"); + } + nextCredentialLeaseGeneration += 1; + activeCredentialLeaseGenerations.set(home, nextCredentialLeaseGeneration); + return nextCredentialLeaseGeneration; +} + +function releaseCredentialLeaseGeneration( + home: string, + ownerGeneration: CredentialLeaseGeneration, +): void { + if (activeCredentialLeaseGenerations.get(home) === ownerGeneration) { + activeCredentialLeaseGenerations.delete(home); + } +} + +async function acquireCredentialHomeLock( + home: string, +): Promise { + // This is deliberately markerless: authority is the live kernel ownership + // of any two candidates. Any two subsets of a three-port set intersect, so + // contenders cannot both reach quorum; one unrelated occupied listener is + // tolerated without probing or trusting the process behind it. + const servers: Server[] = []; + let invalid: Error | null = null; + let released = false; + try { + for (const port of credentialLeasePorts(home)) { + const server = createServer((socket) => socket.destroy()); + try { + await listenForCredentialLease(server, port); + } catch (error) { + await closeCredentialLeaseServer(server); + if (errorCode(error) === "EADDRINUSE") continue; + throw new Error( + "Managed Codex credential ownership could not be established", + { cause: error }, + ); + } + server.on("error", (error) => { + invalid ??= error; + }); + server.on("close", () => { + if (!released) { + invalid ??= new Error( + "Managed Codex credential ownership listener closed unexpectedly", + ); + } + }); + servers.push(server); + if (servers.length === CREDENTIAL_LEASE_QUORUM) break; + } + if (servers.length !== CREDENTIAL_LEASE_QUORUM) { + throw new Error( + "Managed Codex credential home already has an active lease", + ); + } + } catch (error) { + released = true; + await Promise.allSettled(servers.map(closeCredentialLeaseServer)); + throw error; + } + + return Object.freeze({ + assertHeld(): void { + if ( + released || + invalid !== null || + servers.filter((server) => server.listening).length < + CREDENTIAL_LEASE_QUORUM + ) { + throw new Error("Managed Codex credential ownership was lost"); + } + }, + async release(): Promise { + if (released) return; + const outcomes = await Promise.allSettled( + servers.map(closeCredentialLeaseServer), + ); + const listenersStillHeld = servers.filter( + (server) => server.listening, + ).length; + if (listenersStillHeld >= CREDENTIAL_LEASE_QUORUM) { + const failure = outcomes.find( + (outcome): outcome is PromiseRejectedResult => + outcome.status === "rejected", + ); + throw new Error( + "Managed Codex credential ownership could not be released", + { cause: failure?.reason }, + ); + } + // Once fewer than two listeners remain, quorum authority is gone. Never + // throw after that point: a stale quarantine must not touch a successor. + released = true; + }, + }); +} + +function credentialLeasePorts(home: string): readonly number[] { + const userScope = + typeof process.getuid === "function" ? String(process.getuid()) : "win32"; + const digest = createHash("sha256") + .update("paperclip-managed-codex-lease-v2:") + .update(userScope) + .update("\0") + .update(home) + .digest(); + const start = digest.readUInt16BE(0) % CREDENTIAL_LEASE_PORT_COUNT; + const step = (digest.readUInt16BE(2) | 1) % CREDENTIAL_LEASE_PORT_COUNT; + return Array.from( + { length: CREDENTIAL_LEASE_CANDIDATES }, + (_, index) => + CREDENTIAL_LEASE_PORT_MIN + + ((start + index * step) % CREDENTIAL_LEASE_PORT_COUNT), + ); +} + +async function listenForCredentialLease( + server: Server, + port: number, +): Promise { + await new Promise((resolveListen, rejectListen) => { + const onError = (error: Error): void => { + server.off("listening", onListening); + rejectListen(error); + }; + const onListening = (): void => { + server.off("error", onError); + resolveListen(); + }; + server.once("error", onError); + server.once("listening", onListening); + server.listen({ + exclusive: true, + host: CREDENTIAL_LEASE_HOST, + port, + }); + }); +} + +async function closeCredentialLeaseServer(server: Server): Promise { + if (!server.listening) return; + await new Promise((resolveClose, rejectClose) => { + server.close((error) => { + if (error) rejectClose(error); + else resolveClose(); + }); + }); +} +function quarantineCredentialCleanup( + path: string, + home: string, + intentPath: string, + ownerGeneration: CredentialLeaseGeneration, + lock: CredentialHomeLock, +): void { + const existing = quarantinedCredentialCleanups.get(home); + if (existing !== undefined) return; + const cleanup: QuarantinedCredentialCleanup = { + path, + home, + intentPath, + ownerGeneration, + lock, + recovery: null, + }; + quarantinedCredentialCleanups.set(home, cleanup); + startCredentialCleanupRecovery( + cleanup, + MAX_AUTONOMOUS_CREDENTIAL_CLEANUP_ATTEMPTS, + ); +} + +function startCredentialCleanupRecovery( + cleanup: QuarantinedCredentialCleanup, + maxAttempts: number, +): Promise { + if (cleanup.recovery) return cleanup.recovery; + const recovery = (async () => { + let retryDelayMs = 10; + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + try { + // Never let a stale cleanup callback mutate a successor's credential + // after kernel ownership has been lost. + assertCredentialCleanupAuthority(cleanup); + await removeReplaceableCredential(cleanup.path); + await removeReplaceableCredential( + join(cleanup.home, CREDENTIAL_STAGING_FILE), + ); + assertCredentialCleanupAuthority(cleanup); + await syncDirectory(cleanup.home); + assertCredentialCleanupAuthority(cleanup); + await removeCredentialCleanupIntent(cleanup.intentPath, cleanup.home); + assertCredentialCleanupAuthority(cleanup); + await cleanup.lock.release(); + quarantinedCredentialCleanups.delete(cleanup.home); + return; + } catch { + if (attempt === maxAttempts) return; + await new Promise((resolveRetry) => { + const timer = setTimeout(resolveRetry, retryDelayMs); + timer.unref?.(); + }); + retryDelayMs = Math.min(retryDelayMs * 2, 1_000); + } + } + })(); + cleanup.recovery = recovery; + void recovery + .finally(() => { + if (cleanup.recovery === recovery) cleanup.recovery = null; + }) + .catch(() => undefined); + return recovery; +} + +function assertCredentialCleanupAuthority( + cleanup: QuarantinedCredentialCleanup, +): void { + const activeGeneration = activeCredentialLeaseGenerations.get(cleanup.home); + if ( + activeGeneration !== undefined && + activeGeneration !== cleanup.ownerGeneration + ) { + throw new Error( + "Managed Codex credential cleanup was superseded by an active lease", + ); + } + cleanup.lock.assertHeld(); +} + +async function recoverQuarantinedCredentialCleanup( + path: string, + home: string, +): Promise { + const cleanup = quarantinedCredentialCleanups.get(home); + if (cleanup === undefined) return; + await (cleanup.recovery ?? startCredentialCleanupRecovery(cleanup, 1)); + if (!quarantinedCredentialCleanups.has(home)) return; + const admissionRecovery = startCredentialCleanupRecovery(cleanup, 1); + await admissionRecovery; + if (quarantinedCredentialCleanups.has(home)) { + throw new Error( + `Managed Codex credential cleanup remains non-durable for ${path}`, + ); + } +} + +async function recoverPersistedCredentialCleanup( + path: string, + stagingPath: string, + home: string, + intentPath: string, +): Promise { + if (!(await pathExists(intentPath))) return; + await removeCredentialArtifacts([path, stagingPath], home); + await removeCredentialCleanupIntent(intentPath, home); +} + +function credentialLease( + path: string, + home: string, + intentPath: string, + mode: ManagedCodexCredentialMode, + ownerGeneration: CredentialLeaseGeneration, + lock: CredentialHomeLock, +): ManagedCodexCredentialLease { + // Do not admit a provider if kernel ownership was lost while its credential + // was being staged. + lock.assertHeld(); + let closed = false; + let closeAttempt: Promise | null = null; + return Object.freeze({ + path, + mode, + async close(): Promise { + if (closed) return; + if (closeAttempt !== null) return await closeAttempt; + const attempt = (async () => { + const activeGeneration = activeCredentialLeaseGenerations.get(home); + if (activeGeneration !== ownerGeneration) { + // A failed close releases its generation only after publishing a + // quarantine owner. If no successor exists, synchronously join that + // cleanup before acknowledging the retry. If a successor does + // exist, this stale lease has no authority over its credential. + if (activeGeneration === undefined) { + await recoverQuarantinedCredentialCleanup(path, home); + await lock.release(); + } + closed = true; + return; + } + try { + lock.assertHeld(); + await removeCredential(path, home); + await removeCredentialCleanupIntent(intentPath, home); + await lock.release(); + closed = true; + } catch (error) { + quarantineCredentialCleanup( + path, + home, + intentPath, + ownerGeneration, + lock, + ); + throw error; + } finally { + releaseCredentialLeaseGeneration(home, ownerGeneration); + } + })(); + closeAttempt = attempt; + try { + await attempt; + } finally { + if (closeAttempt === attempt) closeAttempt = null; + } + }, + }); +} + +async function createCredentialCleanupIntent( + intentPath: string, + home: string, +): Promise { + let handle: FileHandle | undefined; + try { + handle = await open( + intentPath, + constants.O_WRONLY | + constants.O_CREAT | + constants.O_EXCL | + (constants.O_NOFOLLOW ?? 0), + PRIVATE_FILE_MODE, + ); + await handle.chmod(PRIVATE_FILE_MODE); + await handle.writeFile("paperclip-managed-codex-cleanup-v1\n", "utf8"); + await handle.sync(); + await handle.close(); + handle = undefined; + await syncDirectoryDurably(home); + } finally { + await handle?.close().catch(() => undefined); + } +} + +async function removeCredentialCleanupIntent( + intentPath: string, + home: string, +): Promise { + await removeReplaceableCredential(intentPath); + await syncDirectoryDurably(home); +} + +async function pathExists(path: string): Promise { + try { + await lstat(path); + return true; + } catch (error) { + if (errorCode(error) === "ENOENT") return false; + throw error; + } +} + +function boundedInlineCredential(value: string): Buffer { + const bytes = Buffer.from(value, "utf8"); + if (bytes.length < 1 || bytes.length > MAX_CODEX_CREDENTIAL_BYTES) { + bytes.fill(0); + throw new Error( + "Managed Codex credential document exceeds its bounded size", + ); + } + return bytes; +} + +async function readManagedCredential(sourcePath: string): Promise { + let handle: FileHandle; + try { + handle = await open( + sourcePath, + constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0), + ); + } catch { + throw new Error( + "provider_initialize_protocol_error: provider=acpx stage=credential.stage managed Codex credential missing", + ); + } + try { + const before = await handle.stat({ bigint: true }); + if ( + !before.isFile() || + before.size < 1n || + before.size > BigInt(MAX_CODEX_CREDENTIAL_BYTES) + ) { + throw new Error( + "Managed Codex credential source is not a bounded regular file", + ); + } + if (process.platform !== "win32" && (before.mode & 0o077n) !== 0n) { + throw new Error("Managed Codex credential source permissions are unsafe"); + } + if ( + process.platform !== "win32" && + typeof process.getuid === "function" && + before.uid !== BigInt(process.getuid()) + ) { + throw new Error("Managed Codex credential source ownership is unsafe"); + } + const bytes = await readHandle(handle, Number(before.size)); + const after = await handle.stat({ bigint: true }); + if ( + before.dev !== after.dev || + before.ino !== after.ino || + before.size !== after.size || + before.mtimeNs !== after.mtimeNs || + before.ctimeNs !== after.ctimeNs || + after.size !== BigInt(bytes.length) + ) { + bytes.fill(0); + throw new Error("Managed Codex credential source changed while read"); + } + return bytes; + } finally { + await handle.close(); + } +} + +async function readHandle(handle: FileHandle, size: number): Promise { + const bytes = Buffer.alloc(size); + let offset = 0; + while (offset < size) { + const result = await handle.read(bytes, offset, size - offset, offset); + if (result.bytesRead === 0) break; + offset += result.bytesRead; + } + if (offset !== size) { + bytes.fill(0); + throw new Error("Managed Codex credential source ended while read"); + } + return bytes; +} + +function validateCredentialDocument(bytes: Buffer): void { + let value: unknown; + try { + value = JSON.parse(bytes.toString("utf8")); + } catch { + throw new Error("Managed Codex credential source is malformed"); + } + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error("Managed Codex credential source is malformed"); + } +} + +async function writeCredential( + destination: string, + temporaryPath: string, + home: string, + bytes: Buffer, +): Promise { + let handle: FileHandle; + try { + handle = await open( + temporaryPath, + constants.O_WRONLY | + constants.O_CREAT | + constants.O_EXCL | + (constants.O_NOFOLLOW ?? 0), + PRIVATE_FILE_MODE, + ); + } catch { + throw new Error("Managed Codex credential destination could not be opened"); + } + try { + await handle.chmod(PRIVATE_FILE_MODE); + await handle.writeFile(bytes); + await handle.sync(); + await handle.close(); + try { + await rename(temporaryPath, destination); + } catch (error) { + if ( + process.platform !== "win32" || + !["EACCES", "EEXIST", "ENOTEMPTY", "EPERM"].includes( + errorCode(error) ?? "", + ) + ) { + throw error; + } + // Win32 rename does not replace an existing destination. Remove only + // the already-conflicting pathname (never a real directory), then move + // the fully synced private temporary file into place. + await removeReplaceableCredential(destination); + await rename(temporaryPath, destination); + } + } finally { + await handle.close().catch(() => undefined); + await unlink(temporaryPath).catch(() => undefined); + } + // Do not acknowledge the lease until the namespace update is durable. A + // directory-sync failure is a fail-closed admission condition: retry here so + // neither a returned lease nor a thrown pre-lease error can lose ownership + // of auth.json across a crash. + await syncDirectoryDurably(home); +} + +async function removeReplaceableCredential(path: string): Promise { + try { + const metadata = await lstat(path); + if (metadata.isDirectory() && !metadata.isSymbolicLink()) { + throw new Error("Managed Codex credential destination is a directory"); + } + await unlink(path); + } catch (error) { + if (errorCode(error) !== "ENOENT") throw error; + } +} + +async function removeCredential(path: string, home: string): Promise { + await removeCredentialArtifacts([path], home); +} + +async function removeCredentialArtifacts( + paths: readonly string[], + home: string, +): Promise { + for (const path of paths) { + await removeReplaceableCredential(path); + } + // Sync even after ENOENT: a previous unlink may have succeeded before its + // directory sync failed. Never report cleanup or finish preflight while the + // removal can still be rolled back by a crash. + await syncDirectoryDurably(home); +} + +async function syncDirectoryDurably(directory: string): Promise { + let retryDelayMs = 10; + let lastError: unknown; + let attempts = 0; + for (let attempt = 1; attempt <= MAX_DIRECTORY_SYNC_ATTEMPTS; attempt += 1) { + attempts = attempt; + try { + await syncDirectory(directory); + return; + } catch (error) { + lastError = error; + if ( + attempt === MAX_DIRECTORY_SYNC_ATTEMPTS || + error instanceof DirectorySyncOperationTimeoutError || + error instanceof UnconfirmedDirectorySyncHelperTerminationError + ) { + break; + } + // Keep admission closed during transient failures, while bounding total + // startup/shutdown latency for a persistently unhealthy filesystem. + await new Promise((resolveRetry) => { + setTimeout(resolveRetry, retryDelayMs); + }); + retryDelayMs = Math.min(retryDelayMs * 2, 1_000); + } + } + const attemptNoun = attempts === 1 ? "attempt" : "attempts"; + throw new Error( + `Managed Codex credential directory remained non-durable after ${attempts} ${attemptNoun}`, + { cause: lastError }, + ); +} + +async function syncDirectory(directory: string): Promise { + if (process.platform === "win32") return; + reclaimConfirmedExitedDirectorySyncHelpers(); + // Once an in-process filesystem request times out it cannot be cancelled. + // Keep that single request observed, and permanently use killable helper + // processes for this home so recovery remains available without accumulating + // additional parent-process handles or requests. + if (isolatedDirectorySyncHomes.has(directory)) { + if (failedIsolatedDirectorySyncHomes.has(directory)) { + throw new UnconfirmedDirectorySyncHelperTerminationError(directory); + } + await syncDirectoryInIsolatedProcess(directory); + return; + } + const parentOperation = reserveParentDirectorySyncOperation(); + if (parentOperation === null) { + // Never start a fifth uncancellable parent filesystem request. The helper + // pool has its own global bound and can be killed independently. + isolatedDirectorySyncHomes.add(directory); + await syncDirectoryInIsolatedProcess(directory); + return; + } + let retainParentOperation = false; + try { + const openAttempt = open( + directory, + constants.O_RDONLY | (constants.O_DIRECTORY ?? 0), + ); + let handle: FileHandle; + try { + handle = await waitForDirectorySyncOperation( + openAttempt, + "open", + directory, + ); + } catch (error) { + if (error instanceof DirectorySyncOperationTimeoutError) { + // open(2) cannot be cancelled. Retain this process-global reservation + // for the process lifetime and observe/close a late handle without + // ever starting another parent request for this home. + retainParentOperation = true; + const cleanupAttempt = openAttempt.then( + (lateHandle) => closeDirectoryHandle(lateHandle, directory), + () => undefined, + ); + retainDirectorySyncCleanup(directory, cleanupAttempt); + isolatedDirectorySyncHomes.add(directory); + await syncDirectoryInIsolatedProcess(directory); + return; + } + throw error; + } + + let syncAttempt: Promise; + try { + syncAttempt = handle.sync(); + } catch (error) { + retainParentOperation = await closeDirectoryHandle(handle, directory); + throw error; + } + let syncTimedOut = false; + try { + await waitForDirectorySyncOperation(syncAttempt, "fsync", directory); + } catch (error) { + syncTimedOut = error instanceof DirectorySyncOperationTimeoutError; + if (syncTimedOut) { + // FileHandle.close() waits for outstanding operations. Retain both the + // cleanup observer and the parent-operation slot for process lifetime. + retainParentOperation = true; + const cleanupAttempt = syncAttempt.then( + () => handle.close().catch(() => undefined), + () => handle.close().catch(() => undefined), + ); + retainDirectorySyncCleanup(directory, cleanupAttempt); + isolatedDirectorySyncHomes.add(directory); + await syncDirectoryInIsolatedProcess(directory); + return; + } + throw error; + } finally { + // A completed fsync is the durability boundary. Close is resource + // cleanup; a timed-out close retains the global parent-operation slot. + if (!syncTimedOut) { + retainParentOperation = + (await closeDirectoryHandle(handle, directory)) || + retainParentOperation; + } + } + } finally { + if (!retainParentOperation) { + directorySyncHelperRegistry.activeParentOperations.delete( + parentOperation, + ); + } + } +} + +async function closeDirectoryHandle( + handle: FileHandle, + directory: string, +): Promise { + try { + const closeAttempt = handle.close(); + try { + await waitForDirectorySyncOperation(closeAttempt, "close", directory); + } catch (error) { + // close(2) cannot be cancelled. Keep observing a late rejection without + // leaking additional directory handles. Admission remains closed until + // the real close settles, while a completed fsync stays successful. + if (error instanceof DirectorySyncOperationTimeoutError) { + retainDirectorySyncCleanup(directory, closeAttempt); + isolatedDirectorySyncHomes.add(directory); + return true; + } else { + void closeAttempt.catch(() => undefined); + } + } + } catch { + // Closing cannot invalidate an fsync that already completed, and callers + // with a failed fsync must retain that original durability error. + } + return false; +} + +function reserveParentDirectorySyncOperation(): symbol | null { + if ( + directorySyncHelperRegistry.activeParentOperations.size >= + MAX_PARENT_DIRECTORY_SYNC_OPERATIONS + ) { + return null; + } + const reservation = Symbol("parent-directory-sync"); + directorySyncHelperRegistry.activeParentOperations.add(reservation); + return reservation; +} + +async function syncDirectoryInIsolatedProcess( + directory: string, +): Promise { + reclaimConfirmedExitedDirectorySyncHelpers(); + if (failedIsolatedDirectorySyncHomes.has(directory)) { + throw new UnconfirmedDirectorySyncHelperTerminationError(directory); + } + const activeAttempt = isolatedDirectorySyncAttempts.get(directory); + if (activeAttempt !== undefined) return await activeAttempt; + const attempt = runDirectorySyncHelper(directory); + isolatedDirectorySyncAttempts.set(directory, attempt); + try { + await attempt; + } finally { + if (isolatedDirectorySyncAttempts.get(directory) === attempt) { + isolatedDirectorySyncAttempts.delete(directory); + } + } +} + +async function runDirectorySyncHelper(directory: string): Promise { + reclaimConfirmedExitedDirectorySyncHelpers(); + if ( + directorySyncHelperRegistry.activeChildren.size >= + MAX_DIRECTORY_SYNC_HELPERS + ) { + throw new Error( + `Managed Codex credential directory helper process limit of ${MAX_DIRECTORY_SYNC_HELPERS} was reached`, + ); + } + let child: ChildProcess; + try { + child = spawn( + process.execPath, + [ + "--input-type=module", + "--eval", + DIRECTORY_SYNC_HELPER_SOURCE, + directory, + ], + { + // The helper imports only Node built-ins. Do not inherit loader hooks or + // any credential-bearing process environment into the durability worker. + env: {}, + stdio: "ignore", + windowsHide: true, + }, + ); + directorySyncHelperRegistry.activeChildren.add(child); + directorySyncHelperDirectories.set(child, directory); + child.unref(); + } catch (error) { + throw new Error( + `Managed Codex credential directory helper could not start for ${directory}`, + { cause: error }, + ); + } + await new Promise((resolveHelper, rejectHelper) => { + let timeout: NodeJS.Timeout | undefined; + let killAcknowledgementTimeout: NodeJS.Timeout | undefined; + let timedOut = false; + let settled = false; + let reaped = false; + const settle = (error?: Error): void => { + if (settled) return; + settled = true; + if (timeout !== undefined) clearTimeout(timeout); + if (killAcknowledgementTimeout !== undefined) { + clearTimeout(killAcknowledgementTimeout); + } + if (error === undefined) resolveHelper(); + else rejectHelper(error); + }; + const reap = (): void => { + if (reaped) return; + reaped = true; + directorySyncHelperRegistry.activeChildren.delete(child); + directorySyncHelperDirectories.delete(child); + if (stuckDirectorySyncHelpers.get(directory) === child) { + stuckDirectorySyncHelpers.delete(directory); + failedIsolatedDirectorySyncHomes.delete(directory); + } + child.off("error", onError); + child.off("exit", onExit); + child.off("close", onClose); + }; + const fenceUntilExitConfirmed = (): void => { + failedIsolatedDirectorySyncHomes.add(directory); + stuckDirectorySyncHelpers.set(directory, child); + settle(new UnconfirmedDirectorySyncHelperTerminationError(directory)); + }; + const onError = (): void => { + fenceUntilExitConfirmed(); + // A missing pid proves spawn never created a process, so no retained + // reaper or global capacity slot is necessary. + if (child.pid === undefined) reap(); + }; + const onExit = ( + code: number | null, + signal: NodeJS.Signals | null, + ): void => { + reap(); + if (timedOut) { + settle(new DirectorySyncOperationTimeoutError("helper", directory)); + } else if (code === 0) { + settle(); + } else { + settle( + new Error( + `Managed Codex credential directory helper failed with ${ + signal === null ? `code ${String(code)}` : `signal ${signal}` + }`, + ), + ); + } + }; + const onClose = (): void => reap(); + child.on("error", onError); + child.once("exit", onExit); + child.once("close", onClose); + timeout = setTimeout(() => { + timedOut = true; + // Wait for the exit event after SIGKILL before permitting another helper; + // this is the resource-release acknowledgement the in-process API lacks. + try { + if (!child.kill("SIGKILL")) { + fenceUntilExitConfirmed(); + reclaimConfirmedExitedDirectorySyncHelpers(); + return; + } + killAcknowledgementTimeout = setTimeout(() => { + // A child that does not acknowledge SIGKILL is retained as the sole + // reaper for this home. Fail closed until an event or a kernel probe + // confirms exit, so no later attempt can accumulate another process. + fenceUntilExitConfirmed(); + reclaimConfirmedExitedDirectorySyncHelpers(); + }, DIRECTORY_SYNC_HELPER_KILL_ACK_TIMEOUT_MS); + killAcknowledgementTimeout.unref?.(); + } catch { + fenceUntilExitConfirmed(); + reclaimConfirmedExitedDirectorySyncHelpers(); + } + }, DIRECTORY_SYNC_OPERATION_TIMEOUT_MS); + timeout.unref?.(); + }); +} + +function reclaimConfirmedExitedDirectorySyncHelpers(): void { + for (const child of directorySyncHelperRegistry.activeChildren) { + let exited = + (child.exitCode !== null && child.exitCode !== undefined) || + (child.signalCode !== null && child.signalCode !== undefined); + if (!exited && child.pid !== undefined) { + try { + // ChildProcess events are advisory for capacity accounting: an exited + // helper can fail to deliver `exit`/`close` while its owner is under + // pressure. A signal-0 ESRCH result is the kernel confirmation that + // reclaiming this global slot cannot permit another live helper. + process.kill(child.pid, 0); + } catch (error) { + exited = errorCode(error) === "ESRCH"; + } + } + if (!exited) continue; + directorySyncHelperRegistry.activeChildren.delete(child); + const directory = + directorySyncHelperDirectories.get(child) ?? + [...stuckDirectorySyncHelpers].find(([, owner]) => owner === child)?.[0]; + directorySyncHelperDirectories.delete(child); + if (directory && stuckDirectorySyncHelpers.get(directory) === child) { + stuckDirectorySyncHelpers.delete(directory); + failedIsolatedDirectorySyncHomes.delete(directory); + } + } +} + +function retainDirectorySyncCleanup( + directory: string, + attempt: Promise, +): void { + const observed = attempt.then( + () => undefined, + () => undefined, + ); + const prior = pendingDirectorySyncCleanups.get(directory); + const barrier = + prior === undefined + ? observed + : Promise.allSettled([prior, observed]).then(() => undefined); + pendingDirectorySyncCleanups.set(directory, barrier); + void barrier + .finally(() => { + if (pendingDirectorySyncCleanups.get(directory) === barrier) { + pendingDirectorySyncCleanups.delete(directory); + } + }) + .catch(() => undefined); +} + +class DirectorySyncOperationTimeoutError extends Error { + constructor( + operation: "open" | "fsync" | "close" | "helper", + directory: string, + ) { + super( + `Managed Codex credential directory ${operation} timed out after ${DIRECTORY_SYNC_OPERATION_TIMEOUT_MS}ms for ${directory}`, + ); + this.name = "DirectorySyncOperationTimeoutError"; + } +} + +class UnconfirmedDirectorySyncHelperTerminationError extends Error { + constructor(directory: string) { + super( + `Managed Codex credential directory helper termination was not acknowledged for ${directory}`, + ); + this.name = "UnconfirmedDirectorySyncHelperTerminationError"; + } +} + +async function waitForDirectorySyncOperation( + operationAttempt: Promise, + operation: "open" | "fsync" | "close", + directory: string, +): Promise { + let timeout: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + operationAttempt, + new Promise((_resolve, reject) => { + timeout = setTimeout(() => { + reject(new DirectorySyncOperationTimeoutError(operation, directory)); + }, DIRECTORY_SYNC_OPERATION_TIMEOUT_MS); + timeout.unref?.(); + }), + ]); + } finally { + if (timeout !== undefined) clearTimeout(timeout); + } +} + +function errorCode(error: unknown): string | null { + return typeof error === "object" && error !== null && "code" in error + ? String((error as { code?: unknown }).code) + : null; +}