import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { access, mkdir, mkdtemp, readdir, rm, symlink, writeFile, } from "node:fs/promises"; import { execFileSync } from "node:child_process"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { heartbeatRuns, issues, nativeRunFinalizations, type Db, } from "@paperclipai/db"; import type { NativeExecutionInputV1, PrpEvent, } from "@paperclipai/paperclip-runner"; import { createHash } from "node:crypto"; import { createNativeHarnessBackupStamp, verifyNativeHarnessBackupStamp, } from "./native-harness-backup-stamp.js"; import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js"; type BackendFactoryOptions = { runnerInstanceId?: string; acpxRuntimeDirectory?: string; codexTransportFactory?: () => unknown; dynamicToolHandler?: (call: unknown) => Promise; onSpawn?: (meta: { pid: number; processGroupId: number | null; startedAt: string; }) => Promise; }; type RunnerTransportOptions = { stateDirectory?: string; runnerBinary?: string; prpIdentity?: { runnerInstanceId: string; environmentLeaseId: string; runId: string; }; provider?: "codex" | "opencode" | "acpx"; opencodePermissionMode?: "allow" | "ask" | "deny"; acpxAgent?: "claude" | "codex"; acpxPermissionMode?: "approve-all" | "approve-reads" | "deny-all"; }; const durableControlPlaneState = (identity: Record) => ({ schema: "paperclip.runner.durable.control-plane-state.v1", identity, }); const durableRunnerState = ( identity: Record, lifecycle: string, ) => ({ schema: "paperclip.runner.durable.state.v1", ...identity, lifecycle, }); const state = vi.hoisted(() => ({ execute: vi.fn(), createTransport: vi.fn((_options: RunnerTransportOptions) => ({ transport: {}, })), createBackend: vi.fn( (_input: NativeExecutionInputV1, _options: BackendFactoryOptions) => ({ kind: "test", }), ), cancel: vi.fn(), toolAuthorityDefinitions: vi.fn( async (_binding: Record) => [], ), toolAuthorityExecute: vi.fn(), persistActivity: vi.fn(async (_db: unknown, input: { action: string }) => ({ activity: { id: input.action === "native.cancellation_intent_recorded" ? "native-cancellation-audit" : "native-cancellation-ack-audit", }, publication: { companyId: "company", payload: { action: input.action }, pluginEvent: null, }, })), publishActivity: vi.fn(), resolveRunnerBinary: vi.fn(() => "/tmp/paperclip-runnerd"), release: null as null | (() => void), })); vi.mock("../../vendor/paperclip-runner/index.js", async (importOriginal) => ({ ...(await importOriginal< typeof import("../../vendor/paperclip-runner/index.js") >()), createNativeSessionBackend: state.createBackend, createRunnerdCodexTransport: state.createTransport, executeNativeSession: state.execute, parsePaperclipQuestionSet: (value: unknown) => value, })); vi.mock("./paperclip-runner-tool-authority.js", () => ({ PaperclipRunnerToolAuthority: class { readonly binding: Record; constructor(_db: unknown, binding: Record) { this.binding = binding; } async definitions() { return state.toolAuthorityDefinitions(this.binding); } async execute(call: unknown) { return state.toolAuthorityExecute(this.binding, call); } }, })); vi.mock("../activity-log.js", () => ({ persistActivity: state.persistActivity, publishActivity: state.publishActivity, })); vi.mock("./native-codex-runner.js", () => ({ resolvePaperclipRunnerBinary: state.resolveRunnerBinary, })); import { continuingPendingInteractionIds, buildNativeProviderEnvironment, buildNativeHarnessBackupManifest, cancelNativeSession, createGovernedWaitEventObservation, createRunnerdBackend, executePaperclipNativeSession, getNativeSessionSteeringState, NativeSessionSteeringError, assertRemoteRunnerBuildMetadata, nativeSessionFailureDisposition, nativeSessionFailureSourceCode, nativeSessionRecoveryProjection, nativeGovernedWaitResult, parseRemoteExecutableCandidate, mayUsePreinstalledRunnerArtifact, nativeUsageCostUsd, normalizeNativeUsage, readRemoteProviderPackManifest, providerSessionIdentityTransitionIsAllowed, providerPlanMarkdown, resolveRemoteRunnerTransportMode, renewNativeSessionExecutionLease, runtimeInputLifecycleMetric, runtimeQuestionFallbackFromEvent, resolveNativeRuntimeRequest, resolveNativeHarnessPersistenceProfile, semanticProviderPlanMarkdown, sha256DirectoryTree, stageRemoteRunnerDirectory, steerNativeSession, syncRemoteRunnerDirectoryOut, verifyNativeHarnessBackup, shouldRestoreNativeHarnessBackupIntoSandbox, } from "./native-session-executor.js"; describe("native provider usage normalization", () => { it("reads remote runner run-delta tokens and provider cost", () => { const usage = { total: { inputTokens: 20_000, outputTokens: 500, cacheReadTokens: 8_000, providerCostUsd: 0.12, }, runDelta: { inputTokens: 4_200, outputTokens: 180, cacheReadTokens: 1_500, providerCostUsd: 0.031, }, }; expect(normalizeNativeUsage(usage)).toEqual({ inputTokens: 4_200, outputTokens: 180, cachedInputTokens: 1_500, }); expect(nativeUsageCostUsd(usage)).toBe(0.031); }); it("reads ACPX cumulative usage and a USD cost object", () => { const usage = { cumulative: { inputTokens: 3_000, outputTokens: 240, cachedReadTokens: 900, }, cost: { amount: 0.044, currency: "USD" }, }; expect(normalizeNativeUsage(usage)).toEqual({ inputTokens: 3_000, outputTokens: 240, cachedInputTokens: 900, }); expect(nativeUsageCostUsd(usage)).toBe(0.044); }); it("does not treat a non-USD ACPX amount as dollars", () => { expect( nativeUsageCostUsd({ cost: { amount: 1.25, currency: "EUR" } }), ).toBeUndefined(); }); }); describe("remote provider pack manifest", () => { const canonical = (value: unknown): string => { if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`; if (value && typeof value === "object") { const object = value as Record; return `{${Object.keys(object) .sort() .map((key) => `${JSON.stringify(key)}:${canonical(object[key])}`) .join(",")}}`; } return JSON.stringify(value); }; it("accepts a fully digested pack and rejects artifact tampering", async () => { const root = await mkdtemp(join(tmpdir(), "paperclip-provider-pack-")); await mkdir(join(root, "dist", "cli"), { recursive: true }); await mkdir(join(root, "node_modules", "node", "bin"), { recursive: true }); await mkdir(join(root, "node_modules", ".bin"), { recursive: true }); await mkdir(join(root, "node_modules", "opencode-ai", "bin"), { recursive: true, }); const proxy = "export const proxy = true;\n"; const sidecar = "export const sidecar = true;\n"; const node = "provider-node\n"; const lockfile = "lockfileVersion: '9.0'\n"; const opencodeCommand = "#!/bin/sh\n"; const opencodeExecutable = "opencode-binary\n"; await writeFile( join(root, "dist", "cli", "opencode-app-server-proxy.cjs"), proxy, ); await writeFile( join(root, "dist", "cli", "acpx-runtime-sidecar.cjs"), sidecar, ); await writeFile(join(root, "node_modules", "node", "bin", "node"), node); await writeFile(join(root, "pnpm-lock.yaml"), lockfile); await writeFile( join(root, "node_modules", ".bin", "opencode"), opencodeCommand, ); await writeFile( join(root, "node_modules", "opencode-ai", "bin", "opencode.exe"), opencodeExecutable, ); const digest = (value: string) => `sha256:${createHash("sha256").update(value).digest("hex")}`; const proxySha = `sha256:${createHash("sha256").update(proxy).digest("hex")}`; const sidecarSha = `sha256:${createHash("sha256").update(sidecar).digest("hex")}`; const payload = { pins: { nodeMinimum: "24.11.0", codex: "0.148.0", opencode: "1.18.17", acpx: "0.13.1", claudeAcp: "0.70.0", codexAcp: "1.6.2", }, target: { platform: "linux", architecture: "x64" }, runnerSourceRevision: "1".repeat(40), distDigest: sha256DirectoryTree(join(root, "dist")), bridgeDigest: "", acpxProfileDigests: { claude: "sha256:9d73d1f0f121fb96cc8badb28c22d5bff02d8582eb2e40360a81c189e1b9422a", codex: "sha256:94049b3e3c3aee87de62703786e4fa81d031d7bd979f99bdf516d84f28791a79", }, artifacts: { nodeCommand: { path: "node_modules/node/bin/node", sha256: digest(node), }, productionLock: { path: "pnpm-lock.yaml", sha256: digest(lockfile) }, opencodeCommand: { path: "node_modules/.bin/opencode", sha256: digest(opencodeCommand), }, opencodeExecutable: { path: "node_modules/opencode-ai/bin/opencode.exe", sha256: digest(opencodeExecutable), }, opencodeProxy: { path: "dist/cli/opencode-app-server-proxy.cjs", sha256: proxySha, }, acpxSidecar: { path: "dist/cli/acpx-runtime-sidecar.cjs", sha256: sidecarSha, }, }, }; payload.bridgeDigest = `sha256:${createHash("sha256") .update(proxySha) .update("\n") .update(sidecarSha) .update("\n") .update(payload.distDigest) .digest("hex")}`; const writeManifest = async () => writeFile( join(root, "provider-pack.json"), JSON.stringify({ schema: "paperclip-runner/remote-provider-pack/v1", digest: `sha256:${createHash("sha256").update(canonical(payload)).digest("hex")}`, payload, }), ); await writeManifest(); expect(readRemoteProviderPackManifest(root).payload.pins.opencode).toBe( "1.18.17", ); for (const [artifactName, substituteName] of [ ["nodeCommand", "productionLock"], ["opencodeExecutable", "opencodeCommand"], ["opencodeProxy", "acpxSidecar"], ["acpxSidecar", "opencodeProxy"], ] as const) { const original = payload.artifacts[artifactName]; payload.artifacts[artifactName] = { ...payload.artifacts[substituteName], }; await writeManifest(); expect(() => readRemoteProviderPackManifest(root)).toThrow( /path must be/, ); payload.artifacts[artifactName] = original; } await writeManifest(); await writeFile( join(root, "dist", "cli", "opencode-app-server-proxy.cjs"), "tampered\n", ); expect(() => readRemoteProviderPackManifest(root)).toThrow( "OpenCode proxy digest mismatch", ); await writeFile( join(root, "dist", "cli", "opencode-app-server-proxy.cjs"), proxy, ); await writeFile( join(root, "dist", "cli", "transitive-runtime.js"), "changed transitive module\n", ); expect(() => readRemoteProviderPackManifest(root)).toThrow( "provider dist tree digest mismatch", ); await rm(root, { recursive: true, force: true }); }); }); describe("native harness persistence profiles", () => { const profile = (provider: Record, driverKind: string) => resolveNativeHarnessPersistenceProfile({ provider, session: { driverKind, normalizedSessionId: "session", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, }, } as unknown as NativeExecutionInputV1); it.each([ ["codex", { kind: "codex" }, "codex_app_server", ["runner", "codex-home"]], [ "opencode", { kind: "opencode" }, "opencode_server", ["runner", "opencode"], ], [ "acpx pi", { kind: "acpx", agent: "pi" }, "acpx_runtime", ["runner", "acpx"], ], [ "acpx claude", { kind: "acpx", agent: "claude" }, "acpx_runtime", ["runner", "acpx"], ], [ "acpx codex", { kind: "acpx", agent: "codex" }, "acpx_runtime", ["runner", "acpx"], ], ])( "declares the complete %s recovery state", (_name, provider, driver, directories) => { expect( profile( provider as Record, driver as string, ).directories.map((directory) => directory.name), ).toEqual(directories); }, ); it("excludes disposable Codex scratch trees and launch-time credentials", () => { const codex = profile({ kind: "codex" }, "codex_app_server"); expect( codex.directories.find((directory) => directory.name === "codex-home"), ).toMatchObject({ excludeTopLevelEntries: ["tmp", ".tmp", "auth.json", "config.toml"], }); }); }); describe("verified native harness backups", () => { const backupExecution = { provider: { kind: "codex", model: "gpt-5.6-sol", approvalPolicy: "never" }, binding: { companyId: "company", runId: "run", issueId: "issue", agentId: "agent", executionWorkspaceId: "workspace", }, workspace: { cwd: "/workspace", repoUrl: "https://example.test/repo.git", repoRef: "main", branchName: "paperclip/test", }, session: { normalizedSessionId: "native-session", driverKind: "codex_app_server", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, }, } as unknown as NativeExecutionInputV1; const acpxIdentity = (suffix: string) => ({ providerSessionId: `record-${suffix}`, providerBackendSessionId: `backend-${suffix}`, providerSessionIdentity: { kind: "acpx", normalizedSessionId: "native-session", acpxRecordId: `record-${suffix}`, backendSessionId: `backend-${suffix}`, agentSessionId: `agent-session-${suffix}`, profileDigest: "sha256:profile", workspaceDigest: "sha256:workspace", requestedModel: "claude-sonnet-5", effectiveModel: "claude-sonnet-5", permissionMode: "approve-all", }, }); it("allows only identity-stable ACPX rotation after a governed interaction", () => { const execution = { ...backupExecution, provider: { kind: "acpx", agent: "claude", model: "claude-sonnet-5", }, session: { ...backupExecution.session, driverKind: "acpx_runtime", }, interactionResponses: [{ interactionId: "interaction-1" }], } as unknown as NativeExecutionInputV1; const previous = acpxIdentity("previous"); const current = acpxIdentity("current"); expect( providerSessionIdentityTransitionIsAllowed({ execution, previous, current, }), ).toBe(true); expect( providerSessionIdentityTransitionIsAllowed({ execution: { ...execution, interactionResponses: [], } as unknown as NativeExecutionInputV1, previous, current, }), ).toBe(false); expect( providerSessionIdentityTransitionIsAllowed({ execution, previous, current: { ...current, providerSessionIdentity: { ...current.providerSessionIdentity, workspaceDigest: "sha256:different-workspace", }, }, }), ).toBe(false); expect( providerSessionIdentityTransitionIsAllowed({ execution, previous, current: { ...current, providerBackendSessionId: "unbound-backend", }, }), ).toBe(false); }); it("restores a verified continuation into an intentionally fresh non-reusable sandbox", () => { expect( shouldRestoreNativeHarnessBackupIntoSandbox({ acquisitionOutcome: "created", reusableLeaseConfigured: false, backupAvailable: true, }), ).toBe(true); expect( shouldRestoreNativeHarnessBackupIntoSandbox({ acquisitionOutcome: "created", reusableLeaseConfigured: true, backupAvailable: true, }), ).toBe(false); expect( shouldRestoreNativeHarnessBackupIntoSandbox({ acquisitionOutcome: "created", reusableLeaseConfigured: false, backupAvailable: false, }), ).toBe(false); }); it("accepts a complete digest-matched backup and rejects corruption", async () => { const root = await mkdtemp(join(tmpdir(), "paperclip-harness-backup-")); try { const current = join(root, "failover-backups", "current"); await mkdir(join(current, "runner"), { recursive: true }); await mkdir(join(current, "codex-home", "sessions"), { recursive: true }); await writeFile( join(current, "runner", "runner-state.json"), "runner-state", ); await writeFile( join(current, "codex-home", "sessions", "thread.jsonl"), "thread-state", ); const manifest = buildNativeHarnessBackupManifest({ backupRoot: current, execution: backupExecution, runnerInstanceId: "runner-1", providerSessionIdentity: { providerSessionId: "thread-1", providerBackendSessionId: "session-1", providerSessionIdentity: null, }, sourceProviderLeaseId: "sandbox-1", completedAt: "2026-08-26T00:00:00.000Z", }); await writeFile(join(current, "manifest.json"), JSON.stringify(manifest)); expect( verifyNativeHarnessBackup({ root, execution: backupExecution, runnerInstanceId: "runner-1", }), ).toMatchObject({ root: current, manifest: { sourceProviderLeaseId: "sandbox-1", directories: [ expect.objectContaining({ name: "runner" }), expect.objectContaining({ name: "codex-home" }), ], }, }); const continuationExecution = { ...backupExecution, binding: { ...backupExecution.binding, runId: "run-2", executionWorkspaceId: "run-2", }, } as NativeExecutionInputV1; expect( verifyNativeHarnessBackup({ root, execution: continuationExecution, runnerInstanceId: "runner-1", }), ).not.toBeNull(); await writeFile( join(current, "codex-home", "sessions", "thread.jsonl"), "corrupt", ); expect( verifyNativeHarnessBackup({ root, execution: backupExecution, runnerInstanceId: "runner-1", }), ).toBeNull(); } finally { await rm(root, { recursive: true, force: true }); } }); it("rejects a backup whose provider identity or harness contract changed", async () => { const root = await mkdtemp( join(tmpdir(), "paperclip-harness-backup-identity-"), ); try { const current = join(root, "failover-backups", "current"); await mkdir(join(current, "runner"), { recursive: true }); await mkdir(join(current, "codex-home"), { recursive: true }); await writeFile( join(current, "runner", "runner-state.json"), "runner-state", ); expect(() => buildNativeHarnessBackupManifest({ backupRoot: current, execution: backupExecution, runnerInstanceId: "runner-1", providerSessionIdentity: { providerSessionId: null, providerBackendSessionId: null, providerSessionIdentity: null, }, sourceProviderLeaseId: "sandbox-1", }), ).toThrow("runner_harness_state_mismatch"); } finally { await rm(root, { recursive: true, force: true }); } }); it("verifies the lease stamp and all backup directory digests before replacement", async () => { const stateBase = await mkdtemp(join(tmpdir(), "paperclip-harness-stamp-")); const previousStateDirectory = process.env.PAPERCLIP_RUNNER_STATE_DIR; process.env.PAPERCLIP_RUNNER_STATE_DIR = stateBase; try { const sessionRoot = join( stateBase, createHash("sha256").update("native-session").digest("hex"), ); const current = join(sessionRoot, "failover-backups", "current"); await mkdir(join(current, "runner"), { recursive: true }); await mkdir(join(current, "codex-home", "sessions"), { recursive: true }); await writeFile( join(current, "runner", "runner-state.json"), "runner-state", ); await writeFile( join(current, "codex-home", "sessions", "thread.jsonl"), "thread-state", ); const manifest = buildNativeHarnessBackupManifest({ backupRoot: current, execution: backupExecution, runnerInstanceId: "runner-1", providerSessionIdentity: { providerSessionId: "thread-1", providerBackendSessionId: "session-1", providerSessionIdentity: null, }, sourceProviderLeaseId: "sandbox-1", }); const manifestPath = join(current, "manifest.json"); await writeFile(manifestPath, JSON.stringify(manifest)); const stamp = createNativeHarnessBackupStamp({ manifestPath, normalizedSessionId: "native-session", runnerInstanceId: "runner-1", completedAt: manifest.completedAt, }); expect(verifyNativeHarnessBackupStamp(stamp, "sandbox-1")).toBe(true); expect(verifyNativeHarnessBackupStamp(stamp, "sandbox-2")).toBe(false); await writeFile(join(current, "runner", "runner-state.json"), "corrupt"); expect(verifyNativeHarnessBackupStamp(stamp, "sandbox-1")).toBe(false); } finally { if (previousStateDirectory === undefined) { delete process.env.PAPERCLIP_RUNNER_STATE_DIR; } else { process.env.PAPERCLIP_RUNNER_STATE_DIR = previousStateDirectory; } await rm(stateBase, { recursive: true, force: true }); } }); }); describe("remote provider checkpoint snapshots", () => { it("excludes Codex scratch and credential state without mutating the live provider home", async () => { const execute = vi .fn() .mockResolvedValueOnce({ exitCode: 0, timedOut: false, stdout: "", stderr: "", }) .mockResolvedValueOnce({ exitCode: 0, timedOut: false, stdout: "", stderr: "", }) .mockResolvedValueOnce({ exitCode: 0, timedOut: false, stdout: "", stderr: "", }); const syncOut = vi.fn( async ( _operations: Array<{ files: Array<{ sourcePath: string; targetPath: string; kind: "file" | "directory"; mode?: number; }>; }>, ) => undefined, ); await syncRemoteRunnerDirectoryOut({ runner: { execute, syncOut } as never, sourcePath: "/remote/session/filesystem/codex-home", targetPath: "/tmp/paperclip-checkpoint-test-codex-home", mode: 0o700, excludeTopLevelEntries: ["tmp", ".tmp", "auth.json", "config.toml"], }); expect(execute).toHaveBeenNthCalledWith( 1, expect.objectContaining({ args: ["-c", "test -d '/remote/session/filesystem/codex-home'"], }), ); const snapshotCommand = String(execute.mock.calls[1]?.[0]?.args?.[1]); expect(snapshotCommand).toContain("'--exclude=./tmp'"); expect(snapshotCommand).toContain("'--exclude=./.tmp'"); expect(snapshotCommand).toContain("'--exclude=./auth.json'"); expect(snapshotCommand).toContain("'--exclude=./config.toml'"); expect(snapshotCommand).toContain( "-C '/remote/session/filesystem/codex-home'", ); expect(snapshotCommand).not.toContain( "rm -rf -- '/remote/session/filesystem/codex-home'", ); const batch = syncOut.mock.calls[0]?.[0]?.[0]; expect(batch?.files[0]).toMatchObject({ sourcePath: expect.stringMatching( /^\/remote\/session\/filesystem\/\.paperclip-checkpoint-/, ), targetPath: "/tmp/paperclip-checkpoint-test-codex-home", kind: "directory", mode: 0o700, }); expect(String(execute.mock.calls[2]?.[0]?.args?.[1])).toMatch( /^rm -rf -- '\/remote\/session\/filesystem\/\.paperclip-checkpoint-/, ); }); it("rejects non-top-level checkpoint exclusions", async () => { const execute = vi.fn().mockResolvedValue({ exitCode: 0, timedOut: false, stdout: "", stderr: "", }); await expect( syncRemoteRunnerDirectoryOut({ runner: { execute, syncOut: vi.fn() } as never, sourcePath: "/remote/codex-home", targetPath: "/tmp/paperclip-checkpoint-invalid-codex-home", mode: 0o700, excludeTopLevelEntries: ["../outside"], }), ).rejects.toThrow("runner_remote_checkpoint_exclusion_invalid"); }); it("rejects unsafe fallback archives without replacing durable state", async () => { const root = await mkdtemp(join(tmpdir(), "paperclip-checkpoint-unsafe-")); const archiveSource = join(root, "archive-source"); const targetPath = join(root, "durable-target"); try { await mkdir(archiveSource, { recursive: true }); await mkdir(targetPath, { recursive: true }); await writeFile(join(targetPath, "preserved.txt"), "preserved"); await symlink("/etc/passwd", join(archiveSource, "host-secret")); const archive = execFileSync( "tar", ["-czf", "-", "-C", archiveSource, "."], { maxBuffer: 8 * 1024 * 1024 }, ); const execute = vi .fn() .mockResolvedValueOnce({ exitCode: 0, timedOut: false, stdout: "", stderr: "", }) .mockResolvedValueOnce({ exitCode: 0, timedOut: false, stdout: archive.toString("base64"), stderr: "", }); await expect( syncRemoteRunnerDirectoryOut({ runner: { execute } as never, sourcePath: "/remote/codex-home", targetPath, mode: 0o700, }), ).rejects.toThrow("runner_remote_checkpoint_archive_unsafe_entry"); await expect( access(join(targetPath, "preserved.txt")), ).resolves.toBeUndefined(); } finally { await rm(root, { recursive: true, force: true }); } }); }); describe("remote provider checkpoint restores", () => { it("does not upload excluded Codex scratch trees or credentials", async () => { const sourcePath = await mkdtemp( join(tmpdir(), "paperclip-codex-restore-source-"), ); try { await mkdir(join(sourcePath, "sessions"), { recursive: true }); await mkdir(join(sourcePath, ".tmp"), { recursive: true }); await writeFile( join(sourcePath, "sessions", "thread.jsonl"), "durable session", ); await writeFile( join(sourcePath, ".tmp", "scratch.bin"), "disposable scratch", ); await writeFile(join(sourcePath, "auth.json"), "credential"); await writeFile(join(sourcePath, "config.toml"), "bearer token"); const syncIn = vi.fn( async ( operations: Array<{ files: Array<{ sourcePath: string }>; }>, ) => { const stagedPath = operations[0]!.files[0]!.sourcePath; expect(stagedPath).not.toBe(sourcePath); await expect( access(join(stagedPath, "sessions", "thread.jsonl")), ).resolves.toBeUndefined(); await expect( access(join(stagedPath, ".tmp", "scratch.bin")), ).rejects.toThrow(); await expect(access(join(stagedPath, "auth.json"))).rejects.toThrow(); await expect( access(join(stagedPath, "config.toml")), ).rejects.toThrow(); }, ); await stageRemoteRunnerDirectory({ target: { kind: "remote", transport: "provider", remoteCwd: "/remote", runner: { syncIn } as never, } as never, runner: { syncIn } as never, sourcePath, targetPath: "/remote/codex-home", mode: 0o700, excludeTopLevelEntries: ["tmp", ".tmp", "auth.json", "config.toml"], }); expect(syncIn).toHaveBeenCalledOnce(); } finally { await rm(sourcePath, { recursive: true, force: true }); } }); }); describe("remote preinstalled executable discovery", () => { it("accepts one normalized absolute executable path", () => { expect( parseRemoteExecutableCandidate( "/home/daytona/.local/bin/paperclip-runnerd\n", ), ).toBe("/home/daytona/.local/bin/paperclip-runnerd"); }); it.each([ "paperclip-runnerd\n", "/safe/path\n/unexpected/second-line\n", "/safe/path with spaces\n", "/safe/path;touch-bad\n", ])("rejects ambiguous or shell-active output: %j", (stdout) => { expect(parseRemoteExecutableCandidate(stdout)).toBeNull(); }); it("does not accept a merely contract-compatible runnerd when a build-owned artifact is configured", () => { expect( mayUsePreinstalledRunnerArtifact("/artifacts/paperclip-runnerd"), ).toBe(false); expect(mayUsePreinstalledRunnerArtifact(" ")).toBe(true); expect(mayUsePreinstalledRunnerArtifact(undefined)).toBe(true); }); }); describe("remote runner build metadata", () => { const current = { schema: "paperclip-runner/runnerd-build-metadata/v1", binaryName: "paperclip-runnerd", packageName: "@paperclipai/paperclip-runner", binaryContractVersion: 2, prpTransportModes: ["dial_ws_loopback", "dial_wss", "listen_ws"], }; it("accepts the current contract with the required transport", () => { expect(() => assertRemoteRunnerBuildMetadata(current, "listen_ws"), ).not.toThrow(); }); it("fails before dispatch when a preinstalled runner uses the stale contract", () => { expect(() => assertRemoteRunnerBuildMetadata( { ...current, binaryContractVersion: 1, }, "listen_ws", ), ).toThrow("runner_remote_artifact_contract_incompatible"); }); it("requires the selected transport without falling through", () => { expect(() => assertRemoteRunnerBuildMetadata( { ...current, prpTransportModes: ["dial_wss"], }, "listen_ws", ), ).toThrow("runner_remote_transport_capability_missing:listen_ws"); }); }); describe("remote runner transport authorization", () => { const ingressTarget = { kind: "remote", transport: "sandbox", providerKey: "daytona", remoteCwd: "/workspace", leaseId: "lease-1", effectiveCapabilities: { runnerWebSocketIngress: true }, } as const; it("fails before selecting sandbox ingress for an unauthorized run", () => { expect(() => resolveRemoteRunnerTransportMode({ target: ingressTarget as never, runnerIngressAuthorized: false, }), ).toThrow("runner_ingress_unavailable"); }); it("selects sandbox ingress for a resolved native run", () => { expect( resolveRemoteRunnerTransportMode({ target: ingressTarget as never, runnerIngressAuthorized: true, }), ).toBe("listen_ws"); }); }); describe("runtime question fallback", () => { const questionSet = { schema: "paperclip.question_set.v1" as const, title: "Configure deployment", description: "These answers are required before work can continue.", submitLabel: "Continue", questions: [ { id: "region", prompt: "Which region?", required: true, answerMode: "single_select" as const, options: [ { id: "us", label: "US" }, { id: "eu", label: "Europe" }, ], }, { id: "replicas", prompt: "How many replicas?", required: true, answerMode: "text" as const, textValidation: { inputType: "integer" as const, minimum: 1 }, }, ], }; it.each(["provider_process_lost", "durable_handoff"])( "materializes one idempotent durable interaction after %s", (reason) => { const fallback = runtimeQuestionFallbackFromEvent({ eventType: "runtime_request.expired", runId: "00000000-0000-4000-8000-000000000001", payload: { requestId: "elicitation-1", requestKind: "runtime", requestType: "input", reason, replayAllowed: false, request: { schema: "paperclip.runtime_request.v2", requestKind: "runtime", requestId: "elicitation-1", type: "input", status: "pending", prompt: "Configure deployment", turnId: "turn-1", itemId: "item-1", input: questionSet, }, }, }); expect(fallback).toMatchObject({ kind: "ask_user_questions", idempotencyKey: "runtime-input-durable:v1:00000000-0000-4000-8000-000000000001:elicitation-1", sourceRunId: "00000000-0000-4000-8000-000000000001", continuationPolicy: "wake_assignee", payload: { runtimeRequestId: "elicitation-1", questionSet, supersedeOnUserComment: false, questions: [ { id: "region", selectionMode: "single", options: [ { id: "us", label: "US" }, { id: "eu", label: "Europe" }, ], }, { id: "replicas", selectionMode: "single", options: [{ id: "__paperclip_text__", freeText: true }], }, ], }, }); }, ); it.each([ ["runtime_request.resolved", "provider_process_lost", false], ["runtime_request.cancelled", "provider_process_lost", false], ["runtime_request.expired", "explicit_cancellation", false], ["runtime_request.expired", "provider_process_lost", true], ])( "does not fall back for %s / %s / replay=%s", (eventType, reason, replayAllowed) => { expect( runtimeQuestionFallbackFromEvent({ eventType: eventType as never, runId: "00000000-0000-4000-8000-000000000001", payload: { reason, replayAllowed, request: { schema: "paperclip.runtime_request.v2", requestKind: "runtime", requestId: "elicitation-1", type: "input", status: "pending", turnId: "turn-1", itemId: "item-1", input: questionSet, }, }, }), ).toBeNull(); }, ); it("emits content-free lifecycle metric dimensions", () => { expect( runtimeInputLifecycleMetric({ eventType: "runtime_request.created", payload: { request: { type: "input", requestId: "input-1", origin: { adapter: "codex-app-server" }, input: questionSet, }, }, }), ).toEqual({ outcome: "normalized", adapter: "codex-app-server", requestId: "input-1", }); expect( runtimeInputLifecycleMetric({ eventType: "runtime_request.expired", payload: { requestId: "input-1", requestType: "input", reason: "durable_handoff", adapter: "codex-app-server", }, }), ).toEqual({ outcome: "durable_handoff", adapter: "codex-app-server", requestId: "input-1", }); expect( runtimeInputLifecycleMetric({ eventType: "runtime_request.expired", payload: { requestId: "input-1", requestType: "input", reason: "provider_process_lost", adapter: "codex-app-server", }, }), ).toEqual({ outcome: "provider_loss_handoff", adapter: "codex-app-server", requestId: "input-1", }); }); }); describe("native provider bootstrap environment", () => { it("inherits the host executable and credential-home context", () => { expect( buildNativeProviderEnvironment( {}, { PATH: "/opt/homebrew/bin:/usr/bin", HOME: "/Users/runner", CODEX_HOME: "/Users/runner/.codex", PAPERCLIP_INTERNAL_SECRET: "must-not-leak", }, ), ).toEqual({ PATH: "/opt/homebrew/bin:/usr/bin", HOME: "/Users/runner", CODEX_HOME: "/Users/runner/.codex", }); }); it("lets explicitly configured agent env override host defaults", () => { expect( buildNativeProviderEnvironment( { PATH: "/agent/bin", OPENAI_API_KEY: "configured-provider-key", }, { PATH: "/host/bin", HOME: "/Users/runner", }, ), ).toEqual({ PATH: "/agent/bin", HOME: "/Users/runner", OPENAI_API_KEY: "configured-provider-key", }); }); it("pins the server-assigned workspace over configured environment input", () => { expect( buildNativeProviderEnvironment( { PAPERCLIP_WORKSPACE_CWD: "/untrusted/configured-workspace", }, { HOME: "/Users/runner" }, "/Users/runner/.paperclip/instances/default/workspaces/agent-1", ), ).toEqual({ HOME: "/Users/runner", PAPERCLIP_WORKSPACE_CWD: "/Users/runner/.paperclip/instances/default/workspaces/agent-1", }); }); }); const execution = { schema: "paperclip.native-execution-input.v1", provider: { kind: "codex", model: null }, binding: { companyId: "company", runId: "run-native-cancel", issueId: "issue", agentId: "agent", executionWorkspaceId: "workspace", }, task: { identifier: "PAP-NATIVE", title: "Exercise the native session", description: null, prompt: "Complete the native session test task.", workMode: "standard", }, workspace: { cwd: "/tmp/paperclip-native-session-test", repoUrl: null, repoRef: null, branchName: null, }, session: { normalizedSessionId: "session-native-cancel", driverKind: "codex_app_server", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, }, completionContract: { id: "contract", sha256: "sha", schemaVersion: "paperclip.completion-contract.v1", contract: { revision: "1", objective: "Exercise the native session.", criteria: [{ id: "objective", requirement: "The session completes." }], }, }, interactionResponses: [], credentialBindings: [], } as NativeExecutionInputV1; describe("provider plan synchronization", () => { it("prefers the provider's completed Markdown when it is available", () => { expect( providerPlanMarkdown({ markdown: "# Release plan\n\n1. Prepare\n2. Deploy", explanation: "This fallback must not replace the completed plan.", steps: [{ body: "Fallback", status: "pending" }], }), ).toBe("# Release plan\n\n1. Prepare\n2. Deploy"); }); it("extracts a completed plan from the semantic result artifact", () => { expect( semanticProviderPlanMarkdown({ artifacts: [ { kind: "native_provider_plan", ref: "\n# Health check\n\n1. Add endpoint\n2. Verify it\n", }, ], }), ).toBe("# Health check\n\n1. Add endpoint\n2. Verify it"); }); it("decodes the native provider's compact plan reference into readable Markdown", () => { expect( semanticProviderPlanMarkdown({ artifacts: [ { kind: "native_provider_plan", ref: "native-provider-plan:health-check-endpoint-v1#1-register-GET-health-return-200-json-status-ok;2-add-API-tests", }, ], }), ).toBe( [ "# Health check endpoint", "", "1. Register GET /health return 200 JSON status ok", "2. Add API tests", ].join("\n"), ); }); it("decodes a task-scoped native plan URI", () => { expect( semanticProviderPlanMarkdown({ artifacts: [ { kind: "native_provider_plan", ref: "native-plan://DOT-13/health-check#1-add-GET-health;2-add-tests", }, ], }), ).toBe("# Health check\n\n1. Add GET /health\n2. Add tests"); }); it("retains readable Markdown embedded after a native provider plan reference", () => { expect( semanticProviderPlanMarkdown({ artifacts: [ { kind: "native_provider_plan", ref: "native-provider-plan:DOT-14-health-check-v1\n1. Add `GET /health`.\n2. Add tests.", }, ], }), ).toBe("# Health check\n\n1. Add `GET /health`.\n2. Add tests."); }); it("normalizes a plain numbered native provider plan", () => { expect( semanticProviderPlanMarkdown({ artifacts: [ { kind: "native_provider_plan", ref: "1. Add GET /health. | 2. Add tests. | 3. Document it.", }, ], }), ).toBe("# Plan\n\n1. Add GET /health.\n2. Add tests.\n3. Document it."); }); it("normalizes a task-labelled inline numbered plan", () => { expect( semanticProviderPlanMarkdown({ artifacts: [ { kind: "native_provider_plan", ref: "DOT-16 plan: (1) add GET /health; (2) add tests; (3) document it.", }, ], }), ).toBe("# Plan\n\n1. add GET /health\n2. add tests\n3. document it."); }); it("uses an explicitly numbered semantic summary when the artifact is opaque", () => { expect( semanticProviderPlanMarkdown({ summary: "Native provider plan completed: 1) add GET /health; 2) add tests; 3) document it.", artifacts: [ { kind: "native_provider_plan", ref: "native-provider-plan:DOT-18:health-check", }, ], }), ).toBe("# Plan\n\n1. add GET /health\n2. add tests\n3. document it."); }); it("renders a bounded Markdown checklist without embedding provenance", () => { const markdown = providerPlanMarkdown({ explanation: "Release safely", steps: [ { body: "Prepare", status: "completed" }, { body: "Deploy", status: "in_progress" }, { body: "Verify", status: "blocked" }, ], runId: "must-not-appear", providerThreadId: "native-secret", }); expect(markdown).toBe( [ "Release safely", "", "- [x] Prepare", "- [ ] Deploy _(in progress)_", "- [ ] Verify _(blocked)_", ].join("\n"), ); expect(markdown).not.toContain("must-not-appear"); expect(markdown).not.toContain("native-secret"); }); }); describe("native governed waits", () => { it("turns a durable pending interaction into a response-wake result", () => { expect( nativeGovernedWaitResult({ interaction: { id: "interaction-1", title: "Choose an output format", summary: null, }, completionContract: { revision: "contract-v3", objective: "Create the requested output", criteria: [{ id: "objective", requirement: "The output is created" }], }, }), ).toEqual( expect.objectContaining({ schema: "paperclip.run_result.v1", reportedWorkDisposition: "yielded", summary: "Waiting for Choose an output format.", completionClaim: expect.objectContaining({ contractRevision: "contract-v3", objectiveSatisfied: false, criteria: [ { criterionId: "objective", status: "unknown", evidenceRefs: ["interaction:interaction-1"], }, ], }), evidence: [{ ref: "interaction:interaction-1" }], attentionRequests: [], continuation: { kind: "response_wake", summary: "Resume from the resolved interaction response without repeating prior work.", idempotencyKey: "interaction-response:interaction-1", }, }), ); }); it("keeps an authority-checked partial item-verdict interaction as the wait target", () => { const partial = structuredClone(execution); partial.interactionResponses = [ { interactionId: "interaction-partial", kind: "request_item_verdicts", response: { status: "pending", result: { version: 1, complete: false, items: [{ id: "alpha", verdict: "approve" }], }, }, }, ]; expect(continuingPendingInteractionIds(partial)).toEqual([ "interaction-partial", ]); partial.interactionResponses[0]!.response.status = "answered"; expect(continuingPendingInteractionIds(partial)).toEqual([]); }); it("consumes an exact replay observation once without leaking stale state", async () => { const waitResult = nativeGovernedWaitResult({ interaction: { id: "interaction-replayed", title: "Approve the replayed operation", summary: null, }, completionContract: { revision: "contract-v3", objective: "Complete the approved operation", criteria: [{ id: "objective", requirement: "Complete it" }], }, }); const observation = createGovernedWaitEventObservation( async () => waitResult, ); const replayedEvent: PrpEvent = { schema: "paperclip.prp.event.v1" as const, sourceInstanceId: "runner-recovered", sourceEventId: "runner-recovered:item:7", sourceSeq: 7, sourceKind: "runner" as const, runId: "run-recovered", normalizedSessionId: "session-recovered", turnId: "turn-recovered", eventType: "item.completed" as const, schemaVersion: 1, priority: 0 as const, emittedAt: "2026-08-31T00:00:00.000Z", payload: {}, }; await observation.observe(replayedEvent, true); expect(observation.consume(replayedEvent)).toEqual(waitResult); expect(observation.consume(replayedEvent)).toBeNull(); await observation.observe(replayedEvent, true); expect( observation.consume({ ...replayedEvent, sourceEventId: "runner-recovered:item:8", sourceSeq: 8, }), ).toBeNull(); expect(observation.consume(replayedEvent)).toBeNull(); let resolveLookup!: (value: typeof waitResult) => void; const delayedObservation = createGovernedWaitEventObservation( () => new Promise((resolve) => { resolveLookup = resolve; }), ); const observing = delayedObservation.observe(replayedEvent, true); expect(delayedObservation.consume(replayedEvent)).toBeNull(); resolveLookup(waitResult); await observing; expect(delayedObservation.consume(replayedEvent)).toBeNull(); }); }); type LeaseCoordinator = { runId: string; companyId: string; issueId: string; phase: string; attempt: number; leaseOwner: string | null; leaseExpiresAt: Date | null; resultId: string | null; }; function leaseDb( boundExecution: NativeExecutionInputV1 = execution, coordinatorOverrides: Partial = {}, runResultJson: Record = {}, ): Db { const coordinator: LeaseCoordinator = { runId: boundExecution.binding.runId, companyId: boundExecution.binding.companyId, issueId: boundExecution.binding.issueId, phase: "observed", attempt: 0, leaseOwner: null, leaseExpiresAt: null, resultId: null, ...coordinatorOverrides, }; const update = () => ({ set: () => ({ where: () => { const result = Promise.resolve([]) as unknown as Promise & { returning: () => Promise>; }; result.returning = () => Promise.resolve([{ runId: coordinator.runId }]); return result; }, }), }); const tx = { select: () => ({ from: (table: unknown) => ({ where: () => ({ for: () => ({ limit: () => Promise.resolve([ table === nativeRunFinalizations ? coordinator : { agentId: boundExecution.binding.agentId, companyId: boundExecution.binding.companyId, nativeIssueId: boundExecution.binding.issueId, resultJson: runResultJson, runtimeMode: "native", }, ]), }), }), }), }), update, }; return { transaction: async (operation: (transaction: Db) => Promise) => operation(tx as unknown as Db), update, } as unknown as Db; } function cancellationDb(options?: { coordinator?: { runId: string; assessmentId: string | null; decisionId?: string | null; } | null; failResultJsonUpdateAt?: number; }) { const initialRun = { id: execution.binding.runId, agentId: execution.binding.agentId, companyId: execution.binding.companyId, nativeIssueId: execution.binding.issueId, runtimeMode: "native", contextSnapshot: { issueId: "untrusted-context-issue" }, resultJson: { staleSnapshot: true }, }; let currentResultJson: Record = { durableReceipt: { operationId: "operation-1" }, }; const issue = { status: "in_progress", statusVersion: 3, lastStatusDecisionId: null, }; const coordinator = options && "coordinator" in options ? options.coordinator : { runId: execution.binding.runId, assessmentId: null }; let forUpdateCount = 0; let resultJsonUpdateCount = 0; const updates: Array<{ table: unknown; values: Record }> = []; const select = vi.fn(() => ({ from: (table: unknown) => { const rows = table === heartbeatRuns ? [{ ...initialRun, resultJson: currentResultJson }] : table === issues ? [issue] : table === nativeRunFinalizations && coordinator ? [coordinator] : []; const result = Promise.resolve(rows); type Query = { where: () => Query; for: () => Query; limit: () => Promise; }; const query = {} as Query; Object.assign(query, { where: () => query, for: () => { forUpdateCount += 1; return query; }, limit: () => result, }); return query; }, })); const update = vi.fn((table: unknown) => ({ set: (values: Record) => ({ where: () => { updates.push({ table, values }); const updatesResultJson = "resultJson" in values; if (updatesResultJson) resultJsonUpdateCount += 1; const shouldFail = updatesResultJson && resultJsonUpdateCount === options?.failResultJsonUpdateAt; if (updatesResultJson && !shouldFail) { currentResultJson = values.resultJson as Record; } const result = Promise.resolve([]) as unknown as Promise & { returning: () => Promise>; }; result.returning = () => shouldFail ? Promise.reject(new Error("post_dispatch_db_failure")) : Promise.resolve([{ id: execution.binding.runId }]); return result; }, }), })); const tx = { select, update }; const db = { select, update, transaction: async (operation: (transaction: Db) => Promise) => operation(tx as unknown as Db), } as unknown as Db; return { db, updates, getForUpdateCount: () => forUpdateCount, getResultJson: () => currentResultJson, getResultJsonUpdateCount: () => resultJsonUpdateCount, tx, }; } describe("native session cancellation", () => { beforeEach(() => { state.cancel.mockReset().mockReturnValue({ cleanup: Promise.resolve() }); state.persistActivity.mockClear(); state.publishActivity.mockClear(); state.release = null; state.execute.mockReset().mockImplementation(async (options) => { options.onSession?.({ cancel: state.cancel }); await new Promise((resolve) => { state.release = resolve; }); options.onSession?.(null); return { result: { summary: "cancelled" }, terminal: { runTerminalState: "cancelled" }, turnId: "turn", normalizedSessionId: "session", providerSessionId: null, driverKind: "test", driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, }; }); }); it("routes control-plane cancellation to the active normalized session and removes the handle", async () => { const running = executePaperclipNativeSession({ db: leaseDb(), execution, runnerInstanceId: "runner", }); await vi.waitFor(() => expect(state.release).toBeTypeOf("function")); await expect( cancelNativeSession(execution.binding.runId, "budget hard stop"), ).resolves.toBe(true); await expect( cancelNativeSession(execution.binding.runId, "duplicate budget stop"), ).resolves.toBe(true); expect(state.cancel).toHaveBeenCalledWith({ reason: "budget hard stop", signal: expect.any(AbortSignal), }); expect(state.cancel).toHaveBeenCalledTimes(1); state.release?.(); await running; await expect( cancelNativeSession(execution.binding.runId, "late cancel"), ).resolves.toBe(false); }); it("allows cancellation to be retried when the session dispatch fails", async () => { state.cancel.mockImplementationOnce(() => { throw new Error("transport unavailable"); }); const running = executePaperclipNativeSession({ db: leaseDb(), execution, runnerInstanceId: "runner", }); await vi.waitFor(() => expect(state.release).toBeTypeOf("function")); await expect( cancelNativeSession(execution.binding.runId, "budget hard stop"), ).rejects.toThrow("transport unavailable"); await expect( cancelNativeSession(execution.binding.runId, "retry budget stop"), ).resolves.toBe(true); expect(state.cancel).toHaveBeenNthCalledWith(2, { reason: "retry budget stop", signal: expect.any(AbortSignal), }); state.release?.(); await running; }); it("observes cleanup failure after cancellation authority is committed", async () => { state.cancel.mockImplementationOnce(() => ({ cleanup: Promise.reject(new Error("provider cleanup failed")), })); const running = executePaperclipNativeSession({ db: leaseDb(), execution, runnerInstanceId: "runner", }); await vi.waitFor(() => expect(state.release).toBeTypeOf("function")); await expect( cancelNativeSession(execution.binding.runId, "budget hard stop"), ).resolves.toBe(true); state.release?.(); await running; }); it("binds cancellation to nativeIssueId and merges metadata under a row lock", async () => { const persistence = cancellationDb(); await expect( cancelNativeSession(execution.binding.runId, "budget hard stop", { db: persistence.db, scope: "run", }), ).resolves.toMatchObject({ dispatched: false, decision: expect.any(Object), auditId: "native-cancellation-audit", }); expect(persistence.getForUpdateCount()).toBe(2); const cancellationUpdate = persistence.updates .filter((entry) => "resultJson" in entry.values) .at(-1); expect(cancellationUpdate?.values.resultJson).toMatchObject({ durableReceipt: { operationId: "operation-1" }, nativeCancellation: { schema: "paperclip.native-cancellation.v1", dispatchState: "acknowledged", scope: "run", dispatched: false, intentAuditId: "native-cancellation-audit", acknowledgementAuditId: "native-cancellation-ack-audit", }, }); expect(state.persistActivity).toHaveBeenCalledWith( persistence.tx, expect.objectContaining({ companyId: execution.binding.companyId, issueId: execution.binding.issueId, runId: execution.binding.runId, }), ); expect(state.publishActivity).toHaveBeenCalledTimes(2); }); it("recovers a post-dispatch persistence failure without cancelling the provider twice", async () => { const persistence = cancellationDb({ failResultJsonUpdateAt: 2 }); const running = executePaperclipNativeSession({ db: leaseDb(), execution, runnerInstanceId: "runner", }); await vi.waitFor(() => expect(state.release).toBeTypeOf("function")); await expect( cancelNativeSession(execution.binding.runId, "budget hard stop", { db: persistence.db, scope: "run", }), ).rejects.toThrow("post_dispatch_db_failure"); expect(state.cancel).toHaveBeenCalledTimes(1); expect(persistence.getResultJson()).toMatchObject({ nativeCancellation: { dispatchState: "pending", dispatched: false, intentAuditId: "native-cancellation-audit", }, }); await expect( cancelNativeSession(execution.binding.runId, "budget hard stop", { db: persistence.db, scope: "run", }), ).resolves.toMatchObject({ dispatched: true, auditId: "native-cancellation-audit", }); expect(state.cancel).toHaveBeenCalledTimes(1); expect(persistence.getResultJsonUpdateCount()).toBe(3); expect(persistence.getResultJson()).toMatchObject({ nativeCancellation: { dispatchState: "acknowledged", dispatched: true, intentAuditId: "native-cancellation-audit", acknowledgementAuditId: "native-cancellation-ack-audit", }, }); expect( state.persistActivity.mock.calls.filter( ([, input]) => (input as { action?: string }).action === "native.cancellation_intent_recorded", ), ).toHaveLength(1); const persistedActivities = state.persistActivity.mock.calls.length; await expect( cancelNativeSession(execution.binding.runId, "budget hard stop", { db: persistence.db, scope: "run", }), ).resolves.toMatchObject({ dispatched: true, auditId: "native-cancellation-audit", }); expect(state.cancel).toHaveBeenCalledTimes(1); expect(persistence.getResultJsonUpdateCount()).toBe(3); expect(state.persistActivity).toHaveBeenCalledTimes(persistedActivities); state.release?.(); await running; }); it("fails closed when the persisted native binding has no coordinator", async () => { const persistence = cancellationDb({ coordinator: null }); await expect( cancelNativeSession(execution.binding.runId, "budget hard stop", { db: persistence.db, scope: "run", }), ).rejects.toThrow("native_cancellation_coordinator_missing"); expect(persistence.updates).toEqual([]); expect(state.persistActivity).not.toHaveBeenCalled(); }); }); describe("native session execution lease fencing", () => { it("renews only when the exact fenced owner remains current", async () => { const returning = vi .fn() .mockResolvedValueOnce([{ runId: "run-lease" }]) .mockResolvedValueOnce([]); const where = vi.fn(() => ({ returning })); const set = vi.fn(() => ({ where })); const db = { update: vi.fn(() => ({ set })) } as unknown as Db; const input = { db, runId: "run-lease", companyId: "company-lease", issueId: "issue-lease", leaseOwner: "owner-lease", attempt: 4, leaseTtlMs: 60_000, }; await expect( renewNativeSessionExecutionLease(input), ).resolves.toBeUndefined(); await expect(renewNativeSessionExecutionLease(input)).rejects.toThrow( "native_session_lease_lost", ); expect(returning).toHaveBeenCalledTimes(2); }); it("does not reacquire a provider after a durable result exists", async () => { state.execute.mockClear(); state.createBackend.mockClear(); state.createTransport.mockClear(); await expect( executePaperclipNativeSession({ db: leaseDb(execution, { phase: "workspace_finalizing", resultId: "native-result-1", }), execution, runnerInstanceId: "runner", }), ).rejects.toThrow("native_result_pending_finalization"); expect(state.execute).not.toHaveBeenCalled(); expect(state.createBackend).not.toHaveBeenCalled(); expect(state.createTransport).not.toHaveBeenCalled(); }); it.each(["pending", "acknowledged"] as const)( "does not reacquire a provider while durable cancellation is %s", async (dispatchState) => { state.execute.mockClear(); state.createBackend.mockClear(); state.createTransport.mockClear(); await expect( executePaperclipNativeSession({ db: leaseDb( execution, {}, { nativeCancellation: { schema: "paperclip.native-cancellation.v1", intentId: "native-cancellation:intent-1", intentAuditId: "native-cancellation-audit", companyId: execution.binding.companyId, runId: execution.binding.runId, issueId: execution.binding.issueId, scope: "run", reasonCode: "cancellation_run_only", effects: ["release_run_resources"], dispatchState, dispatched: dispatchState === "acknowledged", decisionId: null, }, }, ), execution, runnerInstanceId: "runner", }), ).rejects.toThrow("native_cancellation_pending_recovery"); expect(state.execute).not.toHaveBeenCalled(); expect(state.createBackend).not.toHaveBeenCalled(); expect(state.createTransport).not.toHaveBeenCalled(); }, ); }); describe("native runtime request resolution", () => { const capabilities = vi.fn(); const snapshot = vi.fn(); const resolveRuntimeRequest = vi.fn(); beforeEach(() => { state.release = null; capabilities.mockReset().mockResolvedValue({ runtimeRequestResolution: true, }); snapshot.mockReset().mockResolvedValue({ activeTurnId: "provider-turn-1" }); resolveRuntimeRequest.mockReset().mockResolvedValue(undefined); state.execute.mockReset().mockImplementation(async (options) => { options.onSession?.({ capabilities, snapshot, resolveRuntimeRequest, cancel: vi.fn(), }); await new Promise((resolve) => { state.release = resolve; }); options.onSession?.(null); return { result: { summary: "completed" }, terminal: { runTerminalState: "succeeded" }, turnId: "provider-turn-1", normalizedSessionId: "session", providerSessionId: null, driverKind: "test", driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, }; }); }); it("revalidates lifecycle after provider reads and blocks stale dispatch", async () => { const running = executePaperclipNativeSession({ db: leaseDb(), execution, runnerInstanceId: "runner", }); await vi.waitFor(() => expect(state.release).toBeTypeOf("function")); const authorizeBeforeDispatch = vi.fn(async () => { expect(capabilities).toHaveBeenCalledTimes(1); expect(snapshot).toHaveBeenCalledTimes(1); throw new Error("runtime_request_no_longer_pending"); }); await expect( resolveNativeRuntimeRequest({ runId: execution.binding.runId, requestId: "runtime-request-1", turnId: "provider-turn-1", resolution: { action: "decline" }, authorizeBeforeDispatch, }), ).rejects.toThrow("runtime_request_no_longer_pending"); expect(authorizeBeforeDispatch).toHaveBeenCalledTimes(1); expect(resolveRuntimeRequest).not.toHaveBeenCalled(); state.release?.(); await running; }); it("atomically joins duplicate responses and rejects a concurrent conflict", async () => { const running = executePaperclipNativeSession({ db: leaseDb(), execution, runnerInstanceId: "runner", }); await vi.waitFor(() => expect(state.release).toBeTypeOf("function")); let releaseAuthorization!: () => void; const authorization = new Promise((resolve) => { releaseAuthorization = resolve; }); const authorizeBeforeDispatch = vi.fn(() => authorization); const first = resolveNativeRuntimeRequest({ runId: execution.binding.runId, requestId: "runtime-request-concurrent", turnId: "provider-turn-1", resolution: { action: "decline" }, authorizeBeforeDispatch, }); await vi.waitFor(() => expect(authorizeBeforeDispatch).toHaveBeenCalledTimes(1), ); const duplicate = resolveNativeRuntimeRequest({ runId: execution.binding.runId, requestId: "runtime-request-concurrent", turnId: "provider-turn-1", resolution: { action: "decline" }, authorizeBeforeDispatch, }); await vi.waitFor(() => expect(snapshot).toHaveBeenCalledTimes(2)); await expect( resolveNativeRuntimeRequest({ runId: execution.binding.runId, requestId: "runtime-request-concurrent", turnId: "provider-turn-1", resolution: { action: "cancel" }, authorizeBeforeDispatch, }), ).rejects.toMatchObject({ code: "runtime_request_resolution_conflict", }); expect(authorizeBeforeDispatch).toHaveBeenCalledTimes(1); expect(resolveRuntimeRequest).not.toHaveBeenCalled(); releaseAuthorization(); const [firstResult, duplicateResult] = await Promise.all([ first, duplicate, ]); expect(duplicateResult.commandId).toBe(firstResult.commandId); expect(resolveRuntimeRequest).toHaveBeenCalledTimes(1); state.release?.(); await running; }); it("clears completed response reservations when the session tears down", async () => { const firstSession = executePaperclipNativeSession({ db: leaseDb(), execution, runnerInstanceId: "runner", }); await vi.waitFor(() => expect(state.release).toBeTypeOf("function")); const first = await resolveNativeRuntimeRequest({ runId: execution.binding.runId, requestId: "runtime-request-reused", turnId: "provider-turn-1", resolution: { action: "decline" }, authorizeBeforeDispatch: vi.fn().mockResolvedValue(undefined), }); state.release?.(); await firstSession; state.release = null; const secondSession = executePaperclipNativeSession({ db: leaseDb(), execution, runnerInstanceId: "runner", }); await vi.waitFor(() => expect(state.release).toBeTypeOf("function")); const second = await resolveNativeRuntimeRequest({ runId: execution.binding.runId, requestId: "runtime-request-reused", turnId: "provider-turn-1", resolution: { action: "decline" }, authorizeBeforeDispatch: vi.fn().mockResolvedValue(undefined), }); expect(second.commandId).not.toBe(first.commandId); expect(resolveRuntimeRequest).toHaveBeenCalledTimes(2); (state.release as (() => void) | null)?.(); await secondSession; }); }); describe("native session same-turn steering", () => { const capabilities = vi.fn(); const snapshot = vi.fn(); const steer = vi.fn(); beforeEach(() => { state.release = null; capabilities.mockReset().mockResolvedValue({ steering: true }); snapshot.mockReset().mockResolvedValue({ activeTurnId: "provider-turn-1" }); steer.mockReset().mockResolvedValue(undefined); state.execute.mockReset().mockImplementation(async (options) => { options.onSession?.({ capabilities, snapshot, steer, cancel: vi.fn() }); await new Promise((resolve) => { state.release = resolve; }); options.onSession?.(null); return { result: { summary: "completed" }, terminal: { runTerminalState: "succeeded" }, turnId: "provider-turn-1", normalizedSessionId: "session", providerSessionId: null, driverKind: "test", driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, }; }); }); async function startActiveSession() { const running = executePaperclipNativeSession({ db: leaseDb(), execution, runnerInstanceId: "runner", }); await vi.waitFor(() => expect(state.release).toBeTypeOf("function")); return { running }; } it("correlates the queued comment with the active provider turn acknowledgement", async () => { const { running } = await startActiveSession(); await expect( getNativeSessionSteeringState(execution.binding.runId), ).resolves.toEqual({ disposition: "available", activeTurnId: "provider-turn-1", }); await expect( steerNativeSession({ runId: execution.binding.runId, message: "Check mobile overflow first.", correlationId: "queued-comment-1", }), ).resolves.toEqual({ turnId: "provider-turn-1" }); expect(steer).toHaveBeenCalledWith({ turnId: "provider-turn-1", message: { role: "user", text: "Check mobile overflow first." }, correlationId: "queued-comment-1", }); state.release?.(); await running; }); it.each([ { label: "unsupported provider", prepare: () => capabilities.mockResolvedValue({ steering: false }), code: "steering_unsupported", }, { label: "stale turn", prepare: () => snapshot.mockResolvedValue({ activeTurnId: null }), code: "steering_stale_turn", }, { label: "provider rejection", prepare: () => steer.mockRejectedValue(new Error("request rejected")), code: "steering_rejected", }, ])("keeps $label retryable with a stable code", async ({ prepare, code }) => { prepare(); const { running } = await startActiveSession(); const error = await steerNativeSession({ runId: execution.binding.runId, message: "Retryable steering", correlationId: "queued-comment-error", }).catch((value) => value); expect(error).toBeInstanceOf(NativeSessionSteeringError); expect(error.code).toBe(code); state.release?.(); await running; }); it("bounds the provider acknowledgement wait", async () => { steer.mockReturnValue(new Promise(() => undefined)); const { running } = await startActiveSession(); const error = await steerNativeSession({ runId: execution.binding.runId, message: "Do not wait forever", correlationId: "queued-comment-timeout", timeoutMs: 5, }).catch((value) => value); expect(error).toBeInstanceOf(NativeSessionSteeringError); expect(error.code).toBe("steering_timeout"); state.release?.(); await running; }); }); describe("native warm session supervision", () => { it("reuses one session across distinct governed runs and closes it after idle expiry", async () => { const close = vi.fn(async () => undefined); const sharedSession = { close }; const base = { ...execution, binding: { ...execution.binding, executionWorkspaceId: "workspace", }, workspace: { cwd: "/tmp/warm-native", repoUrl: null, repoRef: null, branchName: null, }, session: { normalizedSessionId: "session-warm-native", driverKind: "codex_app_server" as const, protocolVersion: 1 as const, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 20 }, }, } as NativeExecutionInputV1; const second = { ...base, binding: { ...base.binding, runId: "run-native-warm-second" }, }; const result = { result: { summary: "completed" }, terminal: { runTerminalState: "succeeded" }, turnId: "turn", normalizedSessionId: "session-warm-native", providerSessionId: "provider-warm-native", driverKind: "test", driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null, }; state.execute .mockReset() .mockImplementationOnce(async (options) => { expect(options.existingSession).toBeUndefined(); options.onSession?.(sharedSession); return result; }) .mockImplementationOnce(async (options) => { expect(options.existingSession).toBe(sharedSession); return result; }); await executePaperclipNativeSession({ db: leaseDb(base), execution: base, runnerInstanceId: "runner", }); await executePaperclipNativeSession({ db: leaseDb(second), execution: second, runnerInstanceId: "runner", }); expect(close).not.toHaveBeenCalled(); await vi.waitFor( () => expect(close).toHaveBeenCalledWith({ reason: "warm native session idle timeout", }), { timeout: 500 }, ); }); it("rehydrates a runnerd warm session from its checkpoint under a fresh run authority", async () => { const stateBase = await mkdtemp( join(tmpdir(), "paperclip-runnerd-warm-authority-"), ); const previousStateDirectory = process.env.PAPERCLIP_RUNNER_STATE_DIR; const previousPaperclipHome = process.env.PAPERCLIP_HOME; process.env.PAPERCLIP_RUNNER_STATE_DIR = stateBase; process.env.PAPERCLIP_HOME = stateBase; const firstClose = vi.fn(async () => undefined); const secondClose = vi.fn(async () => undefined); const firstSession = { close: firstClose }; const secondSession = { close: secondClose }; const first = { ...execution, binding: { ...execution.binding, runId: "run-runnerd-warm-first", executionWorkspaceId: "workspace-runnerd-warm", }, workspace: { cwd: "/tmp/runnerd-warm-authority", repoUrl: null, repoRef: null, branchName: null, }, session: { normalizedSessionId: "session-runnerd-warm-authority", driverKind: "codex_app_server" as const, protocolVersion: 1 as const, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 20 }, }, } as NativeExecutionInputV1; const second = { ...first, binding: { ...first.binding, runId: "run-runnerd-warm-second" }, } as NativeExecutionInputV1; const result = { result: { summary: "completed" }, terminal: { runTerminalState: "succeeded" }, turnId: "turn", normalizedSessionId: first.session.normalizedSessionId, providerSessionId: "provider-runnerd-warm", driverKind: "test", driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null, }; state.execute .mockReset() .mockImplementationOnce(async (options) => { expect(options.existingSession).toBeUndefined(); await options.onCheckpoint?.({ identity: { runId: first.binding.runId, sessionId: first.session.normalizedSessionId, companyId: first.binding.companyId, issueId: first.binding.issueId, agentId: first.binding.agentId, }, providerSessionId: "provider-runnerd-warm", }); options.onSession?.(firstSession); return result; }) .mockImplementationOnce(async (options) => { expect(options.existingSession).toBeUndefined(); expect(options.persistedSession).toEqual( expect.objectContaining({ identity: expect.objectContaining({ runId: second.binding.runId, sessionId: second.session.normalizedSessionId, }), providerSessionId: "provider-runnerd-warm", }), ); options.onSession?.(secondSession); return result; }); try { await executePaperclipNativeSession({ db: leaseDb(first), execution: first, runnerInstanceId: "runner-runnerd-warm", useRunnerd: true, }); const scopedRoots = (await readdir(stateBase, { withFileTypes: true })) .filter( (entry) => entry.isDirectory() && /^[a-f0-9]{64}$/.test(entry.name), ) .map((entry) => join(stateBase, entry.name)); expect(scopedRoots).toHaveLength(1); const durableRoot = scopedRoots[0]!; const durableIdentity = { runId: first.binding.runId, normalizedSessionId: first.session.normalizedSessionId, runnerInstanceId: "runner-runnerd-warm", environmentLeaseId: first.binding.executionWorkspaceId, }; await mkdir(join(durableRoot, "control-plane"), { recursive: true }); await mkdir(join(durableRoot, "runner"), { recursive: true }); await writeFile( join(durableRoot, "control-plane", "control-plane-state.json"), JSON.stringify(durableControlPlaneState(durableIdentity)), ); await writeFile( join(durableRoot, "runner", "runner-state.json"), JSON.stringify(durableRunnerState(durableIdentity, "suspended")), ); const continuationDb = { ...leaseDb(second), select: () => ({ from: () => ({ where: () => ({ limit: () => Promise.resolve([ { status: "succeeded", runnerProfileJson: { nativeExecutionInput: first }, }, ]), }), }), }), } as unknown as Db; await executePaperclipNativeSession({ db: continuationDb, execution: second, runnerInstanceId: "runner-runnerd-warm", useRunnerd: true, }); expect(firstClose).toHaveBeenCalledWith({ reason: "warm native session authority epoch rotated", }); await vi.waitFor(() => expect(secondClose).toHaveBeenCalled(), { timeout: 500, }); } finally { if (previousStateDirectory === undefined) { delete process.env.PAPERCLIP_RUNNER_STATE_DIR; } else { process.env.PAPERCLIP_RUNNER_STATE_DIR = previousStateDirectory; } if (previousPaperclipHome === undefined) { delete process.env.PAPERCLIP_HOME; } else { process.env.PAPERCLIP_HOME = previousPaperclipHome; } await rm(stateBase, { recursive: true, force: true }); } }); it("does not replace a different company's warm session with the same normalized id", async () => { const firstClose = vi.fn(async () => undefined); const secondClose = vi.fn(async () => undefined); const base = { ...execution, binding: { ...execution.binding, companyId: "company-warm-first", runId: "run-warm-first", executionWorkspaceId: "workspace", }, workspace: { cwd: "/tmp/warm-native-company-isolation", repoUrl: null, repoRef: null, branchName: null, }, session: { normalizedSessionId: "shared-company-warm-session", driverKind: "codex_app_server" as const, protocolVersion: 1 as const, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 20 }, }, } as NativeExecutionInputV1; const second = { ...base, binding: { ...base.binding, companyId: "company-warm-second", runId: "run-warm-second", }, }; const result = { result: { summary: "completed" }, terminal: { runTerminalState: "succeeded" }, turnId: "turn", normalizedSessionId: "shared-company-warm-session", providerSessionId: "provider-warm-native", driverKind: "test", driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null, }; state.execute .mockReset() .mockImplementationOnce(async (options) => { expect(options.existingSession).toBeUndefined(); options.onSession?.({ close: firstClose }); return result; }) .mockImplementationOnce(async (options) => { expect(options.existingSession).toBeUndefined(); options.onSession?.({ close: secondClose }); return result; }); await executePaperclipNativeSession({ db: leaseDb(base), execution: base, runnerInstanceId: "runner-first", }); await executePaperclipNativeSession({ db: leaseDb(second), execution: second, runnerInstanceId: "runner-second", }); await vi.waitFor(() => expect(firstClose).toHaveBeenCalled(), { timeout: 500, }); await vi.waitFor(() => expect(secondClose).toHaveBeenCalled(), { timeout: 500, }); expect(firstClose).toHaveBeenCalledWith({ reason: "warm native session idle timeout", }); expect(secondClose).toHaveBeenCalledWith({ reason: "warm native session idle timeout", }); }); it("replaces an idle warm provider session when its pinned permission mode changes", async () => { const firstClose = vi.fn(async () => undefined); const secondClose = vi.fn(async () => undefined); const firstSession = { close: firstClose }; const secondSession = { close: secondClose }; const base = { ...execution, schema: "paperclip.native-execution-input.v4", provider: { kind: "codex", model: null, approvalPolicy: "never" }, binding: { ...execution.binding, runId: "run-permission-never", executionWorkspaceId: "workspace", }, workspace: { cwd: "/tmp/warm-native-permission", repoUrl: null, repoRef: null, branchName: null, }, session: { normalizedSessionId: "session-warm-permission", driverKind: "codex_app_server" as const, protocolVersion: 1 as const, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 20 }, }, runtimeContext: { aggregateDigest: "runtime-context" }, } as unknown as NativeExecutionInputV1; const lowered = { ...base, provider: { kind: "codex", model: null, approvalPolicy: "on-request" }, binding: { ...base.binding, runId: "run-permission-on-request" }, } as NativeExecutionInputV1; const result = { result: { summary: "completed" }, terminal: { runTerminalState: "succeeded" }, turnId: "turn", normalizedSessionId: "session-warm-permission", providerSessionId: "provider-warm-permission", driverKind: "test", driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null, }; state.execute .mockReset() .mockImplementationOnce(async (options) => { expect(options.existingSession).toBeUndefined(); options.onSession?.(firstSession); return result; }) .mockImplementationOnce(async (options) => { expect(options.existingSession).toBeUndefined(); options.onSession?.(secondSession); return result; }); await executePaperclipNativeSession({ db: leaseDb(base), execution: base, runnerInstanceId: "runner", }); await executePaperclipNativeSession({ db: leaseDb(lowered), execution: lowered, runnerInstanceId: "runner", }); expect(firstClose).toHaveBeenCalledWith({ reason: "warm native session configuration changed", }); await vi.waitFor( () => expect(secondClose).toHaveBeenCalledWith({ reason: "warm native session idle timeout", }), { timeout: 500 }, ); }); }); describe("native session bounded recovery", () => { it("preserves stable provider and runner failure causes", () => { expect( nativeSessionFailureSourceCode( new Error( "provider_frame_too_large: harness stdout frame exceeded 4194304 bytes", ), ), ).toBe("provider_frame_too_large"); expect( nativeSessionFailureSourceCode( new Error( "native_runner_process_exited: runnerd exited unexpectedly with code 1", ), ), ).toBe("native_runner_process_exited"); expect( nativeSessionFailureSourceCode( new Error("provider_transport_failed: invalid JSON-RPC"), ), ).toBe("provider_transport_failed"); expect( nativeSessionFailureSourceCode( new Error( "planning_mode_unsupported: installed Codex app-server did not confirm plan mode", ), ), ).toBe("planning_mode_unsupported"); expect( nativeSessionFailureSourceCode( new Error( "native_event_replay_conflict: source sequence 41 contained different bytes", ), ), ).toBe("native_event_replay_conflict"); expect( nativeSessionFailureSourceCode( new Error( "provider_process_exited: provider=codex stage=initialize exitCode=1", ), ), ).toBe("provider_process_exited"); expect( nativeSessionFailureSourceCode( new Error("provider_stdout_closed: provider=codex stage=initialize"), ), ).toBe("provider_stdout_closed"); expect( nativeSessionFailureSourceCode( new Error( "provider_process_status_failed: provider=codex stage=session.open", ), ), ).toBe("provider_process_status_failed"); expect( nativeSessionFailureSourceCode( new Error( "provider_initialize_timeout: provider=codex stage=initialize", ), ), ).toBe("provider_initialize_timeout"); expect( nativeSessionFailureSourceCode( new Error( "provider_initialize_protocol_error: provider=codex stage=initialize", ), ), ).toBe("provider_initialize_protocol_error"); expect( nativeSessionFailureSourceCode( new Error("provider_request_timeout: provider=codex stage=turn.start"), ), ).toBe("provider_request_timeout"); expect( nativeSessionFailureSourceCode( new Error( "runner_remote_provider_artifact_incompatible: OpenCode version mismatch", ), ), ).toBe("runner_remote_provider_artifact_incompatible"); }); it("retries the same run twice and stops at the third failed attempt", () => { const now = new Date("2026-08-09T00:00:00.000Z"); expect(nativeSessionFailureDisposition(1, now)).toEqual({ phase: "retryable_failure", failureCode: "native_session_interrupted", nextAttemptAt: new Date("2026-08-09T00:00:30.000Z"), }); expect(nativeSessionFailureDisposition(2, now)).toEqual({ phase: "retryable_failure", failureCode: "native_session_interrupted", nextAttemptAt: new Date("2026-08-09T00:00:30.000Z"), }); expect(nativeSessionFailureDisposition(3, now)).toEqual({ phase: "terminal_failure", failureCode: "native_session_retry_exhausted", nextAttemptAt: null, }); expect( nativeSessionFailureDisposition(1, now, "native_event_replay_conflict"), ).toEqual({ phase: "terminal_failure", failureCode: "native_event_replay_conflict", nextAttemptAt: null, }); expect( nativeSessionFailureDisposition( 1, now, "runner_remote_provider_artifact_incompatible", ), ).toEqual({ phase: "terminal_failure", failureCode: "runner_remote_provider_artifact_incompatible", nextAttemptAt: null, }); }); it("escalates exhausted result-less sessions to board review instead of leaving the provider as its own owner", () => { expect( nativeSessionRecoveryProjection({ phase: "retryable_failure", failureCode: "native_session_interrupted", agentId: "agent-low-capability", }), ).toEqual({ exhausted: false, issueStatus: null, recoveryOwner: { kind: "agent", agentId: "agent-low-capability" }, recoveryActionOwnerType: "agent", recoveryActionOwnerAgentId: "agent-low-capability", recoveryActionCause: "native_session_interrupted", supersedeOnIdentityChange: true, }); expect( nativeSessionRecoveryProjection({ phase: "terminal_failure", failureCode: "native_session_retry_exhausted", agentId: "agent-low-capability", }), ).toEqual({ exhausted: true, issueStatus: "in_review", recoveryOwner: { kind: "board" }, recoveryActionOwnerType: "board", recoveryActionOwnerAgentId: null, recoveryActionCause: "native_session_retry_exhausted", supersedeOnIdentityChange: true, }); }); }); describe("native process ownership", () => { it("forwards the app-server PID and process group through the production backend seam", async () => { const processMetadata = { pid: 42_001, processGroupId: 42_001, startedAt: "2026-08-18T18:00:00.000Z", }; const onSpawn = vi.fn(async () => undefined); state.createBackend.mockClear(); state.execute.mockReset().mockImplementation(async (options) => { await options.backend.onSpawn(processMetadata); return { result: { summary: "completed" }, terminal: { runTerminalState: "succeeded" }, turnId: "turn", normalizedSessionId: "session", providerSessionId: null, driverKind: "test", driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, }; }); state.createBackend.mockImplementationOnce((_input, options) => ({ kind: "test", onSpawn: options.onSpawn, })); await executePaperclipNativeSession({ db: leaseDb(), execution, runnerInstanceId: "runner", onSpawn, }); expect(state.createBackend).toHaveBeenCalledWith( execution, expect.objectContaining({ runnerInstanceId: "runner", onSpawn, }), ); expect(onSpawn).toHaveBeenCalledWith(processMetadata); }); it.each([ [ "OpenCode", { kind: "opencode", model: "openrouter/deepseek/deepseek-v4-flash-0731", permissionMode: "deny", }, "opencode_server", ], [ "Claude ACPX", { kind: "acpx", agent: "claude", model: "claude-sonnet-5", permissionMode: "approve-all", }, "acpx_runtime", ], [ "Codex ACPX", { kind: "acpx", agent: "codex", model: "gpt-5.6-sol", permissionMode: "deny-all", }, "acpx_runtime", ], ])( "admits the qualified %s provider", async (_name, provider, driverKind) => { const providerExecution = { ...execution, binding: { ...execution.binding, runId: `run-${String(provider.kind)}-${"agent" in provider ? provider.agent : "native"}`, }, provider, session: { ...execution.session, driverKind }, } as unknown as NativeExecutionInputV1; state.createBackend.mockClear(); state.execute.mockReset().mockResolvedValue({ result: { summary: "completed" }, terminal: { runTerminalState: "succeeded" }, turnId: "turn", normalizedSessionId: "session", providerSessionId: null, driverKind, driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, }); await executePaperclipNativeSession({ db: leaseDb(providerExecution), execution: providerExecution, runnerInstanceId: "runner", }); expect(state.createBackend).toHaveBeenCalledWith( providerExecution, expect.any(Object), ); }, ); it("rejects ACPX Pi before constructing a backend", async () => { const piExecution = { ...execution, binding: { ...execution.binding, runId: "run-acpx-pi-rejected" }, provider: { kind: "acpx", agent: "pi", model: "pi-model" }, session: { ...execution.session, driverKind: "acpx_runtime" }, } as unknown as NativeExecutionInputV1; state.createBackend.mockClear(); await expect( executePaperclipNativeSession({ db: leaseDb(piExecution), execution: piExecution, runnerInstanceId: "runner", }), ).rejects.toThrow("descriptor-confined verified launch"); expect(state.createBackend).not.toHaveBeenCalled(); }); }); describe("runnerd provider runtime wiring", () => { let isolatedStateDirectory: string; let previousStateDirectory: string | undefined; beforeEach(async () => { previousStateDirectory = process.env.PAPERCLIP_RUNNER_STATE_DIR; isolatedStateDirectory = await mkdtemp( join(tmpdir(), "paperclip-runnerd-wiring-"), ); process.env.PAPERCLIP_RUNNER_STATE_DIR = isolatedStateDirectory; }); afterEach(async () => { if (previousStateDirectory === undefined) { delete process.env.PAPERCLIP_RUNNER_STATE_DIR; } else { process.env.PAPERCLIP_RUNNER_STATE_DIR = previousStateDirectory; } await rm(isolatedStateDirectory, { recursive: true, force: true }); }); it("rejects overlapping runs for the same runnerd provider session scope", async () => { const first = { ...execution, binding: { ...execution.binding, runId: "run-runnerd-overlap-first", executionWorkspaceId: "workspace-runnerd-overlap", }, session: { ...execution.session, normalizedSessionId: "session-runnerd-overlap", }, } as NativeExecutionInputV1; const second = { ...first, binding: { ...first.binding, runId: "run-runnerd-overlap-second" }, } as NativeExecutionInputV1; let release!: () => void; state.execute.mockReset().mockImplementation( () => new Promise((resolve) => { release = () => resolve({ result: { summary: "completed" }, terminal: { runTerminalState: "succeeded" }, turnId: "turn", normalizedSessionId: first.session.normalizedSessionId, providerSessionId: "provider-runnerd-overlap", driverKind: "test", driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null, }); }), ); const active = executePaperclipNativeSession({ db: leaseDb(first), execution: first, runnerInstanceId: "runner-runnerd-overlap", useRunnerd: true, }); await vi.waitFor(() => expect(release).toBeTypeOf("function")); await expect( executePaperclipNativeSession({ db: leaseDb(second), execution: second, runnerInstanceId: "runner-runnerd-overlap", useRunnerd: true, }), ).rejects.toThrow("native_session_supervisor_busy"); release(); await expect(active).resolves.toBeDefined(); }); it("carries the verified runner and lease binding into a projectless continuation", async () => { const stateBase = await mkdtemp( join(tmpdir(), "paperclip-runner-binding-"), ); const previousStateDirectory = process.env.PAPERCLIP_RUNNER_STATE_DIR; process.env.PAPERCLIP_RUNNER_STATE_DIR = stateBase; const prior = { ...execution, binding: { ...execution.binding, companyId: "company-projectless-continuation", runId: "run-projectless-prior", executionWorkspaceId: "run-projectless-prior", }, session: { ...execution.session, normalizedSessionId: "session-projectless-continuation", }, } as NativeExecutionInputV1; const continuation = { ...prior, binding: { ...prior.binding, runId: "run-projectless-next", executionWorkspaceId: "run-projectless-next", }, } as NativeExecutionInputV1; try { state.createBackend.mockClear(); state.createTransport.mockClear(); await createRunnerdBackend({ db: leaseDb(prior), execution: prior, runnerInstanceId: "runner-projectless-stable", }); state.createBackend.mock.calls[0]![1].codexTransportFactory!(); const scopedRoot = state.createTransport.mock.calls[0]![0].stateDirectory!; await mkdir(join(scopedRoot, "control-plane"), { recursive: true }); await mkdir(join(scopedRoot, "runner"), { recursive: true }); const priorIdentity = { runId: "run-projectless-prior", normalizedSessionId: continuation.session.normalizedSessionId, runnerInstanceId: "runner-projectless-stable", environmentLeaseId: "lease-projectless-stable", }; await writeFile( join(scopedRoot, "control-plane", "control-plane-state.json"), JSON.stringify(durableControlPlaneState(priorIdentity)), ); await writeFile( join(scopedRoot, "runner", "runner-state.json"), JSON.stringify(durableRunnerState(priorIdentity, "suspended")), ); state.createBackend.mockClear(); state.createTransport.mockClear(); state.execute.mockReset().mockResolvedValue({ result: { summary: "completed" }, terminal: { runTerminalState: "succeeded" }, turnId: "turn", normalizedSessionId: continuation.session.normalizedSessionId, providerSessionId: null, driverKind: "test", driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, }); const continuationDb = { ...leaseDb(continuation), select: () => ({ from: () => ({ where: () => ({ limit: () => Promise.resolve([ { status: "succeeded", runnerProfileJson: { nativeExecutionInput: prior }, }, ]), }), }), }), } as unknown as Db; await executePaperclipNativeSession({ db: continuationDb, execution: continuation, runnerInstanceId: "runner-new-heartbeat", useRunnerd: true, }); const backendOptions = state.createBackend.mock.calls[0]![1]; backendOptions.codexTransportFactory!(); expect(state.createTransport).toHaveBeenCalledWith( expect.objectContaining({ stateDirectory: scopedRoot, prpIdentity: expect.objectContaining({ runnerInstanceId: "runner-projectless-stable", environmentLeaseId: "lease-projectless-stable", runId: "run-projectless-next", }), }), ); } finally { if (previousStateDirectory === undefined) { delete process.env.PAPERCLIP_RUNNER_STATE_DIR; } else { process.env.PAPERCLIP_RUNNER_STATE_DIR = previousStateDirectory; } await rm(stateBase, { recursive: true, force: true }); } }); it("uses the native execution workspace as the local provider containment root", async () => { state.createBackend.mockClear(); await createRunnerdBackend({ db: leaseDb(execution), execution, runnerInstanceId: "runner-local-workspace", runnerEnvironment: { HOME: "/home/runner", PAPERCLIP_WORKSPACE_CWD: "/untrusted/configured-workspace", }, }); const backendOptions = state.createBackend.mock.calls[0]![1]; state.createTransport.mockClear(); backendOptions.codexTransportFactory!(); expect(state.createTransport).toHaveBeenCalledWith( expect.objectContaining({ environment: expect.objectContaining({ PAPERCLIP_WORKSPACE_CWD: execution.workspace.cwd, }), }), ); }); it("atomically migrates legacy unscoped state only for its exact durable run identity", async () => { const stateBase = await mkdtemp( join(tmpdir(), "paperclip-legacy-runner-state-"), ); const previousStateDirectory = process.env.PAPERCLIP_RUNNER_STATE_DIR; process.env.PAPERCLIP_RUNNER_STATE_DIR = stateBase; const legacyExecution = { ...execution, binding: { ...execution.binding, companyId: "company-legacy-state", runId: "run-legacy-state", }, session: { ...execution.session, normalizedSessionId: "session-legacy-state", }, } as NativeExecutionInputV1; const legacyRoot = join( stateBase, createHash("sha256").update("session-legacy-state").digest("hex"), ); try { await mkdir(join(legacyRoot, "control-plane"), { recursive: true }); await mkdir(join(legacyRoot, "runner"), { recursive: true }); const legacyIdentity = { runId: "run-legacy-state", normalizedSessionId: "session-legacy-state", runnerInstanceId: "runner-legacy-state", environmentLeaseId: "lease-legacy-state", }; await writeFile( join(legacyRoot, "control-plane", "control-plane-state.json"), JSON.stringify(durableControlPlaneState(legacyIdentity)), ); await writeFile( join(legacyRoot, "runner", "runner-state.json"), JSON.stringify(durableRunnerState(legacyIdentity, "ready")), ); state.createBackend.mockClear(); await createRunnerdBackend({ db: leaseDb(legacyExecution), execution: legacyExecution, runnerInstanceId: "runner-legacy-state", }); state.createTransport.mockClear(); state.createBackend.mock.calls[0]![1].codexTransportFactory!(); const migratedRoot = state.createTransport.mock.calls[0]![0].stateDirectory!; expect(migratedRoot).not.toBe(legacyRoot); await expect(access(legacyRoot)).rejects.toThrow(); await expect( access(join(migratedRoot, "control-plane", "control-plane-state.json")), ).resolves.toBeUndefined(); expect(state.createTransport.mock.calls[0]![0].prpIdentity).toEqual( expect.objectContaining({ runnerInstanceId: "runner-legacy-state", environmentLeaseId: "lease-legacy-state", runId: "run-legacy-state", }), ); expect(state.createTransport.mock.calls[0]![0].runnerBinary).toBe( "/tmp/paperclip-runnerd", ); expect(state.resolveRunnerBinary).toHaveBeenCalled(); const unrelatedExecution = { ...legacyExecution, binding: { ...legacyExecution.binding, companyId: "company-unrelated-state", runId: "run-unrelated-state", }, } as NativeExecutionInputV1; await createRunnerdBackend({ db: leaseDb(unrelatedExecution), execution: unrelatedExecution, runnerInstanceId: "runner-unrelated-state", }); state.createBackend.mock.calls[1]![1].codexTransportFactory!(); expect(state.createTransport.mock.calls[1]![0].stateDirectory).not.toBe( legacyRoot, ); } finally { if (previousStateDirectory === undefined) { delete process.env.PAPERCLIP_RUNNER_STATE_DIR; } else { process.env.PAPERCLIP_RUNNER_STATE_DIR = previousStateDirectory; } await rm(stateBase, { recursive: true, force: true }); } }); it("migrates the former company/session scope into the full native session scope", async () => { const stateBase = await mkdtemp( join(tmpdir(), "paperclip-company-session-state-"), ); const previousStateDirectory = process.env.PAPERCLIP_RUNNER_STATE_DIR; process.env.PAPERCLIP_RUNNER_STATE_DIR = stateBase; const legacyExecution = { ...execution, binding: { ...execution.binding, companyId: "company-former-scope", runId: "run-former-scope", agentId: "agent-former-scope", executionWorkspaceId: "workspace-former-scope", }, session: { ...execution.session, normalizedSessionId: "session-former-scope", }, } as NativeExecutionInputV1; const legacyRoot = join( stateBase, createHash("sha256") .update( JSON.stringify([ legacyExecution.binding.companyId, legacyExecution.session.normalizedSessionId, ]), ) .digest("hex"), ); try { await mkdir(join(legacyRoot, "control-plane"), { recursive: true }); await mkdir(join(legacyRoot, "runner"), { recursive: true }); const legacyIdentity = { runId: legacyExecution.binding.runId, normalizedSessionId: legacyExecution.session.normalizedSessionId, runnerInstanceId: "runner-former-scope", environmentLeaseId: "lease-former-scope", }; await writeFile( join(legacyRoot, "control-plane", "control-plane-state.json"), JSON.stringify(durableControlPlaneState(legacyIdentity)), ); await writeFile( join(legacyRoot, "runner", "runner-state.json"), JSON.stringify(durableRunnerState(legacyIdentity, "ready")), ); state.createBackend.mockClear(); state.createTransport.mockClear(); await createRunnerdBackend({ db: leaseDb(legacyExecution), execution: legacyExecution, runnerInstanceId: "runner-former-scope", }); state.createBackend.mock.calls[0]![1].codexTransportFactory!(); const migratedRoot = state.createTransport.mock.calls[0]![0].stateDirectory!; expect(migratedRoot).not.toBe(legacyRoot); await expect(access(legacyRoot)).rejects.toThrow(); await expect( access(join(migratedRoot, "control-plane", "control-plane-state.json")), ).resolves.toBeUndefined(); } finally { if (previousStateDirectory === undefined) { delete process.env.PAPERCLIP_RUNNER_STATE_DIR; } else { process.env.PAPERCLIP_RUNNER_STATE_DIR = previousStateDirectory; } await rm(stateBase, { recursive: true, force: true }); } }); it("migrates a suspended prior-run authority only when its persisted execution has the same full session scope", async () => { const stateBase = await mkdtemp( join(tmpdir(), "paperclip-prior-run-session-state-"), ); const previousStateDirectory = process.env.PAPERCLIP_RUNNER_STATE_DIR; process.env.PAPERCLIP_RUNNER_STATE_DIR = stateBase; const priorExecution = { ...execution, binding: { ...execution.binding, companyId: "company-prior-run-scope", runId: "run-prior-run-scope", agentId: "agent-prior-run-scope", executionWorkspaceId: "workspace-prior-run-scope", }, session: { ...execution.session, normalizedSessionId: "session-prior-run-scope", }, } as NativeExecutionInputV1; const currentExecution = { ...priorExecution, binding: { ...priorExecution.binding, runId: "run-current-run-scope", }, } as NativeExecutionInputV1; const legacyRoot = join( stateBase, createHash("sha256") .update( JSON.stringify([ currentExecution.binding.companyId, currentExecution.session.normalizedSessionId, ]), ) .digest("hex"), ); const priorRunDb = { select: () => ({ from: () => ({ where: () => ({ limit: () => Promise.resolve([ { status: "succeeded", runnerProfileJson: { nativeExecutionInput: priorExecution, }, }, ]), }), }), }), } as unknown as Db; try { await mkdir(join(legacyRoot, "control-plane"), { recursive: true }); await mkdir(join(legacyRoot, "runner"), { recursive: true }); await writeFile( join(legacyRoot, "control-plane", "control-plane-state.json"), JSON.stringify( durableControlPlaneState({ runId: priorExecution.binding.runId, normalizedSessionId: priorExecution.session.normalizedSessionId, runnerInstanceId: "runner-prior-run-scope", environmentLeaseId: "lease-prior-run-scope", }), ), ); await writeFile( join(legacyRoot, "runner", "runner-state.json"), JSON.stringify( durableRunnerState( { runId: priorExecution.binding.runId, normalizedSessionId: priorExecution.session.normalizedSessionId, runnerInstanceId: "runner-prior-run-scope", environmentLeaseId: "lease-prior-run-scope", }, "suspended", ), ), ); state.createBackend.mockClear(); state.createTransport.mockClear(); await createRunnerdBackend({ db: priorRunDb, execution: currentExecution, runnerInstanceId: "runner-current-run-scope", }); state.createBackend.mock.calls[0]![1].codexTransportFactory!(); const migratedRoot = state.createTransport.mock.calls[0]![0].stateDirectory!; expect(migratedRoot).not.toBe(legacyRoot); await expect(access(legacyRoot)).rejects.toThrow(); expect(state.createTransport.mock.calls[0]![0].prpIdentity).toEqual( expect.objectContaining({ runId: currentExecution.binding.runId, runnerInstanceId: "runner-prior-run-scope", environmentLeaseId: "lease-prior-run-scope", }), ); } finally { if (previousStateDirectory === undefined) { delete process.env.PAPERCLIP_RUNNER_STATE_DIR; } else { process.env.PAPERCLIP_RUNNER_STATE_DIR = previousStateDirectory; } await rm(stateBase, { recursive: true, force: true }); } }); it("quarantines legacy prior-run state only after the database proves a terminal owner in the same full scope", async () => { const stateBase = await mkdtemp( join(tmpdir(), "paperclip-legacy-terminal-unsuspended-state-"), ); const previousStateDirectory = process.env.PAPERCLIP_RUNNER_STATE_DIR; process.env.PAPERCLIP_RUNNER_STATE_DIR = stateBase; const priorExecution = { ...execution, binding: { ...execution.binding, companyId: "company-legacy-terminal-unsuspended", runId: "run-legacy-terminal-unsuspended", agentId: "agent-legacy-terminal-unsuspended", executionWorkspaceId: "workspace-legacy-terminal-unsuspended", }, session: { ...execution.session, normalizedSessionId: "session-legacy-terminal-unsuspended", }, } as NativeExecutionInputV1; const currentExecution = { ...priorExecution, binding: { ...priorExecution.binding, runId: "run-after-legacy-terminal-unsuspended", }, } as NativeExecutionInputV1; const legacyRoot = join( stateBase, createHash("sha256") .update( JSON.stringify([ currentExecution.binding.companyId, currentExecution.session.normalizedSessionId, ]), ) .digest("hex"), ); const terminalPriorRunDb = { select: () => ({ from: () => ({ where: () => ({ limit: () => Promise.resolve([ { status: "succeeded", runnerProfileJson: { nativeExecutionInput: priorExecution, }, }, ]), }), }), }), } as unknown as Db; const identity = { runId: priorExecution.binding.runId, normalizedSessionId: priorExecution.session.normalizedSessionId, runnerInstanceId: "runner-legacy-terminal-unsuspended", environmentLeaseId: "lease-legacy-terminal-unsuspended", }; try { await mkdir(join(legacyRoot, "control-plane"), { recursive: true }); await mkdir(join(legacyRoot, "runner"), { recursive: true }); await writeFile( join(legacyRoot, "control-plane", "control-plane-state.json"), JSON.stringify(durableControlPlaneState(identity)), ); await writeFile( join(legacyRoot, "runner", "runner-state.json"), JSON.stringify(durableRunnerState(identity, "ready")), ); state.createBackend.mockClear(); state.createTransport.mockClear(); await expect( createRunnerdBackend({ db: terminalPriorRunDb, execution: currentExecution, runnerInstanceId: "runner-after-legacy-terminal-unsuspended", }), ).rejects.toThrow("runner_state_identity_mismatch"); await expect(access(legacyRoot)).rejects.toThrow(); const quarantineEntries = await readdir(join(stateBase, "quarantine")); expect(quarantineEntries).toHaveLength(1); expect(quarantineEntries[0]).toContain(".identity_indeterminate."); expect(state.createBackend).not.toHaveBeenCalled(); expect(state.createTransport).not.toHaveBeenCalled(); } finally { if (previousStateDirectory === undefined) { delete process.env.PAPERCLIP_RUNNER_STATE_DIR; } else { process.env.PAPERCLIP_RUNNER_STATE_DIR = previousStateDirectory; } await rm(stateBase, { recursive: true, force: true }); } }); it("rejects scoped prior-run state after restart while its heartbeat is still running", async () => { const stateBase = await mkdtemp( join(tmpdir(), "paperclip-running-prior-run-state-"), ); const previousStateDirectory = process.env.PAPERCLIP_RUNNER_STATE_DIR; process.env.PAPERCLIP_RUNNER_STATE_DIR = stateBase; const priorExecution = { ...execution, binding: { ...execution.binding, companyId: "company-running-prior-scope", runId: "run-running-prior-scope", agentId: "agent-running-prior-scope", executionWorkspaceId: "workspace-running-prior-scope", }, session: { ...execution.session, normalizedSessionId: "session-running-prior-scope", }, } as NativeExecutionInputV1; const currentExecution = { ...priorExecution, binding: { ...priorExecution.binding, runId: "run-after-running-prior-scope", }, } as NativeExecutionInputV1; const runningPriorRunDb = { select: () => ({ from: () => ({ where: () => ({ limit: () => Promise.resolve([ { status: "running", runnerProfileJson: { nativeExecutionInput: priorExecution, }, }, ]), }), }), }), } as unknown as Db; const identity = { runId: priorExecution.binding.runId, normalizedSessionId: priorExecution.session.normalizedSessionId, runnerInstanceId: "runner-running-prior-scope", environmentLeaseId: "lease-running-prior-scope", }; try { state.createBackend.mockClear(); state.createTransport.mockClear(); await createRunnerdBackend({ db: leaseDb(priorExecution), execution: priorExecution, runnerInstanceId: identity.runnerInstanceId, }); state.createBackend.mock.calls[0]![1].codexTransportFactory!(); const scopedRoot = state.createTransport.mock.calls[0]![0].stateDirectory!; await mkdir(join(scopedRoot, "control-plane"), { recursive: true }); await mkdir(join(scopedRoot, "runner"), { recursive: true }); await writeFile( join(scopedRoot, "control-plane", "control-plane-state.json"), JSON.stringify(durableControlPlaneState(identity)), ); await writeFile( join(scopedRoot, "runner", "runner-state.json"), JSON.stringify(durableRunnerState(identity, "suspended")), ); state.createBackend.mockClear(); state.createTransport.mockClear(); await expect( createRunnerdBackend({ db: runningPriorRunDb, execution: currentExecution, runnerInstanceId: "runner-after-running-prior-scope", }), ).rejects.toThrow("runner_state_identity_mismatch"); await expect(access(scopedRoot)).resolves.toBeUndefined(); await expect(access(join(stateBase, "quarantine"))).rejects.toThrow(); expect(state.createBackend).not.toHaveBeenCalled(); expect(state.createTransport).not.toHaveBeenCalled(); } finally { if (previousStateDirectory === undefined) { delete process.env.PAPERCLIP_RUNNER_STATE_DIR; } else { process.env.PAPERCLIP_RUNNER_STATE_DIR = previousStateDirectory; } await rm(stateBase, { recursive: true, force: true }); } }); it("quarantines scoped prior-run state when the heartbeat is terminal but runnerd is not suspended", async () => { const stateBase = await mkdtemp( join(tmpdir(), "paperclip-terminal-unsuspended-state-"), ); const previousStateDirectory = process.env.PAPERCLIP_RUNNER_STATE_DIR; process.env.PAPERCLIP_RUNNER_STATE_DIR = stateBase; const priorExecution = { ...execution, binding: { ...execution.binding, companyId: "company-terminal-unsuspended", runId: "run-terminal-unsuspended", agentId: "agent-terminal-unsuspended", executionWorkspaceId: "workspace-terminal-unsuspended", }, session: { ...execution.session, normalizedSessionId: "session-terminal-unsuspended", }, } as NativeExecutionInputV1; const currentExecution = { ...priorExecution, binding: { ...priorExecution.binding, runId: "run-after-terminal-unsuspended", }, } as NativeExecutionInputV1; const terminalPriorRunDb = { select: () => ({ from: () => ({ where: () => ({ limit: () => Promise.resolve([ { status: "succeeded", runnerProfileJson: { nativeExecutionInput: priorExecution, }, }, ]), }), }), }), } as unknown as Db; const identity = { runId: priorExecution.binding.runId, normalizedSessionId: priorExecution.session.normalizedSessionId, runnerInstanceId: "runner-terminal-unsuspended", environmentLeaseId: "lease-terminal-unsuspended", }; try { state.createBackend.mockClear(); state.createTransport.mockClear(); await createRunnerdBackend({ db: leaseDb(priorExecution), execution: priorExecution, runnerInstanceId: identity.runnerInstanceId, }); state.createBackend.mock.calls[0]![1].codexTransportFactory!(); const scopedRoot = state.createTransport.mock.calls[0]![0].stateDirectory!; await mkdir(join(scopedRoot, "control-plane"), { recursive: true }); await mkdir(join(scopedRoot, "runner"), { recursive: true }); await writeFile( join(scopedRoot, "control-plane", "control-plane-state.json"), JSON.stringify(durableControlPlaneState(identity)), ); await writeFile( join(scopedRoot, "runner", "runner-state.json"), JSON.stringify(durableRunnerState(identity, "ready")), ); state.createBackend.mockClear(); state.createTransport.mockClear(); await expect( createRunnerdBackend({ db: terminalPriorRunDb, execution: currentExecution, runnerInstanceId: "runner-after-terminal-unsuspended", }), ).rejects.toThrow("runner_state_identity_mismatch"); await expect(access(scopedRoot)).rejects.toThrow(); const quarantineEntries = await readdir(join(stateBase, "quarantine")); expect(quarantineEntries).toHaveLength(1); expect(quarantineEntries[0]).toContain(".identity_indeterminate."); expect(state.createBackend).not.toHaveBeenCalled(); expect(state.createTransport).not.toHaveBeenCalled(); } finally { if (previousStateDirectory === undefined) { delete process.env.PAPERCLIP_RUNNER_STATE_DIR; } else { process.env.PAPERCLIP_RUNNER_STATE_DIR = previousStateDirectory; } await rm(stateBase, { recursive: true, force: true }); } }); it("resumes an existing scoped authority only for the exact current run", async () => { const stateBase = await mkdtemp( join(tmpdir(), "paperclip-current-scoped-state-"), ); const previousStateDirectory = process.env.PAPERCLIP_RUNNER_STATE_DIR; process.env.PAPERCLIP_RUNNER_STATE_DIR = stateBase; const currentExecution = { ...execution, binding: { ...execution.binding, companyId: "company-current-scoped-state", runId: "run-current-scoped-state", agentId: "agent-current-scoped-state", executionWorkspaceId: "workspace-current-scoped-state", }, session: { ...execution.session, normalizedSessionId: "session-current-scoped-state", }, } as NativeExecutionInputV1; const identity = { runId: currentExecution.binding.runId, normalizedSessionId: currentExecution.session.normalizedSessionId, runnerInstanceId: "runner-current-scoped-state", environmentLeaseId: "lease-current-scoped-state", }; try { state.createBackend.mockClear(); state.createTransport.mockClear(); await createRunnerdBackend({ db: leaseDb(currentExecution), execution: currentExecution, runnerInstanceId: identity.runnerInstanceId, }); state.createBackend.mock.calls[0]![1].codexTransportFactory!(); const scopedRoot = state.createTransport.mock.calls[0]![0].stateDirectory!; await mkdir(join(scopedRoot, "control-plane"), { recursive: true }); await mkdir(join(scopedRoot, "runner"), { recursive: true }); await writeFile( join(scopedRoot, "control-plane", "control-plane-state.json"), JSON.stringify(durableControlPlaneState(identity)), ); await writeFile( join(scopedRoot, "runner", "runner-state.json"), JSON.stringify(durableRunnerState(identity, "ready")), ); state.createBackend.mockClear(); state.createTransport.mockClear(); await expect( createRunnerdBackend({ db: leaseDb(currentExecution), execution: currentExecution, runnerInstanceId: "runner-restart-placeholder", }), ).resolves.toBeDefined(); state.createBackend.mock.calls[0]![1].codexTransportFactory!(); expect(state.createTransport).toHaveBeenCalledWith( expect.objectContaining({ stateDirectory: scopedRoot, prpIdentity: expect.objectContaining(identity), }), ); } finally { if (previousStateDirectory === undefined) { delete process.env.PAPERCLIP_RUNNER_STATE_DIR; } else { process.env.PAPERCLIP_RUNNER_STATE_DIR = previousStateDirectory; } await rm(stateBase, { recursive: true, force: true }); } }); it.each(["unknown_schema", "unknown_lifecycle"] as const)( "quarantines an exact-run runner state with %s", async (caseName) => { const stateBase = await mkdtemp( join(tmpdir(), `paperclip-${caseName}-runner-state-`), ); const previousStateDirectory = process.env.PAPERCLIP_RUNNER_STATE_DIR; process.env.PAPERCLIP_RUNNER_STATE_DIR = stateBase; const currentExecution = { ...execution, binding: { ...execution.binding, companyId: `company-${caseName}-runner-state`, runId: `run-${caseName}-runner-state`, agentId: `agent-${caseName}-runner-state`, executionWorkspaceId: `workspace-${caseName}-runner-state`, }, session: { ...execution.session, normalizedSessionId: `session-${caseName}-runner-state`, }, } as NativeExecutionInputV1; const identity = { runId: currentExecution.binding.runId, normalizedSessionId: currentExecution.session.normalizedSessionId, runnerInstanceId: `runner-${caseName}-runner-state`, environmentLeaseId: currentExecution.binding.executionWorkspaceId, }; try { state.createBackend.mockClear(); state.createTransport.mockClear(); await createRunnerdBackend({ db: leaseDb(currentExecution), execution: currentExecution, runnerInstanceId: identity.runnerInstanceId, }); state.createBackend.mock.calls[0]![1].codexTransportFactory!(); const scopedRoot = state.createTransport.mock.calls[0]![0].stateDirectory!; await mkdir(join(scopedRoot, "control-plane"), { recursive: true }); await mkdir(join(scopedRoot, "runner"), { recursive: true }); await writeFile( join(scopedRoot, "control-plane", "control-plane-state.json"), JSON.stringify(durableControlPlaneState(identity)), ); const runnerState = durableRunnerState( identity, caseName === "unknown_lifecycle" ? "future_lifecycle" : "ready", ); await writeFile( join(scopedRoot, "runner", "runner-state.json"), JSON.stringify( caseName === "unknown_schema" ? { ...runnerState, schema: "paperclip.runner.durable.state.v999", } : runnerState, ), ); state.createBackend.mockClear(); state.createTransport.mockClear(); await expect( createRunnerdBackend({ db: leaseDb(currentExecution), execution: currentExecution, runnerInstanceId: `runner-${caseName}-retry`, }), ).rejects.toThrow("runner_state_identity_mismatch"); await expect(access(scopedRoot)).rejects.toThrow(); const quarantineEntries = await readdir(join(stateBase, "quarantine")); expect(quarantineEntries).toHaveLength(1); expect(quarantineEntries[0]).toContain(".identity_indeterminate."); expect(state.createBackend).not.toHaveBeenCalled(); expect(state.createTransport).not.toHaveBeenCalled(); } finally { if (previousStateDirectory === undefined) { delete process.env.PAPERCLIP_RUNNER_STATE_DIR; } else { process.env.PAPERCLIP_RUNNER_STATE_DIR = previousStateDirectory; } await rm(stateBase, { recursive: true, force: true }); } }, ); it.each(["missing", "malformed", "unknown_schema", "mismatched"] as const)( "fails closed on %s durable identity in an existing scoped root", async (caseName) => { const stateBase = await mkdtemp( join(tmpdir(), `paperclip-${caseName}-scoped-state-`), ); const previousStateDirectory = process.env.PAPERCLIP_RUNNER_STATE_DIR; process.env.PAPERCLIP_RUNNER_STATE_DIR = stateBase; const scopedExecution = { ...execution, binding: { ...execution.binding, companyId: `company-${caseName}-scoped-state`, runId: `run-${caseName}-scoped-state`, agentId: `agent-${caseName}-scoped-state`, executionWorkspaceId: `workspace-${caseName}-scoped-state`, }, session: { ...execution.session, normalizedSessionId: `session-${caseName}-scoped-state`, }, } as NativeExecutionInputV1; try { state.createBackend.mockClear(); state.createTransport.mockClear(); await createRunnerdBackend({ db: leaseDb(scopedExecution), execution: scopedExecution, runnerInstanceId: `runner-${caseName}-scoped-state`, }); state.createBackend.mock.calls[0]![1].codexTransportFactory!(); const scopedRoot = state.createTransport.mock.calls[0]![0].stateDirectory!; await mkdir(join(scopedRoot, "control-plane"), { recursive: true }); if (caseName !== "missing") { await writeFile( join(scopedRoot, "control-plane", "control-plane-state.json"), caseName === "malformed" ? "{" : caseName === "unknown_schema" ? JSON.stringify({ ...durableControlPlaneState({ runId: scopedExecution.binding.runId, normalizedSessionId: scopedExecution.session.normalizedSessionId, runnerInstanceId: `runner-${caseName}-scoped-state`, environmentLeaseId: scopedExecution.binding.executionWorkspaceId, }), schema: "paperclip.runner.durable.control-plane-state.v999", }) : JSON.stringify( durableControlPlaneState({ runId: scopedExecution.binding.runId, normalizedSessionId: "session-owned-by-another-scope", runnerInstanceId: "runner-owned-by-another-scope", environmentLeaseId: "lease-owned-by-another-scope", }), ), ); } state.createBackend.mockClear(); state.createTransport.mockClear(); await expect( createRunnerdBackend({ db: leaseDb(scopedExecution), execution: scopedExecution, runnerInstanceId: `runner-${caseName}-retry`, }), ).rejects.toThrow("runner_state_identity_mismatch"); await expect(access(scopedRoot)).rejects.toThrow(); const quarantineRoot = join(stateBase, "quarantine"); const quarantineEntries = await readdir(quarantineRoot, { withFileTypes: true, }); expect(quarantineEntries).toHaveLength(1); expect(quarantineEntries[0]!.isDirectory()).toBe(true); expect(quarantineEntries[0]!.name).toContain( caseName === "mismatched" ? ".identity_mismatch." : ".identity_indeterminate.", ); const quarantinedControlPlaneRoot = join( quarantineRoot, quarantineEntries[0]!.name, "control-plane", ); await expect( access(quarantinedControlPlaneRoot), ).resolves.toBeUndefined(); if (caseName !== "missing") { await expect( access( join(quarantinedControlPlaneRoot, "control-plane-state.json"), ), ).resolves.toBeUndefined(); } expect(state.createBackend).not.toHaveBeenCalled(); expect(state.createTransport).not.toHaveBeenCalled(); } finally { if (previousStateDirectory === undefined) { delete process.env.PAPERCLIP_RUNNER_STATE_DIR; } else { process.env.PAPERCLIP_RUNNER_STATE_DIR = previousStateDirectory; } await rm(stateBase, { recursive: true, force: true }); } }, ); it("does not quarantine an unsafe scoped-root symlink", async () => { const stateBase = await mkdtemp( join(tmpdir(), "paperclip-symlink-scoped-state-"), ); const previousStateDirectory = process.env.PAPERCLIP_RUNNER_STATE_DIR; process.env.PAPERCLIP_RUNNER_STATE_DIR = stateBase; const scopedExecution = { ...execution, binding: { ...execution.binding, companyId: "company-symlink-scoped-state", runId: "run-symlink-scoped-state", agentId: "agent-symlink-scoped-state", executionWorkspaceId: "workspace-symlink-scoped-state", }, session: { ...execution.session, normalizedSessionId: "session-symlink-scoped-state", }, } as NativeExecutionInputV1; try { state.createBackend.mockClear(); state.createTransport.mockClear(); await createRunnerdBackend({ db: leaseDb(scopedExecution), execution: scopedExecution, runnerInstanceId: "runner-symlink-scoped-state", }); state.createBackend.mock.calls[0]![1].codexTransportFactory!(); const scopedRoot = state.createTransport.mock.calls[0]![0].stateDirectory!; const symlinkTarget = join(stateBase, "symlink-target"); await rm(scopedRoot, { recursive: true, force: true }); await mkdir(symlinkTarget, { recursive: true }); await writeFile(join(symlinkTarget, "must-remain"), "retained"); await symlink(symlinkTarget, scopedRoot); state.createBackend.mockClear(); state.createTransport.mockClear(); await expect( createRunnerdBackend({ db: leaseDb(scopedExecution), execution: scopedExecution, runnerInstanceId: "runner-symlink-retry", }), ).rejects.toThrow("runner_state_directory_unsafe"); await expect(access(scopedRoot)).resolves.toBeUndefined(); await expect( access(join(symlinkTarget, "must-remain")), ).resolves.toBeUndefined(); await expect(access(join(stateBase, "quarantine"))).rejects.toThrow(); expect(state.createBackend).not.toHaveBeenCalled(); expect(state.createTransport).not.toHaveBeenCalled(); } finally { if (previousStateDirectory === undefined) { delete process.env.PAPERCLIP_RUNNER_STATE_DIR; } else { process.env.PAPERCLIP_RUNNER_STATE_DIR = previousStateDirectory; } await rm(stateBase, { recursive: true, force: true }); } }); it("rejects a suspended prior-run authority whose persisted execution belongs to another full session scope", async () => { const stateBase = await mkdtemp( join(tmpdir(), "paperclip-prior-run-mismatched-state-"), ); const previousStateDirectory = process.env.PAPERCLIP_RUNNER_STATE_DIR; process.env.PAPERCLIP_RUNNER_STATE_DIR = stateBase; const currentExecution = { ...execution, binding: { ...execution.binding, companyId: "company-prior-run-mismatch", runId: "run-current-prior-mismatch", agentId: "agent-current-prior-mismatch", executionWorkspaceId: "workspace-prior-mismatch", }, session: { ...execution.session, normalizedSessionId: "session-prior-run-mismatch", }, } as NativeExecutionInputV1; const priorExecution = { ...currentExecution, binding: { ...currentExecution.binding, runId: "run-prior-mismatched-scope", agentId: "agent-other-prior-mismatch", }, } as NativeExecutionInputV1; const legacyRoot = join( stateBase, createHash("sha256") .update( JSON.stringify([ currentExecution.binding.companyId, currentExecution.session.normalizedSessionId, ]), ) .digest("hex"), ); const priorRunDb = { select: () => ({ from: () => ({ where: () => ({ limit: () => Promise.resolve([ { status: "succeeded", runnerProfileJson: { nativeExecutionInput: priorExecution, }, }, ]), }), }), }), } as unknown as Db; try { await mkdir(join(legacyRoot, "control-plane"), { recursive: true }); await mkdir(join(legacyRoot, "runner"), { recursive: true }); const identity = { runId: priorExecution.binding.runId, normalizedSessionId: currentExecution.session.normalizedSessionId, runnerInstanceId: "runner-prior-run-mismatch", environmentLeaseId: "lease-prior-run-mismatch", }; await writeFile( join(legacyRoot, "control-plane", "control-plane-state.json"), JSON.stringify(durableControlPlaneState(identity)), ); await writeFile( join(legacyRoot, "runner", "runner-state.json"), JSON.stringify(durableRunnerState(identity, "suspended")), ); await expect( createRunnerdBackend({ db: priorRunDb, execution: currentExecution, runnerInstanceId: "runner-current-prior-mismatch", }), ).rejects.toThrow("runner_state_identity_mismatch"); await expect(access(legacyRoot)).resolves.toBeUndefined(); await expect(access(join(stateBase, "quarantine"))).rejects.toThrow(); } finally { if (previousStateDirectory === undefined) { delete process.env.PAPERCLIP_RUNNER_STATE_DIR; } else { process.env.PAPERCLIP_RUNNER_STATE_DIR = previousStateDirectory; } await rm(stateBase, { recursive: true, force: true }); } }); it("fails closed instead of claiming a mismatched former session scope", async () => { const stateBase = await mkdtemp( join(tmpdir(), "paperclip-mismatched-session-state-"), ); const previousStateDirectory = process.env.PAPERCLIP_RUNNER_STATE_DIR; process.env.PAPERCLIP_RUNNER_STATE_DIR = stateBase; const currentExecution = { ...execution, binding: { ...execution.binding, companyId: "company-mismatched-scope", runId: "run-current-scope", agentId: "agent-current-scope", executionWorkspaceId: "workspace-current-scope", }, session: { ...execution.session, normalizedSessionId: "session-mismatched-scope", }, } as NativeExecutionInputV1; const legacyRoot = join( stateBase, createHash("sha256") .update( JSON.stringify([ currentExecution.binding.companyId, currentExecution.session.normalizedSessionId, ]), ) .digest("hex"), ); try { await mkdir(join(legacyRoot, "control-plane"), { recursive: true }); await writeFile( join(legacyRoot, "control-plane", "control-plane-state.json"), JSON.stringify( durableControlPlaneState({ runId: "run-unrelated-scope", normalizedSessionId: currentExecution.session.normalizedSessionId, runnerInstanceId: "runner-unrelated-scope", environmentLeaseId: "lease-unrelated-scope", }), ), ); await expect( createRunnerdBackend({ db: leaseDb(currentExecution), execution: currentExecution, runnerInstanceId: "runner-current-scope", }), ).rejects.toThrow("runner_state_identity_mismatch"); await expect(access(legacyRoot)).resolves.toBeUndefined(); } finally { if (previousStateDirectory === undefined) { delete process.env.PAPERCLIP_RUNNER_STATE_DIR; } else { process.env.PAPERCLIP_RUNNER_STATE_DIR = previousStateDirectory; } await rm(stateBase, { recursive: true, force: true }); } }); it("isolates durable state and tool authority for equal session ids in different companies", async () => { const scopedExecution = (companyId: string, runId: string) => ({ ...execution, schema: "paperclip.native-execution-input.v4", binding: { ...execution.binding, companyId, runId, executionWorkspaceId: "workspace", }, task: { identifier: "DOT-ISOLATION", title: "Isolation test", description: null, prompt: "Verify session isolation.", workMode: "standard", }, workspace: { cwd: "/tmp/native-session-isolation", repoUrl: null, repoRef: null, branchName: null, }, session: { normalizedSessionId: "shared-normalized-session", driverKind: "codex_app_server", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, }, provider: { kind: "codex", model: null, approvalPolicy: "never" }, executionMode: "default", planningContext: null, interactionResponses: [], credentialBindings: [], runtimeContext: nativeRuntimeContextFixture(), }) as unknown as NativeExecutionInputV1; const firstExecution = scopedExecution("company-first", "run-first"); const secondExecution = scopedExecution("company-second", "run-second"); state.createBackend.mockClear(); state.toolAuthorityExecute .mockReset() .mockImplementation((binding: Record) => Promise.resolve({ runId: binding.runId }), ); await createRunnerdBackend({ db: leaseDb(firstExecution), execution: firstExecution, runnerInstanceId: "runner-first", }); await createRunnerdBackend({ db: leaseDb(secondExecution), execution: secondExecution, runnerInstanceId: "runner-second", }); const firstOptions = state.createBackend.mock.calls[0]![1]; const secondOptions = state.createBackend.mock.calls[1]![1]; state.createTransport.mockClear(); firstOptions.codexTransportFactory!(); secondOptions.codexTransportFactory!(); expect(state.createTransport.mock.calls[0]![0].stateDirectory).not.toBe( state.createTransport.mock.calls[1]![0].stateDirectory, ); await expect(firstOptions.dynamicToolHandler!({})).resolves.toEqual({ runId: "run-first", }); await expect(secondOptions.dynamicToolHandler!({})).resolves.toEqual({ runId: "run-second", }); }); it("scopes local durable sessions by agent, workspace, and provider profile while reusing them across runs", async () => { const stateBase = await mkdtemp(join(tmpdir(), "paperclip-session-scope-")); const previousStateDirectory = process.env.PAPERCLIP_RUNNER_STATE_DIR; process.env.PAPERCLIP_RUNNER_STATE_DIR = stateBase; const scopedExecution = (input: { runId: string; agentId?: string; workspaceId?: string; providerKind?: "codex" | "opencode"; }) => ({ ...execution, schema: "paperclip.native-execution-input.v4", binding: { ...execution.binding, companyId: "company-session-scope", runId: input.runId, issueId: "issue-session-scope", agentId: input.agentId ?? "agent-session-scope", executionWorkspaceId: input.workspaceId ?? "workspace-session-scope", }, workspace: { cwd: "/tmp/native-session-scope", repoUrl: "https://example.test/paperclip.git", repoRef: "refs/heads/main", branchName: "main", }, session: { normalizedSessionId: "shared-scoped-session", driverKind: input.providerKind === "opencode" ? "opencode_server" : "codex_app_server", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, }, provider: input.providerKind === "opencode" ? { kind: "opencode", model: "openrouter/deepseek/deepseek-v4-flash-0731", permissionMode: "ask", } : { kind: "codex", model: null, approvalPolicy: "never", }, executionMode: "default", planningContext: null, interactionResponses: [], credentialBindings: [], runtimeContext: nativeRuntimeContextFixture(), }) as unknown as NativeExecutionInputV1; const first = scopedExecution({ runId: "run-session-scope-first" }); const continuation = scopedExecution({ runId: "run-session-scope-continuation", }); const differentAgent = scopedExecution({ runId: "run-session-scope-agent", agentId: "agent-session-scope-other", }); const differentWorkspace = scopedExecution({ runId: "run-session-scope-workspace", workspaceId: "workspace-session-scope-other", }); const differentProviderProfile = scopedExecution({ runId: "run-session-scope-provider", providerKind: "opencode", }); try { state.createBackend.mockClear(); state.createTransport.mockClear(); state.toolAuthorityExecute .mockReset() .mockImplementation((binding: Record) => Promise.resolve({ runId: binding.runId }), ); let firstScopedRoot: string | undefined; for (const candidate of [ first, continuation, differentAgent, differentWorkspace, differentProviderProfile, ]) { const candidateDb = candidate === continuation ? ({ ...leaseDb(candidate), select: () => ({ from: () => ({ where: () => ({ limit: () => Promise.resolve([ { status: "succeeded", runnerProfileJson: { nativeExecutionInput: first, }, }, ]), }), }), }), } as unknown as Db) : leaseDb(candidate); await createRunnerdBackend({ db: candidateDb, execution: candidate, runnerInstanceId: `runner-${candidate.binding.runId}`, }); state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!(); if (candidate === first) { firstScopedRoot = state.createTransport.mock.calls.at(-1)![0].stateDirectory!; const identity = { runId: first.binding.runId, normalizedSessionId: first.session.normalizedSessionId, runnerInstanceId: `runner-${first.binding.runId}`, environmentLeaseId: first.binding.executionWorkspaceId, }; await mkdir(join(firstScopedRoot, "control-plane"), { recursive: true, }); await mkdir(join(firstScopedRoot, "runner"), { recursive: true }); await writeFile( join(firstScopedRoot, "control-plane", "control-plane-state.json"), JSON.stringify(durableControlPlaneState(identity)), ); await writeFile( join(firstScopedRoot, "runner", "runner-state.json"), JSON.stringify(durableRunnerState(identity, "suspended")), ); } } const stateDirectories = state.createTransport.mock.calls.map( ([options]) => options.stateDirectory, ); expect(stateDirectories[1]).toBe(stateDirectories[0]); expect(stateDirectories[0]).toBe(firstScopedRoot); expect( new Set([ stateDirectories[0], stateDirectories[2], stateDirectories[3], stateDirectories[4], ]).size, ).toBe(4); const firstOptions = state.createBackend.mock.calls[0]![1]; const continuationOptions = state.createBackend.mock.calls[1]![1]; await expect(firstOptions.dynamicToolHandler!({})).rejects.toThrow( "native_tool_authority_epoch_revoked", ); await expect( continuationOptions.dynamicToolHandler!({}), ).resolves.toEqual({ runId: continuation.binding.runId }); } finally { if (previousStateDirectory === undefined) { delete process.env.PAPERCLIP_RUNNER_STATE_DIR; } else { process.env.PAPERCLIP_RUNNER_STATE_DIR = previousStateDirectory; } await rm(stateBase, { recursive: true, force: true }); } }); it("rejects a concurrent first-use backend for the same provider session scope", async () => { const first = { ...execution, binding: { ...execution.binding, runId: "run-session-concurrent-first", executionWorkspaceId: "workspace-session-concurrent", }, session: { ...execution.session, normalizedSessionId: "session-concurrent-first-use", }, } as NativeExecutionInputV1; const second = { ...first, binding: { ...first.binding, runId: "run-session-concurrent-second" }, } as NativeExecutionInputV1; let concurrentAttempt: Promise | null = null; state.createBackend.mockImplementationOnce(() => { // Re-enter only after definitions have resolved, at the actual backend // construction boundary. The session claim must still be held here. concurrentAttempt = createRunnerdBackend({ db: leaseDb(second), execution: second, runnerInstanceId: "runner-session-concurrent-second", }); return { kind: "test" }; }); await expect( createRunnerdBackend({ db: leaseDb(first), execution: first, runnerInstanceId: "runner-session-concurrent-first", }), ).resolves.toBeDefined(); expect(concurrentAttempt).not.toBeNull(); await expect(concurrentAttempt!).rejects.toThrow( "native_session_supervisor_busy", ); }); it("uses the remote workspace for both the runner backend and native session", async () => { const remoteCwd = "/home/daytona/paperclip-workspace"; const remoteExecution = { ...execution, binding: { ...execution.binding, runId: "run-remote-workspace-test" }, task: { identifier: "DOT-REMOTE", title: "Remote workspace test", description: null, prompt: "Verify the remote workspace.", workMode: "standard", }, workspace: { cwd: "/host/paperclip-workspace", repoUrl: null, repoRef: null, branchName: null, }, session: { normalizedSessionId: "remote-workspace-session", driverKind: "codex_app_server", protocolVersion: 2, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, }, provider: { kind: "codex", model: null, approvalPolicy: "never", }, executionMode: "default", planningContext: null, interactionResponses: [], credentialBindings: [], } as unknown as NativeExecutionInputV1; state.createBackend.mockClear(); state.execute.mockReset().mockResolvedValue({ result: { summary: "completed" }, terminal: { runTerminalState: "succeeded" }, turnId: "turn", normalizedSessionId: "session", providerSessionId: null, driverKind: "test", driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, }); await executePaperclipNativeSession({ db: leaseDb(remoteExecution), execution: remoteExecution, runnerInstanceId: "runner", useRunnerd: true, runnerExecutionTarget: { kind: "remote", transport: "ssh", remoteCwd, spec: { host: "runner.internal", port: 22, username: "runner", remoteWorkspacePath: remoteCwd, remoteCwd, privateKey: null, knownHosts: null, strictHostKeyChecking: true, }, }, runnerPublicUrl: "wss://paperclip.example.test", }); expect(state.createBackend).toHaveBeenCalledWith( expect.objectContaining({ workspace: expect.objectContaining({ cwd: remoteCwd }), }), expect.any(Object), ); expect(state.execute).toHaveBeenCalledWith( expect.objectContaining({ input: expect.objectContaining({ workspace: expect.objectContaining({ cwd: remoteCwd }), }), }), ); const backendOptions = state.createBackend.mock.calls[0]![1]; state.createTransport.mockClear(); backendOptions.codexTransportFactory!(); expect(state.createTransport).toHaveBeenCalledWith( expect.objectContaining({ environment: expect.objectContaining({ PAPERCLIP_WORKSPACE_CWD: remoteCwd, }), }), ); }); it.each([ ["opencode", { kind: "opencode", model: null }, "opencode_server"], ["acpx", { kind: "acpx", agent: "codex", model: null }, "acpx_runtime"], ])( "requires the build-owned provider pack before launching remote %s", async (providerKind, provider, driverKind) => { const remoteCwd = "/home/daytona/paperclip-workspace"; const remoteProviderExecution = { ...execution, binding: { ...execution.binding, runId: `run-remote-${providerKind}-rejected`, }, session: { ...execution.session, normalizedSessionId: `remote-${providerKind}-rejected`, driverKind, }, provider, } as unknown as NativeExecutionInputV1; state.createBackend.mockClear(); await expect( createRunnerdBackend({ db: leaseDb(remoteProviderExecution), execution: remoteProviderExecution, runnerInstanceId: "runner", runnerExecutionTarget: { kind: "remote", transport: "ssh", remoteCwd, spec: { host: "runner.internal", port: 22, username: "runner", remoteWorkspacePath: remoteCwd, remoteCwd, privateKey: null, knownHosts: null, strictHostKeyChecking: true, }, }, }), ).rejects.toThrow( "runner_remote_provider_artifact_incompatible: configure PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH", ); expect(state.createBackend).not.toHaveBeenCalled(); }, ); it("passes the isolated ACPX runtime directory to the native backend factory", async () => { const acpxExecution = { ...execution, schema: "paperclip.native-execution-input.v4", task: { identifier: "DOT-ACPX", title: "ACPX task", description: null, prompt: "Complete the ACPX task.", workMode: "standard", }, workspace: { cwd: "/tmp/acpx-native", repoUrl: null, repoRef: null, branchName: null, }, session: { normalizedSessionId: "acpx-session", driverKind: "acpx_runtime", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, }, provider: { kind: "acpx", agent: "codex", model: "gpt-5.6-sol", permissionMode: "approve-reads", profile: { driverKind: "acpx_runtime", protocolVersion: 1, acpxVersion: "0.13.1", agent: "codex", agentProfileVersion: 1, agentServerPackage: "@agentclientprotocol/codex-acp", agentServerVersion: "1.6.2", agentRuntimePackage: null, agentRuntimeVersion: null, commandDigest: "sha256:test", }, }, executionMode: "default", planningContext: null, interactionResponses: [], credentialBindings: [], runtimeContext: nativeRuntimeContextFixture(), } as unknown as NativeExecutionInputV1; state.createBackend.mockClear(); await createRunnerdBackend({ db: leaseDb(acpxExecution), execution: acpxExecution, runnerInstanceId: "runner", }); expect(state.createBackend).toHaveBeenCalledWith( acpxExecution, expect.objectContaining({ acpxRuntimeDirectory: expect.stringContaining( "/runtime/paperclip-runner/acpx", ), acpxDynamicToolHandler: expect.any(Function), }), ); state.createTransport.mockClear(); state.createBackend.mock.calls[0]![1].codexTransportFactory!(); expect(state.createTransport).toHaveBeenCalledWith( expect.objectContaining({ provider: "acpx", acpxAgent: "codex", acpxPermissionMode: "approve-reads", }), ); }); it("passes the persisted OpenCode permission mode to runnerd", async () => { const opencodeExecution = { ...execution, schema: "paperclip.native-execution-input.v4", binding: { ...execution.binding, runId: "run-opencode-permissions" }, session: { ...execution.session, normalizedSessionId: "opencode-permissions-session", driverKind: "opencode_server", }, provider: { kind: "opencode", model: "openrouter/deepseek/deepseek-v4-flash-0731", permissionMode: "deny", }, } as unknown as NativeExecutionInputV1; state.createBackend.mockClear(); await createRunnerdBackend({ db: leaseDb(opencodeExecution), execution: opencodeExecution, runnerInstanceId: "runner", }); state.createTransport.mockClear(); state.createBackend.mock.calls[0]![1].codexTransportFactory!(); expect(state.createTransport).toHaveBeenCalledWith( expect.objectContaining({ provider: "opencode", opencodePermissionMode: "deny", }), ); }); });