From 761ac12cc083d7cafca4e0baf5d1a7f7e44d0850 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 8 Sep 2026 23:20:23 -0500 Subject: [PATCH] fix: preserve native warm attachment across runner upgrades Accept bounded Codex deprecation notices after a settled turn and require that capability before reusing a sandbox image runner. Stage replacement artifacts atomically so existing launchers and image symlink targets survive interrupted uploads. Co-Authored-By: Paperclip --- doc/sandbox-work-folders.md | 10 +++ .../src/bin/fake-codex-app-server.rs | 18 ++++ .../runner-core/src/bin/paperclip-runnerd.rs | 5 ++ .../crates/runner-core/src/codex_provider.rs | 1 + .../runner-core/tests/codex_provider.rs | 18 ++++ .../native-session-executor.test.ts | 85 ++++++++++++++++++ .../native-runtime/native-session-executor.ts | 90 +++++++++++++------ 7 files changed, 199 insertions(+), 28 deletions(-) diff --git a/doc/sandbox-work-folders.md b/doc/sandbox-work-folders.md index 5edac3c9e2..2e7bb88cdb 100644 --- a/doc/sandbox-work-folders.md +++ b/doc/sandbox-work-folders.md @@ -340,3 +340,13 @@ process and turn before the controller can interrupt it. This matches the semantic-tool response path. A shutdown interruption must preserve that exact completed-turn authority so the session can be suspended and checkpointed. Invalid results and conflicting identities still fail validation. + +Warm native Codex attachment drains bounded informational deprecation notices +that arrive after the prior turn and its readiness probe. Notices naming another +turn, new work, and provider requests still block attachment. + +The app checks the runner’s passive-notice capability before reusing an image +binary. An older binary is replaced with the app’s compatible artifact through +a temporary file and atomic rename, preserving image symlink targets and the +previous launcher on interrupted uploads. The durable checkpoint contract stays +at version 2 so existing native session backups remain restorable. diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs index 9054d23f71..51db286917 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs @@ -1263,7 +1263,22 @@ fn run() -> Result<(), Box> { }))?; } if emit_post_completion_passive_statuses { + if let Some(gate) = post_completion_notification_gate.as_ref() { + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while !gate.is_file() { + if std::time::Instant::now() >= deadline { + return Err( + "post-completion notification gate timed out".into() + ); + } + thread::sleep(Duration::from_millis(1)); + } + } for notification in [ + json!({ + "method": "deprecationNotice", + "params": {"summary": "A provider setting is deprecated", "details": null} + }), json!({ "method": "remoteControl/status/changed", "params": {"status": "disabled", "environmentId": null} @@ -1295,6 +1310,9 @@ fn run() -> Result<(), Box> { ] { send(notification)?; } + if let Some(gate) = post_completion_notification_gate.as_ref() { + fs::write(gate.with_extension("emitted"), b"emitted")?; + } } if emit_post_completion_foreign_turn { if let Some(gate) = post_completion_notification_gate.as_ref() { diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs index 81421d8866..a93fb66638 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs @@ -111,6 +111,7 @@ fn build_metadata() -> serde_json::Value { "packageName": "@paperclipai/paperclip-runner", "packageVersion": env!("CARGO_PKG_VERSION"), "binaryContractVersion": 2, + "capabilities": ["codex.warm-attachment.passive-notices.v1"], "nativeExecutionVersion": 1, "harnessDriverVersion": 1, "prp": { @@ -368,6 +369,10 @@ mod tests { let metadata = build_metadata(); assert_eq!(metadata["schema"], RUNNERD_BUILD_METADATA_SCHEMA); assert_eq!(metadata["binaryContractVersion"], 2); + assert_eq!( + metadata["capabilities"], + json!(["codex.warm-attachment.passive-notices.v1"]) + ); assert_eq!( metadata["prpTransportModes"], json!(["dial_ws_loopback", "dial_wss", "listen_ws"]) diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs index 1e7254be4f..1ed190cf0f 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs @@ -931,6 +931,7 @@ impl CodexProvider { let safe_tail_method = matches!( method.as_str(), "warning" + | "deprecationNotice" | "configWarning" | "remoteControl/status/changed" | "mcpServer/startupStatus/updated" diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs index 91b80125fb..823c9ec1af 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs @@ -3237,12 +3237,17 @@ fn durable_backend_rotates_tool_authority_for_fresh_run_attach() { #[test] fn durable_backend_drains_a_bounded_completed_turn_tail_during_warm_attach() { let directory = temporary_directory("durable-warm-attach-tail"); + let notification_gate = directory.join("emit-passive-tail"); let config = provider_config( &directory, &[ "--durable-turn-ids", "--emit-post-completion-warning", "--emit-post-completion-passive-statuses", + "--post-completion-notification-gate", + notification_gate + .to_str() + .expect("notification gate is UTF-8"), ], ); let runner_config = durable_config(&directory); @@ -3300,6 +3305,19 @@ fn durable_backend_drains_a_bounded_completed_turn_tail_during_warm_attach() { assert_eq!(readiness.result["warmAttachReady"], true); assert_eq!(readiness.result["warmAttachBlockers"], json!([])); + // Release passive notices after a successful readiness probe. This closes the + // actual probe-to-attachment race without normal event polling consuming them. + fs::write(¬ification_gate, b"release").expect("release passive provider notices"); + let emitted_gate = notification_gate.with_extension("emitted"); + let emitted_deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while !emitted_gate.is_file() { + assert!( + std::time::Instant::now() < emitted_deadline, + "passive notices must be emitted" + ); + std::thread::sleep(std::time::Duration::from_millis(1)); + } + let attached = executor .execute(&command( "attach", diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 5f706f9539..2993743449 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -1,6 +1,8 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { access, + chmod, + lstat, appendFile, mkdir, mkdtemp, @@ -204,6 +206,7 @@ import { semanticProviderPlanMarkdown, sha256DirectoryTree, stageRemoteRunnerDirectory, + stageRemoteRunnerFile, steerNativeSession, syncRemoteRunnerDirectoryOut, verifyNativeHarnessBackup, @@ -1556,6 +1559,79 @@ describe("remote provider checkpoint snapshots", () => { }); }); +describe("atomic remote runner replacement", () => { + for (const useSyncIn of [true, false]) { + it(`replaces an old launcher symlink without writing through it (${useSyncIn ? "provider" : "shell"})`, async () => { + const root = await mkdtemp(join(tmpdir(), "runner-replace-")); + try { + const sourcePath = join(root, "new runner's bytes"); + const imageBinary = join(root, "image-runner"); + const targetPath = join(root, "paperclip-runnerd"); + await writeFile(sourcePath, "fixed runner"); + await writeFile(imageBinary, "old image runner"); + await symlink(imageBinary, targetPath); + const runner = { + execute: async (command: { command: string; args?: string[]; stdin?: string }) => { + const stdout = execFileSync(command.command, command.args ?? [], { input: command.stdin, encoding: "utf8" }); + return { exitCode: 0, stdout, stderr: "" }; + }, + ...(useSyncIn ? { syncIn: async (operations: Array<{ files: Array<{ sourcePath: string; targetPath: string; mode: number }> }>) => { + const file = operations[0]!.files[0]!; + expect(file.targetPath).not.toBe(targetPath); + await writeFile(file.targetPath, await readFile(file.sourcePath)); + await chmod(file.targetPath, file.mode); + } } : {}), + }; + await stageRemoteRunnerFile({ target: {} as never, runner: runner as never, sourcePath, targetPath, mode: 0o755 }); + expect(await readFile(targetPath, "utf8")).toBe("fixed runner"); + expect((await lstat(targetPath)).isSymbolicLink()).toBe(false); + expect((await lstat(targetPath)).mode & 0o777).toBe(0o755); + expect(await readFile(imageBinary, "utf8")).toBe("old image runner"); + expect((await readdir(root)).filter(name => name.includes(".upload-"))).toEqual([]); + } finally { await rm(root, { recursive: true, force: true }); } + }); + } + + it("retains the existing launcher when a provider upload is interrupted", async () => { + const root = await mkdtemp(join(tmpdir(), "runner-interrupted-")); + try { + const targetPath = join(root, "paperclip-runnerd"); + await writeFile(targetPath, "recoverable old runner"); + const runner = { + syncIn: async (operations: Array<{ files: Array<{ targetPath: string }> }>) => { + await writeFile(operations[0]!.files[0]!.targetPath, "partial upload"); + throw new Error("upload interrupted"); + }, + execute: async (command: { command: string; args?: string[] }) => { + execFileSync(command.command, command.args ?? []); + return { exitCode: 0, stdout: "", stderr: "" }; + }, + }; + await expect(stageRemoteRunnerFile({ target: {} as never, runner: runner as never, sourcePath: "unused", targetPath, mode: 0o755 })).rejects.toThrow("upload interrupted"); + expect(await readFile(targetPath, "utf8")).toBe("recoverable old runner"); + expect(await readdir(root)).toEqual(["paperclip-runnerd"]); + } finally { await rm(root, { recursive: true, force: true }); } + }); + + it("does not publish through a launcher symlink to a directory", async () => { + const root = await mkdtemp(join(tmpdir(), "runner-directory-link-")); + try { + const directory = join(root, "unrelated"); + const targetPath = join(root, "paperclip-runnerd"); + const sourcePath = join(root, "candidate"); + await mkdir(directory); await symlink(directory, targetPath); await writeFile(sourcePath, "runner"); + const runner = { execute: async (command: { command: string; args?: string[]; stdin?: string }) => { + try { execFileSync(command.command, command.args ?? [], { input: command.stdin }); return { exitCode: 0, stdout: "", stderr: "" }; } + catch { return { exitCode: 1, stdout: "", stderr: "failed" }; } + } }; + await expect(stageRemoteRunnerFile({ target: {} as never, runner: runner as never, sourcePath, targetPath, mode: 0o755 })).rejects.toThrow("runner_remote_staging_failed"); + expect(await readdir(directory)).toEqual([]); + expect((await lstat(targetPath)).isSymbolicLink()).toBe(true); + expect((await readdir(root)).filter(name => name.includes(".upload-"))).toEqual([]); + } finally { await rm(root, { recursive: true, force: true }); } + }); +}); + describe("remote provider checkpoint restores", () => { it("does not upload excluded Codex scratch trees or credentials", async () => { const sourcePath = await mkdtemp( @@ -1709,6 +1785,7 @@ describe("remote runner build metadata", () => { binaryName: "paperclip-runnerd", packageName: "@paperclipai/paperclip-runner", binaryContractVersion: 2, + capabilities: ["codex.warm-attachment.passive-notices.v1"], prpTransportModes: ["dial_ws_loopback", "dial_wss", "listen_ws"], }; @@ -1730,6 +1807,13 @@ describe("remote runner build metadata", () => { ).toThrow("runner_remote_artifact_contract_incompatible"); }); + it("rejects preinstalled binaries that lack safe passive-notice attachment", () => { + for (const capabilities of [undefined, [], ["unrelated"]]) { + expect(() => assertRemoteRunnerBuildMetadata({ ...current, capabilities }, "listen_ws")) + .toThrow("runner_remote_capability_missing:codex.warm-attachment.passive-notices.v1"); + } + }); + it("requires the selected transport without falling through", () => { expect(() => assertRemoteRunnerBuildMetadata( @@ -6282,6 +6366,7 @@ describe("runnerd provider runtime wiring", () => { binaryName: "paperclip-runnerd", packageName: "@paperclipai/paperclip-runner", binaryContractVersion: 2, + capabilities: ["codex.warm-attachment.passive-notices.v1"], prpTransportModes: ["listen_ws"], }); } else if (command.args?.[0] === "--version") { diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 382c1b21a3..f5ab407cef 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -5342,6 +5342,17 @@ export function assertRemoteRunnerBuildMetadata( ) { throw new Error("runner_remote_artifact_contract_incompatible"); } + // An older image may implement the same durable protocol but still reject + // passive Codex notices during warm attachment. Stage the current artifact + // without changing the checkpoint contract or replacing the sandbox. + if ( + !Array.isArray(metadata.capabilities) || + !metadata.capabilities.includes("codex.warm-attachment.passive-notices.v1") + ) { + throw new Error( + "runner_remote_capability_missing:codex.warm-attachment.passive-notices.v1", + ); + } const modes = Array.isArray(metadata.prpTransportModes) ? metadata.prpTransportModes : []; @@ -5352,7 +5363,7 @@ export function assertRemoteRunnerBuildMetadata( } } -async function stageRemoteRunnerFile(input: { +export async function stageRemoteRunnerFile(input: { target: Extract; runner: CommandManagedRuntimeRunner; sourcePath: string; @@ -5360,36 +5371,59 @@ async function stageRemoteRunnerFile(input: { mode: number; }): Promise { const runner = input.runner; - if (runner.syncIn) { - await runner.syncIn([ - { - operationId: `runner-stage-${randomUUID()}`, - files: [ - { + // The old launcher can be a symlink into the sandbox image. Publish a new + // file atomically instead of following that link or truncating working bytes. + const temporaryPath = `${input.targetPath}.upload-${randomUUID()}`; + const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; + try { + if (runner.syncIn) { + await runner.syncIn([ + { + operationId: `runner-stage-${randomUUID()}`, + files: [{ sourcePath: input.sourcePath, - targetPath: input.targetPath, + targetPath: temporaryPath, kind: "file", mode: input.mode, - }, - ], - }, - ]); - return; - } - const bytes = readFileSync(input.sourcePath); - const directory = posix.dirname(input.targetPath); - const script = - `umask 077; mkdir -p '${directory.replaceAll("'", "'\\''")}' && ` + - `base64 -d > '${input.targetPath.replaceAll("'", "'\\''")}' && ` + - `chmod ${input.mode.toString(8)} '${input.targetPath.replaceAll("'", "'\\''")}'`; - const result = await runner.execute({ - command: "sh", - args: ["-c", script], - stdin: bytes.toString("base64"), - bypassSession: true, - }); - if (result.exitCode !== 0 || result.timedOut) { - throw new Error("runner_remote_staging_failed"); + }], + }, + ]); + } else { + const bytes = readFileSync(input.sourcePath); + const result = await runner.execute({ + command: "sh", + args: ["-c", [ + `umask 077; mkdir -p ${quote(posix.dirname(input.targetPath))}`, + `base64 -d > ${quote(temporaryPath)}`, + `chmod ${input.mode.toString(8)} ${quote(temporaryPath)}`, + ].join(" && ")], + stdin: bytes.toString("base64"), + bypassSession: true, + }); + if (result.exitCode !== 0 || result.timedOut) { + throw new Error("runner_remote_staging_failed"); + } + } + const published = await runner.execute({ + command: "sh", + args: ["-c", [ + `test -f ${quote(temporaryPath)}`, + `test ! -L ${quote(temporaryPath)}`, + `test ! -d ${quote(input.targetPath)}`, + `mv -f -- ${quote(temporaryPath)} ${quote(input.targetPath)}`, + ].join(" && ")], + bypassSession: true, + }); + if (published.exitCode !== 0 || published.timedOut) { + throw new Error("runner_remote_staging_failed"); + } + } catch (error) { + await runner.execute({ + command: "sh", + args: ["-c", `rm -f -- ${quote(temporaryPath)}`], + bypassSession: true, + }).catch(() => undefined); + throw error; } }