fix: preserve native warm attachment across runner upgrades
Accept bounded Codex deprecation notices after a settled turn and require that capability before reusing a sandbox image runner. Stage replacement artifacts atomically so existing launchers and image symlink targets survive interrupted uploads. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
parent
73c07ae140
commit
761ac12cc0
|
|
@ -340,3 +340,13 @@ process and turn before the controller can interrupt it. This matches the
|
|||
semantic-tool response path. A shutdown interruption must preserve that exact
|
||||
completed-turn authority so the session can be suspended and checkpointed.
|
||||
Invalid results and conflicting identities still fail validation.
|
||||
|
||||
Warm native Codex attachment drains bounded informational deprecation notices
|
||||
that arrive after the prior turn and its readiness probe. Notices naming another
|
||||
turn, new work, and provider requests still block attachment.
|
||||
|
||||
The app checks the runner’s passive-notice capability before reusing an image
|
||||
binary. An older binary is replaced with the app’s compatible artifact through
|
||||
a temporary file and atomic rename, preserving image symlink targets and the
|
||||
previous launcher on interrupted uploads. The durable checkpoint contract stays
|
||||
at version 2 so existing native session backups remain restorable.
|
||||
|
|
|
|||
|
|
@ -1263,7 +1263,22 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
|
|||
}))?;
|
||||
}
|
||||
if emit_post_completion_passive_statuses {
|
||||
if let Some(gate) = post_completion_notification_gate.as_ref() {
|
||||
let deadline = std::time::Instant::now() + Duration::from_secs(5);
|
||||
while !gate.is_file() {
|
||||
if std::time::Instant::now() >= deadline {
|
||||
return Err(
|
||||
"post-completion notification gate timed out".into()
|
||||
);
|
||||
}
|
||||
thread::sleep(Duration::from_millis(1));
|
||||
}
|
||||
}
|
||||
for notification in [
|
||||
json!({
|
||||
"method": "deprecationNotice",
|
||||
"params": {"summary": "A provider setting is deprecated", "details": null}
|
||||
}),
|
||||
json!({
|
||||
"method": "remoteControl/status/changed",
|
||||
"params": {"status": "disabled", "environmentId": null}
|
||||
|
|
@ -1295,6 +1310,9 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
|
|||
] {
|
||||
send(notification)?;
|
||||
}
|
||||
if let Some(gate) = post_completion_notification_gate.as_ref() {
|
||||
fs::write(gate.with_extension("emitted"), b"emitted")?;
|
||||
}
|
||||
}
|
||||
if emit_post_completion_foreign_turn {
|
||||
if let Some(gate) = post_completion_notification_gate.as_ref() {
|
||||
|
|
|
|||
|
|
@ -111,6 +111,7 @@ fn build_metadata() -> serde_json::Value {
|
|||
"packageName": "@paperclipai/paperclip-runner",
|
||||
"packageVersion": env!("CARGO_PKG_VERSION"),
|
||||
"binaryContractVersion": 2,
|
||||
"capabilities": ["codex.warm-attachment.passive-notices.v1"],
|
||||
"nativeExecutionVersion": 1,
|
||||
"harnessDriverVersion": 1,
|
||||
"prp": {
|
||||
|
|
@ -368,6 +369,10 @@ mod tests {
|
|||
let metadata = build_metadata();
|
||||
assert_eq!(metadata["schema"], RUNNERD_BUILD_METADATA_SCHEMA);
|
||||
assert_eq!(metadata["binaryContractVersion"], 2);
|
||||
assert_eq!(
|
||||
metadata["capabilities"],
|
||||
json!(["codex.warm-attachment.passive-notices.v1"])
|
||||
);
|
||||
assert_eq!(
|
||||
metadata["prpTransportModes"],
|
||||
json!(["dial_ws_loopback", "dial_wss", "listen_ws"])
|
||||
|
|
|
|||
|
|
@ -931,6 +931,7 @@ impl CodexProvider {
|
|||
let safe_tail_method = matches!(
|
||||
method.as_str(),
|
||||
"warning"
|
||||
| "deprecationNotice"
|
||||
| "configWarning"
|
||||
| "remoteControl/status/changed"
|
||||
| "mcpServer/startupStatus/updated"
|
||||
|
|
|
|||
|
|
@ -3237,12 +3237,17 @@ fn durable_backend_rotates_tool_authority_for_fresh_run_attach() {
|
|||
#[test]
|
||||
fn durable_backend_drains_a_bounded_completed_turn_tail_during_warm_attach() {
|
||||
let directory = temporary_directory("durable-warm-attach-tail");
|
||||
let notification_gate = directory.join("emit-passive-tail");
|
||||
let config = provider_config(
|
||||
&directory,
|
||||
&[
|
||||
"--durable-turn-ids",
|
||||
"--emit-post-completion-warning",
|
||||
"--emit-post-completion-passive-statuses",
|
||||
"--post-completion-notification-gate",
|
||||
notification_gate
|
||||
.to_str()
|
||||
.expect("notification gate is UTF-8"),
|
||||
],
|
||||
);
|
||||
let runner_config = durable_config(&directory);
|
||||
|
|
@ -3300,6 +3305,19 @@ fn durable_backend_drains_a_bounded_completed_turn_tail_during_warm_attach() {
|
|||
assert_eq!(readiness.result["warmAttachReady"], true);
|
||||
assert_eq!(readiness.result["warmAttachBlockers"], json!([]));
|
||||
|
||||
// Release passive notices after a successful readiness probe. This closes the
|
||||
// actual probe-to-attachment race without normal event polling consuming them.
|
||||
fs::write(¬ification_gate, b"release").expect("release passive provider notices");
|
||||
let emitted_gate = notification_gate.with_extension("emitted");
|
||||
let emitted_deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
|
||||
while !emitted_gate.is_file() {
|
||||
assert!(
|
||||
std::time::Instant::now() < emitted_deadline,
|
||||
"passive notices must be emitted"
|
||||
);
|
||||
std::thread::sleep(std::time::Duration::from_millis(1));
|
||||
}
|
||||
|
||||
let attached = executor
|
||||
.execute(&command(
|
||||
"attach",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
access,
|
||||
chmod,
|
||||
lstat,
|
||||
appendFile,
|
||||
mkdir,
|
||||
mkdtemp,
|
||||
|
|
@ -204,6 +206,7 @@ import {
|
|||
semanticProviderPlanMarkdown,
|
||||
sha256DirectoryTree,
|
||||
stageRemoteRunnerDirectory,
|
||||
stageRemoteRunnerFile,
|
||||
steerNativeSession,
|
||||
syncRemoteRunnerDirectoryOut,
|
||||
verifyNativeHarnessBackup,
|
||||
|
|
@ -1556,6 +1559,79 @@ describe("remote provider checkpoint snapshots", () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe("atomic remote runner replacement", () => {
|
||||
for (const useSyncIn of [true, false]) {
|
||||
it(`replaces an old launcher symlink without writing through it (${useSyncIn ? "provider" : "shell"})`, async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "runner-replace-"));
|
||||
try {
|
||||
const sourcePath = join(root, "new runner's bytes");
|
||||
const imageBinary = join(root, "image-runner");
|
||||
const targetPath = join(root, "paperclip-runnerd");
|
||||
await writeFile(sourcePath, "fixed runner");
|
||||
await writeFile(imageBinary, "old image runner");
|
||||
await symlink(imageBinary, targetPath);
|
||||
const runner = {
|
||||
execute: async (command: { command: string; args?: string[]; stdin?: string }) => {
|
||||
const stdout = execFileSync(command.command, command.args ?? [], { input: command.stdin, encoding: "utf8" });
|
||||
return { exitCode: 0, stdout, stderr: "" };
|
||||
},
|
||||
...(useSyncIn ? { syncIn: async (operations: Array<{ files: Array<{ sourcePath: string; targetPath: string; mode: number }> }>) => {
|
||||
const file = operations[0]!.files[0]!;
|
||||
expect(file.targetPath).not.toBe(targetPath);
|
||||
await writeFile(file.targetPath, await readFile(file.sourcePath));
|
||||
await chmod(file.targetPath, file.mode);
|
||||
} } : {}),
|
||||
};
|
||||
await stageRemoteRunnerFile({ target: {} as never, runner: runner as never, sourcePath, targetPath, mode: 0o755 });
|
||||
expect(await readFile(targetPath, "utf8")).toBe("fixed runner");
|
||||
expect((await lstat(targetPath)).isSymbolicLink()).toBe(false);
|
||||
expect((await lstat(targetPath)).mode & 0o777).toBe(0o755);
|
||||
expect(await readFile(imageBinary, "utf8")).toBe("old image runner");
|
||||
expect((await readdir(root)).filter(name => name.includes(".upload-"))).toEqual([]);
|
||||
} finally { await rm(root, { recursive: true, force: true }); }
|
||||
});
|
||||
}
|
||||
|
||||
it("retains the existing launcher when a provider upload is interrupted", async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "runner-interrupted-"));
|
||||
try {
|
||||
const targetPath = join(root, "paperclip-runnerd");
|
||||
await writeFile(targetPath, "recoverable old runner");
|
||||
const runner = {
|
||||
syncIn: async (operations: Array<{ files: Array<{ targetPath: string }> }>) => {
|
||||
await writeFile(operations[0]!.files[0]!.targetPath, "partial upload");
|
||||
throw new Error("upload interrupted");
|
||||
},
|
||||
execute: async (command: { command: string; args?: string[] }) => {
|
||||
execFileSync(command.command, command.args ?? []);
|
||||
return { exitCode: 0, stdout: "", stderr: "" };
|
||||
},
|
||||
};
|
||||
await expect(stageRemoteRunnerFile({ target: {} as never, runner: runner as never, sourcePath: "unused", targetPath, mode: 0o755 })).rejects.toThrow("upload interrupted");
|
||||
expect(await readFile(targetPath, "utf8")).toBe("recoverable old runner");
|
||||
expect(await readdir(root)).toEqual(["paperclip-runnerd"]);
|
||||
} finally { await rm(root, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
it("does not publish through a launcher symlink to a directory", async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "runner-directory-link-"));
|
||||
try {
|
||||
const directory = join(root, "unrelated");
|
||||
const targetPath = join(root, "paperclip-runnerd");
|
||||
const sourcePath = join(root, "candidate");
|
||||
await mkdir(directory); await symlink(directory, targetPath); await writeFile(sourcePath, "runner");
|
||||
const runner = { execute: async (command: { command: string; args?: string[]; stdin?: string }) => {
|
||||
try { execFileSync(command.command, command.args ?? [], { input: command.stdin }); return { exitCode: 0, stdout: "", stderr: "" }; }
|
||||
catch { return { exitCode: 1, stdout: "", stderr: "failed" }; }
|
||||
} };
|
||||
await expect(stageRemoteRunnerFile({ target: {} as never, runner: runner as never, sourcePath, targetPath, mode: 0o755 })).rejects.toThrow("runner_remote_staging_failed");
|
||||
expect(await readdir(directory)).toEqual([]);
|
||||
expect((await lstat(targetPath)).isSymbolicLink()).toBe(true);
|
||||
expect((await readdir(root)).filter(name => name.includes(".upload-"))).toEqual([]);
|
||||
} finally { await rm(root, { recursive: true, force: true }); }
|
||||
});
|
||||
});
|
||||
|
||||
describe("remote provider checkpoint restores", () => {
|
||||
it("does not upload excluded Codex scratch trees or credentials", async () => {
|
||||
const sourcePath = await mkdtemp(
|
||||
|
|
@ -1709,6 +1785,7 @@ describe("remote runner build metadata", () => {
|
|||
binaryName: "paperclip-runnerd",
|
||||
packageName: "@paperclipai/paperclip-runner",
|
||||
binaryContractVersion: 2,
|
||||
capabilities: ["codex.warm-attachment.passive-notices.v1"],
|
||||
prpTransportModes: ["dial_ws_loopback", "dial_wss", "listen_ws"],
|
||||
};
|
||||
|
||||
|
|
@ -1730,6 +1807,13 @@ describe("remote runner build metadata", () => {
|
|||
).toThrow("runner_remote_artifact_contract_incompatible");
|
||||
});
|
||||
|
||||
it("rejects preinstalled binaries that lack safe passive-notice attachment", () => {
|
||||
for (const capabilities of [undefined, [], ["unrelated"]]) {
|
||||
expect(() => assertRemoteRunnerBuildMetadata({ ...current, capabilities }, "listen_ws"))
|
||||
.toThrow("runner_remote_capability_missing:codex.warm-attachment.passive-notices.v1");
|
||||
}
|
||||
});
|
||||
|
||||
it("requires the selected transport without falling through", () => {
|
||||
expect(() =>
|
||||
assertRemoteRunnerBuildMetadata(
|
||||
|
|
@ -6282,6 +6366,7 @@ describe("runnerd provider runtime wiring", () => {
|
|||
binaryName: "paperclip-runnerd",
|
||||
packageName: "@paperclipai/paperclip-runner",
|
||||
binaryContractVersion: 2,
|
||||
capabilities: ["codex.warm-attachment.passive-notices.v1"],
|
||||
prpTransportModes: ["listen_ws"],
|
||||
});
|
||||
} else if (command.args?.[0] === "--version") {
|
||||
|
|
|
|||
|
|
@ -5342,6 +5342,17 @@ export function assertRemoteRunnerBuildMetadata(
|
|||
) {
|
||||
throw new Error("runner_remote_artifact_contract_incompatible");
|
||||
}
|
||||
// An older image may implement the same durable protocol but still reject
|
||||
// passive Codex notices during warm attachment. Stage the current artifact
|
||||
// without changing the checkpoint contract or replacing the sandbox.
|
||||
if (
|
||||
!Array.isArray(metadata.capabilities) ||
|
||||
!metadata.capabilities.includes("codex.warm-attachment.passive-notices.v1")
|
||||
) {
|
||||
throw new Error(
|
||||
"runner_remote_capability_missing:codex.warm-attachment.passive-notices.v1",
|
||||
);
|
||||
}
|
||||
const modes = Array.isArray(metadata.prpTransportModes)
|
||||
? metadata.prpTransportModes
|
||||
: [];
|
||||
|
|
@ -5352,7 +5363,7 @@ export function assertRemoteRunnerBuildMetadata(
|
|||
}
|
||||
}
|
||||
|
||||
async function stageRemoteRunnerFile(input: {
|
||||
export async function stageRemoteRunnerFile(input: {
|
||||
target: Extract<AdapterExecutionTarget, { kind: "remote" }>;
|
||||
runner: CommandManagedRuntimeRunner;
|
||||
sourcePath: string;
|
||||
|
|
@ -5360,36 +5371,59 @@ async function stageRemoteRunnerFile(input: {
|
|||
mode: number;
|
||||
}): Promise<void> {
|
||||
const runner = input.runner;
|
||||
if (runner.syncIn) {
|
||||
await runner.syncIn([
|
||||
{
|
||||
operationId: `runner-stage-${randomUUID()}`,
|
||||
files: [
|
||||
{
|
||||
// The old launcher can be a symlink into the sandbox image. Publish a new
|
||||
// file atomically instead of following that link or truncating working bytes.
|
||||
const temporaryPath = `${input.targetPath}.upload-${randomUUID()}`;
|
||||
const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`;
|
||||
try {
|
||||
if (runner.syncIn) {
|
||||
await runner.syncIn([
|
||||
{
|
||||
operationId: `runner-stage-${randomUUID()}`,
|
||||
files: [{
|
||||
sourcePath: input.sourcePath,
|
||||
targetPath: input.targetPath,
|
||||
targetPath: temporaryPath,
|
||||
kind: "file",
|
||||
mode: input.mode,
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
return;
|
||||
}
|
||||
const bytes = readFileSync(input.sourcePath);
|
||||
const directory = posix.dirname(input.targetPath);
|
||||
const script =
|
||||
`umask 077; mkdir -p '${directory.replaceAll("'", "'\\''")}' && ` +
|
||||
`base64 -d > '${input.targetPath.replaceAll("'", "'\\''")}' && ` +
|
||||
`chmod ${input.mode.toString(8)} '${input.targetPath.replaceAll("'", "'\\''")}'`;
|
||||
const result = await runner.execute({
|
||||
command: "sh",
|
||||
args: ["-c", script],
|
||||
stdin: bytes.toString("base64"),
|
||||
bypassSession: true,
|
||||
});
|
||||
if (result.exitCode !== 0 || result.timedOut) {
|
||||
throw new Error("runner_remote_staging_failed");
|
||||
}],
|
||||
},
|
||||
]);
|
||||
} else {
|
||||
const bytes = readFileSync(input.sourcePath);
|
||||
const result = await runner.execute({
|
||||
command: "sh",
|
||||
args: ["-c", [
|
||||
`umask 077; mkdir -p ${quote(posix.dirname(input.targetPath))}`,
|
||||
`base64 -d > ${quote(temporaryPath)}`,
|
||||
`chmod ${input.mode.toString(8)} ${quote(temporaryPath)}`,
|
||||
].join(" && ")],
|
||||
stdin: bytes.toString("base64"),
|
||||
bypassSession: true,
|
||||
});
|
||||
if (result.exitCode !== 0 || result.timedOut) {
|
||||
throw new Error("runner_remote_staging_failed");
|
||||
}
|
||||
}
|
||||
const published = await runner.execute({
|
||||
command: "sh",
|
||||
args: ["-c", [
|
||||
`test -f ${quote(temporaryPath)}`,
|
||||
`test ! -L ${quote(temporaryPath)}`,
|
||||
`test ! -d ${quote(input.targetPath)}`,
|
||||
`mv -f -- ${quote(temporaryPath)} ${quote(input.targetPath)}`,
|
||||
].join(" && ")],
|
||||
bypassSession: true,
|
||||
});
|
||||
if (published.exitCode !== 0 || published.timedOut) {
|
||||
throw new Error("runner_remote_staging_failed");
|
||||
}
|
||||
} catch (error) {
|
||||
await runner.execute({
|
||||
command: "sh",
|
||||
args: ["-c", `rm -f -- ${quote(temporaryPath)}`],
|
||||
bypassSession: true,
|
||||
}).catch(() => undefined);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue