From 6c85fa060e968f48779e16a7ccb6f52bbb593616 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sun, 30 Aug 2026 11:31:36 -0500 Subject: [PATCH] feat(runner): durably reconcile Codex tools (#12384) ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - The Paperclip Runner gives an agent a durable execution boundary. > - The Codex transport can now advertise a run-scoped semantic tool catalog. > - The durable backend did not yet persist tool calls or correlate their results. > - A restart could therefore lose the boundary between a provider call and a Paperclip action. > - This pull request binds authorized calls, durable events, results, replay, and cancellation. > - The benefit is safe semantic tool recovery without duplicate Paperclip actions. ## Linked Issues or Issue Description Refs #12382 **What existing behavior does this improve?** This improves the durable Codex provider backend in `@paperclipai/paperclip-runner`. **Current behavior** The Codex transport can project authorized dynamic tools. The durable backend rejects their calls because it cannot persist and recover their identities. **Proposed behavior** The durable backend records each authorized call before it emits the semantic input event. It records each result before it sends the result to Codex. It reconciles exact provider replays without another Paperclip action. **Reason and benefit** This order prevents duplicate semantic actions after a process restart. It also keeps unauthorized, changed, oversized, and late calls closed. **Breaking changes** None. A run without an authorized tool catalog still starts Codex with no dynamic tools. ## What Changed - Persist the authorized tool catalog with the Codex provider state. - Emit correlated and redacted semantic input, reconciliation, and result events. - Reconcile exact pending and completed calls after a provider restart. - Reject catalog drift, changed replay input, malformed results, and unauthorized operations. - Complete pending tool calls with a durable failure when a turn stops. - Bound retained tool values and validate recovered state before provider startup. - Bind production runner events to the active run, session, turn, and item identities. ## Verification - `cargo fmt --all -- --check` - `cargo test --workspace` - `pnpm -r typecheck` - `pnpm build` - Confirmed that the PR changes 9 files against `runner-codex-dynamic-tools`. - Confirmed that dependency installation did not change `pnpm-lock.yaml`. ## Risks The main risk is a mismatch between recovered provider state and the controller tool catalog. Recovery validates the complete catalog and its digest before Codex starts. The backend persists a call before it emits work and persists a result before it returns the result to Codex. This PR does not enable the server adapter or change any direct adapter path. ## Model Used OpenAI Codex with GPT-5 and repository tool use. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --- .../src/bin/fake-codex-app-server.rs | 385 ++- .../runner-core/src/bin/paperclip-runnerd.rs | 42 +- .../crates/runner-core/src/codex_provider.rs | 770 ++++- .../crates/runner-core/src/durable/mod.rs | 4 +- .../crates/runner-core/src/durable/state.rs | 37 +- .../runner-core/src/provider_backend.rs | 2348 +++++++++++++++- .../crates/runner-core/src/provider_bridge.rs | 1557 ++++++++-- .../crates/runner-core/src/provider_events.rs | 56 +- .../runner-core/tests/codex_provider.rs | 2496 ++++++++++++++++- .../runner-core/tests/provider_bridge.rs | 540 +++- 10 files changed, 7645 insertions(+), 590 deletions(-) diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs index 1095e28495..81763e1043 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs @@ -120,6 +120,66 @@ fn emit_ambiguous_turn_evidence( } } +fn emit_ambiguous_turn_item() -> io::Result<()> { + send(json!({ + "method": "item/completed", + "params": {"item": { + "id": "replacement-message-before-terminal", + "type": "agentMessage", + "status": "completed", + "text": "Replacement output before terminal authority." + }} + })) +} + +fn send_question(state: &FakeState) -> io::Result<()> { + let turn_id = state.active_turn_id.as_deref().unwrap_or("provider-turn-1"); + send(json!({ + "id": "runtime-request-1", + "method": "item/tool/requestUserInput", + "params": { + "threadId": state.thread_id, + "turnId": turn_id, + "itemId": "question-item-1", + "isBlocking": true, + "title": "Deployment input", + "questions": [{ + "id": "environment", + "header": "Environment", + "question": "Where should we deploy?", + "options": [ + {"label": "Staging", "description": "Deploy safely."}, + {"label": "Production", "description": "Deploy directly."} + ] + }] + } + })) +} + +fn send_runtime_request_flood(state: &FakeState, interrupt_count: u64) -> io::Result<()> { + let turn_id = state.active_turn_id.as_deref().unwrap_or("provider-turn-1"); + for index in 0..160_u64 { + send(json!({ + "id": format!("runtime-flood-{interrupt_count}-{index}"), + "method": "item/tool/requestUserInput", + "params": { + "threadId": state.thread_id, + "turnId": turn_id, + "itemId": format!("question-item-{interrupt_count}-{index}"), + "isBlocking": true, + "title": "Bounded cleanup input", + "questions": [{ + "id": "environment", + "header": "Environment", + "question": "Where should we deploy?", + "options": [{"label": "Staging", "description": "Deploy safely."}], + }], + }, + }))?; + } + Ok(()) +} + fn run() -> Result<(), Box> { let args = std::env::args().skip(1).collect::>(); let state_path = @@ -136,6 +196,19 @@ fn run() -> Result<(), Box> { let exit_after_tool_call_completion = args .iter() .any(|value| value == "--exit-after-tool-call-completion"); + let emit_tool_call_on_resume = args + .iter() + .any(|value| value == "--emit-tool-call-on-resume"); + let resume_unowned_turn_when_marked = args + .iter() + .any(|value| value == "--resume-unowned-turn-when-marked"); + let replay_completed_tool_call_count = argument(&args, "--replay-completed-tool-call-count") + .map(|value| value.parse::()) + .transpose()? + .unwrap_or_default(); + let finish_turn_with_pending_tool = args + .iter() + .any(|value| value == "--finish-turn-with-pending-tool"); let require_dynamic_tool = args.iter().any(|value| value == "--require-dynamic-tool"); let hold_turn = args.iter().any(|value| value == "--hold-turn"); let exit_after_turn_start = args.iter().any(|value| value == "--exit-after-turn-start"); @@ -163,6 +236,9 @@ fn run() -> Result<(), Box> { let missing_id_second_turn_start = args .iter() .any(|value| value == "--missing-id-second-turn-start"); + let missing_id_live_turn_start = args + .iter() + .any(|value| value == "--missing-id-live-turn-start"); let fail_after_accepting_second_turn_before_response = args .iter() .any(|value| value == "--fail-after-accepting-second-turn-before-response"); @@ -184,32 +260,93 @@ fn run() -> Result<(), Box> { let conflicting_ambiguous_second_turn = args .iter() .any(|value| value == "--conflicting-ambiguous-second-turn"); + let ambiguous_older_reused_turn = args + .iter() + .any(|value| value == "--ambiguous-older-reused-turn"); let omit_ambiguous_turn_started = args .iter() .any(|value| value == "--omit-ambiguous-turn-started"); let fail_after_thread_read = args.iter().any(|value| value == "--fail-after-thread-read"); + let fail_first_interrupt = args.iter().any(|value| value == "--fail-first-interrupt"); + let accept_interrupt_without_terminal_once = args + .iter() + .any(|value| value == "--accept-interrupt-without-terminal-once"); + let accept_interrupt_without_terminal = args + .iter() + .any(|value| value == "--accept-interrupt-without-terminal"); + let flood_runtime_requests_on_interrupt = args + .iter() + .any(|value| value == "--flood-runtime-requests-on-interrupt"); + let interrupt_terminal_delay_ms = argument(&args, "--interrupt-terminal-delay-ms") + .map(|value| value.parse::()) + .transpose()?; let exit_after_thread_read = args.iter().any(|value| value == "--exit-after-thread-read"); let fail_after_turn_completion_delay_ms = argument(&args, "--fail-after-turn-completion-delay-ms") .map(|value| value.parse::()) .transpose()?; + let delayed_tool_after_failed_turn = args + .iter() + .any(|value| value == "--delayed-tool-after-failed-turn"); + let delayed_tool_after_next_turn_start = args + .iter() + .any(|value| value == "--delayed-tool-after-next-turn-start"); + let delayed_tool_after_third_turn_start = args + .iter() + .any(|value| value == "--delayed-tool-after-third-turn-start"); + let delayed_tool_after_second_turn_completion = args + .iter() + .any(|value| value == "--delayed-tool-after-second-turn-completion"); + let tool_after_reused_turn_start = args + .iter() + .any(|value| value == "--tool-after-reused-turn-start"); + let tool_after_older_reused_turn_start = args + .iter() + .any(|value| value == "--tool-after-older-reused-turn-start"); + let question_before_failed_turn = args + .iter() + .any(|value| value == "--question-before-failed-turn"); + let reuse_question_id = args.iter().any(|value| value == "--reuse-question-id"); let pre_response_notification = args .iter() .any(|value| value == "--notification-before-response"); let mut state = load_state(&state_path); let mut turn_start_count = 0_u64; + let mut interrupt_count = 0_u64; + let mut delayed_interrupt_terminal_scheduled = false; + let mut answered_questions = 0u8; + let mut replayed_completed_tool_calls = 0_u64; for line in io::stdin().lock().lines() { let message: Value = serde_json::from_str(&line?)?; + if message.get("method").is_none() + && message + .get("id") + .and_then(Value::as_str) + .is_some_and(|id| id.starts_with("runtime-flood-")) + { + let outcome = if message.get("error").is_some() { + "runtime-response:rejected" + } else { + "runtime-response:cancelled" + }; + log_call(call_log.as_deref(), outcome)?; + continue; + } if message.get("method").is_none() && message.get("id") == Some(&json!("runtime-request-1")) { + if reuse_question_id && answered_questions == 0 { + answered_questions = 1; + send_question(&state)?; + continue; + } finish_turn(&state_path, &mut state, "completed")?; continue; } if message.get("method").is_none() && message.get("id") == Some(&json!("tool-request-1")) { if message.pointer("/result/success") == Some(&json!(false)) { log_call(call_log.as_deref(), "tool-response:failure")?; - if state.active_turn_id.is_some() { + if state.active_turn_id.is_some() && !hold_turn { finish_turn(&state_path, &mut state, "failed")?; } continue; @@ -225,7 +362,9 @@ fn run() -> Result<(), Box> { if result != json!({"ok": true, "task": {"id": "task-1"}}) { return Err("semantic tool response changed the operation result".into()); } - if replay_completed_tool_call { + log_call(call_log.as_deref(), &format!("tool-response:{text}"))?; + if replay_completed_tool_call && replayed_completed_tool_calls == 0 { + replayed_completed_tool_calls += 1; send(json!({ "id": "tool-request-replay", "method": "item/tool/call", @@ -237,9 +376,22 @@ fn run() -> Result<(), Box> { "arguments": {} } }))?; - continue; + } else if replayed_completed_tool_calls < replay_completed_tool_call_count { + replayed_completed_tool_calls += 1; + send(json!({ + "id": "tool-request-1", + "method": "item/tool/call", + "params": { + "threadId": state.thread_id, + "turnId": state.active_turn_id, + "callId": "semantic-call-1", + "tool": "get_task_context", + "arguments": {} + } + }))?; + } else if !hold_turn { + finish_turn(&state_path, &mut state, "completed")?; } - finish_turn(&state_path, &mut state, "completed")?; continue; } let Some(method) = message.get("method").and_then(Value::as_str) else { @@ -275,10 +427,31 @@ fn run() -> Result<(), Box> { if require_dynamic_tool && !has_task_context_tool(&message) { return Err("thread/resume omitted the authorized dynamic tool".into()); } + let unowned_turn_marker = state_path.with_file_name("resume-unowned-turn"); + if resume_unowned_turn_when_marked && unowned_turn_marker.exists() { + state.active_turn_id = Some("provider-turn-unowned".to_owned()); + save_state(&state_path, &state)?; + fs::remove_file(unowned_turn_marker)?; + } send(json!({ "id": id, "result": {"thread": {"id": state.thread_id, "sessionId": "codex-account-session"}} }))?; + if emit_tool_call_on_resume { + if let Some(turn_id) = state.active_turn_id.as_deref() { + send(json!({ + "id": "tool-request-1", + "method": "item/tool/call", + "params": { + "threadId": state.thread_id, + "turnId": turn_id, + "callId": "semantic-call-1", + "tool": "get_task_context", + "arguments": {} + } + }))?; + } + } } "thread/read" => { let turns = state @@ -319,17 +492,31 @@ fn run() -> Result<(), Box> { && (complete_ambiguous_second_turn || complete_ambiguous_second_turn_before_response || conflicting_ambiguous_second_turn); - let provider_turn_id = if emits_ambiguous_turn_evidence + let provider_turn_id = if turn_start_count == 3 && ambiguous_older_reused_turn { + "provider-turn-1".to_owned() + } else if emits_ambiguous_turn_evidence || (turn_start_count == 2 && (retain_ambiguous_second_turn_active || hold_ambiguous_second_turn_after_item)) { - "provider-turn-2" + "provider-turn-2".to_owned() + } else if tool_after_reused_turn_start + || (tool_after_older_reused_turn_start && turn_start_count == 3) + { + "provider-turn-1".to_owned() } else { - "provider-turn-1" + format!("provider-turn-{turn_start_count}") }; - state.active_turn_id = Some(provider_turn_id.to_owned()); + state.active_turn_id = Some(provider_turn_id.clone()); save_state(&state_path, &state)?; + if ambiguous_older_reused_turn && turn_start_count == 3 { + send(json!({ + "method": "turn/started", + "params": {"turn": {"id": provider_turn_id}} + }))?; + send(json!({"id": id, "error": {}}))?; + continue; + } if complete_ambiguous_second_turn_before_response && turn_start_count == 2 { emit_ambiguous_turn_evidence( &state_path, @@ -355,6 +542,9 @@ fn run() -> Result<(), Box> { return Ok(()); } if malformed_error_second_turn_start && turn_start_count == 2 { + if hold_ambiguous_second_turn_after_item { + emit_ambiguous_turn_item()?; + } send(json!({"id": id, "error": {}}))?; if emits_ambiguous_turn_evidence && !complete_ambiguous_second_turn_before_response @@ -374,15 +564,7 @@ fn run() -> Result<(), Box> { "result": {"turn": {"status": "inProgress"}} }))?; if hold_ambiguous_second_turn_after_item { - send(json!({ - "method": "item/completed", - "params": {"item": { - "id": "replacement-message-before-terminal", - "type": "agentMessage", - "status": "completed", - "text": "Replacement output before terminal authority." - }} - }))?; + emit_ambiguous_turn_item()?; continue; } if emits_ambiguous_turn_evidence @@ -397,16 +579,103 @@ fn run() -> Result<(), Box> { } return Err("configured failure after missing turn identity".into()); } + if missing_id_live_turn_start { + send(json!({ + "id": id, + "result": {"turn": {"status": "inProgress"}} + }))?; + continue; + } send(json!({ "id": id, - "result": {"turn": {"id": "provider-turn-1", "status": "inProgress"}} + "result": {"turn": {"id": provider_turn_id, "status": "inProgress"}} }))?; send(json!({ "method": "turn/started", - "params": {"turn": {"id": "provider-turn-1"}} + "params": {"turn": {"id": provider_turn_id}} }))?; if fail_after_second_turn_start && turn_start_count == 2 { return Err("configured failure after second turn start".into()); + } else if question_before_failed_turn { + send_question(&state)?; + send(json!({ + "method": "turn/completed", + "params": { + "threadId": state.thread_id, + "turn": {"id": provider_turn_id, "status": "failed"} + } + }))?; + state.active_turn_id = None; + save_state(&state_path, &state)?; + } else if delayed_tool_after_failed_turn { + send(json!({ + "method": "turn/failed", + "params": { + "threadId": state.thread_id, + "turn": {"id": provider_turn_id, "status": "failed"} + } + }))?; + state.active_turn_id = None; + save_state(&state_path, &state)?; + send(json!({ + "id": "tool-request-delayed", + "method": "item/tool/call", + "params": { + "threadId": state.thread_id, + "turnId": provider_turn_id, + "callId": "semantic-call-delayed", + "tool": "get_task_context", + "arguments": {} + } + }))?; + } else if delayed_tool_after_next_turn_start && turn_start_count == 2 { + send(json!({ + "id": "tool-request-delayed", + "method": "item/tool/call", + "params": { + "threadId": state.thread_id, + "turnId": "provider-turn-1", + "callId": "semantic-call-delayed", + "tool": "get_task_context", + "arguments": {} + } + }))?; + } else if delayed_tool_after_third_turn_start && turn_start_count == 3 { + send(json!({ + "id": "tool-request-two-turns-delayed", + "method": "item/tool/call", + "params": { + "threadId": state.thread_id, + "turnId": "provider-turn-1", + "callId": "semantic-call-two-turns-delayed", + "tool": "get_task_context", + "arguments": {} + } + }))?; + } else if tool_after_reused_turn_start && turn_start_count == 2 { + send(json!({ + "id": "tool-request-reused-turn", + "method": "item/tool/call", + "params": { + "threadId": state.thread_id, + "turnId": "provider-turn-1", + "callId": "semantic-call-reused-turn", + "tool": "get_task_context", + "arguments": {} + } + }))?; + } else if tool_after_older_reused_turn_start && turn_start_count == 3 { + send(json!({ + "id": "tool-request-older-reused-turn", + "method": "item/tool/call", + "params": { + "threadId": state.thread_id, + "turnId": "provider-turn-1", + "callId": "semantic-call-older-reused-turn", + "tool": "get_task_context", + "arguments": {} + } + }))?; } else if exit_after_turn_start { return Ok(()); } else if emit_tool_call { @@ -415,41 +684,35 @@ fn run() -> Result<(), Box> { "method": "item/tool/call", "params": { "threadId": state.thread_id, - "turnId": "provider-turn-1", + "turnId": provider_turn_id, "callId": "semantic-call-1", "tool": "get_task_context", "arguments": {} } }))?; - if complete_after_tool_call { + if complete_after_tool_call || finish_turn_with_pending_tool { finish_turn(&state_path, &mut state, "completed")?; if exit_after_tool_call_completion { return Ok(()); } } } else if emit_question { - send(json!({ - "id": "runtime-request-1", - "method": "item/tool/requestUserInput", - "params": { - "threadId": state.thread_id, - "turnId": "provider-turn-1", - "itemId": "question-item-1", - "isBlocking": true, - "title": "Deployment input", - "questions": [{ - "id": "environment", - "header": "Environment", - "question": "Where should we deploy?", - "options": [ - {"label": "Staging", "description": "Deploy safely."}, - {"label": "Production", "description": "Deploy directly."} - ] - }] - } - }))?; + send_question(&state)?; } else if !hold_turn { finish_turn(&state_path, &mut state, "completed")?; + if delayed_tool_after_second_turn_completion && turn_start_count == 2 { + send(json!({ + "id": "tool-request-idle-two-turns-delayed", + "method": "item/tool/call", + "params": { + "threadId": state.thread_id, + "turnId": "provider-turn-1", + "callId": "semantic-call-idle-two-turns-delayed", + "tool": "get_task_context", + "arguments": {} + } + }))?; + } if emit_post_completion_warning { send(json!({ "method": "warning", @@ -477,8 +740,44 @@ fn run() -> Result<(), Box> { } "turn/steer" => send(json!({"id": id, "result": {"accepted": true}}))?, "turn/interrupt" => { - send(json!({"id": id, "result": {"accepted": true}}))?; - finish_turn(&state_path, &mut state, "interrupted")?; + interrupt_count += 1; + if fail_first_interrupt && interrupt_count == 1 { + send(json!({ + "id": id, + "error": {"code": -32001, "message": "configured interrupt failure"} + }))?; + } else { + send(json!({"id": id, "result": {"accepted": true}}))?; + if flood_runtime_requests_on_interrupt { + send_runtime_request_flood(&state, interrupt_count)?; + } + if !accept_interrupt_without_terminal + && !(accept_interrupt_without_terminal_once + && interrupt_count == if fail_first_interrupt { 2 } else { 1 }) + { + if let Some(delay_ms) = interrupt_terminal_delay_ms { + if !delayed_interrupt_terminal_scheduled { + delayed_interrupt_terminal_scheduled = true; + let delayed_state_path = state_path.clone(); + let mut delayed_state = state.clone(); + thread::spawn(move || { + thread::sleep(Duration::from_millis(delay_ms)); + if let Err(error) = finish_turn( + &delayed_state_path, + &mut delayed_state, + "interrupted", + ) { + eprintln!( + "failed to emit delayed interrupt terminal: {error}" + ); + } + }); + } + } else { + finish_turn(&state_path, &mut state, "interrupted")?; + } + } + } } _ if id.is_some() => send(json!({ "id": id, diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs index 4764ed1e88..62aed56ea1 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs @@ -50,28 +50,26 @@ fn run_durable(args: &[String]) -> Result<(), LocalRunnerError> { optional_u64(args, name).map(|value| Duration::from_millis(value.unwrap_or(default))) }; let state_dir = PathBuf::from(value(args, "--state-dir")?); - run_durable_runner( - DurableRunnerConfig { - connect_url: value(args, "--connect-url")?, - state_dir: state_dir.clone(), - runner_instance_id: value(args, "--runner-id")?, - environment_lease_id: value(args, "--environment-lease-id")?, - run_id: value(args, "--run-id")?, - normalized_session_id: value(args, "--session-id")?, - turn_id: value(args, "--turn-id")?, - item_id: value(args, "--item-id")?, - runner_version: value(args, "--runner-version")?, - runner_digest: value(args, "--runner-digest")?, - max_outbox_bytes: usize_value(args, "--max-outbox-bytes", 16 * 1024 * 1024)?, - p0_reserve_bytes: usize_value(args, "--p0-reserve-bytes", 1024 * 1024)?, - max_frame_bytes: usize_value(args, "--max-frame-bytes", 1024 * 1024)?, - reconnect_delay: duration("--reconnect-delay-ms", 250)?, - max_runtime: duration("--max-runtime-ms", 60 * 60 * 1000)?, - }, - ticket, - CodexCommandExecutor::new(state_dir), - ) - .map_err(|error| LocalRunnerError::invalid(error.to_string())) + let config = DurableRunnerConfig { + connect_url: value(args, "--connect-url")?, + state_dir: state_dir.clone(), + runner_instance_id: value(args, "--runner-id")?, + environment_lease_id: value(args, "--environment-lease-id")?, + run_id: value(args, "--run-id")?, + normalized_session_id: value(args, "--session-id")?, + turn_id: value(args, "--turn-id")?, + item_id: value(args, "--item-id")?, + runner_version: value(args, "--runner-version")?, + runner_digest: value(args, "--runner-digest")?, + max_outbox_bytes: usize_value(args, "--max-outbox-bytes", 16 * 1024 * 1024)?, + p0_reserve_bytes: usize_value(args, "--p0-reserve-bytes", 1024 * 1024)?, + max_frame_bytes: usize_value(args, "--max-frame-bytes", 1024 * 1024)?, + reconnect_delay: duration("--reconnect-delay-ms", 250)?, + max_runtime: duration("--max-runtime-ms", 60 * 60 * 1000)?, + }; + let executor = CodexCommandExecutor::with_runner_config(state_dir, &config); + run_durable_runner(config, ticket, executor) + .map_err(|error| LocalRunnerError::invalid(error.to_string())) } fn run() -> Result<(), LocalRunnerError> { diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs index 322236013d..249c13f0ae 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs @@ -4,18 +4,24 @@ use std::time::Duration; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; use crate::durable::redact_text; use crate::local_runner::LocalRunnerError; use crate::process_supervisor::SupervisedProcess; -use crate::provider_bridge::{AuthorizedTool, ToolResult}; +use crate::provider_bridge::{AuthorizedTool, DurableReplayFilter, ToolResult}; +use crate::provider_events::normalized_codex_terminal_event_type; pub const CODEX_APP_SERVER_MAX_FRAME_BYTES: usize = 4 * 1024 * 1024; const MAX_BUFFERED_MESSAGES: usize = 1_024; +const MAX_BUFFERED_MESSAGE_BYTES: usize = 16 * 1024 * 1024; const MAX_INSTRUCTIONS_BYTES: usize = 1024 * 1024; const MAX_PENDING_TOOL_REQUESTS: usize = 4_096; const MAX_PENDING_TOOL_REQUEST_BYTES: usize = 16 * 1024 * 1024; const MAX_COMPLETED_TOOL_CALL_IDS: usize = 4_096; +const MAX_PENDING_RUNTIME_REQUESTS: usize = 128; +const MAX_PENDING_RUNTIME_REQUEST_BYTES: usize = 4 * 1024 * 1024; +pub(crate) const MAX_SETTLED_PROVIDER_TURN_IDS: usize = 4_096; type QuestionOptionLabels = BTreeMap>; type QuestionSetMapping = (String, Value, QuestionOptionLabels); @@ -135,11 +141,71 @@ struct CompletedTurnAuthority { provider_turn_id: String, } +#[derive(Default)] +struct SettledProviderTurnIds { + ids: BTreeSet, + filter: DurableReplayFilter, +} + +impl SettledProviderTurnIds { + fn insert(&mut self, provider_turn_id: String) -> bool { + if self.contains(&provider_turn_id) { + return true; + } + if self.ids.len() >= MAX_SETTLED_PROVIDER_TURN_IDS { + return false; + } + self.ids.insert(provider_turn_id) + } + + fn contains(&self, provider_turn_id: &str) -> bool { + self.ids.contains(provider_turn_id) + } + + fn at_capacity(&self) -> bool { + self.ids.len() >= MAX_SETTLED_PROVIDER_TURN_IDS || !self.filter.is_empty() + } + + fn restore(&mut self, provider_turn_id: String) -> Result<(), LocalRunnerError> { + if !self.insert(provider_turn_id) { + return Err(LocalRunnerError::invalid( + "Codex restored provider turn identity epoch exceeded its exact capacity", + )); + } + Ok(()) + } + + fn restore_all( + &mut self, + provider_turn_ids: impl IntoIterator, + replay_filter: DurableReplayFilter, + ) -> Result<(), LocalRunnerError> { + replay_filter + .validate() + .map_err(|error| LocalRunnerError::invalid(error.to_string()))?; + self.filter = replay_filter; + for provider_turn_id in provider_turn_ids { + if !self.insert(provider_turn_id) { + return Err(LocalRunnerError::invalid( + "Codex restored provider turn identity epoch exceeded its exact capacity", + )); + } + } + Ok(()) + } +} + enum ProviderRequestError { Rejected(LocalRunnerError), Ambiguous(LocalRunnerError), } +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum RejectedAcceptedTurn { + ReusedIdentity(String), + InvalidIdentity, +} + impl ProviderRequestError { fn into_inner(self) -> LocalRunnerError { match self { @@ -159,10 +225,12 @@ struct PendingToolRequest { #[derive(Clone, Debug, PartialEq)] struct PendingRuntimeRequest { rpc_id: Value, + turn_id: String, method: String, params: Value, question_set: Value, option_labels: QuestionOptionLabels, + retained_bytes: usize, } struct BufferedProviderMessage { @@ -178,22 +246,32 @@ enum AmbiguousTurnMessage { pub struct CodexProvider { process: SupervisedProcess, + config: CodexProviderConfig, + authorized_tools: Vec, next_request_id: u64, thread_id: String, provider_session_id: Option, active_provider_turn_id: Option, pending_messages: VecDeque, deferred_ambiguous_messages: VecDeque, + pending_message_bytes: usize, authorized_tool_ids: BTreeSet, pending_tool_requests: BTreeMap, completed_tool_call_ids: BTreeSet, + durable_tool_call_replays: bool, pending_tool_request_bytes: usize, pending_runtime_requests: BTreeMap, + pending_runtime_request_bytes: usize, + runtime_request_scope: [u8; 16], + next_runtime_request_sequence: u64, expected_shutdown: bool, process_generation: u64, completed_turn_authority: Option, completion_reconciliation_pending: bool, ambiguous_turn_start_pending: bool, + settled_provider_turn_ids: SettledProviderTurnIds, + rejected_accepted_turn: Option, + quarantined: bool, } impl CodexProvider { @@ -212,20 +290,7 @@ impl CodexProvider { Self::start_with_tools_for_generation(config, authorized_tools, resume_thread_id, 1) } - pub(crate) fn start_for_generation( - config: &CodexProviderConfig, - resume_thread_id: Option<&str>, - process_generation: u64, - ) -> Result { - Self::start_with_tools_for_generation( - config, - std::iter::empty(), - resume_thread_id, - process_generation, - ) - } - - fn start_with_tools_for_generation( + pub(crate) fn start_with_tools_for_generation( config: &CodexProviderConfig, authorized_tools: impl IntoIterator, resume_thread_id: Option<&str>, @@ -237,7 +302,9 @@ impl CodexProvider { "Codex process generation must be positive", )); } - let (dynamic_tools, authorized_tool_ids) = codex_dynamic_tools(authorized_tools)?; + let authorized_tools = authorized_tools.into_iter().collect::>(); + let (dynamic_tools, authorized_tool_ids) = + codex_dynamic_tools(authorized_tools.iter().cloned())?; let mut provider = Self { process: SupervisedProcess::spawn( &config.command, @@ -245,22 +312,32 @@ impl CodexProvider { Duration::from_secs(2), CODEX_APP_SERVER_MAX_FRAME_BYTES, )?, + config: config.clone(), + authorized_tools, next_request_id: 1, thread_id: String::new(), provider_session_id: None, active_provider_turn_id: None, pending_messages: VecDeque::new(), deferred_ambiguous_messages: VecDeque::new(), + pending_message_bytes: 0, authorized_tool_ids, pending_tool_requests: BTreeMap::new(), completed_tool_call_ids: BTreeSet::new(), + durable_tool_call_replays: false, pending_tool_request_bytes: 0, pending_runtime_requests: BTreeMap::new(), + pending_runtime_request_bytes: 0, + runtime_request_scope: new_runtime_request_scope()?, + next_runtime_request_sequence: 1, expected_shutdown: false, process_generation, completed_turn_authority: None, completion_reconciliation_pending: false, ambiguous_turn_start_pending: false, + settled_provider_turn_ids: SettledProviderTurnIds::default(), + rejected_accepted_turn: None, + quarantined: false, }; let initialized = provider.request( "initialize", @@ -320,7 +397,7 @@ impl CodexProvider { if resume_thread_id.is_some() { let snapshot = provider.read_thread()?; provider.active_provider_turn_id = latest_active_turn_id(&snapshot) - .map(|turn_id| bounded_identifier(Some(&turn_id), "Codex turn id")) + .map(|provider_turn_id| bounded_provider_turn_id(Some(&provider_turn_id))) .transpose()?; } Ok(provider) @@ -330,6 +407,10 @@ impl CodexProvider { self.process.id() } + pub(crate) fn process_generation(&self) -> u64 { + self.process_generation + } + pub fn thread_id(&self) -> &str { &self.thread_id } @@ -346,12 +427,19 @@ impl CodexProvider { self.ambiguous_turn_start_pending } + pub(crate) fn enable_durable_tool_call_replays(&mut self) { + // The durable backend validates the call id, operation, and input + // against its persisted receipt before returning a stored result. + // Direct provider consumers retain the stricter one-shot behavior. + self.durable_tool_call_replays = true; + } + pub(crate) fn restore_completed_turn_authority( &mut self, authoritative: bool, process_generation: Option, provider_turn_id: Option<&str>, - ) { + ) -> Result<(), LocalRunnerError> { self.completed_turn_authority = authoritative.then(|| CompletedTurnAuthority { // Legacy state did not record the generation. Generation zero is // deliberately older than every supervised process generation. @@ -360,6 +448,10 @@ impl CodexProvider { .unwrap_or("durable-completed-turn") .to_owned(), }); + if let Some(authority) = self.completed_turn_authority.as_ref() { + self.settled_provider_turn_ids + .restore(authority.provider_turn_id.clone())?; + } // Resuming a completed durable thread and reading its provider state // is recovery, not new turn work. Keep the prior terminal authoritative // until start_turn explicitly revokes it. @@ -369,6 +461,16 @@ impl CodexProvider { // of them supersedes a completed result. Only accepting a replacement // turn identity revokes this authority. self.completion_reconciliation_pending = false; + Ok(()) + } + + pub(crate) fn restore_settled_turn_identities( + &mut self, + provider_turn_ids: impl IntoIterator, + replay_filter: DurableReplayFilter, + ) -> Result<(), LocalRunnerError> { + self.settled_provider_turn_ids + .restore_all(provider_turn_ids, replay_filter) } pub(crate) fn completed_turn_authority(&self) -> Option<(u64, &str)> { @@ -380,7 +482,87 @@ impl CodexProvider { }) } + pub(crate) fn take_rejected_accepted_turn(&mut self) -> Option { + self.rejected_accepted_turn.take() + } + + pub(crate) fn restart_idle_identity_epoch(&mut self) -> Result<(), LocalRunnerError> { + if self.active_provider_turn_id.is_some() || self.ambiguous_turn_start_pending { + return Err(LocalRunnerError::invalid( + "Codex provider identity epoch cannot rotate while work is active", + )); + } + + let next_generation = self.process_generation.checked_add(1).ok_or_else(|| { + LocalRunnerError::invalid("Codex process generation exhausted during epoch rollover") + })?; + let config = self.config.clone(); + let authorized_tools = self.authorized_tools.clone(); + let thread_id = self.thread_id.clone(); + let completed_turn_authority = self.completed_turn_authority.clone(); + let completion_reconciliation_pending = self.completion_reconciliation_pending; + let durable_tool_call_replays = self.durable_tool_call_replays; + + // Exact turn identities may be forgotten only after the provider + // process that could emit them is gone. Resume the same thread in a + // fresh process generation, then preserve prior completion authority + // until a replacement turn identity is actually accepted. + self.shutdown()?; + let mut replacement = Self::start_with_tools_for_generation( + &config, + authorized_tools, + Some(&thread_id), + next_generation, + )?; + replacement.durable_tool_call_replays = durable_tool_call_replays; + if replacement.active_provider_turn_id.is_some() { + // A terminal can race the provider's own durable idle-state write. + // This process has work Paperclip never dispatched in the new + // epoch, so revoke its request authority and reap it. Retain the + // exact ledger for diagnostics, but never expose the unexpected + // turn as ordinary active work or admit a replacement. + replacement.settled_provider_turn_ids = + std::mem::take(&mut self.settled_provider_turn_ids); + replacement.active_provider_turn_id = None; + replacement.ambiguous_turn_start_pending = true; + replacement.rejected_accepted_turn = Some(RejectedAcceptedTurn::InvalidIdentity); + replacement.quarantined = true; + replacement.pending_messages.clear(); + replacement.deferred_ambiguous_messages.clear(); + replacement.pending_message_bytes = 0; + let _ = replacement.cancel_pending_requests(); + let _ = replacement.process.terminate_group(); + replacement.expected_shutdown = false; + *self = replacement; + return Err(LocalRunnerError::invalid( + "Codex provider epoch rollover resumed unowned active work; the provider was terminated", + )); + } + if let Some(authority) = completed_turn_authority.as_ref() { + replacement.restore_completed_turn_authority( + true, + Some(authority.process_generation), + Some(&authority.provider_turn_id), + )?; + } + replacement.completion_reconciliation_pending = completion_reconciliation_pending; + *self = replacement; + Ok(()) + } + + fn rollover_settled_turn_epoch_if_needed(&mut self) -> Result<(), LocalRunnerError> { + if !self.settled_provider_turn_ids.at_capacity() { + return Ok(()); + } + self.restart_idle_identity_epoch() + } + pub fn start_turn(&mut self, message: &str, cwd: &str) -> Result { + if self.quarantined { + return Err(LocalRunnerError::invalid( + "Codex provider is quarantined after unsafe recovered work", + )); + } if self.active_provider_turn_id.is_some() { return Err(LocalRunnerError::invalid( "Codex already has an active provider turn", @@ -396,6 +578,7 @@ impl CodexProvider { "Codex turn text is empty or exceeds the 1 MiB limit", )); } + self.rollover_settled_turn_epoch_if_needed()?; // Preserve the prior durable result until a replacement turn identity // is accepted. A rejected, ambiguous, or transport-failed attempt does // not prove that replacement work superseded the completed turn. @@ -403,6 +586,7 @@ impl CodexProvider { self.completion_reconciliation_pending = false; let prior_buffered_message_count = self.pending_messages.len(); self.ambiguous_turn_start_pending = true; + let runtime_request_scope = new_runtime_request_scope()?; let result = match self.request_classified( "turn/start", json!({ @@ -433,16 +617,32 @@ impl CodexProvider { } Err(ProviderRequestError::Ambiguous(error)) => return Err(error), }; - let provider_turn_id = bounded_identifier( - result - .pointer("/turn/id") - .or_else(|| result.get("turnId")) - .and_then(Value::as_str), - "Codex turn id", - )?; + let provider_turn_id = result + .pointer("/turn/id") + .or_else(|| result.get("turnId")) + .and_then(Value::as_str); + let provider_turn_id = match bounded_provider_turn_id(provider_turn_id) { + Ok(provider_turn_id) => provider_turn_id, + Err(error) => { + // A successful turn/start response means the provider may + // already be executing the work. Without a bounded identity, + // runnerd cannot durably bind, interrupt, or reconcile it. + // Terminate the process and let the durable backend close the + // run before returning the protocol error. + self.rejected_accepted_turn = Some(RejectedAcceptedTurn::InvalidIdentity); + self.expected_shutdown = true; + self.completed_turn_authority = None; + let _ = self.cancel_pending_requests(); + let _ = self.process.terminate_group(); + return Err(error); + } + }; + if self.settled_provider_turn_ids.contains(&provider_turn_id) { + return Err(self.reject_accepted_reused_turn_identity(provider_turn_id)); + } // Only a validated provider turn identity proves that replacement // work exists and supersedes the prior completed result. - self.accept_replacement_turn(provider_turn_id); + self.accept_replacement_turn(provider_turn_id, runtime_request_scope); Ok(result) } @@ -471,17 +671,12 @@ impl CodexProvider { )) })?; - if self - .completed_turn_authority - .as_ref() - .is_some_and(|authority| authority.provider_turn_id == provider_turn_id) - { - return Err(LocalRunnerError::invalid(format!( - "Codex {method} notification reused the previously completed turn id while resolving an ambiguous turn start" - ))); + if self.settled_provider_turn_ids.contains(&provider_turn_id) { + return Err(self.reject_accepted_reused_turn_identity(provider_turn_id)); } - self.accept_replacement_turn(provider_turn_id.clone()); + let runtime_request_scope = new_runtime_request_scope()?; + self.accept_replacement_turn(provider_turn_id.clone(), runtime_request_scope); if method == "turn/started" && message.get("id").is_none() { Ok(AmbiguousTurnMessage::ReconciledWithStart) } else { @@ -489,13 +684,39 @@ impl CodexProvider { } } - fn accept_replacement_turn(&mut self, provider_turn_id: String) { + fn accept_replacement_turn( + &mut self, + provider_turn_id: String, + runtime_request_scope: [u8; 16], + ) { self.ambiguous_turn_start_pending = false; self.expected_shutdown = false; self.completed_turn_authority = None; self.completion_reconciliation_pending = false; self.completed_tool_call_ids.clear(); + // Retain the prior settled identity while the next turn runs. Besides + // recognizing delayed prior-turn requests, this fails closed if a + // provider ambiguously reuses the same turn id for fresh work. self.active_provider_turn_id = Some(provider_turn_id); + self.runtime_request_scope = runtime_request_scope; + } + + fn reject_accepted_reused_turn_identity( + &mut self, + provider_turn_id: String, + ) -> LocalRunnerError { + // Both a successful response and identity-bearing evidence after an + // ambiguous response prove the provider accepted work. A settled + // identity cannot durably own that work, so terminate its process + // before returning instead of leaving an untracked turn alive. + self.rejected_accepted_turn = Some(RejectedAcceptedTurn::ReusedIdentity(provider_turn_id)); + self.expected_shutdown = true; + self.completed_turn_authority = None; + let _ = self.cancel_pending_requests(); + let _ = self.process.terminate_group(); + LocalRunnerError::invalid( + "Codex reused a settled provider turn identity after accepting work; the provider was terminated", + ) } pub fn steer_turn(&mut self, message: &str) -> Result { @@ -554,15 +775,84 @@ impl CodexProvider { .ok_or_else(|| { LocalRunnerError::invalid("runtime response has no pending Codex request") })?; + if self.active_provider_turn_id.as_deref() != Some(pending.turn_id.as_str()) { + return Err(LocalRunnerError::invalid( + "runtime response belongs to another Codex turn", + )); + } let result = codex_question_response(&pending, response)?; self.process .send(&json!({"id": pending.rpc_id, "result": result}))?; - self.pending_runtime_requests.remove(request_id); + if let Some(completed) = self.pending_runtime_requests.remove(request_id) { + self.pending_runtime_request_bytes = self + .pending_runtime_request_bytes + .saturating_sub(completed.retained_bytes); + } Ok(()) } + fn reject_post_terminal_request( + &mut self, + rpc_id: Value, + method: &str, + ) -> Result, LocalRunnerError> { + let message = format!( + "ignored delayed {} request after the Codex turn terminated", + bounded_method(method) + ); + let response = if method == "item/tool/call" { + json!({ + "id": rpc_id, + "result": codex_tool_failure("the Codex turn has already terminated"), + }) + } else { + json!({ + "id": rpc_id, + "error": {"code": -32000, "message": "the Codex turn has already terminated"}, + }) + }; + // The terminal notification is already authoritative and may be + // waiting in the durable outbox. A courtesy rejection must not turn a + // provider that has closed stdin into a fatal polling error. + let _ = self.process.send(&response); + Ok(Some(CodexProviderEvent::Notification { + method: "warning".to_owned(), + params: json!({"message": message, "providerMethod": bounded_method(method)}), + })) + } + pub fn poll(&mut self) -> Result, LocalRunnerError> { + if self.quarantined { + // Never interpret provider-originated requests after fail-closed + // quarantine. Drain output only so process termination cannot + // deadlock on a full pipe, then surface an unequivocal failure. + if self + .process + .receive_stdout_line(Duration::from_millis(1))? + .is_some() + { + return Ok(None); + } + return Ok(self + .process + .try_wait()? + .map(|exit| CodexProviderEvent::Exited { + exit_code: exit.exit_code, + success: false, + completed_turn_authoritative: false, + completed_turn_observed_by_process: false, + completion_reconciles_exit: false, + process_generation: self.process_generation, + completed_turn_process_generation: None, + })); + } let buffered = self.pending_messages.pop_front(); + if let Some(buffered) = buffered.as_ref() { + self.pending_message_bytes = self.pending_message_bytes.saturating_sub(json_size( + &buffered.value, + "buffered Codex provider message", + )?); + } let message = if let Some(buffered) = buffered { buffered.value } else { @@ -625,11 +915,24 @@ impl CodexProvider { match self.classify_ambiguous_turn_message(&message)? { AmbiguousTurnMessage::Ready => {} AmbiguousTurnMessage::Deferred => { - if self.deferred_ambiguous_messages.len() >= MAX_BUFFERED_MESSAGES { + if self + .pending_messages + .len() + .saturating_add(self.deferred_ambiguous_messages.len()) + >= MAX_BUFFERED_MESSAGES + { return Err(LocalRunnerError::invalid( "Codex emitted too many messages before resolving an ambiguous turn start", )); } + let retained_bytes = json_size(&message, "buffered Codex provider message")?; + self.pending_message_bytes = + retain_buffered_message_bytes(self.pending_message_bytes, retained_bytes) + .ok_or_else(|| { + LocalRunnerError::invalid( + "Codex buffered messages exceed the 16 MiB aggregate limit", + ) + })?; self.deferred_ambiguous_messages .push_back(BufferedProviderMessage { value: message }); return Ok(None); @@ -641,6 +944,14 @@ impl CodexProvider { } AmbiguousTurnMessage::ReconciledNeedsStart { provider_turn_id } => { let mut replay = std::mem::take(&mut self.deferred_ambiguous_messages); + let retained_bytes = json_size(&message, "buffered Codex provider message")?; + self.pending_message_bytes = + retain_buffered_message_bytes(self.pending_message_bytes, retained_bytes) + .ok_or_else(|| { + LocalRunnerError::invalid( + "Codex buffered messages exceed the 16 MiB aggregate limit", + ) + })?; replay.push_back(BufferedProviderMessage { value: message }); replay.append(&mut self.pending_messages); self.pending_messages = replay; @@ -666,6 +977,13 @@ impl CodexProvider { "Codex tool call named another thread", )); } + if request_targets_non_active_turn( + self.active_provider_turn_id.as_deref(), + &self.settled_provider_turn_ids, + ¶ms, + ) { + return self.reject_post_terminal_request(rpc_id, method); + } let active_turn_id = self.active_provider_turn_id.as_deref().ok_or_else(|| { LocalRunnerError::invalid("Codex tool call arrived outside an active turn") })?; @@ -719,7 +1037,8 @@ impl CodexProvider { input: input.clone(), retained_bytes, }; - if self.completed_tool_call_ids.contains(&call_id) { + let completed_replay = self.completed_tool_call_ids.contains(&call_id); + if completed_replay && !self.durable_tool_call_replays { return Err(LocalRunnerError::invalid( "Codex reused a completed tool call id", )); @@ -746,7 +1065,9 @@ impl CodexProvider { "Codex emitted too many pending tool calls", )); } - if self.completed_tool_call_ids.len() >= MAX_COMPLETED_TOOL_CALL_IDS { + if !completed_replay + && self.completed_tool_call_ids.len() >= MAX_COMPLETED_TOOL_CALL_IDS + { return Err(LocalRunnerError::invalid( "Codex emitted too many completed tool calls in one turn", )); @@ -770,36 +1091,93 @@ impl CodexProvider { "Codex runtime request named another thread", )); } - let active_turn_id = self.active_provider_turn_id.as_deref().ok_or_else(|| { + if request_targets_non_active_turn( + self.active_provider_turn_id.as_deref(), + &self.settled_provider_turn_ids, + ¶ms, + ) { + return self.reject_post_terminal_request(rpc_id, method); + } + let active_turn_id = self.active_provider_turn_id.clone().ok_or_else(|| { LocalRunnerError::invalid( "Codex runtime request arrived outside an active turn", ) })?; - if params.get("turnId").and_then(Value::as_str) != Some(active_turn_id) { + if params.get("turnId").and_then(Value::as_str) != Some(active_turn_id.as_str()) { return Err(LocalRunnerError::invalid( "Codex runtime request named another turn", )); } - let (request_id, question_set, option_labels) = + let (provider_request_id, question_set, option_labels) = codex_question_set(&rpc_id, ¶ms)?; + let retained_bytes = pending_runtime_request_size( + &rpc_id, + &active_turn_id, + method, + ¶ms, + &question_set, + &option_labels, + )?; let pending = PendingRuntimeRequest { - rpc_id, + rpc_id: rpc_id.clone(), + turn_id: active_turn_id.clone(), method: method.to_owned(), params, question_set: question_set.clone(), option_labels, + retained_bytes, }; - if let Some(existing) = self.pending_runtime_requests.get(&request_id) { + if let Some(existing) = self + .pending_runtime_requests + .values() + .find(|existing| existing.rpc_id == rpc_id) + { if existing != &pending { return Err(LocalRunnerError::invalid( "Codex reused a runtime request id with different input", )); } return Ok(None); - } else { - self.pending_runtime_requests - .insert(request_id.clone(), pending); } + let retained_request_bytes = retain_pending_runtime_request_bytes( + self.pending_runtime_request_bytes, + retained_bytes, + ); + if self.pending_runtime_requests.len() >= MAX_PENDING_RUNTIME_REQUESTS + || retained_request_bytes.is_none() + { + self.process.send(&json!({ + "id": rpc_id, + "error": { + "code": -32000, + "message": "Paperclip rejected this runtime request because the pending input capacity was reached", + }, + }))?; + return Ok(Some(CodexProviderEvent::Notification { + method: "warning".to_owned(), + params: json!({ + "message": "rejected a Codex runtime request at the bounded pending-input limit", + "providerMethod": "item/tool/requestUserInput", + }), + })); + } + let request_sequence = self.next_runtime_request_sequence; + self.next_runtime_request_sequence = self + .next_runtime_request_sequence + .checked_add(1) + .ok_or_else(|| { + LocalRunnerError::invalid("Codex runtime request sequence overflowed") + })?; + let request_id = scoped_runtime_request_id( + &self.runtime_request_scope, + &active_turn_id, + &provider_request_id, + request_sequence, + ); + self.pending_runtime_requests + .insert(request_id.clone(), pending); + self.pending_runtime_request_bytes = + retained_request_bytes.expect("bounded runtime request bytes checked above"); return Ok(Some(CodexProviderEvent::RuntimeRequest { request_id, question_set, @@ -817,24 +1195,60 @@ impl CodexProvider { if let Some(method) = message.get("method").and_then(Value::as_str) { let params = message.get("params").cloned().unwrap_or(Value::Null); + let terminal_event_type = normalized_codex_terminal_event_type(method, ¶ms); + let notification_turn_id = params + .get("turnId") + .or_else(|| params.pointer("/turn/id")) + .and_then(Value::as_str); + if terminal_event_type.is_some() + && notification_turn_id.is_some() + && notification_turn_id != self.active_provider_turn_id.as_deref() + && notification_turn_id + .is_some_and(|turn_id| self.settled_provider_turn_ids.contains(turn_id)) + { + return Ok(Some(CodexProviderEvent::Notification { + method: "warning".to_owned(), + params: json!({ + "message": "ignored a terminal notification for a non-active Codex turn", + "providerMethod": bounded_method(method), + }), + })); + } validate_notification_binding( &self.thread_id, self.active_provider_turn_id.as_deref(), ¶ms, )?; - if method == "turn/completed" { - let provider_turn_id = self.active_provider_turn_id.clone().ok_or_else(|| { - LocalRunnerError::invalid( - "Codex completion arrived outside an active provider turn", - ) - })?; + if let Some(terminal_event_type) = terminal_event_type { + if self.active_provider_turn_id.is_none() { + return Err(LocalRunnerError::invalid( + "Codex terminal arrived outside an active provider turn", + )); + } + let provider_turn_id = self + .active_provider_turn_id + .clone() + .expect("active provider turn checked above"); + let completed_turn_authority = if terminal_event_type == "turn.completed" { + Some(CompletedTurnAuthority { + process_generation: self.process_generation, + provider_turn_id: provider_turn_id.clone(), + }) + } else { + None + }; + if !self + .settled_provider_turn_ids + .insert(provider_turn_id.clone()) + { + return Err(LocalRunnerError::invalid( + "Codex provider turn identity epoch reached its exact capacity", + )); + } self.active_provider_turn_id = None; self.expected_shutdown = true; - self.completed_turn_authority = Some(CompletedTurnAuthority { - process_generation: self.process_generation, - provider_turn_id, - }); - self.completion_reconciliation_pending = true; + self.completed_turn_authority = completed_turn_authority; + self.completion_reconciliation_pending = terminal_event_type == "turn.completed"; // The provider terminal is authoritative once received. Clear // local request ownership and attempt courtesy responses, but // a provider that already closed stdin must not turn the @@ -898,6 +1312,7 @@ impl CodexProvider { fn cancel_pending_requests(&mut self) -> Result<(), LocalRunnerError> { let pending_runtime = std::mem::take(&mut self.pending_runtime_requests); let pending = std::mem::take(&mut self.pending_tool_requests); + self.pending_runtime_request_bytes = 0; self.pending_tool_request_bytes = 0; let mut first_error = None; for request in pending_runtime.into_values() { @@ -967,17 +1382,44 @@ impl CodexProvider { } return Ok(message.get("result").cloned().unwrap_or(Value::Null)); } - if self.pending_messages.len() >= MAX_BUFFERED_MESSAGES { + if self + .pending_messages + .len() + .saturating_add(self.deferred_ambiguous_messages.len()) + >= MAX_BUFFERED_MESSAGES + { return Err(ProviderRequestError::Ambiguous(LocalRunnerError::invalid( "Codex emitted too many messages before a request response", ))); } + let retained_bytes = json_size(&message, "buffered Codex provider message") + .map_err(ProviderRequestError::Ambiguous)?; + let next_retained_bytes = + retain_buffered_message_bytes(self.pending_message_bytes, retained_bytes) + .ok_or_else(|| { + ProviderRequestError::Ambiguous(LocalRunnerError::invalid( + "Codex buffered messages exceed the 16 MiB aggregate limit", + )) + })?; self.pending_messages .push_back(BufferedProviderMessage { value: message }); + self.pending_message_bytes = next_retained_bytes; } } } +fn json_size(value: &Value, label: &str) -> Result { + serde_json::to_vec(value) + .map(|bytes| bytes.len()) + .map_err(|error| LocalRunnerError::invalid(format!("{label} is not serializable: {error}"))) +} + +fn retain_buffered_message_bytes(current: usize, incoming: usize) -> Option { + current + .checked_add(incoming) + .filter(|total| *total <= MAX_BUFFERED_MESSAGE_BYTES) +} + fn pending_tool_request_size( parts: impl IntoIterator, ) -> Result { @@ -1002,6 +1444,29 @@ fn retain_pending_tool_request_bytes( }) } +fn pending_runtime_request_size( + rpc_id: &Value, + turn_id: &str, + method: &str, + params: &Value, + question_set: &Value, + option_labels: &QuestionOptionLabels, +) -> Result { + serde_json::to_vec(&(rpc_id, turn_id, method, params, question_set, option_labels)) + .map(|encoded| encoded.len()) + .map_err(|error| { + LocalRunnerError::invalid(format!( + "Codex pending runtime request is not serializable: {error}" + )) + }) +} + +fn retain_pending_runtime_request_bytes(current: usize, incoming: usize) -> Option { + current + .checked_add(incoming) + .filter(|total| *total <= MAX_PENDING_RUNTIME_REQUEST_BYTES) +} + fn codex_dynamic_tools( authorized_tools: impl IntoIterator, ) -> Result<(Vec, BTreeSet), LocalRunnerError> { @@ -1079,12 +1544,31 @@ fn codex_dynamic_tools( fn bounded_identifier(value: Option<&str>, label: &str) -> Result { let value = value.ok_or_else(|| LocalRunnerError::invalid(format!("{label} is required")))?; - if value.is_empty() || value.len() > 160 || value.chars().any(char::is_control) { + let mut characters = value.chars(); + let valid_first = characters + .next() + .is_some_and(|character| character.is_ascii_alphanumeric()); + let valid_rest = characters.all(|character| { + character.is_ascii_alphanumeric() || matches!(character, '_' | '-' | '.' | ':') + }); + if value.len() > 160 || !valid_first || !valid_rest { return Err(LocalRunnerError::invalid(format!("{label} is invalid"))); } Ok(value.to_owned()) } +fn bounded_provider_turn_id(value: Option<&str>) -> Result { + let value = value + .filter(|value| !value.is_empty()) + .ok_or_else(|| LocalRunnerError::invalid("Codex turn/start omitted turn.id"))?; + if value.len() > 240 || value.chars().any(char::is_control) { + return Err(LocalRunnerError::invalid( + "Codex turn/start returned an invalid turn.id", + )); + } + Ok(value.to_owned()) +} + fn codex_tool_failure(message: &str) -> Value { json!({ "success": false, @@ -1160,6 +1644,17 @@ fn notification_turn_id(params: &Value) -> Option<&str> { .filter(|value| !value.is_empty()) } +fn request_targets_non_active_turn( + active_turn_id: Option<&str>, + settled_turn_ids: &SettledProviderTurnIds, + params: &Value, +) -> bool { + let requested_turn_id = params.get("turnId").and_then(Value::as_str); + requested_turn_id.is_some_and(|requested| { + settled_turn_ids.contains(requested) || active_turn_id != Some(requested) + }) +} + fn latest_active_turn_id(snapshot: &Value) -> Option { snapshot .pointer("/thread/turns") @@ -1296,6 +1791,33 @@ fn codex_question_set( )) } +fn new_runtime_request_scope() -> Result<[u8; 16], LocalRunnerError> { + let mut scope = [0u8; 16]; + getrandom::fill(&mut scope).map_err(|error| { + LocalRunnerError::invalid(format!( + "failed to mint Codex runtime request scope: {error}" + )) + })?; + Ok(scope) +} + +fn scoped_runtime_request_id( + scope: &[u8; 16], + turn_id: &str, + provider_request_id: &str, + request_sequence: u64, +) -> String { + let mut digest = Sha256::new(); + digest.update(scope); + digest.update([0]); + digest.update(turn_id.as_bytes()); + digest.update([0]); + digest.update(provider_request_id.as_bytes()); + digest.update([0]); + digest.update(request_sequence.to_be_bytes()); + format!("runtime-request-{:x}", digest.finalize()) +} + fn codex_question_response( pending: &PendingRuntimeRequest, response: &Value, @@ -1436,10 +1958,12 @@ mod tests { assert_eq!(question_set["schema"], "paperclip.question_set.v1"); let pending = PendingRuntimeRequest { rpc_id: json!(41), + turn_id: "turn-1".to_owned(), method: "item/tool/requestUserInput".to_owned(), params: Value::Null, question_set, option_labels: labels, + retained_bytes: 0, }; let native = codex_question_response( &pending, @@ -1461,6 +1985,23 @@ mod tests { }), ) .is_err()); + let scope = [7u8; 16]; + assert_eq!( + scoped_runtime_request_id(&scope, "turn-1", "41", 1), + scoped_runtime_request_id(&scope, "turn-1", "41", 1), + ); + assert_ne!( + scoped_runtime_request_id(&scope, "turn-1", "41", 1), + scoped_runtime_request_id(&scope, "turn-2", "41", 1), + ); + assert_ne!( + scoped_runtime_request_id(&scope, "turn-1", "41", 1), + scoped_runtime_request_id(&[8u8; 16], "turn-1", "41", 1), + ); + assert_ne!( + scoped_runtime_request_id(&scope, "turn-1", "41", 1), + scoped_runtime_request_id(&scope, "turn-1", "41", 2), + ); assert!(codex_question_response( &pending, &json!({ @@ -1503,6 +2044,87 @@ mod tests { .is_ok()); } + #[test] + fn rejects_requests_bound_to_any_non_active_turn_nonfatally() { + let mut turn_one_settled = SettledProviderTurnIds::default(); + turn_one_settled.insert("turn-1".to_owned()); + let mut turn_two_settled = SettledProviderTurnIds::default(); + turn_two_settled.insert("turn-2".to_owned()); + let no_settled_turns = SettledProviderTurnIds::default(); + assert!(request_targets_non_active_turn( + Some("turn-2"), + &turn_one_settled, + &json!({"turnId": "turn-1"}), + )); + assert!(request_targets_non_active_turn( + Some("turn-2"), + &turn_one_settled, + &json!({"turnId": "turn-0"}), + )); + assert!(request_targets_non_active_turn( + None, + &turn_two_settled, + &json!({"turnId": "turn-1"}), + )); + assert!(request_targets_non_active_turn( + Some("turn-1"), + &turn_one_settled, + &json!({"turnId": "turn-1"}), + )); + assert!(!request_targets_non_active_turn( + Some("turn-2"), + &turn_one_settled, + &json!({"turnId": "turn-2"}), + )); + assert!(!request_targets_non_active_turn( + None, + &no_settled_turns, + &json!({}), + )); + } + + #[test] + fn settled_provider_turn_history_never_evicts_exact_identities() { + let mut settled = SettledProviderTurnIds::default(); + for index in 0..MAX_SETTLED_PROVIDER_TURN_IDS { + assert!(settled.insert(format!("turn-{index}"))); + } + + assert_eq!(settled.ids.len(), MAX_SETTLED_PROVIDER_TURN_IDS); + assert!(settled.contains("turn-0")); + assert!(settled.contains("turn-1")); + assert!(settled.at_capacity()); + assert!(!settled.insert(format!("turn-{MAX_SETTLED_PROVIDER_TURN_IDS}"))); + assert!(!settled.contains(&format!("turn-{MAX_SETTLED_PROVIDER_TURN_IDS}"))); + assert_eq!(settled.ids.len(), MAX_SETTLED_PROVIDER_TURN_IDS); + assert!(settled.contains("turn-0")); + assert!(settled.filter.is_empty()); + } + + #[test] + fn restored_provider_turn_identity_is_retained_in_release_builds() { + let mut settled = SettledProviderTurnIds::default(); + + settled.restore("turn-restored".to_owned()).unwrap(); + + assert!(settled.contains("turn-restored")); + } + + #[test] + fn restores_the_complete_durable_provider_turn_ledger() { + let mut settled = SettledProviderTurnIds::default(); + + settled + .restore_all( + ["turn-older".to_owned(), "turn-latest".to_owned()], + DurableReplayFilter::default(), + ) + .unwrap(); + + assert!(settled.contains("turn-older")); + assert!(settled.contains("turn-latest")); + } + #[test] fn bounds_all_retained_pending_tool_request_data_in_aggregate() { let request_bytes = pending_tool_request_size([1, 2, 3, 4]).unwrap(); @@ -1516,4 +2138,30 @@ mod tests { assert!(retain_pending_tool_request_bytes(usize::MAX, 1).is_err()); assert!(pending_tool_request_size([usize::MAX, 1]).is_err()); } + + #[test] + fn bounds_all_retained_runtime_request_data_in_aggregate() { + assert_eq!( + retain_pending_runtime_request_bytes(MAX_PENDING_RUNTIME_REQUEST_BYTES - 10, 10,), + Some(MAX_PENDING_RUNTIME_REQUEST_BYTES) + ); + assert_eq!( + retain_pending_runtime_request_bytes(MAX_PENDING_RUNTIME_REQUEST_BYTES, 1), + None + ); + assert_eq!(retain_pending_runtime_request_bytes(usize::MAX, 1), None); + } + + #[test] + fn bounds_messages_buffered_while_waiting_for_a_response() { + assert_eq!( + retain_buffered_message_bytes(MAX_BUFFERED_MESSAGE_BYTES - 10, 10), + Some(MAX_BUFFERED_MESSAGE_BYTES) + ); + assert_eq!( + retain_buffered_message_bytes(MAX_BUFFERED_MESSAGE_BYTES, 1), + None + ); + assert_eq!(retain_buffered_message_bytes(usize::MAX, 1), None); + } } diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs index ef3ec61456..3bbf2f9311 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs @@ -9,12 +9,14 @@ use std::time::Duration; pub use runner::{run_durable_runner, CommandExecution, CommandExecutor, PolledEvent}; pub(crate) use state::{ - create_private_temporary_file, open_private_regular_file, redact_text, verify_private_directory, + create_private_temporary_file, open_private_regular_file, redact_text, sanitize_value, + verify_private_directory, }; pub use state::{ Command, CommandDisposition, DurableState, DurableStateStore, EventPriority, StoredCommandResult, StoredOutboxEvent, }; +pub(crate) use transport::current_unix_ms; pub const PROTOCOL: &str = "paperclip.runner"; pub const PROTOCOL_VERSION: u64 = 1; diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/state.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/state.rs index 50c6df265c..0e27f28f3c 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/state.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/state.rs @@ -1078,7 +1078,23 @@ fn sensitive_key(key: &str) -> bool { .any(|needle| normalized.contains(needle)) } -fn sanitize_value(value: &Value) -> Value { +fn protocol_authorization_boundary(key: &str, value: &Value) -> bool { + key.eq_ignore_ascii_case("authorizationBoundary") + && value.as_str().is_some_and(|boundary| { + matches!( + boundary, + "company" + | "actor" + | "active_task" + | "grant" + | "governed_action" + | "lock" + | "revision" + ) + }) +} + +pub(crate) fn sanitize_value(value: &Value) -> Value { match value { Value::Object(object) => Value::Object( object @@ -1086,7 +1102,9 @@ fn sanitize_value(value: &Value) -> Value { .map(|(key, value)| { ( key.clone(), - if sensitive_key(key) { + if protocol_authorization_boundary(key, value) { + value.clone() + } else if sensitive_key(key) { Value::String("[REDACTED]".to_owned()) } else { sanitize_value(value) @@ -1323,6 +1341,21 @@ mod tests { ); } + #[test] + fn protocol_authorization_boundary_is_not_redacted_as_a_credential() { + let sanitized = sanitize_value(&json!({ + "authorizationBoundary": "active_task", + "nested": {"authorizationBoundary": "Bearer secret-value"}, + "authorization": "Bearer secret-value", + })); + assert_eq!(sanitized["authorizationBoundary"], json!("active_task")); + assert_eq!(sanitized["authorization"], json!("[REDACTED]")); + assert_eq!( + sanitized["nested"]["authorizationBoundary"], + json!("[REDACTED]") + ); + } + #[test] fn outbox_reserves_capacity_for_p0_and_bounds_frames() { let mut bounds_config = config(PathBuf::from("unused")); diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs index 860362da03..5c037db769 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs @@ -11,17 +11,69 @@ use std::os::unix::fs::PermissionsExt; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; -use crate::codex_provider::{CodexProvider, CodexProviderConfig, CodexProviderEvent}; -use crate::durable::{ - create_private_temporary_file, open_private_regular_file, verify_private_directory, Command, - CommandExecution, CommandExecutor, DurableRunnerError, EventPriority, PolledEvent, +use crate::codex_provider::{ + CodexProvider, CodexProviderConfig, CodexProviderEvent, RejectedAcceptedTurn, + MAX_SETTLED_PROVIDER_TURN_IDS, +}; +use crate::durable::{ + create_private_temporary_file, current_unix_ms, open_private_regular_file, sanitize_value, + verify_private_directory, Command, CommandExecution, CommandExecutor, DurableRunnerConfig, + DurableRunnerError, EventPriority, PolledEvent, +}; +use crate::provider_bridge::{ + authorized_tool_catalog_digest, semantic_value_digest, AuthorizedToolSet, DurableReplayFilter, + PendingToolCall, ProviderBridgeError, ProviderToolBridge, ToolResult, MAX_PENDING_CALLS, + TOOL_SET_SCHEMA, +}; +use crate::provider_events::{ + normalize_codex_notification, normalized_codex_terminal_event_type, NormalizedProviderEvent, }; -use crate::provider_events::{normalize_codex_notification, NormalizedProviderEvent}; const PROVIDER_STATE_SCHEMA: &str = "paperclip.runner.codex-provider-state.v1"; const PROVIDER_STATE_FILE: &str = "codex-provider-state.json"; -const MAX_PROVIDER_STATE_BYTES: u64 = 2 * 1024 * 1024; +const MAX_PROVIDER_STATE_BYTES: u64 = 16 * 1024 * 1024; const MAX_EVENTS_PER_POLL: usize = 128; +// One accepted semantic call can produce an input and a result event. Normal +// traffic cannot consume the additional capacity required to diagnose a +// receipt-limit stop, settle every retained call, and record the provider plus +// run terminal events. +const MAX_REGULAR_QUEUED_PROVIDER_EVENTS: usize = 2 * MAX_PENDING_CALLS + 3; +const MAX_RECEIPT_LIMIT_TERMINAL_RESERVE: usize = MAX_PENDING_CALLS + 4; +// During a receipt-limit stop, provider polling continues even when older +// events remain unacknowledged so an already-buffered authoritative terminal +// wins over the deadline fallback. Reserve one complete poll of cleanup events +// in addition to the semantic-result and terminal envelopes. +const MAX_TERMINAL_SETTLEMENT_EVENTS: usize = MAX_PENDING_CALLS + MAX_EVENTS_PER_POLL + 4; +const MAX_QUEUED_PROVIDER_EVENTS: usize = + MAX_REGULAR_QUEUED_PROVIDER_EVENTS + MAX_TERMINAL_SETTLEMENT_EVENTS; +const MAX_RECEIPT_LIMIT_INTERRUPT_ATTEMPTS: u8 = 3; +const RECEIPT_LIMIT_INTERRUPT_TERMINAL_DEADLINE_MS: u64 = 2_000; +const RECEIPT_LIMIT_ACCEPTED_TERMINAL_DEADLINE_MS: u64 = 30_000; + +fn receipt_limit_deadline_after(timeout_ms: u64) -> Result { + current_unix_ms()?.checked_add(timeout_ms).ok_or_else(|| { + DurableRunnerError::invalid("Codex receipt-limit interruption deadline overflowed") + }) +} + +#[derive(Clone, Debug)] +struct ProviderEventIdentity { + run_id: String, + normalized_session_id: String, + turn_id: String, + item_id: String, +} + +impl ProviderEventIdentity { + fn from_config(config: &DurableRunnerConfig) -> Self { + Self { + run_id: config.run_id.clone(), + normalized_session_id: config.normalized_session_id.clone(), + turn_id: config.turn_id.clone(), + item_id: config.item_id.clone(), + } + } +} #[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] #[serde(rename_all = "camelCase")] @@ -70,6 +122,106 @@ fn completion_contract( Ok(Some(binding)) } +fn authorized_tool_set(payload: &Value) -> Result { + if let Some(value) = payload.get("authorizedTools") { + return serde_json::from_value(value.clone()).map_err(|error| { + DurableRunnerError::invalid(format!("run.prepare authorizedTools is invalid: {error}")) + }); + } + let operations = Vec::new(); + let catalog_digest = authorized_tool_catalog_digest(&operations).map_err(|error| { + DurableRunnerError::invalid(format!("empty authorized tool set is invalid: {error}")) + })?; + Ok(AuthorizedToolSet { + schema: TOOL_SET_SCHEMA.to_owned(), + schema_version: 1, + catalog_digest, + operations, + }) +} + +fn semantic_correlation(identity: &ProviderEventIdentity) -> Value { + json!({ + "runId": identity.run_id, + "normalizedSessionId": identity.normalized_session_id, + "turnId": identity.turn_id, + "itemId": identity.item_id, + }) +} + +fn semantic_input_event( + identity: &ProviderEventIdentity, + call: &PendingToolCall, +) -> NormalizedProviderEvent { + let safe_input = sanitize_value(&call.input); + NormalizedProviderEvent { + event_type: "semantic_tool.input".to_owned(), + priority: EventPriority::P0, + payload: json!({ + "semantic_tool": { + "schema": "paperclip.prp.semantic_tool.v1", + "schemaVersion": 1, + "phase": "input", + "operationId": call.operation_id, + "callId": call.call_id, + "correlation": semantic_correlation(identity), + "idempotencyKey": Value::Null, + "content": { + "digest": semantic_value_digest(&safe_input), + "redactionDisposition": "digest_only", + "references": [], + }, + "input": safe_input, + }, + }), + } +} + +fn semantic_result_event( + identity: &ProviderEventIdentity, + result: &ToolResult, +) -> NormalizedProviderEvent { + let safe_result = sanitize_value(&result.result); + let envelope = safe_result + .get("resultReceipt") + .filter(|receipt| { + receipt.get("schema").and_then(Value::as_str) + == Some("paperclip.prp.semantic_tool.v1") + && receipt.get("phase").and_then(Value::as_str) == Some("result") + && receipt.get("operationId").and_then(Value::as_str) + == Some(result.operation_id.as_str()) + && receipt.get("callId").and_then(Value::as_str) == Some(result.call_id.as_str()) + && receipt.get("correlation") == Some(&semantic_correlation(identity)) + }) + .cloned() + .unwrap_or_else(|| { + json!({ + "schema": "paperclip.prp.semantic_tool.v1", + "schemaVersion": 1, + "phase": "result", + "operationId": result.operation_id, + "callId": result.call_id, + "correlation": semantic_correlation(identity), + "idempotencyKey": Value::Null, + "content": { + "digest": semantic_value_digest(&safe_result), + "redactionDisposition": "digest_only", + "references": [], + }, + "outcome": if result.is_error { "failed" } else { "succeeded" }, + "code": if result.is_error { "semantic_tool_failed" } else { "semantic_tool_succeeded" }, + "retryable": false, + "authorizationBoundary": "active_task", + "operationReceiptId": format!("operation_{}", result.call_id), + }) + }); + NormalizedProviderEvent { + event_type: "semantic_tool.result".to_owned(), + priority: EventPriority::P0, + payload: json!({"semantic_tool": envelope}), + } +} + fn terminal_events(state: &CodexProviderState, event_type: &str) -> Vec { let Some(contract) = state.completion_contract.as_ref() else { return Vec::new(); @@ -158,6 +310,8 @@ struct CodexProviderState { #[serde(default)] completion_contract: Option, #[serde(default)] + tool_bridge: ProviderToolBridge, + #[serde(default)] thread_id: Option, #[serde(default)] provider_session_id: Option, @@ -173,17 +327,69 @@ struct CodexProviderState { completed_turn_process_generation: Option, #[serde(default)] completed_provider_turn_id: Option, + // Unlike the live provider process, the durable run survives restarts. + // Terminal identities stay exact for one process-generation epoch. At the + // bound, an idle process is reaped before this epoch is rotated. + #[serde(default)] + settled_provider_turn_ids: std::collections::BTreeSet, + #[serde(default)] + settled_provider_turn_filter: DurableReplayFilter, + #[serde(default)] + receipt_limit_diagnostic_emitted: bool, + #[serde(default)] + receipt_limit_interrupt_pending: bool, + #[serde(default)] + receipt_limit_interrupt_accepted: bool, + #[serde(default)] + receipt_limit_interrupt_attempts: u8, + #[serde(default)] + receipt_limit_interrupt_deadline_unix_ms: Option, last_agent_message: Option, #[serde(default)] pending_events: VecDeque, + #[serde(default)] + queued_events: VecDeque, #[serde(default = "initial_provider_event_seq")] next_provider_event_seq: u64, } +#[derive(Debug, PartialEq)] +enum ToolCallAdmission { + CompletedReplay(ToolResult), + PendingReplay, + Pending(PendingToolCall), +} + +fn settled_provider_turn_contains( + identities: &std::collections::BTreeSet, + _filter: &DurableReplayFilter, + provider_turn_id: &str, +) -> bool { + identities.contains(provider_turn_id) +} + +fn remember_settled_provider_turn( + identities: &mut std::collections::BTreeSet, + _filter: &mut DurableReplayFilter, + provider_turn_id: String, +) -> Result<(), DurableRunnerError> { + if identities.contains(&provider_turn_id) { + return Ok(()); + } + if identities.len() >= MAX_SETTLED_PROVIDER_TURN_IDS { + return Err(DurableRunnerError::invalid( + "Codex provider turn identity epoch reached its exact capacity", + )); + } + identities.insert(provider_turn_id); + Ok(()) +} + impl CodexProviderState { fn new( config: CodexProviderConfig, completion_contract: Option, + tool_bridge: ProviderToolBridge, ) -> Self { let thread_id = config.provider_session_id.clone(); Self { @@ -191,6 +397,7 @@ impl CodexProviderState { lifecycle: "prepared".to_owned(), config, completion_contract, + tool_bridge, thread_id, provider_session_id: None, active_provider_turn_id: None, @@ -199,8 +406,16 @@ impl CodexProviderState { provider_process_generation: 0, completed_turn_process_generation: None, completed_provider_turn_id: None, + settled_provider_turn_ids: std::collections::BTreeSet::new(), + settled_provider_turn_filter: DurableReplayFilter::default(), + receipt_limit_diagnostic_emitted: false, + receipt_limit_interrupt_pending: false, + receipt_limit_interrupt_accepted: false, + receipt_limit_interrupt_attempts: 0, + receipt_limit_interrupt_deadline_unix_ms: None, last_agent_message: None, pending_events: VecDeque::new(), + queued_events: VecDeque::new(), next_provider_event_seq: initial_provider_event_seq(), } } @@ -209,6 +424,9 @@ impl CodexProviderState { self.config .validate() .map_err(|error| DurableRunnerError::invalid(error.to_string()))?; + self.tool_bridge.validate_recovered().map_err(|error| { + DurableRunnerError::invalid(format!("Codex semantic tool state is invalid: {error}")) + })?; let mut pending_event_ids = HashSet::new(); if self.schema != PROVIDER_STATE_SCHEMA || !matches!( @@ -262,24 +480,59 @@ impl CodexProviderState { || (!self.completed_turn_authoritative && (self.completed_turn_process_generation.is_some() || self.completed_provider_turn_id.is_some())) + || self.settled_provider_turn_ids.len() > MAX_SETTLED_PROVIDER_TURN_IDS + || self.settled_provider_turn_filter.validate().is_err() + || self + .settled_provider_turn_ids + .iter() + .any(|provider_turn_id| { + provider_turn_id.is_empty() + || provider_turn_id.len() > 240 + || provider_turn_id.chars().any(char::is_control) + }) + || self + .active_provider_turn_id + .as_ref() + .is_some_and(|provider_turn_id| { + settled_provider_turn_contains( + &self.settled_provider_turn_ids, + &self.settled_provider_turn_filter, + provider_turn_id, + ) + }) || self .completed_turn_process_generation .is_some_and(|generation| generation > self.provider_process_generation) + || (self.receipt_limit_diagnostic_emitted && self.active_provider_turn_id.is_none()) + || (self.receipt_limit_interrupt_pending + && (!self.receipt_limit_diagnostic_emitted + || self.active_provider_turn_id.is_none())) + || (self.receipt_limit_interrupt_accepted && !self.receipt_limit_interrupt_pending) + || self.receipt_limit_interrupt_attempts > MAX_RECEIPT_LIMIT_INTERRUPT_ATTEMPTS + || (!self.receipt_limit_interrupt_pending && self.receipt_limit_interrupt_attempts != 0) + || self + .receipt_limit_interrupt_deadline_unix_ms + .is_some_and(|deadline| deadline == 0 || !self.receipt_limit_interrupt_pending) || (matches!( self.lifecycle.as_str(), "prepared" | "session_open" | "closed" ) && self.active_provider_turn_id.is_some()) || self.next_provider_event_seq == 0 || self.pending_events.len() > MAX_EVENTS_PER_POLL + 3 - || self.pending_events.iter().any(|event| { - provider_event_sequence(&event.executor_event_id) - .is_none_or(|sequence| sequence >= self.next_provider_event_seq) - || !pending_event_ids.insert(event.executor_event_id.as_str()) - || event.event_type.is_empty() - || event.event_type.len() > 160 - || event.event_type.chars().any(char::is_control) - || !event.payload.is_object() - }) + || self.queued_events.len() > MAX_QUEUED_PROVIDER_EVENTS + || self + .pending_events + .iter() + .chain(self.queued_events.iter()) + .any(|event| { + provider_event_sequence(&event.executor_event_id) + .is_none_or(|sequence| sequence >= self.next_provider_event_seq) + || !pending_event_ids.insert(event.executor_event_id.as_str()) + || event.event_type.is_empty() + || event.event_type.len() > 160 + || event.event_type.chars().any(char::is_control) + || !event.payload.is_object() + }) { return Err(DurableRunnerError::invalid( "Codex provider state is malformed or inconsistent", @@ -288,20 +541,131 @@ impl CodexProviderState { Ok(()) } - fn push_event(&mut self, event: NormalizedProviderEvent) -> Result<(), DurableRunnerError> { + fn push_event_with_limit( + &mut self, + event: NormalizedProviderEvent, + max_queued_events: usize, + ) -> Result<(), DurableRunnerError> { + let queue_event = + !self.queued_events.is_empty() || self.pending_events.len() >= MAX_EVENTS_PER_POLL; + if queue_event && self.queued_events.len() >= max_queued_events { + return Err(DurableRunnerError::invalid( + "Codex provider event backlog exceeds its durable limit", + )); + } let sequence = self.next_provider_event_seq; self.next_provider_event_seq = sequence .checked_add(1) .ok_or_else(|| DurableRunnerError::invalid("provider event sequence exhausted"))?; - self.pending_events.push_back(PolledEvent { + let event = PolledEvent { executor_event_id: provider_event_id(sequence), event_type: event.event_type, priority: event.priority, payload: event.payload, - }); + }; + if queue_event { + self.queued_events.push_back(event); + } else { + self.pending_events.push_back(event); + } Ok(()) } + fn begin_receipt_limit_stop( + &mut self, + call_id: String, + operation_id: String, + deadline_unix_ms: u64, + ) -> Result { + if self.receipt_limit_diagnostic_emitted { + return Ok(self.receipt_limit_interrupt_pending); + } + self.push_terminal_event(NormalizedProviderEvent { + event_type: "harness.diagnostic".to_owned(), + priority: EventPriority::P0, + payload: json!({ + "provider": "codex", + "code": "semantic_tool_turn_receipt_limit", + "operationId": operation_id, + "callId": call_id, + "message": "The active provider turn reached its durable semantic-tool receipt limit and was interrupted", + "paperclipExecuted": false, + }), + })?; + self.receipt_limit_diagnostic_emitted = true; + self.receipt_limit_interrupt_pending = true; + self.receipt_limit_interrupt_attempts = 0; + self.receipt_limit_interrupt_deadline_unix_ms = Some(deadline_unix_ms); + Ok(true) + } + + fn record_receipt_limit_interrupt_attempt(&mut self) -> Result<(), DurableRunnerError> { + if self.receipt_limit_interrupt_attempts >= MAX_RECEIPT_LIMIT_INTERRUPT_ATTEMPTS { + return Err(DurableRunnerError::invalid( + "Codex receipt-limit interruption exceeded its durable retry bound", + )); + } + self.receipt_limit_interrupt_attempts += 1; + Ok(()) + } + + fn mark_receipt_limit_interrupt_accepted(&mut self, accepted_deadline_unix_ms: u64) { + // Provider restoration can reconcile the active turn as already + // settled while an interruption command is in flight. In that case + // `interrupt_turn` returns `already_settled` and recovery has already + // cleared the durable retry marker. Do not recreate an accepted state + // without a pending interruption or active turn. + if !self.receipt_limit_interrupt_pending || self.active_provider_turn_id.is_none() { + return; + } + if !self.receipt_limit_interrupt_accepted { + self.receipt_limit_interrupt_deadline_unix_ms = Some( + self.receipt_limit_interrupt_deadline_unix_ms + .unwrap_or_default() + .max(accepted_deadline_unix_ms), + ); + } + self.receipt_limit_interrupt_accepted = true; + } + + fn push_event(&mut self, event: NormalizedProviderEvent) -> Result<(), DurableRunnerError> { + self.push_event_with_limit(event, MAX_REGULAR_QUEUED_PROVIDER_EVENTS) + } + + fn push_terminal_event( + &mut self, + event: NormalizedProviderEvent, + ) -> Result<(), DurableRunnerError> { + self.push_event_with_limit(event, MAX_QUEUED_PROVIDER_EVENTS) + } + + fn push_receipt_limit_cleanup_event( + &mut self, + event: NormalizedProviderEvent, + ) -> Result<(), DurableRunnerError> { + let queue_event = + !self.queued_events.is_empty() || self.pending_events.len() >= MAX_EVENTS_PER_POLL; + if queue_event + && self.queued_events.len() + >= MAX_QUEUED_PROVIDER_EVENTS - MAX_RECEIPT_LIMIT_TERMINAL_RESERVE + { + // Continue draining the provider so an authoritative terminal can + // still be observed, but never let cleanup chatter consume the + // semantic-result and terminal-event reserve. + return Ok(()); + } + self.push_terminal_event(event) + } + + fn refill_pending_events(&mut self) { + while self.pending_events.len() < MAX_EVENTS_PER_POLL { + let Some(event) = self.queued_events.pop_front() else { + break; + }; + self.pending_events.push_back(event); + } + } + fn extend_events( &mut self, events: impl IntoIterator, @@ -312,6 +676,43 @@ impl CodexProviderState { Ok(()) } + fn admit_tool_call( + &mut self, + call_id: &str, + operation_id: &str, + input: &Value, + ) -> Result { + if let Some(result) = self + .tool_bridge + .replay_result(call_id, operation_id, input)? + { + // An exact completed replay is a transport retry. Its input and + // result receipts are already durable, so recording another event + // would make an otherwise idempotent replay consume bounded event + // capacity and could prevent returning the stored result. + return Ok(ToolCallAdmission::CompletedReplay(result)); + } + + let pending_replay = self + .tool_bridge + .pending_calls() + .any(|pending| pending.call_id == call_id); + let call = self.tool_bridge.begin_call( + call_id.to_owned(), + operation_id.to_owned(), + input.clone(), + )?; + if pending_replay { + // The first input receipt is already durable. An exact pending + // replay is only the provider re-establishing its request after a + // process restart; appending another event would make the retry + // consume bounded backlog capacity without adding information. + Ok(ToolCallAdmission::PendingReplay) + } else { + Ok(ToolCallAdmission::Pending(call)) + } + } + fn reconcile_active_provider_turn(&mut self, active_provider_turn_id: Option) { self.active_provider_turn_id = active_provider_turn_id; if self.active_provider_turn_id.is_some() { @@ -330,12 +731,53 @@ impl CodexProviderState { "session_open".to_owned() }; } + + fn settle_active_provider_turn_identity(&mut self) -> Result<(), DurableRunnerError> { + let provider_turn_id = self.active_provider_turn_id.clone().ok_or_else(|| { + DurableRunnerError::invalid("Codex terminal omitted its active provider turn identity") + })?; + remember_settled_provider_turn( + &mut self.settled_provider_turn_ids, + &mut self.settled_provider_turn_filter, + provider_turn_id, + )?; + Ok(()) + } + + fn recovered_settled_provider_turn_ids( + &self, + ) -> Result<(std::collections::BTreeSet, DurableReplayFilter), DurableRunnerError> { + let mut settled_provider_turn_ids = self.settled_provider_turn_ids.clone(); + let mut settled_provider_turn_filter = self.settled_provider_turn_filter.clone(); + // State written before the durable set was introduced retained only + // the latest completed identity. Fold that legacy authority into the + // new ledger before the provider is allowed to accept replacement work. + if let Some(provider_turn_id) = self.completed_provider_turn_id.clone() { + remember_settled_provider_turn( + &mut settled_provider_turn_ids, + &mut settled_provider_turn_filter, + provider_turn_id, + )?; + } + Ok((settled_provider_turn_ids, settled_provider_turn_filter)) + } + + fn extend_terminal_events( + &mut self, + events: impl IntoIterator, + ) -> Result<(), DurableRunnerError> { + for event in events { + self.push_terminal_event(event)?; + } + Ok(()) + } } pub struct CodexCommandExecutor { state_dir: PathBuf, state: Option, provider: Option, + event_identity: Option, restore_checked: bool, } @@ -345,10 +787,17 @@ impl CodexCommandExecutor { state_dir: state_dir.into(), state: None, provider: None, + event_identity: None, restore_checked: false, } } + pub fn with_runner_config(state_dir: impl Into, config: &DurableRunnerConfig) -> Self { + let mut executor = Self::new(state_dir); + executor.event_identity = Some(ProviderEventIdentity::from_config(config)); + executor + } + fn state_path(&self) -> PathBuf { self.state_dir.join(PROVIDER_STATE_FILE) } @@ -380,9 +829,14 @@ impl CodexCommandExecutor { file.read_to_end(&mut input).map_err(|error| { DurableRunnerError::invalid(format!("failed to read Codex provider state: {error}")) })?; - let state: CodexProviderState = serde_json::from_slice(&input).map_err(|error| { + let mut state: CodexProviderState = serde_json::from_slice(&input).map_err(|error| { DurableRunnerError::invalid(format!("Codex provider state is malformed: {error}")) })?; + state.tool_bridge.attach_existing_run().map_err(|error| { + DurableRunnerError::invalid(format!( + "Codex semantic tool state could not be reattached: {error}" + )) + })?; state.validate()?; self.state = Some(state); self.restore_provider_if_needed() @@ -413,15 +867,139 @@ impl CodexCommandExecutor { let completed_turn_process_generation = state.completed_turn_process_generation; let completed_provider_turn_id = state.completed_provider_turn_id.clone(); let ambiguous_turn_start_pending = state.ambiguous_turn_start_pending; - let mut provider = CodexProvider::start_for_generation( + let tool_replay_history_blocks_admission = + state.tool_bridge.replay_history_blocks_admission(); + let tool_receipt_epoch_has_active_receipts = state.tool_bridge.has_active_receipts(); + let (settled_provider_turn_ids, settled_provider_turn_filter) = + state.recovered_settled_provider_turn_ids()?; + let provider_epoch_requires_rollover = settled_provider_turn_ids.len() + >= MAX_SETTLED_PROVIDER_TURN_IDS + || !settled_provider_turn_filter.is_empty(); + let mut provider = CodexProvider::start_with_tools_for_generation( &state.config, + state.tool_bridge.authorized_tools().cloned(), Some(&thread_id), process_generation, ) .map_err(|error| { DurableRunnerError::invalid(format!("failed to resume Codex provider: {error}")) })?; + provider.enable_durable_tool_call_replays(); + provider + .restore_settled_turn_identities( + settled_provider_turn_ids.iter().cloned(), + settled_provider_turn_filter.clone(), + ) + .map_err(|error| { + DurableRunnerError::invalid(format!( + "failed to restore Codex provider turn identities: {error}" + )) + })?; let recovered_active_turn_id = provider.active_provider_turn_id().map(str::to_owned); + let legacy_epoch_is_ambiguous = (provider_epoch_requires_rollover + && (ambiguous_turn_start_pending || recovered_active_turn_id.is_some())) + || (tool_replay_history_blocks_admission + && (ambiguous_turn_start_pending + || recovered_active_turn_id.is_some() + || (previous_active_turn_id.is_some() + && tool_receipt_epoch_has_active_receipts))); + if legacy_epoch_is_ambiguous { + // A saturated legacy epoch cannot prove that recovered work can + // be identified and settled exactly. Reap the resumed process + // generation and close the run instead of risking duplicate work. + let provider_reported_active = recovered_active_turn_id.is_some(); + let provider_shutdown_failed = provider.shutdown().is_err(); + drop(provider); + let state = self + .state + .as_mut() + .expect("Codex state remains available during legacy recovery"); + state.provider_process_generation = process_generation; + state.settled_provider_turn_ids = settled_provider_turn_ids; + state.settled_provider_turn_filter = settled_provider_turn_filter; + state.active_provider_turn_id = None; + state.ambiguous_turn_start_pending = false; + state.completed_turn_authoritative = false; + state.completed_turn_process_generation = None; + state.completed_provider_turn_id = None; + state.receipt_limit_diagnostic_emitted = false; + state.receipt_limit_interrupt_pending = false; + state.receipt_limit_interrupt_accepted = false; + state.receipt_limit_interrupt_attempts = 0; + state.receipt_limit_interrupt_deadline_unix_ms = None; + state.last_agent_message = None; + state.lifecycle = "closed".to_owned(); + let _ = state.push_terminal_event(NormalizedProviderEvent { + event_type: "harness.diagnostic".to_owned(), + priority: EventPriority::P0, + payload: json!({ + "provider": "codex", + "code": "legacy_provider_turn_epoch_ambiguous", + "message": "Codex recovery could not safely identify and settle active work from a saturated legacy replay epoch; Paperclip terminated the provider and closed the durable run", + "paperclipAccepted": false, + "providerReportedActive": provider_reported_active, + "ambiguousStartPending": ambiguous_turn_start_pending, + "providerShutdownFailed": provider_shutdown_failed, + }), + }); + self.save_state()?; + return Ok(()); + } + if let Some(reused_provider_turn_id) = recovered_active_turn_id + .as_ref() + .filter(|provider_turn_id| { + settled_provider_turn_contains( + &settled_provider_turn_ids, + &settled_provider_turn_filter, + provider_turn_id, + ) + }) + .cloned() + { + // The durable terminal ledger is authoritative. A resumed provider + // that reports one of those identities as active is contradictory + // and may still be mutating the workspace. Terminate that process + // generation and persist the run closed before exposing recovery + // to the controller; otherwise this path would reopen settled work. + let provider_shutdown_failed = provider.shutdown().is_err(); + let state = self + .state + .as_mut() + .expect("Codex state remains available during recovery"); + state.provider_process_generation = process_generation; + state.settled_provider_turn_ids = settled_provider_turn_ids; + state.settled_provider_turn_filter = settled_provider_turn_filter; + state.active_provider_turn_id = None; + state.ambiguous_turn_start_pending = false; + state.completed_turn_authoritative = false; + state.completed_turn_process_generation = None; + state.completed_provider_turn_id = None; + state.receipt_limit_diagnostic_emitted = false; + state.receipt_limit_interrupt_pending = false; + state.receipt_limit_interrupt_accepted = false; + state.receipt_limit_interrupt_attempts = 0; + state.receipt_limit_interrupt_deadline_unix_ms = None; + state.last_agent_message = None; + state.lifecycle = "closed".to_owned(); + // Closing the provider is the safety boundary. Preserve that + // durable transition even when an already-full event backlog has + // no room for an additional diagnostic. + let _ = state.push_terminal_event(NormalizedProviderEvent { + event_type: "harness.diagnostic".to_owned(), + priority: EventPriority::P0, + payload: json!({ + "provider": "codex", + "code": "provider_turn_identity_reused", + "providerTurnId": reused_provider_turn_id, + "message": "Codex recovery reported a previously settled turn identity as active; Paperclip terminated the provider and closed the durable run", + "paperclipAccepted": false, + "providerReportedActive": true, + "providerShutdownFailed": provider_shutdown_failed, + }), + }); + self.save_state()?; + return Ok(()); + } if ambiguous_turn_start_pending { let recovered_turn_id = recovered_active_turn_id.as_deref().ok_or_else(|| { DurableRunnerError::invalid( @@ -434,37 +1012,99 @@ impl CodexCommandExecutor { )); } } - provider.restore_completed_turn_authority( - completed_turn_authoritative - && recovered_active_turn_id.is_none() - && !ambiguous_turn_start_pending, - completed_turn_process_generation, - completed_provider_turn_id.as_deref(), - ); + provider + .restore_completed_turn_authority( + completed_turn_authoritative + && recovered_active_turn_id.is_none() + && !ambiguous_turn_start_pending, + completed_turn_process_generation, + completed_provider_turn_id.as_deref(), + ) + .map_err(|error| { + DurableRunnerError::invalid(format!( + "failed to restore Codex completion authority: {error}" + )) + })?; self.provider = Some(provider); - self.state - .as_mut() - .expect("Codex state remains available during recovery") - .provider_process_generation = process_generation; - if provider_had_exited - || ambiguous_turn_start_pending - || recovered_active_turn_id != previous_active_turn_id { let state = self .state .as_mut() .expect("Codex state remains available during recovery"); + state.provider_process_generation = process_generation; + state.settled_provider_turn_ids = settled_provider_turn_ids; + state.settled_provider_turn_filter = settled_provider_turn_filter; + } + if provider_had_exited + || ambiguous_turn_start_pending + || recovered_active_turn_id != previous_active_turn_id + { + let recovered_turn_ended = + previous_active_turn_id.is_some() && recovered_active_turn_id.is_none(); + let identity = self.event_identity.clone(); + let state = self + .state + .as_mut() + .expect("Codex state remains available during recovery"); + if recovered_turn_ended { + if !provider_epoch_requires_rollover { + state.settle_active_provider_turn_identity()?; + } + let settled = state + .tool_bridge + .settle_turn("provider_turn_terminated") + .map_err(|error| { + DurableRunnerError::invalid(format!( + "failed to settle semantic tools during recovery: {error}" + )) + })?; + if !settled.is_empty() { + let identity = identity.as_ref().ok_or_else(|| { + DurableRunnerError::invalid( + "Codex semantic tool events require the durable runner identity", + ) + })?; + for result in settled { + state.push_terminal_event(semantic_result_event(identity, &result))?; + } + } + state.receipt_limit_diagnostic_emitted = false; + state.receipt_limit_interrupt_pending = false; + state.receipt_limit_interrupt_accepted = false; + state.receipt_limit_interrupt_attempts = 0; + state.receipt_limit_interrupt_deadline_unix_ms = None; + } state.reconcile_active_provider_turn(recovered_active_turn_id.clone()); - state.push_event(NormalizedProviderEvent { + let reconciled = NormalizedProviderEvent { event_type: "session.reconciled".to_owned(), priority: EventPriority::P0, payload: json!({ "provider": "codex", "providerSessionId": thread_id, - "previousProviderTurnId": previous_active_turn_id, - "activeProviderTurnId": recovered_active_turn_id, + "previousProviderTurnId": previous_active_turn_id.clone(), + "activeProviderTurnId": recovered_active_turn_id.clone(), }), - })?; + }; + if recovered_turn_ended { + state.push_terminal_event(reconciled)?; + // A turn that disappeared while runnerd was offline has no + // trustworthy success notification to replay. Terminate it + // conservatively so the controller cannot wait forever or + // mistake an unknown outcome for success. + state.push_terminal_event(NormalizedProviderEvent { + event_type: "turn.failed".to_owned(), + priority: EventPriority::P0, + payload: json!({ + "provider": "codex", + "providerTurnId": previous_active_turn_id, + "status": "failed", + "providerTerminalObserved": false, + }), + })?; + state.extend_terminal_events(terminal_events(state, "turn.failed"))?; + } else { + state.push_event(reconciled)?; + } } self.save_state()?; Ok(()) @@ -502,7 +1142,7 @@ impl CodexCommandExecutor { })?; if bytes.len() as u64 > MAX_PROVIDER_STATE_BYTES { return Err(DurableRunnerError::invalid( - "Codex provider state exceeds the 2 MiB limit", + "Codex provider state exceeds the 16 MiB limit", )); } let (temporary, mut file) = create_private_temporary_file(&path)?; @@ -542,7 +1182,8 @@ impl CodexCommandExecutor { .validate() .map_err(|error| DurableRunnerError::invalid(error.to_string()))?; let completion_contract = completion_contract(payload)?; - if let Some(state) = &self.state { + let tool_set = authorized_tool_set(payload)?; + if let Some(state) = self.state.as_mut() { if state.config != config || state.completion_contract != completion_contract { return Err(DurableRunnerError::invalid( "Codex provider or completion contract changed across the durable run", @@ -553,8 +1194,33 @@ impl CodexCommandExecutor { "Codex provider session is already closed", )); } + if state.tool_bridge.has_catalog() { + state + .tool_bridge + .verify_tool_set(&tool_set) + .map_err(|error| { + DurableRunnerError::invalid(format!( + "run.prepare tool contract changed: {error}" + )) + })?; + } else { + state.tool_bridge.prepare(tool_set).map_err(|error| { + DurableRunnerError::invalid(format!( + "run.prepare tool contract rejected: {error}" + )) + })?; + self.save_state()?; + } } else { - self.state = Some(CodexProviderState::new(config, completion_contract)); + let mut tool_bridge = ProviderToolBridge::default(); + tool_bridge.prepare(tool_set).map_err(|error| { + DurableRunnerError::invalid(format!("run.prepare tool contract rejected: {error}")) + })?; + self.state = Some(CodexProviderState::new( + config, + completion_contract, + tool_bridge, + )); self.save_state()?; } Ok(CommandExecution::result(json!({ @@ -579,24 +1245,50 @@ impl CodexCommandExecutor { .provider_process_generation .checked_add(1) .ok_or_else(|| DurableRunnerError::invalid("Codex process generation exhausted"))?; - let mut provider = CodexProvider::start_for_generation( + let (settled_provider_turn_ids, settled_provider_turn_filter) = + state.recovered_settled_provider_turn_ids()?; + let mut provider = CodexProvider::start_with_tools_for_generation( &state.config, + state.tool_bridge.authorized_tools().cloned(), state.thread_id.as_deref(), process_generation, ) .map_err(|error| { DurableRunnerError::invalid(format!("failed to start Codex provider: {error}")) })?; - provider.restore_completed_turn_authority( - state.completed_turn_authoritative && provider.active_provider_turn_id().is_none(), - state.completed_turn_process_generation, - state.completed_provider_turn_id.as_deref(), - ); + provider.enable_durable_tool_call_replays(); + provider + .restore_settled_turn_identities( + settled_provider_turn_ids.iter().cloned(), + settled_provider_turn_filter.clone(), + ) + .map_err(|error| { + DurableRunnerError::invalid(format!( + "failed to restore Codex provider turn identities: {error}" + )) + })?; + provider + .restore_completed_turn_authority( + state.completed_turn_authoritative + && provider.active_provider_turn_id().is_none(), + state.completed_turn_process_generation, + state.completed_provider_turn_id.as_deref(), + ) + .map_err(|error| { + DurableRunnerError::invalid(format!( + "failed to restore Codex completion authority: {error}" + )) + })?; self.provider = Some(provider); - self.state - .as_mut() - .expect("Codex state remains available after provider start") - .provider_process_generation = process_generation; + { + let state = self + .state + .as_mut() + .expect("Codex state remains available after provider start"); + state.provider_process_generation = process_generation; + state.settled_provider_turn_ids = settled_provider_turn_ids; + state.settled_provider_turn_filter = settled_provider_turn_filter; + } self.save_state()?; } self.provider @@ -604,6 +1296,21 @@ impl CodexCommandExecutor { .ok_or_else(|| DurableRunnerError::invalid("Codex provider is unavailable")) } + fn verify_attached_tools(&self, payload: &Value) -> Result<(), DurableRunnerError> { + if payload.get("authorizedTools").is_none() { + return Ok(()); + } + let tool_set = authorized_tool_set(payload)?; + self.state + .as_ref() + .ok_or_else(|| DurableRunnerError::invalid("Codex provider has not been prepared"))? + .tool_bridge + .verify_tool_set(&tool_set) + .map_err(|error| { + DurableRunnerError::invalid(format!("run.attach tool contract changed: {error}")) + }) + } + fn open_session(&mut self) -> Result { self.restore_provider_if_needed()?; if self @@ -636,6 +1343,11 @@ impl CodexCommandExecutor { state.thread_id = Some(thread_id.clone()); state.provider_session_id = provider_session_id.clone(); state.active_provider_turn_id = None; + state.receipt_limit_diagnostic_emitted = false; + state.receipt_limit_interrupt_pending = false; + state.receipt_limit_interrupt_accepted = false; + state.receipt_limit_interrupt_attempts = 0; + state.receipt_limit_interrupt_deadline_unix_ms = None; state.lifecycle = "session_open".to_owned(); state.config.provider_version.clone() }; @@ -667,8 +1379,156 @@ impl CodexCommandExecutor { }) } + fn close_after_rejected_provider_acceptance( + &mut self, + rejected_accepted_turn: &RejectedAcceptedTurn, + ) -> Result<(), DurableRunnerError> { + let provider_process_generation = self + .provider + .as_ref() + .map(CodexProvider::process_generation); + // The provider has already been terminated. Drop its quarantined + // handle before persisting the closure so recovery can never resume + // work that Codex accepted without Paperclip accepting its identity. + self.provider = None; + let state = self + .state + .as_mut() + .expect("Codex state remains available after rejected provider acceptance"); + if let Some(provider_process_generation) = provider_process_generation { + state.provider_process_generation = provider_process_generation; + } + state.active_provider_turn_id = None; + state.ambiguous_turn_start_pending = false; + state.completed_turn_authoritative = false; + state.completed_turn_process_generation = None; + state.completed_provider_turn_id = None; + state.receipt_limit_diagnostic_emitted = false; + state.receipt_limit_interrupt_pending = false; + state.receipt_limit_interrupt_accepted = false; + state.receipt_limit_interrupt_attempts = 0; + state.receipt_limit_interrupt_deadline_unix_ms = None; + state.last_agent_message = None; + state.lifecycle = "closed".to_owned(); + // Closure is the safety boundary. Preserve it even if a saturated + // event queue cannot retain this additional diagnostic. + let _ = state.push_terminal_event(NormalizedProviderEvent { + event_type: "harness.diagnostic".to_owned(), + priority: EventPriority::P0, + payload: json!({ + "provider": "codex", + "code": match rejected_accepted_turn { + RejectedAcceptedTurn::ReusedIdentity(_) => "provider_turn_identity_reused", + RejectedAcceptedTurn::InvalidIdentity => "provider_turn_identity_invalid", + }, + "providerTurnId": match rejected_accepted_turn { + RejectedAcceptedTurn::ReusedIdentity(provider_turn_id) => json!(provider_turn_id), + RejectedAcceptedTurn::InvalidIdentity => Value::Null, + }, + "message": match rejected_accepted_turn { + RejectedAcceptedTurn::ReusedIdentity(_) => "Codex accepted work with a previously settled turn identity; Paperclip terminated the provider and closed the durable run", + RejectedAcceptedTurn::InvalidIdentity => "Codex accepted work without a valid bounded turn identity; Paperclip terminated the provider and closed the durable run", + }, + "paperclipAccepted": false, + "providerAccepted": true, + }), + }); + self.save_state() + } + + fn rollover_provider_identity_epochs_if_needed(&mut self) -> Result<(), DurableRunnerError> { + let tool_rollover_required = self + .state + .as_ref() + .is_some_and(|state| state.tool_bridge.replay_history_blocks_admission()); + let rollover_required = self.state.as_ref().is_some_and(|state| { + state.settled_provider_turn_ids.len() >= MAX_SETTLED_PROVIDER_TURN_IDS + || !state.settled_provider_turn_filter.is_empty() + || tool_rollover_required + }); + if !rollover_required { + return Ok(()); + } + let rollover_is_safe = self.state.as_ref().is_some_and(|state| { + state.active_provider_turn_id.is_none() + && !state.ambiguous_turn_start_pending + && state.lifecycle == "session_open" + }); + if !rollover_is_safe { + return Err(DurableRunnerError::invalid( + "Codex provider identity epoch cannot rotate while work is active", + )); + } + + let (restart_result, process_generation, rejected_accepted_turn) = { + let provider = self.provider.as_mut().ok_or_else(|| { + DurableRunnerError::invalid( + "Codex provider identity epoch cannot rotate without an attached process", + ) + })?; + let restart_result = provider.restart_idle_identity_epoch(); + ( + restart_result, + provider.process_generation(), + provider.take_rejected_accepted_turn(), + ) + }; + if let Err(error) = restart_result { + if let Some(rejected_accepted_turn) = rejected_accepted_turn { + let failure_kind = match &rejected_accepted_turn { + RejectedAcceptedTurn::ReusedIdentity(_) => "accepted identity reuse", + RejectedAcceptedTurn::InvalidIdentity => "an invalid accepted identity", + }; + self.close_after_rejected_provider_acceptance(&rejected_accepted_turn)?; + return Err(DurableRunnerError::invalid(format!( + "Codex identity epoch rollover failed closed after {failure_kind}: {error}" + ))); + } + return Err(DurableRunnerError::invalid(format!( + "failed to rotate the completed Codex identity epoch: {error}" + ))); + }; + let state = self + .state + .as_mut() + .expect("Codex state remains available during identity epoch rollover"); + state.provider_process_generation = process_generation; + state.settled_provider_turn_ids.clear(); + if let Some(completed_provider_turn_id) = state.completed_provider_turn_id.clone() { + // The replacement process restored this still-authoritative + // terminal into its fresh epoch. Mirror that one tombstone in the + // durable ledger until accepting replacement work revokes the + // completion authority. + state + .settled_provider_turn_ids + .insert(completed_provider_turn_id); + } + state.settled_provider_turn_filter = DurableReplayFilter::default(); + if tool_rollover_required { + state + .tool_bridge + .rollover_replay_epoch_after_provider_restart() + .map_err(|error| { + DurableRunnerError::invalid(format!( + "failed to rotate Codex semantic tool replay authority: {error}" + )) + })?; + } + self.save_state()?; + Ok(()) + } + fn start_turn(&mut self, payload: &Value) -> Result { self.restore_provider_if_needed()?; + if self + .state + .as_ref() + .is_some_and(|state| state.lifecycle == "closed") + { + return Err(DurableRunnerError::invalid( + "Codex provider session is closed", + )); + } if self .state .as_ref() @@ -678,6 +1538,35 @@ impl CodexCommandExecutor { "Codex already has an active provider turn", )); } + if self + .state + .as_ref() + .is_some_and(|state| state.ambiguous_turn_start_pending) + { + return Err(DurableRunnerError::invalid( + "Codex has an unresolved ambiguous provider turn start", + )); + } + if self + .state + .as_ref() + .is_some_and(|state| state.queued_events.len() >= MAX_REGULAR_QUEUED_PROVIDER_EVENTS) + { + return Err(DurableRunnerError::invalid( + "cannot start a new Codex turn until terminal events are acknowledged", + )); + } + self.state + .as_mut() + .expect("Codex state remains available before turn receipt preparation") + .tool_bridge + .prepare_turn() + .map_err(|error| { + DurableRunnerError::invalid(format!( + "Codex semantic tool receipts could not prepare the next turn: {error}" + )) + })?; + self.rollover_provider_identity_epochs_if_needed()?; let text = payload .get("text") .and_then(Value::as_str) @@ -698,16 +1587,36 @@ impl CodexCommandExecutor { state.ambiguous_turn_start_pending = true; } self.save_state()?; - let (start_result, completion_authority_retained, ambiguous_turn_start_pending) = { + let ( + start_result, + completion_authority_retained, + ambiguous_turn_start_pending, + rejected_accepted_turn, + ) = { let provider = self.ensure_provider()?; let result = provider.start_turn(text, &cwd); ( result, provider.completed_turn_authority().is_some(), provider.ambiguous_turn_start_pending(), + provider.take_rejected_accepted_turn(), ) }; if let Err(error) = start_result { + if let Some(rejected_accepted_turn) = rejected_accepted_turn { + // Codex accepted this work before disclosing a usable durable + // identity. The provider has already been terminated; close + // this run before returning so recovery cannot resume the + // untracked turn from the provider's thread snapshot. + self.close_after_rejected_provider_acceptance(&rejected_accepted_turn)?; + let failure_kind = match &rejected_accepted_turn { + RejectedAcceptedTurn::ReusedIdentity(_) => "accepted identity reuse", + RejectedAcceptedTurn::InvalidIdentity => "an invalid accepted identity", + }; + return Err(DurableRunnerError::invalid(format!( + "Codex turn/start failed closed after {failure_kind}: {error}" + ))); + } let state = self .state .as_mut() @@ -748,6 +1657,11 @@ impl CodexCommandExecutor { state.completed_turn_authoritative = false; state.completed_turn_process_generation = None; state.completed_provider_turn_id = None; + state.receipt_limit_diagnostic_emitted = false; + state.receipt_limit_interrupt_pending = false; + state.receipt_limit_interrupt_accepted = false; + state.receipt_limit_interrupt_attempts = 0; + state.receipt_limit_interrupt_deadline_unix_ms = None; state.last_agent_message = None; state.lifecycle = "turn_active".to_owned(); self.save_state()?; @@ -773,6 +1687,30 @@ impl CodexCommandExecutor { "reason": reason, }))); } + let has_pending_tools = self + .state + .as_ref() + .is_some_and(|state| state.tool_bridge.pending_calls().next().is_some()); + if has_pending_tools { + let identity = self.event_identity()?; + let mut next_state = self + .state + .clone() + .expect("Codex state remains available during interruption"); + let cancelled = next_state + .tool_bridge + .cancel_pending_calls("provider_turn_stopped") + .map_err(|error| { + DurableRunnerError::invalid(format!( + "failed to cancel pending semantic tools: {error}" + )) + })?; + for result in cancelled { + next_state.push_terminal_event(semantic_result_event(&identity, &result))?; + } + self.persist_state(&next_state)?; + self.state = Some(next_state); + } self.ensure_provider()?.interrupt_turn().map_err(|error| { DurableRunnerError::invalid(format!("Codex turn interrupt failed: {error}")) })?; @@ -799,6 +1737,15 @@ impl CodexCommandExecutor { .get("requestId") .and_then(Value::as_str) .ok_or_else(|| DurableRunnerError::invalid("request.resolve requires requestId"))?; + if self + .state + .as_ref() + .is_none_or(|state| state.active_provider_turn_id.is_none()) + { + return Err(DurableRunnerError::invalid( + "cannot resolve a Codex runtime request outside an active turn", + )); + } let response = payload .get("response") .ok_or_else(|| DurableRunnerError::invalid("request.resolve requires response"))?; @@ -817,6 +1764,377 @@ impl CodexCommandExecutor { }) } + fn event_identity(&self) -> Result { + self.event_identity.clone().ok_or_else(|| { + DurableRunnerError::invalid( + "Codex semantic tool events require the durable runner identity", + ) + }) + } + + fn reject_tool_call( + &mut self, + call_id: String, + operation_id: String, + reason: String, + ) -> Result<(), DurableRunnerError> { + let state = self + .state + .as_mut() + .expect("Codex state remains available for a rejected tool call"); + let event = NormalizedProviderEvent { + event_type: "harness.diagnostic".to_owned(), + priority: EventPriority::P0, + payload: json!({ + "provider": "codex", + "code": "semantic_tool_denied", + "operationId": operation_id, + "callId": call_id, + "message": reason, + "paperclipExecuted": false, + }), + }; + if state.receipt_limit_interrupt_pending { + state.push_receipt_limit_cleanup_event(event)?; + } else { + state.push_event(event)?; + } + self.save_state()?; + let rejection = ToolResult { + call_id, + operation_id, + result: json!({ + "error": { + "code": "invalid_tool_call", + "message": "Paperclip rejected this semantic tool call", + "retryable": false, + }, + }), + is_error: true, + }; + self.provider + .as_mut() + .expect("provider remains present while rejecting its tool call") + .deliver_tool_result(&rejection) + .map_err(|delivery_error| { + DurableRunnerError::invalid(format!( + "failed to return the semantic tool rejection: {delivery_error}" + )) + }) + } + + fn stop_turn_at_tool_receipt_limit( + &mut self, + call_id: String, + operation_id: String, + ) -> Result<(), DurableRunnerError> { + let deadline_unix_ms = + receipt_limit_deadline_after(RECEIPT_LIMIT_INTERRUPT_TERMINAL_DEADLINE_MS)?; + let state = self + .state + .as_mut() + .expect("Codex state remains available at its tool receipt limit"); + let interrupt_pending = state.begin_receipt_limit_stop( + call_id.clone(), + operation_id.clone(), + deadline_unix_ms, + )?; + let first_interrupt_attempt = + interrupt_pending && state.receipt_limit_interrupt_attempts == 0; + if interrupt_pending { + self.save_state()?; + } + // The durable diagnostic owns the turn-level failure, while every + // buffered JSON-RPC call still receives an explicit provider error. + // Deliver the rejection before requesting interruption: Codex may + // close the transport as part of the interrupt, and a failed courtesy + // RPC must never abort runnerd's terminal polling loop. + let rejection = ToolResult { + call_id, + operation_id, + result: json!({ + "error": { + "code": "semantic_tool_turn_receipt_limit", + "message": "Paperclip stopped this turn at its durable semantic-tool receipt limit", + "retryable": false, + }, + }), + is_error: true, + }; + if let Some(provider) = self.provider.as_mut() { + let _ = provider.deliver_tool_result(&rejection); + } + if first_interrupt_attempt { + // Keep the durable retry marker until a terminal notification is + // observed. Provider acceptance acknowledges only this RPC; it + // does not prove that the turn stopped. Later buffered calls may + // therefore retry the idempotent interruption instead of leaving + // a still-active receipt-exhausted turn permanently unstopped. + self.state + .as_mut() + .expect("Codex state remains available before receipt-limit interruption") + .record_receipt_limit_interrupt_attempt()?; + self.save_state()?; + match self.interrupt_turn("semantic_tool_turn_receipt_limit") { + Ok(_) => { + let accepted_deadline_unix_ms = + receipt_limit_deadline_after(RECEIPT_LIMIT_ACCEPTED_TERMINAL_DEADLINE_MS)?; + self.state + .as_mut() + .expect("Codex state remains available after receipt-limit interruption") + .mark_receipt_limit_interrupt_accepted(accepted_deadline_unix_ms); + self.save_state()?; + } + // The first interruption attempt is also best-effort. Its + // durable retry marker is already saved, and propagating the + // transport error here would terminate runnerd before it can + // poll the provider's terminal notification. + Err(_) => {} + } + } + Ok(()) + } + + fn settle_receipt_limit_interrupt_after_deadline(&mut self) -> Result<(), DurableRunnerError> { + let interrupt_accepted = self + .state + .as_ref() + .is_some_and(|state| state.receipt_limit_interrupt_accepted); + let provider_shutdown_failed = self + .provider + .as_mut() + .is_some_and(|provider| provider.shutdown().is_err()); + self.provider = None; + let identity = self.event_identity()?; + let state = self + .state + .as_mut() + .expect("Codex state remains available at its receipt-limit retry bound"); + state.settle_active_provider_turn_identity()?; + let settled = state + .tool_bridge + .settle_turn("semantic_tool_turn_receipt_limit") + .map_err(|error| { + DurableRunnerError::invalid(format!( + "failed to settle semantic tools at the receipt-limit retry bound: {error}" + )) + })?; + for result in settled { + state.push_terminal_event(semantic_result_event(&identity, &result))?; + } + state.active_provider_turn_id = None; + state.completed_turn_authoritative = false; + state.receipt_limit_diagnostic_emitted = false; + state.receipt_limit_interrupt_pending = false; + state.receipt_limit_interrupt_accepted = false; + state.receipt_limit_interrupt_attempts = 0; + state.receipt_limit_interrupt_deadline_unix_ms = None; + state.lifecycle = "provider_exited".to_owned(); + let terminal_event_type = if interrupt_accepted { + "turn.interrupted" + } else { + "turn.failed" + }; + state.push_terminal_event(NormalizedProviderEvent { + event_type: terminal_event_type.to_owned(), + priority: EventPriority::P0, + payload: json!({ + "provider": "codex", + "code": if interrupt_accepted { + "semantic_tool_turn_receipt_limit_interrupt_deadline" + } else { + "semantic_tool_turn_receipt_limit_interrupt_unconfirmed" + }, + "message": if interrupt_accepted { + "Codex accepted the receipt-limit interruption but did not emit its terminal before the bounded shutdown deadline" + } else { + "Codex did not confirm terminal state after the bounded receipt-limit interruption attempts" + }, + "interruptAccepted": interrupt_accepted, + "providerTerminalObserved": false, + "providerShutdownFailed": provider_shutdown_failed, + }), + })?; + let terminal = terminal_events(state, terminal_event_type); + state.extend_terminal_events(terminal)?; + self.save_state() + } + + fn retry_receipt_limit_interrupt(&mut self) -> Result<(), DurableRunnerError> { + let (should_retry, attempts, persisted_deadline) = + self.state.as_ref().map_or((false, 0, None), |state| { + ( + state.receipt_limit_interrupt_pending + && state.active_provider_turn_id.is_some(), + state.receipt_limit_interrupt_attempts, + state.receipt_limit_interrupt_deadline_unix_ms, + ) + }); + if !should_retry { + return Ok(()); + } + let now_unix_ms = current_unix_ms()?; + let deadline_unix_ms = match persisted_deadline { + Some(deadline) => deadline, + None => { + // Older durable state did not record this additive field. Give + // an already-pending interruption one complete bounded window + // after recovery rather than falling back on poll count alone. + let timeout_ms = if self + .state + .as_ref() + .is_some_and(|state| state.receipt_limit_interrupt_accepted) + { + RECEIPT_LIMIT_ACCEPTED_TERMINAL_DEADLINE_MS + } else { + RECEIPT_LIMIT_INTERRUPT_TERMINAL_DEADLINE_MS + }; + let deadline = receipt_limit_deadline_after(timeout_ms)?; + self.state + .as_mut() + .expect("Codex state remains available while adding its receipt-limit deadline") + .receipt_limit_interrupt_deadline_unix_ms = Some(deadline); + self.save_state()?; + deadline + } + }; + // Attempts bound network traffic, while the durable wall-clock deadline + // gives an accepted asynchronous interruption time to deliver its + // authoritative terminal. The provider is always polled once more below + // before an elapsed deadline is converted into the conservative fallback. + if attempts >= MAX_RECEIPT_LIMIT_INTERRUPT_ATTEMPTS || now_unix_ms >= deadline_unix_ms { + return Ok(()); + } + // Polling is the autonomous recovery path until a terminal notification + // clears the durable marker. RPC acceptance alone does not establish + // that Codex stopped the turn, so accepted-but-unsettled interruptions + // remain idempotently retryable across polls and process restarts. + self.state + .as_mut() + .expect("Codex state remains available before receipt-limit retry") + .record_receipt_limit_interrupt_attempt()?; + self.save_state()?; + if self + .interrupt_turn("semantic_tool_turn_receipt_limit_retry") + .is_ok() + && self.state.as_ref().is_some_and(|state| { + state.receipt_limit_interrupt_pending && state.active_provider_turn_id.is_some() + }) + { + let accepted_deadline_unix_ms = + receipt_limit_deadline_after(RECEIPT_LIMIT_ACCEPTED_TERMINAL_DEADLINE_MS)?; + self.state + .as_mut() + .expect("Codex state remains available after receipt-limit retry") + .mark_receipt_limit_interrupt_accepted(accepted_deadline_unix_ms); + self.save_state()?; + } + Ok(()) + } + + fn settle_receipt_limit_interrupt_if_deadline_elapsed( + &mut self, + ) -> Result<(), DurableRunnerError> { + let deadline = self.state.as_ref().and_then(|state| { + (state.receipt_limit_interrupt_pending && state.active_provider_turn_id.is_some()) + .then_some(state.receipt_limit_interrupt_deadline_unix_ms) + .flatten() + }); + let Some(deadline) = deadline else { + return Ok(()); + }; + if current_unix_ms()? >= deadline { + self.settle_receipt_limit_interrupt_after_deadline()?; + } + Ok(()) + } + + fn handle_tool_call( + &mut self, + call_id: String, + operation_id: String, + input: Value, + ) -> Result<(), DurableRunnerError> { + let identity = self.event_identity()?; + let admission = self + .state + .as_mut() + .ok_or_else(|| DurableRunnerError::invalid("Codex provider is not prepared"))? + .admit_tool_call(&call_id, &operation_id, &input); + match admission { + Ok(ToolCallAdmission::CompletedReplay(result)) => { + self.provider + .as_mut() + .expect("provider remains present while handling its tool call") + .deliver_tool_result(&result) + .map_err(|error| { + DurableRunnerError::invalid(format!( + "failed to replay a durable semantic tool result: {error}" + )) + })?; + return Ok(()); + } + Ok(ToolCallAdmission::PendingReplay) => return Ok(()), + Err(error) => { + if error.is_active_turn_receipt_limit() { + return self.stop_turn_at_tool_receipt_limit(call_id, operation_id); + } + return self.reject_tool_call(call_id, operation_id, error.to_string()); + } + Ok(ToolCallAdmission::Pending(call)) => { + self.state + .as_mut() + .expect("Codex state remains available while accepting a tool call") + .push_event(semantic_input_event(&identity, &call))?; + self.save_state() + } + } + } + + fn deliver_semantic_result( + &mut self, + payload: &Value, + ) -> Result { + let result: ToolResult = serde_json::from_value(payload.clone()).map_err(|error| { + DurableRunnerError::invalid(format!("semantic tool result is invalid: {error}")) + })?; + let identity = self.event_identity()?; + let was_completed = self + .state + .as_ref() + .is_some_and(|state| state.tool_bridge.has_completed_call(&result.call_id)); + let mut next_state = self + .state + .clone() + .ok_or_else(|| DurableRunnerError::invalid("Codex provider is not prepared"))?; + next_state + .tool_bridge + .apply_result(result.clone()) + .map_err(|error| { + DurableRunnerError::invalid(format!("semantic tool result was rejected: {error}")) + })?; + if was_completed { + return Ok(CommandExecution::result(json!({ + "status": "duplicate", + "callId": result.call_id, + }))); + } + next_state.push_event(semantic_result_event(&identity, &result))?; + self.persist_state(&next_state)?; + self.state = Some(next_state); + self.ensure_provider()? + .deliver_tool_result(&result) + .map_err(|error| { + DurableRunnerError::invalid(format!( + "failed to return semantic tool result to Codex: {error}" + )) + })?; + Ok(CommandExecution::result(json!({ + "status": "delivered", + "callId": result.call_id, + }))) + } + fn close_session(&mut self) -> Result { if let Some(provider) = self.provider.as_mut() { provider.shutdown().map_err(|error| { @@ -830,6 +2148,11 @@ impl CodexCommandExecutor { .ok_or_else(|| DurableRunnerError::invalid("Codex provider is not prepared"))?; state.active_provider_turn_id = None; state.ambiguous_turn_start_pending = false; + state.receipt_limit_diagnostic_emitted = false; + state.receipt_limit_interrupt_pending = false; + state.receipt_limit_interrupt_accepted = false; + state.receipt_limit_interrupt_attempts = 0; + state.receipt_limit_interrupt_deadline_unix_ms = None; state.lifecycle = "closed".to_owned(); let thread_id = state.thread_id.clone(); self.save_state()?; @@ -865,15 +2188,24 @@ impl CodexCommandExecutor { // an ambiguous-start failure cannot degrade into an empty successful // poll on the same executor. self.restore_provider_if_needed()?; - if self - .state - .as_ref() - .is_some_and(|state| !state.pending_events.is_empty()) + // Receipt-limit interruption is autonomous recovery. It must advance + // even while older durable events await acknowledgement, otherwise a + // slow or disconnected controller can keep an exhausted provider turn + // alive forever. Terminal settlement uses the reserved event capacity. + self.retry_receipt_limit_interrupt()?; + let receipt_limit_terminal_poll = self.state.as_ref().is_some_and(|state| { + state.receipt_limit_interrupt_pending && state.active_provider_turn_id.is_some() + }); + if !receipt_limit_terminal_poll + && self + .state + .as_ref() + .is_some_and(|state| !state.pending_events.is_empty()) { return Ok(()); } if self.provider.is_none() { - return Ok(()); + return self.settle_receipt_limit_interrupt_if_deadline_elapsed(); } for _ in 0..MAX_EVENTS_PER_POLL { let event = self @@ -889,11 +2221,9 @@ impl CodexCommandExecutor { CodexProviderEvent::ToolCall { call_id, operation_id, - .. + input, } => { - return Err(DurableRunnerError::invalid(format!( - "Codex emitted semantic tool call {call_id} for {operation_id} before the durable tool bridge was attached" - ))); + self.handle_tool_call(call_id, operation_id, input)?; } CodexProviderEvent::Notification { method, params } => { let active_provider_turn_id = if method == "turn/started" { @@ -904,14 +2234,17 @@ impl CodexCommandExecutor { } else { None }; - let completed_turn_authority = if method == "turn/completed" { - self.provider - .as_ref() - .and_then(CodexProvider::completed_turn_authority) - .map(|(generation, turn_id)| (generation, turn_id.to_owned())) - } else { - None - }; + let normalized_terminal_type = + normalized_codex_terminal_event_type(&method, ¶ms); + let completed_turn_authority = + if normalized_terminal_type == Some("turn.completed") { + self.provider + .as_ref() + .and_then(CodexProvider::completed_turn_authority) + .map(|(generation, turn_id)| (generation, turn_id.to_owned())) + } else { + None + }; let normalized = normalize_codex_notification(&method, ¶ms); let terminal_event_type = normalized .iter() @@ -926,6 +2259,7 @@ impl CodexCommandExecutor { | "turn.interrupted" ) }); + let identity = self.event_identity.clone(); let state = self .state .as_mut() @@ -948,23 +2282,63 @@ impl CodexCommandExecutor { })?; state.reconcile_active_provider_turn(Some(provider_turn_id)); } - if method == "turn/completed" { - let (process_generation, provider_turn_id) = completed_turn_authority - .ok_or_else(|| { + if terminal_event_type.is_some() { + state.settle_active_provider_turn_identity()?; + let settled = state + .tool_bridge + .settle_turn("provider_turn_terminated") + .map_err(|error| { + DurableRunnerError::invalid(format!( + "failed to settle semantic tools at turn termination: {error}" + )) + })?; + if !settled.is_empty() { + let identity = identity.as_ref().ok_or_else(|| { + DurableRunnerError::invalid( + "Codex semantic tool events require the durable runner identity", + ) + })?; + for result in settled { + state.push_terminal_event(semantic_result_event( + identity, &result, + ))?; + } + } + state.active_provider_turn_id = None; + if terminal_event_type.as_deref() == Some("turn.completed") { + let (process_generation, provider_turn_id) = completed_turn_authority + .ok_or_else(|| { DurableRunnerError::invalid( "Codex completion omitted process and turn authority", ) })?; - state.active_provider_turn_id = None; - state.completed_turn_authoritative = true; - state.completed_turn_process_generation = Some(process_generation); - state.completed_provider_turn_id = Some(provider_turn_id); + state.completed_turn_authoritative = true; + state.completed_turn_process_generation = Some(process_generation); + state.completed_provider_turn_id = Some(provider_turn_id); + } else { + state.completed_turn_authoritative = false; + state.completed_turn_process_generation = None; + state.completed_provider_turn_id = None; + } + state.receipt_limit_diagnostic_emitted = false; + state.receipt_limit_interrupt_pending = false; + state.receipt_limit_interrupt_accepted = false; + state.receipt_limit_interrupt_attempts = 0; + state.receipt_limit_interrupt_deadline_unix_ms = None; state.ambiguous_turn_start_pending = false; state.lifecycle = "session_open".to_owned(); } - state.extend_events(normalized)?; + if terminal_event_type.is_some() { + state.extend_terminal_events(normalized)?; + } else if receipt_limit_terminal_poll { + for event in normalized { + state.push_receipt_limit_cleanup_event(event)?; + } + } else { + state.extend_events(normalized)?; + } if let Some(event_type) = terminal_event_type { - state.extend_events(terminal_events(state, &event_type))?; + state.extend_terminal_events(terminal_events(state, &event_type))?; } self.save_state()?; } @@ -977,29 +2351,35 @@ impl CodexCommandExecutor { .or_else(|| question_set.pointer("/questions/0/prompt")) .and_then(Value::as_str) .unwrap_or("Codex needs your input"); - self.state + let state = self + .state .as_mut() - .expect("Codex state remains available while polling") - .push_event(NormalizedProviderEvent { - event_type: "runtime_request.created".to_owned(), - priority: EventPriority::P0, - payload: json!({ - "request": { - "schema": "paperclip.runtime_request.v2", - "requestKind": "runtime", - "requestId": request_id, - "type": "input", - "status": "pending", - "prompt": prompt, - "input": question_set, - "origin": { - "adapter": "codex-app-server", - "provider": "codex", - "method": "item/tool/requestUserInput", - }, + .expect("Codex state remains available while polling"); + let event = NormalizedProviderEvent { + event_type: "runtime_request.created".to_owned(), + priority: EventPriority::P0, + payload: json!({ + "request": { + "schema": "paperclip.runtime_request.v2", + "requestKind": "runtime", + "requestId": request_id, + "type": "input", + "status": "pending", + "prompt": prompt, + "input": question_set, + "origin": { + "adapter": "codex-app-server", + "provider": "codex", + "method": "item/tool/requestUserInput", }, - }), - })?; + }, + }), + }; + if receipt_limit_terminal_poll { + state.push_receipt_limit_cleanup_event(event)?; + } else { + state.push_event(event)?; + } self.save_state()?; } CodexProviderEvent::Exited { @@ -1022,7 +2402,7 @@ impl CodexCommandExecutor { // whether this separate session exit belongs to that // completion or is a later idle-provider failure. state.lifecycle = "provider_exited".to_owned(); - state.push_event(NormalizedProviderEvent { + state.push_terminal_event(NormalizedProviderEvent { // A completed turn remains authoritative, while the // reusable provider session independently becomes // unavailable. Avoid emitting session.failed for @@ -1053,7 +2433,11 @@ impl CodexCommandExecutor { } } } - Ok(()) + // Check the durable deadline only after a complete provider poll. A + // terminal that arrived after interruption acceptance but before this + // observation remains authoritative even when several fast controller + // polls have already exhausted the interruption-attempt budget. + self.settle_receipt_limit_interrupt_if_deadline_elapsed() } } @@ -1066,6 +2450,7 @@ impl CommandExecutor for CodexCommandExecutor { if self.state.is_none() && command.payload.get("provider").is_some() { self.prepare(&command.payload)?; } + self.verify_attached_tools(&command.payload)?; let mut execution = self.open_session()?; execution.events.push(( "run.attached".to_owned(), @@ -1081,6 +2466,7 @@ impl CommandExecutor for CodexCommandExecutor { self.interrupt_turn(&command.command_type) } "request.resolve" => self.resolve_request(&command.payload), + "semantic_tool.result" => self.deliver_semantic_result(&command.payload), "session.snapshot" => self.snapshot(), "session.close" | "session.destroy" => self.close_session(), "runner.drain" | "runner.suspend" | "runner.shutdown" => { @@ -1119,6 +2505,7 @@ impl CommandExecutor for CodexCommandExecutor { )); } next_state.pending_events.drain(..count); + next_state.refill_pending_events(); self.persist_state(&next_state)?; self.state = Some(next_state); Ok(()) @@ -1160,6 +2547,7 @@ mod tests { approval_policy: "never".to_owned(), }, completion_contract: None, + tool_bridge: ProviderToolBridge::default(), thread_id: Some("thread-1".to_owned()), provider_session_id: None, active_provider_turn_id: None, @@ -1168,8 +2556,16 @@ mod tests { provider_process_generation: 0, completed_turn_process_generation: None, completed_provider_turn_id: None, + settled_provider_turn_ids: std::collections::BTreeSet::new(), + settled_provider_turn_filter: DurableReplayFilter::default(), + receipt_limit_diagnostic_emitted: false, + receipt_limit_interrupt_pending: false, + receipt_limit_interrupt_accepted: false, + receipt_limit_interrupt_attempts: 0, + receipt_limit_interrupt_deadline_unix_ms: None, last_agent_message: None, pending_events: VecDeque::new(), + queued_events: VecDeque::new(), next_provider_event_seq: initial_provider_event_seq(), }; assert!(state.validate().is_err()); @@ -1194,6 +2590,7 @@ mod tests { approval_policy: "never".to_owned(), }, None, + ProviderToolBridge::default(), ); state.thread_id = Some("thread-1".to_owned()); state.lifecycle = "session_open".to_owned(); @@ -1236,6 +2633,7 @@ mod tests { revision: "1".to_owned(), criterion_ids: vec!["objective".to_owned()], }), + ProviderToolBridge::default(), ); state.last_agent_message = Some("Finished the requested work.".to_owned()); let events = terminal_events(&state, "turn.completed"); @@ -1244,4 +2642,752 @@ mod tests { assert_eq!(events[1].event_type, "run.terminal"); assert_eq!(events[1].payload["runTerminalState"], "succeeded"); } + + #[test] + fn semantic_input_digest_covers_the_transmitted_redacted_value() { + let identity = ProviderEventIdentity { + run_id: "run-1".to_owned(), + normalized_session_id: "session-1".to_owned(), + turn_id: "turn-1".to_owned(), + item_id: "item-1".to_owned(), + }; + let call = PendingToolCall { + call_id: "call-1".to_owned(), + operation_id: "get_task_context".to_owned(), + input: json!({"password": "do-not-persist", "safe": true}), + }; + let event = semantic_input_event(&identity, &call); + let transmitted = &event.payload["semantic_tool"]["input"]; + assert_eq!(transmitted["password"], "[REDACTED]"); + assert_eq!( + event.payload["semantic_tool"]["content"]["digest"], + semantic_value_digest(transmitted) + ); + } + + #[test] + fn receipt_limit_diagnostic_is_durable_and_turn_idempotent() { + let mut state = CodexProviderState::new( + CodexProviderConfig { + provider: "codex".to_owned(), + driver: "codex_app_server".to_owned(), + provider_version: "test".to_owned(), + command: PathBuf::from("codex"), + args: vec!["app-server".to_owned()], + cwd: std::env::current_dir() + .unwrap() + .to_string_lossy() + .into_owned(), + model: None, + provider_session_id: None, + instructions: String::new(), + approval_policy: "never".to_owned(), + }, + None, + ProviderToolBridge::default(), + ); + state.thread_id = Some("thread-1".to_owned()); + state.active_provider_turn_id = Some("turn-1".to_owned()); + state.lifecycle = "turn_active".to_owned(); + + assert!(state + .begin_receipt_limit_stop("call-first".to_owned(), "tool.first".to_owned(), 10_000) + .unwrap()); + state.mark_receipt_limit_interrupt_accepted(50_000); + assert!(state + .begin_receipt_limit_stop("call-second".to_owned(), "tool.second".to_owned(), 20_000) + .unwrap()); + assert_eq!(state.pending_events.len(), 1); + assert_eq!(state.pending_events[0].payload["callId"], "call-first"); + assert_eq!(state.receipt_limit_interrupt_deadline_unix_ms, Some(50_000)); + + let mut recovered: CodexProviderState = + serde_json::from_slice(&serde_json::to_vec(&state).unwrap()).unwrap(); + recovered.validate().unwrap(); + assert!(recovered.receipt_limit_interrupt_accepted); + assert_eq!( + recovered.receipt_limit_interrupt_deadline_unix_ms, + Some(50_000) + ); + assert!(recovered + .begin_receipt_limit_stop( + "call-after-restart".to_owned(), + "tool.third".to_owned(), + 30_000, + ) + .unwrap()); + assert_eq!(recovered.pending_events.len(), 1); + // Only a terminal notification clears the retry marker. Accepting an + // interrupt request does not prove that the provider stopped. + assert!(recovered + .begin_receipt_limit_stop( + "call-after-success".to_owned(), + "tool.fourth".to_owned(), + 40_000, + ) + .unwrap()); + assert_eq!(recovered.pending_events.len(), 1); + } + + #[test] + fn settled_receipt_limit_interrupt_cannot_be_marked_accepted() { + let mut state = CodexProviderState::new( + CodexProviderConfig { + provider: "codex".to_owned(), + driver: "codex_app_server".to_owned(), + provider_version: "test".to_owned(), + command: PathBuf::from("codex"), + args: vec!["app-server".to_owned()], + cwd: std::env::current_dir() + .unwrap() + .to_string_lossy() + .into_owned(), + model: None, + provider_session_id: None, + instructions: String::new(), + approval_policy: "never".to_owned(), + }, + None, + ProviderToolBridge::default(), + ); + state.thread_id = Some("thread-1".to_owned()); + state.active_provider_turn_id = Some("turn-1".to_owned()); + state.lifecycle = "turn_active".to_owned(); + state + .begin_receipt_limit_stop("call-1".to_owned(), "tool.one".to_owned(), 10_000) + .unwrap(); + state.record_receipt_limit_interrupt_attempt().unwrap(); + + // Recovery observed that the turn ended before the interrupt RPC was + // issued and cleared the receipt-limit interruption state. + state.settle_active_provider_turn_identity().unwrap(); + state.active_provider_turn_id = None; + state.lifecycle = "session_open".to_owned(); + state.receipt_limit_diagnostic_emitted = false; + state.receipt_limit_interrupt_pending = false; + state.receipt_limit_interrupt_accepted = false; + state.receipt_limit_interrupt_attempts = 0; + state.receipt_limit_interrupt_deadline_unix_ms = None; + + state.mark_receipt_limit_interrupt_accepted(50_000); + + assert!(!state.receipt_limit_interrupt_accepted); + assert!(state.receipt_limit_interrupt_deadline_unix_ms.is_none()); + state.validate().unwrap(); + } + + #[test] + fn regular_backlog_preserves_receipt_limit_and_terminal_settlement_capacity() { + let mut state = CodexProviderState::new( + CodexProviderConfig { + provider: "codex".to_owned(), + driver: "codex_app_server".to_owned(), + provider_version: "test".to_owned(), + command: PathBuf::from("codex"), + args: vec!["app-server".to_owned()], + cwd: std::env::current_dir() + .unwrap() + .to_string_lossy() + .into_owned(), + model: None, + provider_session_id: None, + instructions: String::new(), + approval_policy: "never".to_owned(), + }, + None, + ProviderToolBridge::default(), + ); + let identity = ProviderEventIdentity { + run_id: "run-1".to_owned(), + normalized_session_id: "session-1".to_owned(), + turn_id: "turn-1".to_owned(), + item_id: "item-1".to_owned(), + }; + let ordinary_event = || NormalizedProviderEvent { + event_type: "harness.diagnostic".to_owned(), + priority: EventPriority::P1, + payload: json!({"code": "ordinary_backlog"}), + }; + for _ in 0..(MAX_EVENTS_PER_POLL + MAX_REGULAR_QUEUED_PROVIDER_EVENTS) { + state.push_event(ordinary_event()).unwrap(); + } + assert_eq!(state.pending_events.len(), MAX_EVENTS_PER_POLL); + assert_eq!( + state.queued_events.len(), + MAX_REGULAR_QUEUED_PROVIDER_EVENTS + ); + assert!(state.push_event(ordinary_event()).is_err()); + + for _ in 0..MAX_EVENTS_PER_POLL { + state + .push_receipt_limit_cleanup_event(ordinary_event()) + .unwrap(); + } + let cleanup_boundary = state.queued_events.len(); + state + .push_receipt_limit_cleanup_event(ordinary_event()) + .expect("cleanup overflow is dropped while preserving terminal capacity"); + assert_eq!(state.queued_events.len(), cleanup_boundary); + + for index in 0..MAX_PENDING_CALLS { + state + .push_terminal_event(semantic_result_event( + &identity, + &ToolResult { + call_id: format!("call-{index}"), + operation_id: "get_task_context".to_owned(), + result: json!({"error": {"code": "provider_turn_terminated"}}), + is_error: true, + }, + )) + .unwrap(); + } + for event_type in [ + "harness.diagnostic", + "turn.completed", + "run.result.proposed", + "run.terminal", + ] { + state + .push_terminal_event(NormalizedProviderEvent { + event_type: event_type.to_owned(), + priority: EventPriority::P0, + payload: json!({"terminal": true}), + }) + .unwrap(); + } + + assert_eq!(state.pending_events.len(), MAX_EVENTS_PER_POLL); + assert_eq!(state.queued_events.len(), MAX_QUEUED_PROVIDER_EVENTS); + state.validate().unwrap(); + assert!(state.push_terminal_event(ordinary_event()).is_err()); + assert!(serde_json::to_vec(&state).unwrap().len() as u64 <= MAX_PROVIDER_STATE_BYTES); + } + + #[test] + fn completed_replays_are_read_only_at_the_regular_event_boundary() { + let operation = crate::provider_bridge::AuthorizedTool { + operation_id: "get_task_context".to_owned(), + version: 1, + description: "Read the active task context.".to_owned(), + input_schema: json!({"type": "object"}), + response_schema: json!({"type": "object"}), + }; + let mut bridge = ProviderToolBridge::default(); + bridge + .prepare(AuthorizedToolSet { + schema: TOOL_SET_SCHEMA.to_owned(), + schema_version: 1, + catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation)) + .unwrap(), + operations: vec![operation], + }) + .unwrap(); + bridge + .begin_call( + "call-replayed".to_owned(), + "get_task_context".to_owned(), + json!({}), + ) + .unwrap(); + let replayed_result = ToolResult { + call_id: "call-replayed".to_owned(), + operation_id: "get_task_context".to_owned(), + result: json!({"ok": true}), + is_error: false, + }; + bridge.apply_result(replayed_result.clone()).unwrap(); + + let mut state = CodexProviderState::new( + CodexProviderConfig { + provider: "codex".to_owned(), + driver: "codex_app_server".to_owned(), + provider_version: "test".to_owned(), + command: PathBuf::from("codex"), + args: vec!["app-server".to_owned()], + cwd: std::env::current_dir() + .unwrap() + .to_string_lossy() + .into_owned(), + model: None, + provider_session_id: None, + instructions: String::new(), + approval_policy: "never".to_owned(), + }, + None, + bridge, + ); + let identity = ProviderEventIdentity { + run_id: "run-1".to_owned(), + normalized_session_id: "session-1".to_owned(), + turn_id: "turn-1".to_owned(), + item_id: "item-1".to_owned(), + }; + + // Keep the pending window occupied, then model the input and result + // receipts retained by the maximum 4,096 completed calls. Only three + // regular queued-event slots remain at this boundary. + for index in 0..MAX_EVENTS_PER_POLL { + state + .push_event(NormalizedProviderEvent { + event_type: "provider.notice.recorded".to_owned(), + priority: EventPriority::P1, + payload: json!({"index": index}), + }) + .unwrap(); + } + for index in 0..MAX_PENDING_CALLS { + let call = PendingToolCall { + call_id: format!("call-{index}"), + operation_id: "get_task_context".to_owned(), + input: json!({}), + }; + state + .push_event(semantic_input_event(&identity, &call)) + .unwrap(); + state + .push_event(semantic_result_event( + &identity, + &ToolResult { + call_id: call.call_id, + operation_id: call.operation_id, + result: json!({"ok": true}), + is_error: false, + }, + )) + .unwrap(); + } + assert_eq!( + MAX_REGULAR_QUEUED_PROVIDER_EVENTS - state.queued_events.len(), + 3 + ); + let pending_len = state.pending_events.len(); + let queued_len = state.queued_events.len(); + let next_sequence = state.next_provider_event_seq; + + for _ in 0..4 { + assert_eq!( + state + .admit_tool_call("call-replayed", "get_task_context", &json!({})) + .unwrap(), + ToolCallAdmission::CompletedReplay(replayed_result.clone()) + ); + } + assert_eq!(state.pending_events.len(), pending_len); + assert_eq!(state.queued_events.len(), queued_len); + assert_eq!(state.next_provider_event_seq, next_sequence); + state.validate().unwrap(); + } + + #[test] + fn pending_replays_are_read_only_at_the_regular_event_boundary() { + let operation = crate::provider_bridge::AuthorizedTool { + operation_id: "get_task_context".to_owned(), + version: 1, + description: "Read the active task context.".to_owned(), + input_schema: json!({"type": "object"}), + response_schema: json!({"type": "object"}), + }; + let mut bridge = ProviderToolBridge::default(); + bridge + .prepare(AuthorizedToolSet { + schema: TOOL_SET_SCHEMA.to_owned(), + schema_version: 1, + catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation)) + .unwrap(), + operations: vec![operation], + }) + .unwrap(); + bridge + .begin_call( + "call-pending".to_owned(), + "get_task_context".to_owned(), + json!({}), + ) + .unwrap(); + + let mut state = CodexProviderState::new( + CodexProviderConfig { + provider: "codex".to_owned(), + driver: "codex_app_server".to_owned(), + provider_version: "test".to_owned(), + command: PathBuf::from("codex"), + args: vec!["app-server".to_owned()], + cwd: std::env::current_dir() + .unwrap() + .to_string_lossy() + .into_owned(), + model: None, + provider_session_id: None, + instructions: String::new(), + approval_policy: "never".to_owned(), + }, + None, + bridge, + ); + for index in 0..(MAX_EVENTS_PER_POLL + MAX_REGULAR_QUEUED_PROVIDER_EVENTS) { + state + .push_event(NormalizedProviderEvent { + event_type: "provider.notice.recorded".to_owned(), + priority: EventPriority::P1, + payload: json!({"index": index}), + }) + .unwrap(); + } + assert_eq!(state.pending_events.len(), MAX_EVENTS_PER_POLL); + assert_eq!( + state.queued_events.len(), + MAX_REGULAR_QUEUED_PROVIDER_EVENTS + ); + let next_sequence = state.next_provider_event_seq; + + for _ in 0..4 { + assert_eq!( + state + .admit_tool_call("call-pending", "get_task_context", &json!({})) + .unwrap(), + ToolCallAdmission::PendingReplay + ); + } + assert_eq!(state.pending_events.len(), MAX_EVENTS_PER_POLL); + assert_eq!( + state.queued_events.len(), + MAX_REGULAR_QUEUED_PROVIDER_EVENTS + ); + assert_eq!(state.next_provider_event_seq, next_sequence); + state.validate().unwrap(); + } + + #[test] + fn exact_regular_backlog_capacity_rejects_turn_admission() { + let mut state = CodexProviderState::new( + CodexProviderConfig { + provider: "codex".to_owned(), + driver: "codex_app_server".to_owned(), + provider_version: "test".to_owned(), + command: PathBuf::from("codex"), + args: vec!["app-server".to_owned()], + cwd: std::env::current_dir() + .unwrap() + .to_string_lossy() + .into_owned(), + model: None, + provider_session_id: None, + instructions: String::new(), + approval_policy: "never".to_owned(), + }, + None, + ProviderToolBridge::default(), + ); + state.lifecycle = "prepared".to_owned(); + for _ in 0..(MAX_EVENTS_PER_POLL + MAX_REGULAR_QUEUED_PROVIDER_EVENTS) { + state + .push_event(NormalizedProviderEvent { + event_type: "harness.diagnostic".to_owned(), + priority: EventPriority::P1, + payload: json!({"code": "admission_boundary"}), + }) + .unwrap(); + } + assert_eq!( + state.queued_events.len(), + MAX_REGULAR_QUEUED_PROVIDER_EVENTS, + ); + let mut executor = CodexCommandExecutor::new(PathBuf::from("unused-test-state")); + executor.state = Some(state); + executor.restore_checked = true; + + let error = executor + .start_turn(&json!({"text": "must not reach the provider"})) + .expect_err("the exact regular backlog limit must reject admission"); + assert!(error + .to_string() + .contains("until terminal events are acknowledged")); + assert!(executor.provider.is_none()); + } + + #[test] + fn transient_receipt_limit_clears_before_a_later_turn() { + let operation = crate::provider_bridge::AuthorizedTool { + operation_id: "get_task_context".to_owned(), + version: 1, + description: "Read the active task context.".to_owned(), + input_schema: json!({"type": "object"}), + response_schema: json!({"type": "object"}), + }; + let mut bridge = ProviderToolBridge::default(); + bridge + .prepare(AuthorizedToolSet { + schema: TOOL_SET_SCHEMA.to_owned(), + schema_version: 1, + catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation)) + .unwrap(), + operations: vec![operation], + }) + .unwrap(); + let mut encoded = serde_json::to_value(&bridge).unwrap(); + encoded["durableRunReceiptLimitReached"] = Value::Bool(true); + let mut bridge: ProviderToolBridge = serde_json::from_value(encoded).unwrap(); + bridge.attach_existing_run().unwrap(); + + assert!(bridge.durable_run_receipt_limit_reached()); + bridge.prepare_turn().unwrap(); + assert!(!bridge.durable_run_receipt_limit_reached()); + } + + #[test] + fn restore_reattaches_the_durable_tool_result_byte_counter() { + let directory = std::env::temp_dir().join(format!( + "paperclip-provider-tool-byte-restore-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&directory); + fs::create_dir_all(&directory).unwrap(); + let operation = crate::provider_bridge::AuthorizedTool { + operation_id: "get_task_context".to_owned(), + version: 1, + description: "Read the active task context.".to_owned(), + input_schema: json!({"type": "object"}), + response_schema: json!({"type": "object"}), + }; + let mut bridge = ProviderToolBridge::default(); + bridge + .prepare(AuthorizedToolSet { + schema: TOOL_SET_SCHEMA.to_owned(), + schema_version: 1, + catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation)) + .unwrap(), + operations: vec![operation], + }) + .unwrap(); + bridge + .begin_call( + "call-1".to_owned(), + "get_task_context".to_owned(), + json!({}), + ) + .unwrap(); + bridge + .apply_result(ToolResult { + call_id: "call-1".to_owned(), + operation_id: "get_task_context".to_owned(), + result: json!({"ok": true}), + is_error: false, + }) + .unwrap(); + bridge.settle_turn("provider_turn_terminated").unwrap(); + assert!(bridge.retained_result_bytes_for_test() > 0); + + let state = CodexProviderState::new( + CodexProviderConfig { + provider: "codex".to_owned(), + driver: "codex_app_server".to_owned(), + provider_version: "test".to_owned(), + command: PathBuf::from("codex"), + args: vec!["app-server".to_owned()], + cwd: std::env::current_dir() + .unwrap() + .to_string_lossy() + .into_owned(), + model: None, + provider_session_id: None, + instructions: String::new(), + approval_policy: "never".to_owned(), + }, + None, + bridge, + ); + let writer = CodexCommandExecutor::new(&directory); + writer.persist_state(&state).unwrap(); + + let mut recovered = CodexCommandExecutor::new(&directory); + recovered.restore().unwrap(); + assert!( + recovered + .state + .as_ref() + .unwrap() + .tool_bridge + .retained_result_bytes_for_test() + > 0 + ); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn durable_provider_turn_ledger_backfills_legacy_completion_authority() { + let mut state = CodexProviderState::new( + CodexProviderConfig { + provider: "codex".to_owned(), + driver: "codex_app_server".to_owned(), + provider_version: "test".to_owned(), + command: PathBuf::from("codex"), + args: vec!["app-server".to_owned()], + cwd: std::env::current_dir() + .unwrap() + .to_string_lossy() + .into_owned(), + model: None, + provider_session_id: None, + instructions: String::new(), + approval_policy: "never".to_owned(), + }, + None, + ProviderToolBridge::default(), + ); + state.completed_turn_authoritative = true; + state.completed_turn_process_generation = Some(1); + state.completed_provider_turn_id = Some("provider-turn-legacy".to_owned()); + state.provider_process_generation = 1; + + let (recovered, recovered_filter) = state.recovered_settled_provider_turn_ids().unwrap(); + + assert!(settled_provider_turn_contains( + &recovered, + &recovered_filter, + "provider-turn-legacy" + )); + } + + #[test] + fn durable_provider_turn_ledger_never_evicts_within_an_epoch() { + let mut state = CodexProviderState::new( + CodexProviderConfig { + provider: "codex".to_owned(), + driver: "codex_app_server".to_owned(), + provider_version: "test".to_owned(), + command: PathBuf::from("codex"), + args: vec!["app-server".to_owned()], + cwd: std::env::current_dir() + .unwrap() + .to_string_lossy() + .into_owned(), + model: None, + provider_session_id: None, + instructions: String::new(), + approval_policy: "never".to_owned(), + }, + None, + ProviderToolBridge::default(), + ); + state.thread_id = Some("thread-1".to_owned()); + state.lifecycle = "turn_active".to_owned(); + for index in 0..MAX_SETTLED_PROVIDER_TURN_IDS - 1 { + state + .settled_provider_turn_ids + .insert(format!("provider-turn-{index:04}")); + } + state.active_provider_turn_id = Some("provider-turn-final".to_owned()); + + state.settle_active_provider_turn_identity().unwrap(); + state.active_provider_turn_id = None; + state.lifecycle = "session_open".to_owned(); + + assert_eq!( + state.settled_provider_turn_ids.len(), + MAX_SETTLED_PROVIDER_TURN_IDS + ); + assert!(state.settled_provider_turn_filter.is_empty()); + assert!(state + .settled_provider_turn_ids + .contains("provider-turn-final")); + assert!(state + .settled_provider_turn_ids + .contains("provider-turn-0000")); + + state.lifecycle = "turn_active".to_owned(); + state.active_provider_turn_id = Some("provider-turn-overflow".to_owned()); + assert!(state.settle_active_provider_turn_identity().is_err()); + assert!(!state + .settled_provider_turn_ids + .contains("provider-turn-overflow")); + state.active_provider_turn_id = None; + state.lifecycle = "session_open".to_owned(); + state.validate().unwrap(); + let recovered: CodexProviderState = + serde_json::from_str(&serde_json::to_string(&state).unwrap()).unwrap(); + assert_eq!( + recovered.settled_provider_turn_ids, + state.settled_provider_turn_ids + ); + assert_eq!( + recovered.settled_provider_turn_filter, + state.settled_provider_turn_filter + ); + recovered.validate().unwrap(); + } + + #[test] + fn receipt_limit_deadline_settlement_preserves_unacknowledged_events() { + let directory = std::env::temp_dir().join(format!( + "paperclip-provider-receipt-deadline-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&directory); + fs::create_dir_all(&directory).unwrap(); + let mut state = CodexProviderState::new( + CodexProviderConfig { + provider: "codex".to_owned(), + driver: "codex_app_server".to_owned(), + provider_version: "test".to_owned(), + command: PathBuf::from("codex"), + args: vec!["app-server".to_owned()], + cwd: std::env::current_dir() + .unwrap() + .to_string_lossy() + .into_owned(), + model: None, + provider_session_id: None, + instructions: String::new(), + approval_policy: "never".to_owned(), + }, + None, + ProviderToolBridge::default(), + ); + state.thread_id = Some("thread-1".to_owned()); + state.active_provider_turn_id = Some("turn-1".to_owned()); + state.lifecycle = "turn_active".to_owned(); + state.receipt_limit_diagnostic_emitted = true; + state.receipt_limit_interrupt_pending = true; + state.receipt_limit_interrupt_attempts = MAX_RECEIPT_LIMIT_INTERRUPT_ATTEMPTS; + state.receipt_limit_interrupt_deadline_unix_ms = Some(1); + state + .push_event(NormalizedProviderEvent { + event_type: "provider.notice.recorded".to_owned(), + priority: EventPriority::P1, + payload: json!({"message": "awaiting acknowledgement"}), + }) + .unwrap(); + let mut executor = CodexCommandExecutor::new(&directory); + executor.state = Some(state); + executor.event_identity = Some(ProviderEventIdentity { + run_id: "run-1".to_owned(), + normalized_session_id: "session-1".to_owned(), + turn_id: "turn-1".to_owned(), + item_id: "item-1".to_owned(), + }); + executor.restore_checked = true; + + // Exercise receipt-limit recovery directly. `poll_provider` also + // restores a missing provider by design, while this unit test + // intentionally injects state without constructing a provider. + executor.retry_receipt_limit_interrupt().unwrap(); + executor + .settle_receipt_limit_interrupt_if_deadline_elapsed() + .unwrap(); + + let state = executor.state.as_ref().unwrap(); + assert_eq!(state.lifecycle, "provider_exited"); + assert!(state.active_provider_turn_id.is_none()); + assert!(!state.receipt_limit_interrupt_pending); + assert!(state.pending_events.iter().any(|event| { + event.event_type == "provider.notice.recorded" + && event.payload == json!({"message": "awaiting acknowledgement"}) + })); + assert!(state + .pending_events + .iter() + .any(|event| event.event_type == "turn.failed")); + fs::remove_dir_all(directory).unwrap(); + } } diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_bridge.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_bridge.rs index f5ef365baf..62213d3378 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_bridge.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_bridge.rs @@ -1,9 +1,9 @@ -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::error::Error; use std::fmt::{self, Display, Formatter}; use serde::de::{self, MapAccess, Visitor}; -use serde::{Deserialize, Deserializer, Serialize}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde_json::Value; use sha2::{Digest, Sha256}; @@ -12,18 +12,32 @@ pub const TOOL_CALL_SCHEMA: &str = "paperclip.prp.semantic_tool.v1"; pub const TOOL_RESULT_COMMAND: &str = "semantic_tool.result"; const MAX_AUTHORIZED_TOOLS: usize = 256; const MAX_DESCRIPTION_BYTES: usize = 16 * 1024; -const MAX_SCHEMA_BYTES: usize = 1024 * 1024; -const MAX_TOOL_SET_BYTES: usize = 4 * 1024 * 1024; -const MAX_TOOL_VALUE_BYTES: usize = 1024 * 1024; -// Settled results are authoritative replay receipts and live for the durable -// run. Bound their complete encoded map, while reserving enough room for every -// active call to later produce a maximum-sized result. The 1 KiB allowance -// covers the map key, bounded call/operation identities, JSON field names, and -// escaping around a 1 MiB result value. +const MAX_SCHEMA_BYTES: usize = 512 * 1024; +// Leave room for the authenticated PRP command or event envelope inside the +// default 1 MiB transport frame. +const MAX_TOOL_SET_BYTES: usize = 768 * 1024; +const MAX_TOOL_VALUE_BYTES: usize = 768 * 1024; +const MAX_ACCEPTED_TOOL_VALUE_BYTES: usize = 4 * 1024 * 1024; +const MAX_RETAINED_TOOL_VALUE_BYTES: usize = 8 * 1024 * 1024; +// Exact settled receipts are authoritative for the durable run. Bound their +// complete encoded representation independently of the raw-value budget so a +// long sequence of tiny results cannot accumulate unbounded identity and JSON +// envelope overhead. const MAX_SETTLED_RESULT_BYTES: usize = 8 * 1024 * 1024; -const MAX_SETTLED_RESULT_ENTRY_BYTES: usize = MAX_TOOL_VALUE_BYTES + 1024; -const MAX_RETAINED_CALLS: usize = 4_096; +pub(crate) const MAX_PENDING_CALLS: usize = 4_096; +// Active-turn inputs and results are separately bounded. Settled identities +// remain exact for a complete provider-process epoch so a delayed request +// cannot replay an old call ID after crossing a turn boundary. At the bound, +// the backend must reap the idle process before it rotates this ledger. +const MAX_DURABLE_CALL_RECEIPTS: usize = 4_096; const MAX_SETTLED_CALL_IDS: usize = 65_536; +// Retain the legacy serialized filter shape for recovery compatibility. New +// state never inserts probabilistic identities. A recovered non-empty filter +// makes the durable run fail closed until attach_run because its identities +// cannot safely be distinguished from fresh calls. +const REPLAY_FILTER_WORDS: usize = 32_768; +const ACTIVE_TURN_RECEIPT_LIMIT_MESSAGE: &str = + "durable provider tool receipt limit reached for the active turn"; #[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] #[serde(rename_all = "camelCase")] @@ -62,26 +76,145 @@ pub struct ToolResult { pub is_error: bool, } +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +#[serde(rename_all = "camelCase")] +struct CompletedToolCall { + call: PendingToolCall, + result: ToolResult, +} + +#[derive(Clone, Debug, Default, PartialEq)] +struct SettledCallIds { + order: VecDeque, + members: BTreeSet, +} + +// Preserve the existing on-disk array shape so durable state written before +// this bounded ordering was introduced remains readable. New state records IDs +// from oldest to newest, which makes pruning deterministic across recovery. +impl Serialize for SettledCallIds { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + self.order.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for SettledCallIds { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let values = VecDeque::::deserialize(deserializer)?; + let mut settled = Self::default(); + for value in values { + if settled.members.insert(value.clone()) { + settled.order.push_back(value); + } + } + Ok(settled) + } +} + +impl SettledCallIds { + fn clear(&mut self) { + self.order.clear(); + self.members.clear(); + } + + fn contains(&self, call_id: &str) -> bool { + self.members.contains(call_id) + } + + fn is_empty(&self) -> bool { + self.order.is_empty() + } + + fn len(&self) -> usize { + self.order.len() + } + + fn iter(&self) -> impl Iterator { + self.order.iter() + } + + fn extend_recent( + &mut self, + call_ids: impl IntoIterator, + limit: usize, + ) -> Vec { + for call_id in call_ids { + if self.members.insert(call_id.clone()) { + self.order.push_back(call_id); + } + } + let mut evicted = Vec::new(); + while self.order.len() > limit { + if let Some(call_id) = self.order.pop_front() { + self.members.remove(&call_id); + evicted.push(call_id); + } + } + evicted + } +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)] +pub(crate) struct DurableReplayFilter { + words: Vec, +} + +impl DurableReplayFilter { + pub(crate) fn is_empty(&self) -> bool { + self.words.is_empty() + } + + pub(crate) fn validate(&self) -> Result<(), ProviderBridgeError> { + if self.words.is_empty() || self.words.len() == REPLAY_FILTER_WORDS { + Ok(()) + } else { + Err(ProviderBridgeError::invalid( + "durable replay filter has an invalid size", + )) + } + } +} + #[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)] #[serde(rename_all = "camelCase")] pub struct ProviderToolBridge { authorized: BTreeMap, + #[serde(default)] + catalog_operations: Vec, catalog_digest: Option, pending: BTreeMap, #[serde(deserialize_with = "deserialize_retained_results")] - completed: BTreeMap, + completed: BTreeMap, + // Keep authoritative result bodies through the provider's replay window; + // prepare_turn releases them after settlement. #[serde(default, deserialize_with = "deserialize_retained_results")] - settled_results: BTreeMap, + settled_results: BTreeMap, // Derived from completed + settled results. It is intentionally omitted // from durable JSON and recomputed by attach_existing_run so old state and // tampered counters cannot bypass the byte envelope. #[serde(skip)] retained_result_bytes: usize, - // Compatibility tombstones for state written before settled results were - // retained. They still fail closed on call-id reuse, but cannot replay a - // value that the older state format discarded. + // Exact replay tombstones span every provider turn in the durable run. #[serde(default)] - settled_call_ids: BTreeSet, + settled_call_ids: SettledCallIds, + #[serde(default)] + settled_call_filter: DurableReplayFilter, + // Resource exhaustion stops the active turn. prepare_turn releases bulky + // result payloads and recomputes this marker, but replay ambiguity remains + // fail-closed until attach_run or a verified idle provider-process restart + // establishes fresh replay authority. + #[serde( + default, + alias = "settledHistoryResetPending", + skip_serializing_if = "is_false" + )] + durable_run_receipt_limit_reached: bool, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -91,6 +224,14 @@ impl ProviderBridgeError { fn invalid(message: impl Into) -> Self { Self(message.into()) } + + fn active_turn_receipt_limit() -> Self { + Self::invalid(ACTIVE_TURN_RECEIPT_LIMIT_MESSAGE) + } + + pub fn is_active_turn_receipt_limit(&self) -> bool { + self.0 == ACTIVE_TURN_RECEIPT_LIMIT_MESSAGE + } } impl Display for ProviderBridgeError { @@ -101,6 +242,10 @@ impl Display for ProviderBridgeError { impl Error for ProviderBridgeError {} +fn is_false(value: &bool) -> bool { + !*value +} + impl ProviderToolBridge { pub fn prepare(&mut self, tool_set: AuthorizedToolSet) -> Result<(), ProviderBridgeError> { self.prepare_internal(tool_set, false) @@ -117,6 +262,8 @@ impl ProviderToolBridge { self.settled_results.clear(); self.retained_result_bytes = 0; self.settled_call_ids.clear(); + self.settled_call_filter = DurableReplayFilter::default(); + self.durable_run_receipt_limit_reached = false; Ok(()) } @@ -125,92 +272,9 @@ impl ProviderToolBridge { // preserve them so an interrupted dispatcher can resume or replay the // authoritative result. `attach_run` remains the boundary that rejects // carrying pending calls into a different run. - if self - .authorized - .iter() - .any(|(operation_id, tool)| operation_id != &tool.operation_id) - { - return Err(ProviderBridgeError::invalid( - "recovered authorized tool identities are inconsistent", - )); - } - let Some(catalog_digest) = self.catalog_digest.clone() else { - if self.authorized.is_empty() - && self.pending.is_empty() - && self.completed.is_empty() - && self.settled_results.is_empty() - && self.settled_call_ids.is_empty() - { - self.retained_result_bytes = 0; - return Ok(()); - } - return Err(ProviderBridgeError::invalid( - "recovered authorized tools omit the catalog digest", - )); - }; - let recovered_tool_set = AuthorizedToolSet { - schema: TOOL_SET_SCHEMA.to_owned(), - schema_version: 1, - catalog_digest, - operations: self.authorized.values().cloned().collect(), - }; - validate_authorized_tool_set(&recovered_tool_set).map_err(|error| { - ProviderBridgeError::invalid(format!( - "recovered authorized tool catalog is invalid: {error}" - )) - })?; - if self - .settled_call_ids - .len() - .checked_add(self.settled_results.len()) - .and_then(|total| total.checked_add(self.pending.len())) - .and_then(|total| total.checked_add(self.completed.len())) - .is_none_or(|total| total > MAX_SETTLED_CALL_IDS) - || self.pending.len().saturating_add(self.completed.len()) > MAX_RETAINED_CALLS - || self - .settled_call_ids - .iter() - .any(|call_id| !is_stable_call_id(call_id)) - || self.settled_results.iter().any(|(call_id, result)| { - !is_stable_call_id(call_id) - || call_id != &result.call_id - || self.settled_call_ids.contains(call_id) - || validate_retained_result(result).is_err() - || validate_tool_result_contract(&self.authorized, result).is_err() - }) - || self.settled_call_ids.iter().any(|call_id| { - self.pending.contains_key(call_id) || self.completed.contains_key(call_id) - }) - || self.settled_results.keys().any(|call_id| { - self.pending.contains_key(call_id) || self.completed.contains_key(call_id) - }) - || self.pending.iter().any(|(call_id, call)| { - !is_stable_call_id(call_id) - || call_id != &call.call_id - || self.completed.contains_key(call_id) - || self.settled_call_ids.contains(call_id) - || self.settled_results.contains_key(call_id) - || validate_pending_tool_call(&self.authorized, call).is_err() - }) - || self.completed.iter().any(|(call_id, result)| { - !is_stable_call_id(call_id) - || call_id != &result.call_id - || self.pending.contains_key(call_id) - || validate_retained_result(result).is_err() - || validate_tool_result_contract(&self.authorized, result).is_err() - }) - { - return Err(ProviderBridgeError::invalid( - "recovered provider tool call state is invalid", - )); - } + self.validate_recovered()?; self.retained_result_bytes = retained_result_bytes(self.settled_results.iter().chain(self.completed.iter()))?; - self.ensure_settled_result_capacity(0).map_err(|_| { - ProviderBridgeError::invalid( - "recovered provider tool results exceed the durable byte limit", - ) - })?; Ok(()) } @@ -235,6 +299,7 @@ impl ProviderToolBridge { } } self.catalog_digest = Some(tool_set.catalog_digest); + self.catalog_operations = tool_set.operations.clone(); self.authorized = tool_set .operations .into_iter() @@ -244,7 +309,245 @@ impl ProviderToolBridge { } pub fn authorized_tools(&self) -> impl Iterator { - self.authorized.values() + self.catalog_operations.iter() + } + + pub fn validate_recovered(&self) -> Result<(), ProviderBridgeError> { + let Some(catalog_digest) = self.catalog_digest.clone() else { + return if self.authorized.is_empty() + && self.catalog_operations.is_empty() + && self.pending.is_empty() + && self.completed.is_empty() + && self.settled_results.is_empty() + && self.settled_call_ids.is_empty() + && self.settled_call_filter.is_empty() + && !self.durable_run_receipt_limit_reached + { + Ok(()) + } else { + Err(ProviderBridgeError::invalid( + "recovered provider tool bridge omitted its catalog identity", + )) + }; + }; + let mut expected = ProviderToolBridge::default(); + expected.prepare(AuthorizedToolSet { + schema: TOOL_SET_SCHEMA.to_owned(), + schema_version: 1, + catalog_digest, + operations: self.catalog_operations.clone(), + })?; + if self.authorized != expected.authorized { + return Err(ProviderBridgeError::invalid( + "recovered provider tool bridge changed its authorized catalog", + )); + } + let retained_receipts = self.pending.len().saturating_add(self.completed.len()); + if self.pending.len() > MAX_PENDING_CALLS || retained_receipts > MAX_DURABLE_CALL_RECEIPTS { + return Err(ProviderBridgeError::invalid( + "recovered provider tool bridge exceeds its call limit", + )); + } + self.validate_retained_value_bytes()?; + self.settled_call_filter.validate()?; + if self.settled_call_ids.len() > MAX_SETTLED_CALL_IDS { + return Err(ProviderBridgeError::invalid( + "recovered provider tool bridge exceeds its settled call identity limit", + )); + } + if !self.settled_identity_capacity_allows(0) { + return Err(ProviderBridgeError::invalid( + "recovered provider tool bridge cannot settle its active call identities", + )); + } + for call_id in self.settled_call_ids.iter() { + validate_stable_id(call_id, "settled tool call id")?; + if self.pending.contains_key(call_id) || self.completed.contains_key(call_id) { + return Err(ProviderBridgeError::invalid( + "recovered settled provider tool call identity is inconsistent", + )); + } + } + let mut pending_validator = expected.clone(); + for (call_id, call) in &self.pending { + if call_id != &call.call_id + || self.completed.contains_key(call_id) + || self.settled_results.contains_key(call_id) + || self.has_settled_call_id(call_id) + { + return Err(ProviderBridgeError::invalid( + "recovered provider tool call identity is inconsistent", + )); + } + pending_validator.begin_call( + call.call_id.clone(), + call.operation_id.clone(), + call.input.clone(), + )?; + } + for (call_id, completed) in &self.completed { + if call_id != &completed.call.call_id + || call_id != &completed.result.call_id + || completed.call.operation_id != completed.result.operation_id + || self.settled_results.contains_key(call_id) + || self.has_settled_call_id(call_id) + { + return Err(ProviderBridgeError::invalid( + "recovered completed tool call identity is inconsistent", + )); + } + let mut completed_validator = expected.clone(); + completed_validator.begin_call( + completed.call.call_id.clone(), + completed.call.operation_id.clone(), + completed.call.input.clone(), + )?; + completed_validator.apply_result(completed.result.clone())?; + } + for (call_id, settled) in &self.settled_results { + if call_id != &settled.call.call_id + || call_id != &settled.result.call_id + || settled.call.operation_id != settled.result.operation_id + || !self.settled_call_ids.contains(call_id) + || self.pending.contains_key(call_id) + || self.completed.contains_key(call_id) + { + return Err(ProviderBridgeError::invalid( + "recovered settled tool result identity is inconsistent", + )); + } + let mut settled_validator = expected.clone(); + settled_validator.begin_call( + settled.call.call_id.clone(), + settled.call.operation_id.clone(), + settled.call.input.clone(), + )?; + settled_validator.apply_result(settled.result.clone())?; + } + let retained_bytes = + retained_result_bytes(self.settled_results.iter().chain(self.completed.iter()))?; + ensure_settled_result_capacity(retained_bytes, self.pending.values()).map_err(|_| { + ProviderBridgeError::invalid( + "recovered provider tool results exceed the durable byte limit", + ) + })?; + Ok(()) + } + + pub fn verify_tool_set(&self, tool_set: &AuthorizedToolSet) -> Result<(), ProviderBridgeError> { + let mut expected = ProviderToolBridge::default(); + expected.prepare(tool_set.clone())?; + if self.catalog_digest != expected.catalog_digest + || self.catalog_operations != expected.catalog_operations + || self.authorized != expected.authorized + { + return Err(ProviderBridgeError::invalid( + "authorized tool set changed across a durable session", + )); + } + Ok(()) + } + + pub fn has_catalog(&self) -> bool { + self.catalog_digest.is_some() + } + + pub fn durable_run_receipt_limit_reached(&self) -> bool { + self.durable_run_receipt_limit_reached + } + + fn has_settled_call_id(&self, call_id: &str) -> bool { + self.settled_call_ids.contains(call_id) + } + + pub(crate) fn replay_history_blocks_admission(&self) -> bool { + self.settled_call_ids.len() >= MAX_SETTLED_CALL_IDS || !self.settled_call_filter.is_empty() + } + + pub(crate) fn has_active_receipts(&self) -> bool { + !self.pending.is_empty() || !self.completed.is_empty() + } + + /// Forget an exact replay epoch only after its provider process has been + /// reaped and an idle replacement generation has been established. The + /// process boundary is what prevents an old provider request from being + /// admitted after its tombstone is released; callers must not use this as + /// ordinary capacity eviction. + pub(crate) fn rollover_replay_epoch_after_provider_restart( + &mut self, + ) -> Result<(), ProviderBridgeError> { + if !self.replay_history_blocks_admission() { + return Err(ProviderBridgeError::invalid( + "cannot rotate provider tool replay authority before its exact capacity boundary", + )); + } + if !self.pending.is_empty() + || !self.completed.is_empty() + || !self.settled_results.is_empty() + || self.retained_result_bytes != 0 + { + return Err(ProviderBridgeError::invalid( + "cannot rotate provider tool replay authority while receipts are retained", + )); + } + self.settled_call_ids.clear(); + self.settled_call_filter = DurableReplayFilter::default(); + self.durable_run_receipt_limit_reached = false; + Ok(()) + } + + fn settled_identity_capacity_allows(&self, additional_calls: usize) -> bool { + self.settled_call_ids + .len() + .checked_add(self.total_call_receipts()) + .and_then(|total| total.checked_add(additional_calls)) + .is_some_and(|total| total <= MAX_SETTLED_CALL_IDS) + } + + pub fn prepare_turn(&mut self) -> Result<(), ProviderBridgeError> { + if !self.pending.is_empty() || !self.completed.is_empty() { + return Err(ProviderBridgeError::invalid( + "cannot prepare the next provider turn while tool calls are active", + )); + } + // The authoritative result bodies are needed only while the provider + // can replay the just-settled turn. Release that bulky data at the + // verified turn boundary, but retain every call-ID tombstone and any + // recovered legacy filter until the owning provider process is reaped. + self.settled_results.clear(); + self.retained_result_bytes = 0; + self.durable_run_receipt_limit_reached = self.replay_history_blocks_admission(); + Ok(()) + } + + #[cfg(test)] + pub(crate) fn retained_result_bytes_for_test(&self) -> usize { + self.retained_result_bytes + } + + pub fn replay_result( + &self, + call_id: &str, + operation_id: &str, + input: &Value, + ) -> Result, ProviderBridgeError> { + if let Some(completed) = self + .completed + .get(call_id) + .or_else(|| self.settled_results.get(call_id)) + { + if completed.call.operation_id != operation_id || &completed.call.input != input { + return Err(ProviderBridgeError::invalid( + "provider replayed a completed tool call with different input", + )); + } + return Ok(Some(completed.result.clone())); + } + Ok(None) + } + + pub fn has_completed_call(&self, call_id: &str) -> bool { + self.completed.contains_key(call_id) || self.has_settled_call_id(call_id) } pub fn begin_call( @@ -253,6 +556,24 @@ impl ProviderToolBridge { operation_id: String, input: Value, ) -> Result { + validate_stable_id(&call_id, "tool call id")?; + validate_operation_id(&operation_id)?; + let authorized = self.authorized.get(&operation_id).ok_or_else(|| { + ProviderBridgeError::invalid(format!( + "provider requested unauthorized tool {operation_id}" + )) + })?; + let validator = jsonschema::validator_for(&authorized.input_schema).map_err(|_| { + ProviderBridgeError::invalid(format!( + "tool {operation_id} has an invalid durable input JSON Schema" + )) + })?; + if !validator.is_valid(&input) { + return Err(ProviderBridgeError::invalid(format!( + "provider arguments for {operation_id} failed JSON Schema validation" + ))); + } + bounded_json(&input, MAX_TOOL_VALUE_BYTES, "provider tool input")?; let call = PendingToolCall { call_id: call_id.clone(), operation_id, @@ -270,55 +591,93 @@ impl ProviderToolBridge { } if self.completed.contains_key(&call_id) || self.settled_results.contains_key(&call_id) - || self.settled_call_ids.contains(&call_id) + || self.has_settled_call_id(&call_id) { return Err(ProviderBridgeError::invalid( "provider reused a completed tool call id", )); } - if self.pending.len().saturating_add(self.completed.len()) >= MAX_RETAINED_CALLS { + // A durable receipt ledger may be exactly full or legacy state may + // contain only a probabilistic summary of evicted identities. Neither + // state can admit more work safely in this provider-process epoch. The + // backend reaps an idle process before establishing a fresh epoch. + if self.replay_history_blocks_admission() { + self.durable_run_receipt_limit_reached = true; + return Err(ProviderBridgeError::active_turn_receipt_limit()); + } + if self.durable_run_receipt_limit_reached { + return Err(ProviderBridgeError::active_turn_receipt_limit()); + } + // Pending and completed calls become exact settled identities when the + // turn terminates. Reserve that identity capacity before dispatch so + // every admitted call can be settled without stranding active state. + if !self.settled_identity_capacity_allows(1) { + self.durable_run_receipt_limit_reached = true; + return Err(ProviderBridgeError::active_turn_receipt_limit()); + } + if self.pending.len() >= MAX_PENDING_CALLS { return Err(ProviderBridgeError::invalid( - "provider tool receipt limit reached for the active turn", + "concurrent provider tool call limit reached", )); } - // Reserve durable identity space before accepting work. Settlement can - // then never fail merely because earlier turns filled the ledger and - // leave completed receipts stranded in the active-turn budget. - if self - .settled_call_ids - .len() - .saturating_add(self.settled_results.len()) - .saturating_add(self.pending.len()) - .saturating_add(self.completed.len()) - >= MAX_SETTLED_CALL_IDS + if self.total_call_receipts() >= MAX_DURABLE_CALL_RECEIPTS { + return Err(ProviderBridgeError::active_turn_receipt_limit()); + } + let input_bytes = json_size(&call.input, "provider tool input")?; + let pending_bytes = self.pending_value_bytes()?; + if pending_bytes + .checked_add(input_bytes) + .is_none_or(|total| total > MAX_ACCEPTED_TOOL_VALUE_BYTES) { return Err(ProviderBridgeError::invalid( - "durable provider tool call identity limit reached", + "retained provider tool values exceed the 4 MiB acceptance limit", )); } - // Reserve the worst-case encoded result before accepting the call. - // This makes apply_result and settlement infallible with respect to - // durable result capacity: accepted work can always retain its exact - // authoritative replay value. - self.ensure_settled_result_capacity(1)?; + let result_reserve = self + .pending + .len() + .saturating_add(1) + .checked_mul(MAX_TOOL_VALUE_BYTES) + .ok_or_else(|| ProviderBridgeError::invalid("provider tool result reserve overflow"))?; + let retained_value_bytes = self.retained_value_bytes()?; + if retained_value_bytes + .checked_add(input_bytes) + .and_then(|total| total.checked_add(MAX_TOOL_VALUE_BYTES)) + .is_none_or(|total| total > MAX_RETAINED_TOOL_VALUE_BYTES) + { + self.durable_run_receipt_limit_reached = true; + return Err(ProviderBridgeError::active_turn_receipt_limit()); + } + if retained_value_bytes + .checked_add(input_bytes) + .and_then(|total| total.checked_add(result_reserve)) + .is_none_or(|total| total > MAX_RETAINED_TOOL_VALUE_BYTES) + { + return Err(ProviderBridgeError::active_turn_receipt_limit()); + } + // Reserve the complete encoded receipt, including this exact input and + // a maximum-sized result, before accepting work. An admitted call can + // therefore always retain its authoritative result at settlement. + if ensure_settled_result_capacity(self.retained_result_bytes, std::iter::once(&call)) + .is_err() + { + self.durable_run_receipt_limit_reached = true; + return Err(ProviderBridgeError::active_turn_receipt_limit()); + } + if self.ensure_settled_result_capacity(Some(&call)).is_err() { + return Err(ProviderBridgeError::active_turn_receipt_limit()); + } self.pending.insert(call_id, call.clone()); Ok(call) } pub fn apply_result(&mut self, result: ToolResult) -> Result { - if result.call_id.is_empty() - || result.call_id.len() > 160 - || result.call_id.chars().any(char::is_control) - { - return Err(ProviderBridgeError::invalid( - "tool result call id is invalid", - )); - } + validate_stable_id(&result.call_id, "tool result call id")?; validate_operation_id(&result.operation_id)?; bounded_json(&result.result, MAX_TOOL_VALUE_BYTES, "provider tool result")?; if let Some(existing) = self.completed.get(&result.call_id) { - return if existing == &result { - Ok(existing.result.clone()) + return if existing.result == result { + Ok(existing.result.result.clone()) } else { Err(ProviderBridgeError::invalid( "conflicting duplicate tool result", @@ -326,20 +685,20 @@ impl ProviderToolBridge { }; } if let Some(existing) = self.settled_results.get(&result.call_id) { - return if existing == &result { - Ok(existing.result.clone()) + return if existing.result == result { + Ok(existing.result.result.clone()) } else { Err(ProviderBridgeError::invalid( "conflicting duplicate settled tool result", )) }; } - if self.settled_call_ids.contains(&result.call_id) { + if self.has_settled_call_id(&result.call_id) { return Err(ProviderBridgeError::invalid( "legacy settled tool result cannot be replayed", )); } - let pending = self.pending.get(&result.call_id).ok_or_else(|| { + let pending = self.pending.get(&result.call_id).cloned().ok_or_else(|| { ProviderBridgeError::invalid("tool result does not match a pending provider call") })?; if pending.operation_id != result.operation_id { @@ -347,89 +706,218 @@ impl ProviderToolBridge { "tool result operation does not match its call", )); } - validate_tool_result_contract(&self.authorized, &result)?; - let result_bytes = retained_result_entry_bytes(&result.call_id, &result)?; - let next_retained_bytes = self - .retained_result_bytes + let authorized = self.authorized.get(&result.operation_id).ok_or_else(|| { + ProviderBridgeError::invalid("tool result operation is no longer authorized") + })?; + let validator = jsonschema::validator_for(&authorized.response_schema).map_err(|_| { + ProviderBridgeError::invalid(format!( + "tool {} has an invalid durable response JSON Schema", + result.operation_id + )) + })?; + let response = semantic_response_value(&result)?; + if !result.is_error { + // Paperclip semantic dispatchers return an authoritative envelope; + // provider contracts describe the operation-specific value inside + // `result`. Direct values remain valid for compatibility with v1 + // peers that do not wrap their semantic result. + if let Some(response) = response { + if !validator.is_valid(response) { + return Err(ProviderBridgeError::invalid(format!( + "tool result for {} failed JSON Schema validation", + result.operation_id + ))); + } + } + } + let result_bytes = json_size(&result.result, "provider tool result")?; + let retained_bytes = self.retained_value_bytes()?; + if retained_bytes .checked_add(result_bytes) - .ok_or_else(|| ProviderBridgeError::invalid("durable provider result size overflow"))?; - let remaining_pending_reserve = self - .pending - .len() - .saturating_sub(1) - .checked_mul(MAX_SETTLED_RESULT_ENTRY_BYTES) - .ok_or_else(|| ProviderBridgeError::invalid("durable provider result size overflow"))?; - if next_retained_bytes - .checked_add(remaining_pending_reserve) - .is_none_or(|bytes| bytes > MAX_SETTLED_RESULT_BYTES) + .is_none_or(|total| total > MAX_RETAINED_TOOL_VALUE_BYTES) { return Err(ProviderBridgeError::invalid( - "durable provider tool result byte limit reached", + "retained provider tool values exceed the 8 MiB aggregate limit", )); } + let completed = CompletedToolCall { + call: pending, + result: result.clone(), + }; + let receipt_bytes = retained_result_entry_bytes(&result.call_id, &completed)?; + let next_retained_bytes = self + .retained_result_bytes + .checked_add(receipt_bytes) + .ok_or_else(|| ProviderBridgeError::invalid("durable provider result size overflow"))?; + ensure_settled_result_capacity( + next_retained_bytes, + self.pending + .iter() + .filter(|(call_id, _)| *call_id != &result.call_id) + .map(|(_, call)| call), + )?; self.pending.remove(&result.call_id); self.retained_result_bytes = next_retained_bytes; - self.completed - .insert(result.call_id.clone(), result.clone()); + self.completed.insert(result.call_id.clone(), completed); Ok(result.result) } - pub fn settle_turn(&mut self) -> Result<(), ProviderBridgeError> { - if !self.pending.is_empty() { - return Err(ProviderBridgeError::invalid( - "cannot settle provider tool receipts while calls are pending", - )); - } - self.retained_result_bytes = - retained_result_bytes(self.settled_results.iter().chain(self.completed.iter()))?; - self.ensure_settled_result_capacity(0)?; - // The identity capacity was reserved in `begin_call`, so moving the - // authoritative receipts cannot fail a valid admitted turn. The check - // above rejects only recovered state that bypassed attach validation. - self.settled_results.append(&mut self.completed); - Ok(()) - } - pub fn pending_calls(&self) -> impl Iterator { self.pending.values() } - fn ensure_settled_result_capacity( - &self, - additional_pending: usize, - ) -> Result<(), ProviderBridgeError> { - let pending_count = self + pub fn cancel_pending_calls( + &mut self, + code: &str, + ) -> Result, ProviderBridgeError> { + let mut next = self.clone(); + let results = next.cancel_pending_calls_internal(code)?; + *self = next; + Ok(results) + } + + pub fn settle_turn(&mut self, code: &str) -> Result, ProviderBridgeError> { + validate_stable_id(code, "tool cancellation code")?; + let results = self .pending - .len() - .checked_add(additional_pending) - .ok_or_else(|| { - ProviderBridgeError::invalid("durable provider result count overflow") - })?; - let pending_reserve = pending_count - .checked_mul(MAX_SETTLED_RESULT_ENTRY_BYTES) - .ok_or_else(|| ProviderBridgeError::invalid("durable provider result size overflow"))?; + .values() + .map(|call| cancelled_tool_result(call, code)) + .collect::>(); + let mut settled_entries = self.completed.clone(); + for (call_id, call) in &self.pending { + settled_entries.insert( + call_id.clone(), + CompletedToolCall { + call: call.clone(), + result: cancelled_tool_result(call, code), + }, + ); + } + let next_retained_bytes = + retained_result_bytes(self.settled_results.iter().chain(settled_entries.iter()))?; + ensure_settled_result_capacity(next_retained_bytes, std::iter::empty())?; + + // Byte capacity was reserved at admission. Keep every identity exact + // for the lifetime of this provider-process epoch; prepare_turn + // releases only the bulky result bodies. + let new_identity_count = settled_entries + .keys() + .filter(|call_id| !self.settled_call_ids.contains(call_id)) + .count(); if self - .retained_result_bytes - .checked_add(pending_reserve) - .is_none_or(|bytes| bytes > MAX_SETTLED_RESULT_BYTES) + .settled_call_ids + .len() + .checked_add(new_identity_count) + .is_none_or(|total| total > MAX_SETTLED_CALL_IDS) { + return Err(ProviderBridgeError::active_turn_receipt_limit()); + } + let evicted = self + .settled_call_ids + .extend_recent(settled_entries.keys().cloned(), MAX_SETTLED_CALL_IDS); + debug_assert!(evicted.is_empty()); + self.pending.clear(); + self.completed.clear(); + self.settled_results.append(&mut settled_entries); + self.retained_result_bytes = next_retained_bytes; + Ok(results) + } + + fn cancel_pending_calls_internal( + &mut self, + code: &str, + ) -> Result, ProviderBridgeError> { + validate_stable_id(code, "tool cancellation code")?; + let pending = self.pending.values().cloned().collect::>(); + let mut results = Vec::with_capacity(pending.len()); + for call in pending { + let result = cancelled_tool_result(&call, code); + self.apply_result(result.clone())?; + results.push(result); + } + Ok(results) + } + + fn retained_value_bytes(&self) -> Result { + let pending = self + .pending + .values() + .map(|call| json_size(&call.input, "retained provider tool input")); + let completed = self.completed.values().flat_map(|entry| { + [ + json_size(&entry.call.input, "retained provider tool input"), + json_size(&entry.result.result, "retained provider tool result"), + ] + }); + let settled = self.settled_results.values().flat_map(|entry| { + [ + json_size(&entry.call.input, "retained provider tool input"), + json_size(&entry.result.result, "retained provider tool result"), + ] + }); + pending + .chain(completed) + .chain(settled) + .try_fold(0usize, |total, bytes| { + total.checked_add(bytes?).ok_or_else(|| { + ProviderBridgeError::invalid("retained provider tool values overflow") + }) + }) + } + + fn pending_value_bytes(&self) -> Result { + self.pending.values().try_fold(0usize, |total, call| { + total + .checked_add(json_size(&call.input, "retained provider tool input")?) + .ok_or_else(|| { + ProviderBridgeError::invalid("retained provider tool values overflow") + }) + }) + } + + fn total_call_receipts(&self) -> usize { + self.pending.len().saturating_add(self.completed.len()) + } + + fn validate_retained_value_bytes(&self) -> Result<(), ProviderBridgeError> { + if self.retained_value_bytes()? > MAX_RETAINED_TOOL_VALUE_BYTES { return Err(ProviderBridgeError::invalid( - "durable provider tool result byte limit reached", + "retained provider tool values exceed the 8 MiB aggregate limit", )); } Ok(()) } + + fn ensure_settled_result_capacity( + &self, + additional_pending: Option<&PendingToolCall>, + ) -> Result<(), ProviderBridgeError> { + ensure_settled_result_capacity( + self.retained_result_bytes, + self.pending.values().chain(additional_pending.into_iter()), + ) + } } -fn validate_retained_result(result: &ToolResult) -> Result<(), ProviderBridgeError> { - if !is_stable_call_id(&result.call_id) { +fn validate_retained_result(entry: &CompletedToolCall) -> Result<(), ProviderBridgeError> { + validate_stable_id(&entry.call.call_id, "retained tool call id")?; + validate_stable_id(&entry.result.call_id, "retained tool result call id")?; + validate_operation_id(&entry.call.operation_id)?; + validate_operation_id(&entry.result.operation_id)?; + if entry.call.call_id != entry.result.call_id + || entry.call.operation_id != entry.result.operation_id + { return Err(ProviderBridgeError::invalid( - "retained tool result call id is invalid", + "retained provider tool receipt identity is inconsistent", )); } - validate_operation_id(&result.operation_id)?; bounded_json( - &result.result, + &entry.call.input, + MAX_TOOL_VALUE_BYTES, + "retained provider tool input", + )?; + bounded_json( + &entry.result.result, MAX_TOOL_VALUE_BYTES, "retained provider tool result", ) @@ -439,9 +927,7 @@ fn validate_pending_tool_call( authorized: &BTreeMap, call: &PendingToolCall, ) -> Result<(), ProviderBridgeError> { - if !is_stable_call_id(&call.call_id) { - return Err(ProviderBridgeError::invalid("tool call id is invalid")); - } + validate_stable_id(&call.call_id, "tool call id")?; validate_operation_id(&call.operation_id)?; let tool = authorized.get(&call.operation_id).ok_or_else(|| { ProviderBridgeError::invalid(format!( @@ -571,16 +1057,16 @@ fn validate_authorized_tool_set(tool_set: &AuthorizedToolSet) -> Result<(), Prov fn retained_result_entry_bytes( call_id: &str, - result: &ToolResult, + result: &CompletedToolCall, ) -> Result { // A two-item tuple has the same delimiter cost as a one-entry JSON map. // Summing tuples therefore equals one entry exactly and conservatively // overcounts a multi-entry map by one byte per additional receipt. - encoded_json_bytes(&(call_id, result), "retained provider tool result") + json_size(&(call_id, result), "retained provider tool result") } fn retained_result_bytes<'a>( - results: impl IntoIterator, + results: impl IntoIterator, ) -> Result { results .into_iter() @@ -593,16 +1079,53 @@ fn retained_result_bytes<'a>( }) } +fn pending_result_reserve_bytes(call: &PendingToolCall) -> Result { + let placeholder = CompletedToolCall { + call: call.clone(), + result: ToolResult { + call_id: call.call_id.clone(), + operation_id: call.operation_id.clone(), + result: Value::Null, + is_error: false, + }, + }; + retained_result_entry_bytes(&call.call_id, &placeholder)? + .checked_sub(json_size(&Value::Null, "provider tool result reserve")?) + .and_then(|bytes| bytes.checked_add(MAX_TOOL_VALUE_BYTES)) + .ok_or_else(|| ProviderBridgeError::invalid("durable provider result size overflow")) +} + +fn ensure_settled_result_capacity<'a>( + retained_bytes: usize, + pending: impl IntoIterator, +) -> Result<(), ProviderBridgeError> { + let reserved_bytes = pending + .into_iter() + .try_fold(retained_bytes, |total, call| { + total + .checked_add(pending_result_reserve_bytes(call)?) + .ok_or_else(|| { + ProviderBridgeError::invalid("durable provider result size overflow") + }) + })?; + if reserved_bytes > MAX_SETTLED_RESULT_BYTES { + return Err(ProviderBridgeError::invalid( + "durable provider tool result byte limit reached", + )); + } + Ok(()) +} + fn deserialize_retained_results<'de, D>( deserializer: D, -) -> Result, D::Error> +) -> Result, D::Error> where D: Deserializer<'de>, { struct RetainedResultsVisitor; impl<'de> Visitor<'de> for RetainedResultsVisitor { - type Value = BTreeMap; + type Value = BTreeMap; fn expecting(&self, formatter: &mut Formatter<'_>) -> fmt::Result { formatter.write_str("a bounded map of retained provider tool results") @@ -622,14 +1145,14 @@ where } let mut results = BTreeMap::new(); let mut retained_bytes = 0usize; - while let Some((call_id, result)) = map.next_entry::()? { + while let Some((call_id, result)) = map.next_entry::()? { if results.len() >= MAX_SETTLED_CALL_IDS { return Err(de::Error::custom( "retained provider tool result count exceeds the durable limit", )); } validate_retained_result(&result).map_err(de::Error::custom)?; - if call_id != result.call_id { + if call_id != result.call.call_id || call_id != result.result.call_id { return Err(de::Error::custom( "retained provider tool result identity is inconsistent", )); @@ -658,8 +1181,19 @@ where deserializer.deserialize_map(RetainedResultsVisitor) } -fn is_stable_call_id(value: &str) -> bool { - !value.is_empty() && value.len() <= 160 && !value.chars().any(char::is_control) +fn cancelled_tool_result(call: &PendingToolCall, code: &str) -> ToolResult { + ToolResult { + call_id: call.call_id.clone(), + operation_id: call.operation_id.clone(), + result: serde_json::json!({ + "error": { + "code": code, + "message": "The provider turn stopped before this semantic tool completed", + "retryable": false, + }, + }), + is_error: true, + } } pub fn authorized_tool_catalog_digest( @@ -678,11 +1212,16 @@ pub fn authorized_tool_catalog_digest( Ok(format!("sha256:{digest:x}")) } +pub fn semantic_value_digest(value: &Value) -> String { + let digest = Sha256::digest(canonical_json(value).as_bytes()); + format!("sha256:{digest:x}") +} + fn canonical_json(value: &Value) -> String { match value { Value::Null => "null".to_owned(), Value::Bool(value) => value.to_string(), - Value::Number(value) => value.to_string(), + Value::Number(value) => canonical_json_number(value), Value::String(value) => { serde_json::to_string(value).expect("serializing an in-memory JSON string cannot fail") } @@ -696,7 +1235,7 @@ fn canonical_json(value: &Value) -> String { ), Value::Object(object) => { let mut entries = object.iter().collect::>(); - entries.sort_by_key(|(key, _)| *key); + entries.sort_by(|(left, _), (right, _)| left.encode_utf16().cmp(right.encode_utf16())); format!( "{{{}}}", entries @@ -714,6 +1253,82 @@ fn canonical_json(value: &Value) -> String { } } +fn canonical_json_number(value: &serde_json::Number) -> String { + if value.is_i64() || value.is_u64() { + return value.to_string(); + } + let Some(float) = value.as_f64() else { + return value.to_string(); + }; + if float == 0.0 { + return "0".to_owned(); + } + + // serde_json preserves valid lexical spellings such as `1.0`, whereas + // JSON.stringify canonicalizes JavaScript numbers. Normalize the shortest + // serde representation to the ECMAScript decimal/exponent thresholds so + // the server and runner hash the same schema value. + let encoded = value.to_string().to_ascii_lowercase(); + let (negative, unsigned) = encoded + .strip_prefix('-') + .map_or((false, encoded.as_str()), |rest| (true, rest)); + let (coefficient, explicit_exponent) = unsigned + .split_once('e') + .map_or((unsigned, 0_i32), |(coefficient, exponent)| { + (coefficient, exponent.parse::().unwrap_or(0)) + }); + let fraction_digits = coefficient + .split_once('.') + .map_or(0_i32, |(_, fraction)| fraction.len() as i32); + let mut digits = coefficient + .bytes() + .filter(|byte| *byte != b'.') + .map(char::from) + .collect::(); + let mut decimal_position = digits.len() as i32 + explicit_exponent - fraction_digits; + + let leading_zeros = digits.bytes().take_while(|byte| *byte == b'0').count(); + digits.drain(..leading_zeros); + decimal_position -= leading_zeros as i32; + while digits.ends_with('0') { + digits.pop(); + } + if digits.is_empty() { + return "0".to_owned(); + } + + let body = if (1e-6..1e21).contains(&float.abs()) { + if decimal_position <= 0 { + format!("0.{}{}", "0".repeat((-decimal_position) as usize), digits) + } else if decimal_position >= digits.len() as i32 { + format!( + "{}{}", + digits, + "0".repeat((decimal_position - digits.len() as i32) as usize) + ) + } else { + let split = decimal_position as usize; + format!("{}.{}", &digits[..split], &digits[split..]) + } + } else { + let exponent = decimal_position - 1; + let coefficient = if digits.len() == 1 { + digits + } else { + format!("{}.{}", &digits[..1], &digits[1..]) + }; + format!( + "{coefficient}e{}{exponent}", + if exponent >= 0 { "+" } else { "" } + ) + }; + if negative { + format!("-{body}") + } else { + body + } +} + fn semantic_response_value(result: &ToolResult) -> Result, ProviderBridgeError> { let Some(envelope) = result.result.as_object() else { return Ok(Some(&result.result)); @@ -732,9 +1347,11 @@ fn semantic_response_value(result: &ToolResult) -> Result, Provid )); } if ok { - envelope.get("result").map(Some).ok_or_else(|| { - ProviderBridgeError::invalid("successful semantic result omitted result") - }) + envelope + .get("result") + .or_else(|| envelope.get("value")) + .map(Some) + .ok_or_else(|| ProviderBridgeError::invalid("successful semantic result omitted value")) } else if envelope.get("denial").is_some() || envelope.get("error").is_some() { Ok(None) } else { @@ -745,6 +1362,10 @@ fn semantic_response_value(result: &ToolResult) -> Result, Provid } fn validate_operation_id(value: &str) -> Result<(), ProviderBridgeError> { + validate_stable_id(value, "tool operation id") +} + +fn validate_stable_id(value: &str, label: &str) -> Result<(), ProviderBridgeError> { let mut chars = value.chars(); let first = chars .next() @@ -755,7 +1376,7 @@ fn validate_operation_id(value: &str) -> Result<(), ProviderBridgeError> { if first && rest && value.len() <= 160 { Ok(()) } else { - Err(ProviderBridgeError::invalid("tool operation id is invalid")) + Err(ProviderBridgeError::invalid(format!("{label} is invalid"))) } } @@ -774,7 +1395,7 @@ fn bounded_json( max_bytes: usize, label: &str, ) -> Result<(), ProviderBridgeError> { - let bytes = encoded_json_bytes(value, label)?; + let bytes = json_size(value, label)?; if bytes > max_bytes { return Err(ProviderBridgeError::invalid(format!( "{label} exceeds the {max_bytes} byte limit" @@ -783,7 +1404,7 @@ fn bounded_json( Ok(()) } -fn encoded_json_bytes(value: &impl Serialize, label: &str) -> Result { +fn json_size(value: &impl Serialize, label: &str) -> Result { serde_json::to_vec(value) .map(|bytes| bytes.len()) .map_err(|_| ProviderBridgeError::invalid(format!("{label} is not serializable"))) @@ -792,24 +1413,524 @@ fn encoded_json_bytes(value: &impl Serialize, label: &str) -> Result Option<&'static str> { + let status = match method { + "turn/failed" => "failed", + "turn/cancelled" => "cancelled", + "turn/interrupted" => "interrupted", + "turn/completed" => string( + params + .pointer("/turn/status") + .or_else(|| params.get("status")), + ), + _ => return None, + }; + Some(match status { + "failed" | "error" => "turn.failed", + "cancelled" | "canceled" => "turn.cancelled", + "interrupted" | "aborted" => "turn.interrupted", + _ => "turn.completed", + }) +} + fn bounded_text(value: &str, max_chars: usize) -> String { redact_text(value).chars().take(max_chars).collect() } @@ -118,18 +141,19 @@ pub fn normalize_codex_notification(method: &str, params: &Value) -> Vec { - let status = string( - params - .pointer("/turn/status") - .or_else(|| params.get("status")), - ); - let event_type = match status { - "failed" | "error" => "turn.failed", - "cancelled" | "canceled" => "turn.cancelled", - "interrupted" | "aborted" => "turn.interrupted", - _ => "turn.completed", + "turn/completed" | "turn/failed" | "turn/cancelled" | "turn/interrupted" => { + let status = match method { + "turn/failed" => "failed", + "turn/cancelled" => "cancelled", + "turn/interrupted" => "interrupted", + _ => string( + params + .pointer("/turn/status") + .or_else(|| params.get("status")), + ), }; + let event_type = normalized_codex_terminal_event_type(method, params) + .expect("matched Codex terminal method has a normalized terminal type"); push( &mut events, event_type, @@ -366,6 +390,16 @@ mod tests { ); assert_eq!(terminal[0].event_type, "turn.failed"); assert_eq!(terminal[0].priority, EventPriority::P0); + for (method, expected) in [ + ("turn/failed", "turn.failed"), + ("turn/cancelled", "turn.cancelled"), + ("turn/interrupted", "turn.interrupted"), + ] { + let terminal = + normalize_codex_notification(method, &json!({"turnId": "provider-turn"})); + assert_eq!(terminal[0].event_type, expected); + assert_eq!(terminal[0].priority, EventPriority::P0); + } let usage = normalize_codex_notification( "thread/tokenUsage/updated", diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs index 08c3380360..4e8050f1b1 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs @@ -5,9 +5,14 @@ use std::sync::atomic::{AtomicU64, Ordering}; use paperclip_runner_core::codex_provider::{ CodexProvider, CodexProviderConfig, CodexProviderEvent, }; -use paperclip_runner_core::durable::{Command, CommandExecutor, DurableRunnerError, PolledEvent}; +use paperclip_runner_core::durable::{ + Command, CommandExecutor, DurableRunnerConfig, DurableRunnerError, PolledEvent, +}; use paperclip_runner_core::provider_backend::CodexCommandExecutor; -use paperclip_runner_core::provider_bridge::{AuthorizedTool, ToolResult}; +use paperclip_runner_core::provider_bridge::{ + authorized_tool_catalog_digest, AuthorizedTool, AuthorizedToolSet, ProviderToolBridge, + ToolResult, TOOL_SET_SCHEMA, +}; use paperclip_runner_core::provider_events::normalize_codex_notification; use serde_json::{json, Value}; @@ -62,6 +67,36 @@ fn task_context_tool() -> AuthorizedTool { } } +fn task_context_tool_set() -> AuthorizedToolSet { + let operations = vec![task_context_tool()]; + AuthorizedToolSet { + schema: TOOL_SET_SCHEMA.to_owned(), + schema_version: 1, + catalog_digest: authorized_tool_catalog_digest(&operations).unwrap(), + operations, + } +} + +fn durable_config(directory: &Path) -> DurableRunnerConfig { + DurableRunnerConfig { + connect_url: "ws://127.0.0.1:3000/runner".to_owned(), + state_dir: directory.to_path_buf(), + runner_instance_id: "runner-1".to_owned(), + environment_lease_id: "lease-1".to_owned(), + run_id: "run-1".to_owned(), + normalized_session_id: "session-1".to_owned(), + turn_id: "turn-1".to_owned(), + item_id: "item-1".to_owned(), + runner_version: "test-1".to_owned(), + runner_digest: format!("sha256:{}", "a".repeat(64)), + max_outbox_bytes: 16 * 1024 * 1024, + p0_reserve_bytes: 1024 * 1024, + max_frame_bytes: 1024 * 1024, + reconnect_delay: std::time::Duration::from_millis(1), + max_runtime: std::time::Duration::from_secs(5), + } +} + fn command(id: &str, sequence: u64, command_type: &str, payload: Value) -> Command { Command { schema: "paperclip.prp.command.v1".to_owned(), @@ -83,6 +118,14 @@ fn call_count(directory: &Path, method: &str) -> usize { .count() } +fn recorded_tool_responses(directory: &Path) -> Vec { + fs::read_to_string(directory.join("calls.log")) + .unwrap_or_default() + .lines() + .filter_map(|line| line.strip_prefix("tool-response:").map(str::to_owned)) + .collect() +} + fn poll_and_ack( executor: &mut CodexCommandExecutor, ) -> Result, DurableRunnerError> { @@ -91,6 +134,132 @@ fn poll_and_ack( Ok(events) } +fn wait_for_notification(provider: &mut CodexProvider, expected_method: &str) -> Value { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while std::time::Instant::now() < deadline { + match provider.poll().expect("poll provider notification") { + Some(CodexProviderEvent::Notification { method, params }) => { + if method == expected_method { + return params; + } + } + Some(_) => {} + None => std::thread::sleep(std::time::Duration::from_millis(1)), + } + } + panic!("did not observe Codex {expected_method} notification before the deadline"); +} + +fn wait_for_provider_error(provider: &mut CodexProvider) -> String { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while std::time::Instant::now() < deadline { + match provider.poll() { + Err(error) => return error.to_string(), + Ok(Some(_)) => {} + Ok(None) => std::thread::sleep(std::time::Duration::from_millis(1)), + } + } + panic!("did not observe the expected Codex provider error before the deadline"); +} + +fn wait_for_executor_event( + executor: &mut CodexCommandExecutor, + expected_event_type: &str, +) -> PolledEvent { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while std::time::Instant::now() < deadline { + let events = poll_and_ack(executor).expect("poll Codex executor event"); + if let Some(event) = events + .into_iter() + .find(|event| event.event_type == expected_event_type) + { + return event; + } + std::thread::sleep(std::time::Duration::from_millis(1)); + } + panic!("did not observe Codex {expected_event_type} event before the deadline"); +} + +fn wait_for_provider_exit(provider: &mut CodexProvider) { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while std::time::Instant::now() < deadline { + if matches!( + provider.poll().expect("poll terminated provider"), + Some(CodexProviderEvent::Exited { .. }) + ) { + return; + } + std::thread::sleep(std::time::Duration::from_millis(1)); + } + panic!("the provider accepted a reused turn identity but remained live"); +} + +fn wait_for_reused_identity_reap(provider: &mut CodexProvider) { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while std::time::Instant::now() < deadline { + match provider.poll() { + Err(error) => assert!( + error.to_string().contains("reused a settled"), + "unexpected error while reaping reused provider identity: {error}" + ), + Ok(Some(CodexProviderEvent::Exited { + completed_turn_authoritative, + .. + })) => { + assert!( + !completed_turn_authoritative, + "accepted identity reuse must revoke prior completion authority" + ); + return; + } + Ok(Some(_)) => {} + Ok(None) => std::thread::sleep(std::time::Duration::from_millis(1)), + } + } + panic!("the provider accepted a reused turn identity but was not reaped"); +} + +fn wait_for_fake_provider_idle(directory: &Path) { + let state_path = directory.join("fake-state.json"); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while std::time::Instant::now() < deadline { + let active_turn_id = fs::read(&state_path) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + .and_then(|state| state.get("activeTurnId").cloned()); + if active_turn_id == Some(Value::Null) { + return; + } + std::thread::sleep(std::time::Duration::from_millis(1)); + } + panic!("the fake provider did not persist its idle turn state before the deadline"); +} + +fn saturate_provider_tool_receipts(directory: &Path) { + let mut bridge = ProviderToolBridge::default(); + bridge.prepare(task_context_tool_set()).unwrap(); + for index in 0..4_096 { + let call_id = format!("retained-call-{index}"); + bridge + .begin_call(call_id.clone(), "get_task_context".into(), json!({})) + .unwrap(); + bridge + .apply_result(ToolResult { + call_id, + operation_id: "get_task_context".into(), + result: json!({"ok": true}), + is_error: false, + }) + .unwrap(); + } + let state_path = directory.join("codex-provider-state.json"); + let mut persisted: Value = + serde_json::from_slice(&fs::read(&state_path).expect("read provider state")).unwrap(); + persisted["toolBridge"] = serde_json::to_value(bridge).unwrap(); + fs::write(&state_path, serde_json::to_vec_pretty(&persisted).unwrap()) + .expect("write saturated provider state"); +} + #[test] fn codex_transport_buffers_notifications_while_waiting_for_responses() { let directory = temporary_directory("buffering"); @@ -505,6 +674,285 @@ fn clean_provider_exit_does_not_refail_a_completed_turn() { fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); } +#[test] +fn durable_backend_reaps_before_rotating_full_provider_identity_epochs() { + let directory = temporary_directory("provider-identity-epoch-rollover"); + let config = provider_config(&directory, &["--require-dynamic-tool", "--emit-tool-call"]); + let runner_config = durable_config(&directory); + let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + first + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({ + "provider": config, + "authorizedTools": task_context_tool_set(), + }), + )) + .expect("prepare Codex provider"); + first + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open Codex session"); + drop(first); + + let state_path = directory.join("codex-provider-state.json"); + let mut persisted: Value = serde_json::from_slice( + &fs::read(&state_path).expect("read provider state before epoch rollover"), + ) + .expect("parse provider state before epoch rollover"); + let prior_generation = persisted["providerProcessGeneration"] + .as_u64() + .expect("provider generation is persisted"); + persisted["settledProviderTurnIds"] = Value::Array( + (0..4_096) + .map(|index| Value::String(format!("provider-turn-{index}"))) + .collect(), + ); + persisted["completedTurnAuthoritative"] = Value::Bool(true); + persisted["completedTurnProcessGeneration"] = json!(prior_generation); + persisted["completedProviderTurnId"] = json!("provider-turn-4095"); + persisted["toolBridge"]["settledCallIds"] = Value::Array( + (0..65_536) + .map(|index| Value::String(format!("semantic-call-{index}"))) + .collect(), + ); + persisted["toolBridge"]["durableRunReceiptLimitReached"] = Value::Bool(true); + fs::write(&state_path, serde_json::to_vec_pretty(&persisted).unwrap()) + .expect("write full provider identity epochs"); + + let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + recovered + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Start after a verified provider epoch rollover."}), + )) + .expect("roll over the idle provider generation and start fresh work"); + + let rolled: Value = serde_json::from_slice( + &fs::read(&state_path).expect("read provider state after epoch rollover"), + ) + .expect("parse provider state after epoch rollover"); + assert!(rolled["providerProcessGeneration"].as_u64().unwrap() > prior_generation); + assert_eq!( + rolled["settledProviderTurnIds"], + json!(["provider-turn-4095"]) + ); + assert_eq!(rolled["settledProviderTurnFilter"], json!({"words": []})); + assert_eq!(rolled["toolBridge"]["settledCallIds"], json!([])); + assert_eq!( + rolled["toolBridge"]["settledCallFilter"], + json!({"words": []}) + ); + assert!(rolled["toolBridge"]["durableRunReceiptLimitReached"].is_null()); + + let mut event_types = Vec::new(); + let semantic_deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while std::time::Instant::now() < semantic_deadline { + event_types.extend( + poll_and_ack(&mut recovered) + .expect("poll semantic work in the fresh provider epoch") + .into_iter() + .map(|event| event.event_type), + ); + if event_types + .iter() + .any(|event| event == "semantic_tool.input") + { + break; + } + std::thread::sleep(std::time::Duration::from_millis(1)); + } + assert!(event_types + .iter() + .any(|event| event == "semantic_tool.input")); + + recovered.shutdown().expect("stop rolled provider process"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn durable_backend_closes_when_identity_rollover_resumes_unowned_work() { + let directory = temporary_directory("provider-identity-rollover-unowned-work"); + let config = provider_config(&directory, &["--resume-unowned-turn-when-marked"]); + let mut first = CodexCommandExecutor::new(&directory); + first + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({"provider": config}), + )) + .expect("prepare Codex provider"); + first + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open Codex session"); + drop(first); + + let state_path = directory.join("codex-provider-state.json"); + let mut persisted: Value = serde_json::from_slice( + &fs::read(&state_path).expect("read provider state before unsafe rollover"), + ) + .expect("parse provider state before unsafe rollover"); + persisted["settledProviderTurnIds"] = Value::Array( + (0..4_096) + .map(|index| Value::String(format!("provider-turn-{index}"))) + .collect(), + ); + fs::write(&state_path, serde_json::to_vec_pretty(&persisted).unwrap()) + .expect("write full provider identity epoch"); + + let mut recovered = CodexCommandExecutor::new(&directory); + poll_and_ack(&mut recovered).expect("attach an idle provider before rollover"); + wait_for_fake_provider_idle(&directory); + let attached: Value = + serde_json::from_slice(&fs::read(&state_path).expect("read attached provider state")) + .expect("parse attached provider state"); + let attached_generation = attached["providerProcessGeneration"] + .as_u64() + .expect("attached provider generation is persisted"); + + // Race the provider's idle snapshot with work Paperclip never dispatched. + // The replacement process observes this turn during thread/resume and must + // close the durable run instead of leaving a quarantined session open. + fs::write(directory.join("resume-unowned-turn"), b"armed") + .expect("arm unowned provider work before rollover"); + let resumes_before_rollover = call_count(&directory, "thread/resume"); + let error = recovered + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Never overlap the unowned provider turn."}), + )) + .expect_err("identity rollover fails closed after resuming unowned work"); + assert!(error + .to_string() + .contains("identity epoch rollover failed closed after an invalid accepted identity")); + assert_eq!( + call_count(&directory, "thread/resume"), + resumes_before_rollover + 1, + ); + + let closed: Value = + serde_json::from_slice(&fs::read(&state_path).expect("read fail-closed provider state")) + .expect("parse fail-closed provider state"); + assert_eq!(closed["lifecycle"], "closed"); + assert!(closed["activeProviderTurnId"].is_null()); + assert_eq!(closed["ambiguousTurnStartPending"], false); + assert_eq!(closed["completedTurnAuthoritative"], false); + assert!(closed["providerProcessGeneration"].as_u64().unwrap() > attached_generation); + + let events = poll_and_ack(&mut recovered).expect("read fail-closed rollover diagnostic"); + assert!(events.iter().any(|event| { + event.event_type == "harness.diagnostic" + && event.payload["code"] == "provider_turn_identity_invalid" + && event.payload["paperclipAccepted"] == false + && event.payload["providerAccepted"] == true + })); + + let resumes_before_retry = call_count(&directory, "thread/resume"); + assert!(recovered + .execute(&command( + "turn-retry", + 4, + "turn.start", + json!({"text": "Do not resume the quarantined provider."}), + )) + .unwrap_err() + .to_string() + .contains("provider session is closed")); + assert_eq!( + call_count(&directory, "thread/resume"), + resumes_before_retry, + "closed rollover state must never resume the unowned provider turn", + ); + + recovered.shutdown().expect("close fail-closed executor"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn direct_provider_reaps_before_rotating_a_full_turn_identity_epoch() { + let directory = temporary_directory("direct-turn-identity-epoch-rollover"); + let config = provider_config( + &directory, + &[ + "--require-dynamic-tool", + "--resume-unowned-turn-when-marked", + ], + ); + let mut provider = CodexProvider::start_with_tools(&config, [task_context_tool()], None) + .expect("start Codex provider"); + let initial_process_id = provider.process_id(); + + // Keep every identity exact until the process boundary makes it safe to + // forget them. The next turn must transparently resume in a new process + // generation instead of permanently rejecting this session. + for index in 0..4_096 { + provider + .start_turn(&format!("Complete provider turn {index}."), &config.cwd) + .expect("start provider turn before identity rollover"); + wait_for_notification(&mut provider, "turn/completed"); + } + + provider + .start_turn( + "Continue after the exact identity epoch fills.", + &config.cwd, + ) + .expect("roll over the provider process and start fresh work"); + assert_ne!(provider.process_id(), initial_process_id); + assert_eq!(call_count(&directory, "thread/resume"), 1); + wait_for_notification(&mut provider, "turn/completed"); + let rolled_process_id = provider.process_id(); + + // Fill the next process epoch and force its resume probe to observe an + // active turn. Rollover must retain that work instead of dispatching a + // concurrent replacement after forgetting the old exact identities. The + // prior generation's authoritative completion remains as one tombstone in + // this epoch, so 4,094 additional completions fill the remaining slots. + for index in 1..4_095 { + provider + .start_turn( + &format!("Complete rolled provider turn {index}."), + &config.cwd, + ) + .expect("start provider turn in the rolled identity epoch"); + wait_for_notification(&mut provider, "turn/completed"); + } + assert_eq!(provider.process_id(), rolled_process_id); + // The terminal notification is flushed before the fake provider persists + // its idle state. Wait for that write, then arm a one-shot resume race so + // only the replacement generation reports unowned active work. + wait_for_fake_provider_idle(&directory); + fs::write(directory.join("resume-unowned-turn"), b"armed") + .expect("arm unowned work for the replacement provider resume"); + let error = provider + .start_turn( + "Do not overlap the turn recovered during epoch rollover.", + &config.cwd, + ) + .expect_err("a resumed active turn is reaped before replacement work"); + assert!(error + .to_string() + .contains("resumed unowned active work; the provider was terminated")); + assert_eq!(provider.active_provider_turn_id(), None); + assert!(provider + .start_turn( + "Never admit replacement work after quarantine.", + &config.cwd, + ) + .unwrap_err() + .to_string() + .contains("quarantined after unsafe recovered work")); + wait_for_provider_exit(&mut provider); + + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + #[test] fn post_completion_observation_does_not_hide_same_or_resumed_process_failure() { let directory = temporary_directory("completion-then-nonzero-exit"); @@ -650,7 +1098,11 @@ fn rejected_replacement_turn_start_preserves_result_and_exit_authority() { .start_turn("Reject replacement work.", &config.cwd) .expect_err("the replacement turn/start returns a definite rejection"); let mut buffered_notification_seen = false; - let rejected_start_exit = (0..64).find_map(|_| { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + let rejected_start_exit = loop { + if std::time::Instant::now() >= deadline { + break None; + } match provider .poll() .expect("poll exit after rejected replacement start") @@ -661,21 +1113,23 @@ fn rejected_replacement_turn_start_preserves_result_and_exit_authority() { == Some("buffered before replacement rejection") => { buffered_notification_seen = true; - None } Some(CodexProviderEvent::Exited { success, completed_turn_authoritative, completion_reconciles_exit, .. - }) => Some(( - success, - completed_turn_authoritative, - completion_reconciles_exit, - )), - _ => None, + }) => { + break Some(( + success, + completed_turn_authoritative, + completion_reconciles_exit, + )); + } + _ => {} } - }); + std::thread::sleep(std::time::Duration::from_millis(1)); + }; assert!(buffered_notification_seen); assert_eq!(rejected_start_exit, Some((false, true, false))); @@ -727,7 +1181,11 @@ fn rejected_replacement_turn_start_does_not_hide_contradictory_turn_evidence() { "unexpected duplicate-start error: {duplicate_error}" ); let mut contradictory_turn_seen = false; - let rejected_start_exit = (0..64).find_map(|_| { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + let rejected_start_exit = loop { + if std::time::Instant::now() >= deadline { + break None; + } match provider .poll() .expect("poll exit after contradictory replacement rejection") @@ -738,21 +1196,23 @@ fn rejected_replacement_turn_start_does_not_hide_contradictory_turn_evidence() { == Some("provider-turn-contradiction") => { contradictory_turn_seen = true; - None } Some(CodexProviderEvent::Exited { success, completed_turn_authoritative, completion_reconciles_exit, .. - }) => Some(( - success, - completed_turn_authoritative, - completion_reconciles_exit, - )), - _ => None, + }) => { + break Some(( + success, + completed_turn_authoritative, + completion_reconciles_exit, + )); + } + _ => {} } - }); + std::thread::sleep(std::time::Duration::from_millis(1)); + }; assert!(contradictory_turn_seen); assert_eq!( provider.active_provider_turn_id(), @@ -771,7 +1231,6 @@ fn ambiguous_or_dead_replacement_start_preserves_result_not_exit_authority() { "--fail-after-accepting-second-turn-before-response", ), ("malformed-error", "--malformed-error-second-turn-start"), - ("missing-turn-id", "--missing-id-second-turn-start"), ] { let directory = temporary_directory(label); let config = provider_config(&directory, &[switch]); @@ -822,6 +1281,63 @@ fn ambiguous_or_dead_replacement_start_preserves_result_not_exit_authority() { } } +#[test] +fn durable_backend_closes_after_accepted_turn_omits_its_identity() { + let directory = temporary_directory("accepted-turn-missing-identity"); + let config = provider_config(&directory, &["--missing-id-live-turn-start"]); + let runner_config = durable_config(&directory); + let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + executor + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({"provider": config}), + )) + .expect("prepare Codex provider"); + executor + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open Codex session"); + + let error = executor + .execute(&command( + "turn-without-id", + 3, + "turn.start", + json!({"text": "Accept work but omit its durable identity."}), + )) + .expect_err("accepted work without a turn identity must fail closed"); + assert!(error.to_string().contains("omitted turn.id")); + + let persisted: Value = serde_json::from_slice( + &fs::read(directory.join("codex-provider-state.json")) + .expect("read fail-closed provider state"), + ) + .expect("parse fail-closed provider state"); + assert_eq!(persisted["lifecycle"], "closed"); + assert!(persisted["activeProviderTurnId"].is_null()); + + drop(executor); + let resumes_before_closed_restore = call_count(&directory, "thread/resume"); + let mut closed = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + let events = closed + .poll_events() + .expect("closed invalid-identity state remains readable without resuming Codex"); + assert!(events.iter().any(|event| { + event.event_type == "harness.diagnostic" + && event.payload["code"] == "provider_turn_identity_invalid" + && event.payload["providerTurnId"].is_null() + })); + assert_eq!( + call_count(&directory, "thread/resume"), + resumes_before_closed_restore, + "recovery must not resume provider work accepted without a durable identity" + ); + + closed.shutdown().expect("close fail-closed executor"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + #[test] fn ambiguous_replacement_turn_adopts_one_later_completion_identity() { for (label, switch, omit_started) in [ @@ -940,7 +1456,7 @@ fn ambiguous_replacement_turn_rejects_conflicting_later_identity() { let config = provider_config( &directory, &[ - "--missing-id-second-turn-start", + "--malformed-error-second-turn-start", "--conflicting-ambiguous-second-turn", ], ); @@ -948,41 +1464,23 @@ fn ambiguous_replacement_turn_rejects_conflicting_later_identity() { provider .start_turn("Complete the first turn.", &config.cwd) .expect("start first provider turn"); - let first_completed = (0..32).any(|_| { - matches!( - provider.poll().expect("poll first turn"), - Some(CodexProviderEvent::Notification { method, .. }) - if method == "turn/completed" - ) - }); - assert!( - first_completed, - "observe the authoritative first completion" - ); + wait_for_notification(&mut provider, "turn/completed"); provider .start_turn("Accept replacement work ambiguously.", &config.cwd) - .expect_err("the replacement response omits its turn identity"); - let replacement_started = provider - .poll() - .expect("poll replacement start") - .expect("replacement start is available"); - assert!(matches!( - replacement_started, - CodexProviderEvent::Notification { method, params } - if method == "turn/started" - && params.pointer("/turn/id").and_then(Value::as_str) - == Some("provider-turn-2") - )); + .expect_err("the replacement response is transport-ambiguous"); + let replacement_started = wait_for_notification(&mut provider, "turn/started"); + assert_eq!( + replacement_started + .pointer("/turn/id") + .and_then(Value::as_str), + Some("provider-turn-2") + ); assert_eq!(provider.active_provider_turn_id(), Some("provider-turn-2")); - let conflicting_completion = provider - .poll() - .expect_err("a second replacement identity must fail closed"); + let conflicting_completion = wait_for_provider_error(&mut provider); assert!( - conflicting_completion - .to_string() - .contains("another active turn"), + conflicting_completion.contains("another active turn"), "unexpected conflicting-identity error: {conflicting_completion}" ); assert_eq!(provider.active_provider_turn_id(), Some("provider-turn-2")); @@ -990,6 +1488,53 @@ fn ambiguous_replacement_turn_rejects_conflicting_later_identity() { fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); } +#[test] +fn ambiguous_replacement_turn_rejects_an_older_settled_identity() { + let directory = temporary_directory("ambiguous-older-settled-turn-identity"); + let config = provider_config(&directory, &["--ambiguous-older-reused-turn"]); + let mut provider = CodexProvider::start(&config, None).expect("start Codex provider"); + for message in ["Complete turn one.", "Complete turn two."] { + provider + .start_turn(message, &config.cwd) + .expect("start completed provider turn"); + wait_for_notification(&mut provider, "turn/completed"); + } + provider + .start_turn("Ambiguously reuse the first turn identity.", &config.cwd) + .expect_err("the replacement response is transport-ambiguous"); + let error = wait_for_provider_error(&mut provider); + assert!( + error.contains("reused a settled provider turn identity"), + "unexpected older-identity error: {error}" + ); + assert_eq!(provider.active_provider_turn_id(), None); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + let completed_turn_authoritative = (0..) + .take_while(|_| std::time::Instant::now() < deadline) + .find_map(|_| { + match provider + .poll() + .expect("poll provider after rejected identity") + { + Some(CodexProviderEvent::Exited { + completed_turn_authoritative, + .. + }) => Some(completed_turn_authoritative), + Some(_) => None, + None => { + std::thread::sleep(std::time::Duration::from_millis(1)); + None + } + } + }); + assert!( + completed_turn_authoritative == Some(false), + "accepted work with an older settled identity must terminate the provider and revoke completion authority" + ); + + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + #[test] fn clean_exit_after_ambiguous_replacement_start_fails_the_durable_session() { let directory = temporary_directory("durable-clean-exit-after-ambiguous-turn-start"); @@ -1161,7 +1706,7 @@ fn replacement_item_is_not_persisted_before_ambiguous_turn_identity() { let config = provider_config( &directory, &[ - "--missing-id-second-turn-start", + "--malformed-error-second-turn-start", "--hold-ambiguous-second-turn-after-item", ], ); @@ -1185,15 +1730,7 @@ fn replacement_item_is_not_persisted_before_ambiguous_turn_identity() { json!({"text": "Complete the first turn."}), )) .expect("start first provider turn"); - for _ in 0..32 { - if poll_and_ack(&mut executor) - .expect("poll first turn") - .iter() - .any(|event| event.event_type == "turn.completed") - { - break; - } - } + wait_for_executor_event(&mut executor, "turn.completed"); executor .execute(&command( @@ -1202,7 +1739,7 @@ fn replacement_item_is_not_persisted_before_ambiguous_turn_identity() { "turn.start", json!({"text": "Emit replacement output before terminal authority."}), )) - .expect_err("replacement response omits its identity"); + .expect_err("replacement response is transport-ambiguous"); assert!( poll_and_ack(&mut executor) .expect("defer identity-less replacement output") @@ -1252,7 +1789,7 @@ fn completed_ambiguous_replacement_fails_closed_after_process_loss() { let config = provider_config( &directory, &[ - "--missing-id-second-turn-start", + "--malformed-error-second-turn-start", "--complete-ambiguous-second-turn-before-response", "--omit-ambiguous-turn-started", ], @@ -1277,15 +1814,7 @@ fn completed_ambiguous_replacement_fails_closed_after_process_loss() { json!({"text": "Complete the first turn."}), )) .expect("start first provider turn"); - for _ in 0..32 { - if poll_and_ack(&mut executor) - .expect("poll first turn") - .iter() - .any(|event| event.event_type == "turn.completed") - { - break; - } - } + wait_for_executor_event(&mut executor, "turn.completed"); executor .execute(&command( @@ -1294,7 +1823,7 @@ fn completed_ambiguous_replacement_fails_closed_after_process_loss() { "turn.start", json!({"text": "Complete replacement work before returning an invalid response."}), )) - .expect_err("replacement response omits its identity"); + .expect_err("replacement response is transport-ambiguous"); executor.shutdown().expect("stop first provider process"); drop(executor); @@ -1335,7 +1864,7 @@ fn ambiguous_replacement_completion_replaces_durable_turn_authority() { let config = provider_config( &directory, &[ - "--missing-id-second-turn-start", + "--malformed-error-second-turn-start", "--complete-ambiguous-second-turn", ], ); @@ -1360,19 +1889,7 @@ fn ambiguous_replacement_completion_replaces_durable_turn_authority() { )) .expect("start first provider turn"); - let mut first_events = Vec::new(); - for _ in 0..32 { - first_events.extend( - poll_and_ack(&mut executor) - .expect("poll first turn") - .into_iter() - .map(|event| event.event_type), - ); - if first_events.iter().any(|event| event == "turn.completed") { - break; - } - } - assert!(first_events.iter().any(|event| event == "turn.completed")); + wait_for_executor_event(&mut executor, "turn.completed"); executor .execute(&command( @@ -1381,10 +1898,11 @@ fn ambiguous_replacement_completion_replaces_durable_turn_authority() { "turn.start", json!({"text": "Complete replacement work after the malformed response."}), )) - .expect_err("accepted replacement response omits its turn identity"); + .expect_err("accepted replacement response is transport-ambiguous"); let mut replacement_events = Vec::new(); - for _ in 0..64 { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while std::time::Instant::now() < deadline { replacement_events.extend( poll_and_ack(&mut executor) .expect("poll accepted replacement evidence") @@ -1397,6 +1915,7 @@ fn ambiguous_replacement_completion_replaces_durable_turn_authority() { { break; } + std::thread::sleep(std::time::Duration::from_millis(1)); } assert!(replacement_events .iter() @@ -1478,6 +1997,461 @@ fn codex_rejects_a_tool_call_that_was_not_advertised() { fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); } +#[test] +fn codex_rejects_delayed_calls_after_every_terminal_notification() { + let directory = temporary_directory("delayed-tool-after-failure"); + let config = provider_config(&directory, &["--delayed-tool-after-failed-turn"]); + let mut provider = CodexProvider::start_with_tools(&config, [task_context_tool()], None) + .expect("start Codex with an authorized tool"); + provider + .start_turn("Fail before invoking a tool.", &config.cwd) + .expect("start provider turn"); + + wait_for_notification(&mut provider, "turn/failed"); + assert_eq!(provider.active_provider_turn_id(), None); + let warning = wait_for_notification(&mut provider, "warning"); + assert!(warning["message"] + .as_str() + .is_some_and(|message| message.contains("after the Codex turn terminated"))); + + let _ = provider.shutdown(); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn codex_rejects_a_delayed_prior_turn_call_while_the_next_turn_is_active() { + let directory = temporary_directory("delayed-tool-after-next-turn-start"); + let config = provider_config(&directory, &["--delayed-tool-after-next-turn-start"]); + let mut provider = CodexProvider::start_with_tools(&config, [task_context_tool()], None) + .expect("start Codex with an authorized tool"); + provider + .start_turn("Complete the first turn.", &config.cwd) + .expect("start first provider turn"); + wait_for_notification(&mut provider, "turn/completed"); + + provider + .start_turn("Keep the second turn active.", &config.cwd) + .expect("start second provider turn"); + assert_eq!(provider.active_provider_turn_id(), Some("provider-turn-2")); + let warning = wait_for_notification(&mut provider, "warning"); + assert!(warning["message"] + .as_str() + .is_some_and(|message| message.contains("after the Codex turn terminated"))); + assert_eq!(provider.active_provider_turn_id(), Some("provider-turn-2")); + + let _ = provider.shutdown(); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn codex_rejects_a_two_turn_old_call_while_fresh_work_is_active() { + let directory = temporary_directory("delayed-tool-after-third-turn-start"); + let config = provider_config(&directory, &["--delayed-tool-after-third-turn-start"]); + let mut provider = CodexProvider::start_with_tools(&config, [task_context_tool()], None) + .expect("start Codex with an authorized tool"); + + for message in ["Complete turn one.", "Complete turn two."] { + provider + .start_turn(message, &config.cwd) + .expect("start completed provider turn"); + wait_for_notification(&mut provider, "turn/completed"); + } + provider + .start_turn("Keep turn three active.", &config.cwd) + .expect("start third provider turn"); + assert_eq!(provider.active_provider_turn_id(), Some("provider-turn-3")); + + let warning = wait_for_notification(&mut provider, "warning"); + assert_eq!(warning["providerMethod"], "item/tool/call"); + assert_eq!(provider.active_provider_turn_id(), Some("provider-turn-3")); + + let _ = provider.shutdown(); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn codex_fails_closed_when_a_provider_reuses_a_settled_turn_id() { + let directory = temporary_directory("tool-after-reused-turn-start"); + let config = provider_config(&directory, &["--tool-after-reused-turn-start"]); + let mut provider = CodexProvider::start_with_tools(&config, [task_context_tool()], None) + .expect("start Codex with an authorized tool"); + provider + .start_turn("Complete the first turn.", &config.cwd) + .expect("start first provider turn"); + wait_for_notification(&mut provider, "turn/completed"); + + let error = provider + .start_turn("Reuse the settled provider identity.", &config.cwd) + .expect_err("reject a provider turn with a reused identity"); + assert!(error.to_string().contains("reused a settled")); + assert_eq!(provider.active_provider_turn_id(), None); + wait_for_reused_identity_reap(&mut provider); + + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn codex_fails_closed_when_a_provider_reuses_an_older_settled_turn_id() { + let directory = temporary_directory("tool-after-older-reused-turn-start"); + let config = provider_config(&directory, &["--tool-after-older-reused-turn-start"]); + let mut provider = CodexProvider::start_with_tools(&config, [task_context_tool()], None) + .expect("start Codex with an authorized tool"); + for message in ["Complete turn one.", "Complete turn two."] { + provider + .start_turn(message, &config.cwd) + .expect("start completed provider turn"); + wait_for_notification(&mut provider, "turn/completed"); + } + + let error = provider + .start_turn("Reuse the older settled provider identity.", &config.cwd) + .expect_err("reject a provider turn with an older reused identity"); + assert!(error.to_string().contains("reused a settled")); + assert_eq!(provider.active_provider_turn_id(), None); + wait_for_reused_identity_reap(&mut provider); + + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn durable_backend_rejects_an_older_provider_turn_id_after_restart() { + let directory = temporary_directory("durable-older-reused-turn-after-restart"); + let config = provider_config(&directory, &[]); + let runner_config = durable_config(&directory); + let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + first + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({"provider": config}), + )) + .expect("prepare Codex provider"); + first + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open Codex session"); + + for (sequence, message) in [(3, "Complete turn one."), (4, "Complete turn two.")] { + first + .execute(&command( + &format!("turn-{sequence}"), + sequence, + "turn.start", + json!({"text": message}), + )) + .expect("start completed provider turn"); + let mut completed = false; + for _ in 0..32 { + completed |= poll_and_ack(&mut first) + .expect("poll completed provider turn") + .iter() + .any(|event| event.event_type == "turn.completed"); + if completed { + break; + } + } + assert!(completed); + } + first.shutdown().expect("stop first provider process"); + drop(first); + + // The fake provider's process-local counter restarts at provider-turn-1. + // Durable state must still remember that older identity, not only the most + // recent provider-turn-2 completion authority. + let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + let error = recovered + .execute(&command( + "turn-reused", + 5, + "turn.start", + json!({"text": "Do not accept an older provider turn identity."}), + )) + .expect_err("reject a provider turn identity retained before restart"); + assert!(error.to_string().contains("reused a settled")); + + let persisted: Value = serde_json::from_slice( + &fs::read(directory.join("codex-provider-state.json")) + .expect("read fail-closed provider state"), + ) + .expect("parse fail-closed provider state"); + assert_eq!(persisted["lifecycle"], "closed"); + assert!(persisted["activeProviderTurnId"].is_null()); + + drop(recovered); + let resumes_before_closed_restore = call_count(&directory, "thread/resume"); + let mut closed = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + let events = closed + .poll_events() + .expect("closed reused-identity state remains readable without resuming Codex"); + assert!(events.iter().any(|event| { + event.event_type == "harness.diagnostic" + && event.payload["code"] == "provider_turn_identity_reused" + })); + assert_eq!( + call_count(&directory, "thread/resume"), + resumes_before_closed_restore, + "recovery must not resume provider work accepted under a reused identity" + ); + + closed.shutdown().expect("close fail-closed executor"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn durable_recovery_closes_a_provider_that_reopens_a_settled_turn() { + let directory = temporary_directory("durable-settled-turn-active-on-recovery"); + let config = provider_config(&directory, &[]); + let runner_config = durable_config(&directory); + let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + first + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({"provider": config}), + )) + .expect("prepare Codex provider"); + first + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open Codex session"); + first + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Complete the durable turn."}), + )) + .expect("start provider turn"); + let mut completed = false; + for _ in 0..32 { + completed |= poll_and_ack(&mut first) + .expect("poll completed provider turn") + .iter() + .any(|event| event.event_type == "turn.completed"); + if completed { + break; + } + } + assert!(completed); + first.shutdown().expect("stop first provider process"); + drop(first); + + // Contradict the durable terminal ledger with a resumed provider snapshot + // that reports the exact settled identity as active again. + fs::write( + directory.join("fake-state.json"), + serde_json::to_vec_pretty(&json!({ + "threadId": "codex-thread-1", + "activeTurnId": "provider-turn-1", + })) + .unwrap(), + ) + .expect("write contradictory fake provider state"); + + let resumes_before_recovery = call_count(&directory, "thread/resume"); + let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + let events = recovered + .poll_events() + .expect("fail-closed recovery remains observable"); + assert!(events.iter().any(|event| { + event.event_type == "harness.diagnostic" + && event.payload["code"] == "provider_turn_identity_reused" + && event.payload["providerTurnId"] == "provider-turn-1" + })); + assert_eq!( + call_count(&directory, "thread/resume"), + resumes_before_recovery + 1, + ); + let persisted: Value = serde_json::from_slice( + &fs::read(directory.join("codex-provider-state.json")) + .expect("read fail-closed provider state"), + ) + .expect("parse fail-closed provider state"); + assert_eq!(persisted["lifecycle"], "closed"); + assert!(persisted["activeProviderTurnId"].is_null()); + assert_eq!(persisted["completedTurnAuthoritative"], false); + + drop(recovered); + let resumes_before_closed_restore = call_count(&directory, "thread/resume"); + let mut closed = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + closed + .poll_events() + .expect("closed recovery state remains readable"); + assert_eq!( + call_count(&directory, "thread/resume"), + resumes_before_closed_restore, + "closed recovery must not resume the contradictory provider turn again", + ); + + closed.shutdown().expect("close fail-closed executor"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn codex_rejects_a_two_turn_old_call_while_idle() { + let directory = temporary_directory("delayed-tool-after-second-turn-completion"); + let config = provider_config(&directory, &["--delayed-tool-after-second-turn-completion"]); + let mut provider = CodexProvider::start_with_tools(&config, [task_context_tool()], None) + .expect("start Codex with an authorized tool"); + + for message in ["Complete turn one.", "Complete turn two."] { + provider + .start_turn(message, &config.cwd) + .expect("start completed provider turn"); + wait_for_notification(&mut provider, "turn/completed"); + } + assert_eq!(provider.active_provider_turn_id(), None); + + let warning = wait_for_notification(&mut provider, "warning"); + assert_eq!(warning["providerMethod"], "item/tool/call"); + assert_eq!(provider.active_provider_turn_id(), None); + + let _ = provider.shutdown(); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn durable_backend_delivers_a_terminal_before_rejecting_a_delayed_tool_call() { + let directory = temporary_directory("durable-delayed-tool-after-terminal"); + let config = provider_config(&directory, &["--delayed-tool-after-failed-turn"]); + let runner_config = durable_config(&directory); + let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + executor + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({ + "provider": config, + "authorizedTools": task_context_tool_set(), + "completionContract": { + "revision": "sha256:delayed-tool-contract", + "criterionIds": ["criterion_delayed_tool"] + }, + }), + )) + .unwrap(); + executor + .execute(&command("open", 2, "session.open", json!({}))) + .unwrap(); + executor + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Fail before invoking a delayed tool."}), + )) + .unwrap(); + + let mut observed = Vec::new(); + for _ in 0..32 { + observed.extend(poll_and_ack(&mut executor).expect("poll durable provider events")); + if observed + .iter() + .any(|event| event.event_type == "provider.notice.recorded") + { + break; + } + } + let terminal = observed + .iter() + .position(|event| event.event_type == "run.terminal") + .expect("the durable terminal is delivered"); + let rejection = observed + .iter() + .position(|event| event.event_type == "provider.notice.recorded") + .expect("the delayed request is rejected non-fatally"); + assert!(terminal < rejection); + + executor.shutdown().unwrap(); + fs::remove_dir_all(directory).unwrap(); +} + +#[test] +fn codex_rejects_a_runtime_response_after_its_turn_terminates() { + let directory = temporary_directory("delayed-question-response"); + let config = provider_config(&directory, &["--question-before-failed-turn"]); + let mut provider = CodexProvider::start(&config, None).expect("start Codex provider"); + provider + .start_turn("Ask and then fail.", &config.cwd) + .expect("start provider turn"); + + let mut request_id = None; + let mut terminal_seen = false; + for _ in 0..16 { + match provider.poll().expect("poll question and terminal") { + Some(CodexProviderEvent::RuntimeRequest { + request_id: observed, + .. + }) => request_id = Some(observed), + Some(CodexProviderEvent::Notification { method, .. }) if method == "turn/completed" => { + terminal_seen = true; + } + _ => {} + } + if request_id.is_some() && terminal_seen { + break; + } + } + let request_id = request_id.expect("observe the runtime request before termination"); + assert!(terminal_seen); + let error = provider + .resolve_runtime_request( + &request_id, + &json!({ + "schema": "paperclip.question_response.v1", + "answers": {"environment": {"selectedOptionIds": ["option-1"]}} + }), + ) + .expect_err("terminal requests must not remain resolvable"); + assert!(error.to_string().contains("no pending Codex request")); + + let _ = provider.shutdown(); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn reused_provider_question_ids_get_unique_controller_identities() { + let directory = temporary_directory("reused-question-id"); + let config = provider_config(&directory, &["--emit-question", "--reuse-question-id"]); + let mut provider = CodexProvider::start(&config, None).expect("start Codex provider"); + provider + .start_turn("Ask twice with one provider request id.", &config.cwd) + .expect("start provider turn"); + + let first_request_id = (0..16) + .find_map(|_| match provider.poll().expect("poll first question") { + Some(CodexProviderEvent::RuntimeRequest { request_id, .. }) => Some(request_id), + _ => None, + }) + .expect("observe first runtime request"); + let response = json!({ + "schema": "paperclip.question_response.v1", + "answers": {"environment": {"selectedOptionIds": ["option-1"]}} + }); + provider + .resolve_runtime_request(&first_request_id, &response) + .expect("resolve first runtime request"); + let second_request_id = (0..16) + .find_map(|_| match provider.poll().expect("poll second question") { + Some(CodexProviderEvent::RuntimeRequest { request_id, .. }) => Some(request_id), + _ => None, + }) + .expect("observe second runtime request"); + + assert_ne!(first_request_id, second_request_id); + let stale = provider + .resolve_runtime_request(&first_request_id, &response) + .expect_err("the first controller identity cannot resolve the second question"); + assert!(stale.to_string().contains("no pending Codex request")); + provider + .resolve_runtime_request(&second_request_id, &response) + .expect("resolve second runtime request"); + + let _ = provider.shutdown(); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + #[test] fn codex_resume_advertises_the_same_authorized_tools() { let directory = temporary_directory("dynamic-tool-resume"); @@ -1490,6 +2464,535 @@ fn codex_resume_advertises_the_same_authorized_tools() { fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); } +#[test] +fn durable_backend_routes_a_semantic_tool_result_back_to_codex() { + let directory = temporary_directory("durable-dynamic-tool"); + let config = provider_config(&directory, &["--require-dynamic-tool", "--emit-tool-call"]); + let runner_config = durable_config(&directory); + let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + executor + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({ + "provider": config, + "authorizedTools": task_context_tool_set(), + }), + )) + .expect("prepare the durable Codex tool set"); + executor + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open the Codex session"); + executor + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Inspect the durable fake task."}), + )) + .expect("start the Codex turn"); + + let mut semantic_input = None; + for _ in 0..32 { + let events = poll_and_ack(&mut executor).expect("poll semantic input"); + semantic_input = events + .iter() + .find(|event| event.event_type == "semantic_tool.input") + .cloned() + .or(semantic_input); + if semantic_input.is_some() { + break; + } + } + let semantic_input = semantic_input.expect("durable semantic input is emitted"); + assert_eq!( + semantic_input.payload["semantic_tool"]["correlation"]["runId"], + "run-1" + ); + assert_eq!( + semantic_input.payload["semantic_tool"]["operationId"], + "get_task_context" + ); + + let delivered = executor + .execute(&command( + "tool-result", + 4, + "semantic_tool.result", + json!({ + "callId": "semantic-call-1", + "operationId": "get_task_context", + "result": {"ok": true, "task": {"id": "task-1"}}, + "isError": false, + }), + )) + .expect("deliver the durable semantic result"); + assert_eq!(delivered.result["status"], "delivered"); + + let mut result_seen = false; + let mut terminal_seen = false; + for _ in 0..32 { + let events = poll_and_ack(&mut executor).expect("poll result and completion"); + result_seen |= events + .iter() + .any(|event| event.event_type == "semantic_tool.result"); + terminal_seen |= events + .iter() + .any(|event| event.event_type == "turn.completed"); + if result_seen && terminal_seen { + break; + } + } + assert!(result_seen); + assert!(terminal_seen); + executor.shutdown().expect("stop provider"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn durable_backend_replays_completed_results_without_mutating_the_event_queue() { + let directory = temporary_directory("durable-completed-tool-replay"); + let config = provider_config( + &directory, + &[ + "--require-dynamic-tool", + "--emit-tool-call", + "--hold-turn", + "--replay-completed-tool-call-count", + "4", + ], + ); + let runner_config = durable_config(&directory); + let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + executor + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({ + "provider": config, + "authorizedTools": task_context_tool_set(), + }), + )) + .expect("prepare the durable Codex tool set"); + executor + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open the Codex session"); + executor + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Replay the completed fake tool call."}), + )) + .expect("start the Codex turn"); + + let mut input_seen = false; + for _ in 0..32 { + input_seen |= poll_and_ack(&mut executor) + .expect("poll semantic input") + .iter() + .any(|event| event.event_type == "semantic_tool.input"); + if input_seen { + break; + } + } + assert!(input_seen, "durable semantic input is emitted"); + + executor + .execute(&command( + "tool-result", + 4, + "semantic_tool.result", + json!({ + "callId": "semantic-call-1", + "operationId": "get_task_context", + "result": {"ok": true, "task": {"id": "task-1"}}, + "isError": false, + }), + )) + .expect("deliver the original durable semantic result"); + let result_events = poll_and_ack(&mut executor).expect("acknowledge semantic result"); + assert_eq!( + result_events + .iter() + .filter(|event| event.event_type == "semantic_tool.result") + .count(), + 1 + ); + + let state_path = directory.join("codex-provider-state.json"); + let state_before_replays = fs::read(&state_path).expect("read state before exact replays"); + let expected_response = r#"{"ok":true,"task":{"id":"task-1"}}"#; + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while recorded_tool_responses(&directory).len() < 5 && std::time::Instant::now() < deadline { + let events = poll_and_ack(&mut executor).expect("service completed tool replay"); + assert!( + events.iter().all(|event| { + event.event_type != "semantic_tool.input" + && event.event_type != "semantic_tool.reconciled" + && event.event_type != "semantic_tool.result" + }), + "an exact completed replay must not add semantic reconciliation events" + ); + std::thread::sleep(std::time::Duration::from_millis(1)); + } + + let responses = recorded_tool_responses(&directory); + assert_eq!( + responses.len(), + 5, + "the original result and four replays return" + ); + assert!(responses + .iter() + .all(|response| response == expected_response)); + assert_eq!( + fs::read(&state_path).expect("read state after exact replays"), + state_before_replays, + "exact completed replays must not consume durable event capacity" + ); + + executor.shutdown().expect("stop provider"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn durable_backend_replays_pending_tool_calls_without_mutating_the_event_queue() { + let directory = temporary_directory("durable-tool-recovery"); + let config = provider_config( + &directory, + &[ + "--require-dynamic-tool", + "--emit-tool-call", + "--emit-tool-call-on-resume", + ], + ); + let runner_config = durable_config(&directory); + let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + first + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({ + "provider": config, + "authorizedTools": task_context_tool_set(), + }), + )) + .expect("prepare the recoverable tool bridge"); + first + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open the first provider"); + first + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Hold the tool call across recovery."}), + )) + .expect("start the first turn"); + let mut input_seen = false; + for _ in 0..32 { + input_seen |= poll_and_ack(&mut first) + .expect("poll first tool input") + .iter() + .any(|event| event.event_type == "semantic_tool.input"); + if input_seen { + break; + } + } + assert!(input_seen); + drop(first); + + let state_path = directory.join("codex-provider-state.json"); + let mut recovered = None; + for replay in 0..4 { + let before: Value = serde_json::from_slice( + &fs::read(&state_path).expect("read state before provider recovery"), + ) + .expect("parse state before provider recovery"); + let resume_count = call_count(&directory, "thread/resume"); + let mut next = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + for _ in 0..32 { + let events = poll_and_ack(&mut next).expect("poll exact pending replay"); + assert!(events.iter().all(|event| { + event.event_type != "semantic_tool.input" + && event.event_type != "semantic_tool.reconciled" + && event.event_type != "semantic_tool.result" + })); + if call_count(&directory, "thread/resume") > resume_count { + break; + } + } + assert!( + call_count(&directory, "thread/resume") > resume_count, + "recovery {replay} resumes the active provider turn" + ); + for _ in 0..4 { + let events = poll_and_ack(&mut next).expect("poll exact pending replay"); + assert!(events.iter().all(|event| { + event.event_type != "semantic_tool.input" + && event.event_type != "semantic_tool.reconciled" + && event.event_type != "semantic_tool.result" + })); + } + let after: Value = serde_json::from_slice( + &fs::read(&state_path).expect("read state after pending replay"), + ) + .expect("parse state after pending replay"); + assert_eq!( + after["pendingEvents"], before["pendingEvents"], + "exact pending replay {replay} must not append pending events" + ); + assert_eq!( + after["queuedEvents"], before["queuedEvents"], + "exact pending replay {replay} must not append queued events" + ); + assert_eq!( + after["nextProviderEventSeq"], before["nextProviderEventSeq"], + "exact pending replay {replay} must not consume durable event capacity" + ); + recovered = Some(next); + if replay < 3 { + drop(recovered.take()); + } + } + let mut recovered = recovered.expect("retain the final recovered provider"); + recovered + .execute(&command( + "tool-result", + 4, + "semantic_tool.result", + json!({ + "callId": "semantic-call-1", + "operationId": "get_task_context", + "result": {"ok": true, "task": {"id": "task-1"}}, + "isError": false, + }), + )) + .expect("complete the replayed tool call"); + let events = poll_and_ack(&mut recovered).expect("poll the completed pending call"); + assert_eq!( + events + .iter() + .filter(|event| event.event_type == "semantic_tool.result") + .count(), + 1 + ); + recovered.shutdown().expect("stop recovered provider"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn durable_backend_settles_pending_tools_when_recovery_finds_the_turn_ended() { + let directory = temporary_directory("durable-tool-ended-offline"); + let config = provider_config(&directory, &["--require-dynamic-tool", "--emit-tool-call"]); + let runner_config = durable_config(&directory); + let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + first + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({ + "provider": config, + "authorizedTools": task_context_tool_set(), + "completionContract": { + "revision": "sha256:offline-recovery-contract", + "criterionIds": ["criterion_offline_recovery"] + }, + }), + )) + .expect("prepare the recoverable tool bridge"); + first + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open the first provider"); + first + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "End while the runner is offline."}), + )) + .expect("start the first turn"); + let mut input_seen = false; + for _ in 0..32 { + input_seen |= poll_and_ack(&mut first) + .expect("poll first tool input") + .iter() + .any(|event| event.event_type == "semantic_tool.input"); + if input_seen { + break; + } + } + assert!(input_seen); + drop(first); + + fs::write( + directory.join("fake-state.json"), + serde_json::to_vec_pretty(&json!({ + "threadId": "codex-thread-1", + "activeTurnId": null, + })) + .unwrap(), + ) + .expect("record that the provider turn ended while offline"); + + let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + recovered + .execute(&command("snapshot", 4, "session.snapshot", json!({}))) + .expect("restore the provider session"); + let mut observed = Vec::new(); + for _ in 0..32 { + observed.extend( + poll_and_ack(&mut recovered) + .expect("poll recovered settlement") + .into_iter() + .map(|event| event.event_type), + ); + if observed.iter().any(|event| event == "run.terminal") { + break; + } + } + let semantic_result = observed + .iter() + .position(|event| event == "semantic_tool.result") + .expect("recovery settles the pending semantic tool"); + let reconciled = observed + .iter() + .position(|event| event == "session.reconciled") + .expect("recovery emits a reconciliation event"); + let terminal = observed + .iter() + .position(|event| event == "run.terminal") + .expect("offline turn recovery terminates the run"); + assert!(semantic_result < reconciled); + assert!(reconciled < terminal); + assert!(recovered + .execute(&command( + "late-result", + 5, + "semantic_tool.result", + json!({ + "callId": "semantic-call-1", + "operationId": "get_task_context", + "result": {"ok": true}, + "isError": false, + }), + )) + .is_err()); + + recovered.shutdown().expect("stop recovered provider"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn durable_backend_rejects_tool_catalog_drift_during_attach() { + let directory = temporary_directory("durable-tool-attach-drift"); + let config = provider_config(&directory, &[]); + let runner_config = durable_config(&directory); + let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + executor + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({ + "provider": config, + "authorizedTools": task_context_tool_set(), + }), + )) + .expect("prepare the durable tool catalog"); + + let mut changed = task_context_tool_set(); + changed.operations[0].description = "Changed after recovery.".to_owned(); + changed.catalog_digest = authorized_tool_catalog_digest(&changed.operations).unwrap(); + let error = executor + .execute(&command( + "attach", + 2, + "run.attach", + json!({"authorizedTools": changed}), + )) + .expect_err("attach must reject tool catalog drift"); + assert!(error.to_string().contains("tool contract changed")); + + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn durable_backend_settles_tools_before_a_natural_terminal_event() { + let directory = temporary_directory("durable-tool-terminal"); + let config = provider_config( + &directory, + &[ + "--require-dynamic-tool", + "--emit-tool-call", + "--finish-turn-with-pending-tool", + ], + ); + let runner_config = durable_config(&directory); + let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + executor + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({ + "provider": config, + "authorizedTools": task_context_tool_set(), + }), + )) + .unwrap(); + executor + .execute(&command("open", 2, "session.open", json!({}))) + .unwrap(); + executor + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Terminate with a pending tool."}), + )) + .unwrap(); + + let mut observed = Vec::new(); + for _ in 0..32 { + let events = poll_and_ack(&mut executor).unwrap(); + observed.extend(events.into_iter().map(|event| event.event_type)); + if observed.iter().any(|event| event == "turn.completed") { + break; + } + } + let semantic_result = observed + .iter() + .position(|event| event == "semantic_tool.result") + .expect("terminal settlement emits a failed semantic result"); + let terminal = observed + .iter() + .position(|event| event == "turn.completed") + .expect("provider terminal event is emitted"); + assert!(semantic_result < terminal); + assert!(executor + .execute(&command( + "late-result", + 4, + "semantic_tool.result", + json!({ + "callId": "semantic-call-1", + "operationId": "get_task_context", + "result": {"ok": true}, + "isError": false, + }), + )) + .is_err()); + + executor.shutdown().unwrap(); + fs::remove_dir_all(directory).unwrap(); +} + #[test] fn durable_backend_resumes_the_active_thread_without_restarting_the_turn() { let directory = temporary_directory("resume"); @@ -1548,6 +3051,256 @@ fn durable_backend_resumes_the_active_thread_without_restarting_the_turn() { fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); } +#[test] +fn legacy_full_active_epoch_is_closed_on_recovery() { + let directory = temporary_directory("legacy-full-active-recovery"); + let config = provider_config(&directory, &["--hold-turn"]); + let mut first = CodexCommandExecutor::new(&directory); + first + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({"provider": config}), + )) + .expect("prepare Codex provider"); + first + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open Codex session"); + first + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Hold the legacy full-epoch turn."}), + )) + .expect("start held provider turn"); + first.shutdown().expect("stop first provider process"); + drop(first); + + let state_path = directory.join("codex-provider-state.json"); + let mut persisted: Value = + serde_json::from_slice(&fs::read(&state_path).expect("read active provider state")) + .expect("parse active provider state"); + let prior_generation = persisted["providerProcessGeneration"] + .as_u64() + .expect("provider generation is persisted"); + persisted["settledProviderTurnIds"] = Value::Array( + (0..4_096) + .map(|index| Value::String(format!("legacy-provider-turn-{index}"))) + .collect(), + ); + fs::write(&state_path, serde_json::to_vec_pretty(&persisted).unwrap()) + .expect("write legacy full active state"); + + let mut recovered = CodexCommandExecutor::new(&directory); + let events = recovered + .poll_events() + .expect("legacy full-epoch recovery remains observable"); + assert!(events.iter().any(|event| { + event.event_type == "harness.diagnostic" + && event.payload["code"] == "legacy_provider_turn_epoch_ambiguous" + })); + let closed: Value = + serde_json::from_slice(&fs::read(&state_path).expect("read closed legacy full state")) + .expect("parse closed legacy full state"); + assert_eq!(closed["lifecycle"], "closed"); + assert!(closed["activeProviderTurnId"].is_null()); + assert_eq!(closed["ambiguousTurnStartPending"], false); + assert_eq!(closed["completedTurnAuthoritative"], false); + assert!(closed["settledProviderTurnFilter"]["words"] + .as_array() + .expect("legacy filter shape remains valid") + .is_empty()); + assert_eq!( + closed["settledProviderTurnIds"].as_array().unwrap().len(), + 4_096 + ); + assert!(closed["providerProcessGeneration"].as_u64().unwrap() > prior_generation); + assert!(recovered + .execute(&command( + "replacement-turn", + 4, + "turn.start", + json!({"text": "Do not reopen closed legacy work."}), + )) + .expect_err("closed legacy full-epoch state rejects replacement work") + .to_string() + .contains("closed")); + assert_eq!(call_count(&directory, "turn/start"), 1); + + recovered.shutdown().expect("close recovered executor"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn legacy_filtered_ambiguous_epoch_is_closed_on_recovery() { + let directory = temporary_directory("legacy-filtered-ambiguous-recovery"); + let config = provider_config(&directory, &[]); + let mut first = CodexCommandExecutor::new(&directory); + first + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({"provider": config}), + )) + .expect("prepare Codex provider"); + first + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open Codex session"); + first.shutdown().expect("stop first provider process"); + drop(first); + + let state_path = directory.join("codex-provider-state.json"); + let mut persisted: Value = + serde_json::from_slice(&fs::read(&state_path).expect("read idle provider state")) + .expect("parse idle provider state"); + let mut legacy_words = vec![0_u64; 32_768]; + legacy_words[0] = 1; + persisted["settledProviderTurnFilter"] = json!({"words": legacy_words}); + persisted["ambiguousTurnStartPending"] = Value::Bool(true); + fs::write(&state_path, serde_json::to_vec_pretty(&persisted).unwrap()) + .expect("write legacy-filtered ambiguous state"); + + let mut recovered = CodexCommandExecutor::new(&directory); + let events = recovered + .poll_events() + .expect("legacy ambiguous recovery remains observable"); + let diagnostic = events + .iter() + .find(|event| { + event.event_type == "harness.diagnostic" + && event.payload["code"] == "legacy_provider_turn_epoch_ambiguous" + }) + .expect("legacy ambiguous recovery emits a terminal diagnostic"); + assert_eq!(diagnostic.payload["ambiguousStartPending"], true); + assert_eq!(diagnostic.payload["providerReportedActive"], false); + let closed: Value = + serde_json::from_slice(&fs::read(&state_path).expect("read closed legacy ambiguous state")) + .expect("parse closed legacy ambiguous state"); + assert_eq!(closed["lifecycle"], "closed"); + assert!(closed["activeProviderTurnId"].is_null()); + assert_eq!(closed["ambiguousTurnStartPending"], false); + assert!(!closed["settledProviderTurnFilter"]["words"] + .as_array() + .expect("legacy filter remains durable evidence") + .is_empty()); + assert_eq!(call_count(&directory, "turn/start"), 0); + + recovered.shutdown().expect("close recovered executor"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn legacy_full_ended_epoch_reconciles_before_idle_rollover() { + let directory = temporary_directory("legacy-full-ended-recovery"); + let config = provider_config(&directory, &["--hold-turn"]); + let mut first = CodexCommandExecutor::new(&directory); + first + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({"provider": config}), + )) + .expect("prepare Codex provider"); + first + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open Codex session"); + first + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Let this legacy turn end while runnerd is away."}), + )) + .expect("start held provider turn"); + first.shutdown().expect("stop first provider process"); + drop(first); + + let state_path = directory.join("codex-provider-state.json"); + let mut persisted: Value = + serde_json::from_slice(&fs::read(&state_path).expect("read active provider state")) + .expect("parse active provider state"); + persisted["settledProviderTurnIds"] = Value::Array( + (0..4_096) + .map(|index| Value::String(format!("legacy-provider-turn-{index}"))) + .collect(), + ); + fs::write(&state_path, serde_json::to_vec_pretty(&persisted).unwrap()) + .expect("write full legacy provider epoch"); + fs::write( + directory.join("fake-state.json"), + serde_json::to_vec_pretty(&json!({"threadId": "codex-thread-1"})).unwrap(), + ) + .expect("mark the provider turn ended while runnerd was away"); + + let mut recovered = CodexCommandExecutor::new(&directory); + let events = recovered + .poll_events() + .expect("reconcile an ended turn from a full legacy epoch"); + assert!(events.iter().any(|event| event.event_type == "turn.failed")); + let reconciled: Value = + serde_json::from_slice(&fs::read(&state_path).expect("read reconciled legacy state")) + .expect("parse reconciled legacy state"); + assert_eq!(reconciled["lifecycle"], "session_open"); + assert!(reconciled["activeProviderTurnId"].is_null()); + assert_eq!( + reconciled["settledProviderTurnIds"] + .as_array() + .unwrap() + .len(), + 4_096 + ); + + recovered + .execute(&command( + "replacement-turn", + 4, + "turn.start", + json!({"text": "Start only after the idle epoch rolls over."}), + )) + .expect("roll over the idle full epoch and start replacement work"); + let rolled: Value = + serde_json::from_slice(&fs::read(&state_path).expect("read rolled legacy state")) + .expect("parse rolled legacy state"); + assert!(rolled["settledProviderTurnIds"] + .as_array() + .expect("rolled exact identities remain an array") + .is_empty()); + assert!(rolled["settledProviderTurnFilter"]["words"] + .as_array() + .expect("rolled filter remains valid") + .is_empty()); + assert_eq!(call_count(&directory, "turn/start"), 2); + + recovered.shutdown().expect("stop rolled provider process"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn codex_resume_rejects_an_oversized_active_turn_identity() { + let directory = temporary_directory("resume-oversized-active-turn-id"); + let config = provider_config(&directory, &[]); + fs::write( + directory.join("fake-state.json"), + serde_json::to_vec_pretty(&json!({ + "threadId": "codex-thread-1", + "activeTurnId": "x".repeat(241), + })) + .unwrap(), + ) + .expect("write fake provider state with an oversized active identity"); + + let error = CodexProvider::start(&config, Some("codex-thread-1")) + .err() + .expect("reject an oversized recovered provider turn identity"); + assert!(error.to_string().contains("invalid turn.id")); + + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + #[test] fn provider_exit_preserves_and_reconciles_the_active_turn() { let directory = temporary_directory("exit-active-turn"); @@ -1615,6 +3368,498 @@ fn provider_exit_preserves_and_reconciles_the_active_turn() { fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); } +#[test] +fn receipt_limit_rejects_the_call_and_keeps_polling_when_interrupt_fails() { + let directory = temporary_directory("receipt-limit-interrupt-failure"); + let config = provider_config( + &directory, + &[ + "--require-dynamic-tool", + "--hold-turn", + "--emit-tool-call-on-resume", + "--fail-first-interrupt", + "--accept-interrupt-without-terminal-once", + ], + ); + let runner_config = durable_config(&directory); + let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + first + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({ + "provider": config, + "authorizedTools": task_context_tool_set(), + }), + )) + .expect("prepare Codex provider"); + first + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open Codex session"); + first + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Hold this saturated turn for recovery."}), + )) + .expect("start held provider turn"); + drop(first); + + saturate_provider_tool_receipts(&directory); + + let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + let events = poll_and_ack(&mut recovered) + .expect("a failed first interrupt must not terminate durable provider polling"); + assert!(events.iter().any(|event| { + event.event_type == "harness.diagnostic" + && event.payload["code"] == "semantic_tool_turn_receipt_limit" + })); + assert_eq!(call_count(&directory, "tool-response:failure"), 1); + assert_eq!(call_count(&directory, "turn/interrupt"), 1); + + let mut settled = Vec::new(); + for _ in 0..4 { + settled.extend( + poll_and_ack(&mut recovered) + .expect("polling must autonomously retry the durable receipt-limit interrupt"), + ); + if settled + .iter() + .any(|event| event.event_type == "turn.interrupted") + { + break; + } + } + assert!( + settled + .iter() + .any(|event| event.event_type == "turn.interrupted"), + "the retry must settle the receipt-exhausted turn" + ); + assert_eq!(call_count(&directory, "turn/interrupt"), 3); + + recovered.shutdown().expect("stop recovered provider"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn receipt_limit_retry_preserves_a_turn_settled_during_provider_recovery() { + let directory = temporary_directory("receipt-limit-recovered-settlement"); + let config = provider_config( + &directory, + &[ + "--require-dynamic-tool", + "--hold-turn", + "--emit-tool-call-on-resume", + "--fail-first-interrupt", + ], + ); + let runner_config = durable_config(&directory); + let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + first + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({ + "provider": config, + "authorizedTools": task_context_tool_set(), + }), + )) + .expect("prepare Codex provider"); + first + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open Codex session"); + first + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Recover a provider-settled receipt-limited turn."}), + )) + .expect("start held provider turn"); + drop(first); + + saturate_provider_tool_receipts(&directory); + + let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + let events = poll_and_ack(&mut recovered) + .expect("the first failed interruption must leave a durable retry"); + assert!(events.iter().any(|event| { + event.event_type == "harness.diagnostic" + && event.payload["code"] == "semantic_tool_turn_receipt_limit" + })); + assert_eq!(call_count(&directory, "turn/interrupt"), 1); + + // Lose the live transport while retaining its durable interruption retry, + // then model the provider settling the turn before runnerd reconnects. + recovered + .shutdown() + .expect("stop the provider before retry recovery"); + let fake_state_path = directory.join("fake-state.json"); + let mut fake_state: Value = + serde_json::from_slice(&fs::read(&fake_state_path).expect("read fake provider state")) + .expect("parse fake provider state"); + fake_state["activeTurnId"] = Value::Null; + fs::write( + &fake_state_path, + serde_json::to_vec_pretty(&fake_state).unwrap(), + ) + .expect("settle the fake provider turn before recovery"); + + let events = poll_and_ack(&mut recovered) + .expect("an already-settled retry must preserve the recovered terminal state"); + assert!(events.iter().any(|event| { + event.event_type == "session.reconciled" && event.payload["activeProviderTurnId"].is_null() + })); + assert_eq!( + call_count(&directory, "turn/interrupt"), + 1, + "an already-settled recovered turn must not receive another interrupt RPC" + ); + + let persisted: Value = serde_json::from_slice( + &fs::read(directory.join("codex-provider-state.json")) + .expect("read reconciled provider state"), + ) + .expect("parse reconciled provider state"); + assert!(persisted["activeProviderTurnId"].is_null()); + assert_eq!(persisted["receiptLimitInterruptPending"], false); + assert_eq!(persisted["receiptLimitInterruptAccepted"], false); + assert_eq!(persisted["receiptLimitInterruptAttempts"], 0); + assert!(persisted["receiptLimitInterruptDeadlineUnixMs"].is_null()); + + recovered.shutdown().expect("stop recovered provider"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn receipt_limit_accepts_a_terminal_after_the_initial_interrupt_deadline() { + let directory = temporary_directory("receipt-limit-delayed-terminal"); + let config = provider_config( + &directory, + &[ + "--require-dynamic-tool", + "--hold-turn", + "--emit-tool-call-on-resume", + "--interrupt-terminal-delay-ms", + "2100", + ], + ); + let runner_config = durable_config(&directory); + let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + first + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({ + "provider": config, + "authorizedTools": task_context_tool_set(), + }), + )) + .expect("prepare Codex provider"); + first + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open Codex session"); + first + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Accept a delayed authoritative interruption."}), + )) + .expect("start held provider turn"); + drop(first); + saturate_provider_tool_receipts(&directory); + + let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + let mut emitted = Vec::new(); + for _ in 0..4_096 { + emitted.extend( + poll_and_ack(&mut recovered) + .expect("a delayed accepted interrupt must remain durably pollable"), + ); + if emitted + .iter() + .any(|event| event.event_type == "turn.interrupted") + { + break; + } + } + assert!( + emitted + .iter() + .any(|event| event.event_type == "turn.interrupted"), + "the delayed provider terminal must remain authoritative after two seconds" + ); + assert!(!emitted.iter().any(|event| { + event.event_type == "turn.interrupted" + && event.payload["code"] == "semantic_tool_turn_receipt_limit_interrupt_deadline" + })); + assert!( + !emitted.iter().any(|event| { + event.event_type == "turn.failed" + && event.payload["code"] == "semantic_tool_turn_receipt_limit_interrupt_unconfirmed" + }), + "fast provider polls must not replace a delayed terminal with fallback failure" + ); + + recovered.shutdown().expect("stop recovered provider"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn receipt_limit_polls_an_authoritative_terminal_with_unacknowledged_events() { + let directory = temporary_directory("receipt-limit-terminal-with-unacked-events"); + let config = provider_config( + &directory, + &[ + "--require-dynamic-tool", + "--hold-turn", + "--emit-tool-call-on-resume", + "--accept-interrupt-without-terminal-once", + ], + ); + let runner_config = durable_config(&directory); + let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + first + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({ + "provider": config, + "authorizedTools": task_context_tool_set(), + }), + )) + .expect("prepare Codex provider"); + first + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open Codex session"); + first + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Retain the receipt-limit diagnostic until terminal polling."}), + )) + .expect("start held provider turn"); + drop(first); + saturate_provider_tool_receipts(&directory); + + let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + let pending = recovered + .poll_events() + .expect("begin the durable receipt-limit stop"); + assert!(pending.iter().any(|event| { + event.event_type == "harness.diagnostic" + && event.payload["code"] == "semantic_tool_turn_receipt_limit" + })); + assert!(!pending + .iter() + .any(|event| event.event_type == "turn.interrupted")); + + let terminal = recovered + .poll_events() + .expect("poll the provider terminal before old events are acknowledged"); + assert!(terminal.iter().any(|event| { + event.event_type == "turn.interrupted" && event.payload.get("code").is_none() + })); + assert!(!terminal.iter().any(|event| { + event.event_type == "turn.interrupted" + && event.payload["code"] == "semantic_tool_turn_receipt_limit_interrupt_deadline" + })); + recovered + .acknowledge_events(terminal.len()) + .expect("acknowledge the diagnostic and authoritative terminal together"); + + recovered.shutdown().expect("stop recovered provider"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn receipt_limit_polling_bounds_and_rejects_runtime_request_floods() { + let directory = temporary_directory("receipt-limit-runtime-request-flood"); + let config = provider_config( + &directory, + &[ + "--require-dynamic-tool", + "--hold-turn", + "--emit-tool-call-on-resume", + "--accept-interrupt-without-terminal", + "--flood-runtime-requests-on-interrupt", + ], + ); + let runner_config = durable_config(&directory); + let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + first + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({ + "provider": config, + "authorizedTools": task_context_tool_set(), + }), + )) + .expect("prepare Codex provider"); + first + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open Codex session"); + first + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Bound runtime requests while stopping this turn."}), + )) + .expect("start held provider turn"); + drop(first); + saturate_provider_tool_receipts(&directory); + + let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + let mut rejected_at_capacity = false; + while !rejected_at_capacity && std::time::Instant::now() < deadline { + let events = recovered + .poll_events() + .expect("runtime-request cleanup remains bounded across repeated polls"); + assert!(events.len() <= 128); + rejected_at_capacity = events.iter().any(|event| { + event.event_type == "provider.notice.recorded" + && event.payload["summary"] + == "rejected a Codex runtime request at the bounded pending-input limit" + }); + recovered + .acknowledge_events(events.len()) + .expect("advance the bounded durable event prefix"); + if events.is_empty() { + std::thread::sleep(std::time::Duration::from_millis(1)); + } + } + assert!( + rejected_at_capacity, + "production records that requests above the pending count/byte envelope were rejected" + ); + + recovered.shutdown().expect("stop recovered provider"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn receipt_limit_synthesizes_interrupted_after_an_accepted_terminal_deadline() { + let directory = temporary_directory("receipt-limit-missing-terminal"); + let config = provider_config( + &directory, + &[ + "--require-dynamic-tool", + "--hold-turn", + "--emit-tool-call-on-resume", + "--accept-interrupt-without-terminal", + ], + ); + let runner_config = durable_config(&directory); + let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + first + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({ + "provider": config, + "authorizedTools": task_context_tool_set(), + }), + )) + .expect("prepare Codex provider"); + first + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open Codex session"); + first + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Bound a missing interrupt terminal."}), + )) + .expect("start held provider turn"); + drop(first); + saturate_provider_tool_receipts(&directory); + + let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + let mut emitted = Vec::new(); + for _ in 0..8 { + emitted.extend( + poll_and_ack(&mut recovered) + .expect("receipt-limit fallback must remain durably pollable"), + ); + if call_count(&directory, "turn/interrupt") == 3 { + break; + } + } + assert!(!emitted + .iter() + .any(|event| event.event_type == "turn.failed")); + recovered + .shutdown() + .expect("pause the provider before expiring its durable deadline"); + drop(recovered); + + let state_path = directory.join("codex-provider-state.json"); + let mut persisted: Value = + serde_json::from_slice(&fs::read(&state_path).expect("read pending receipt-limit state")) + .expect("parse pending receipt-limit state"); + persisted["receiptLimitInterruptDeadlineUnixMs"] = json!(1); + fs::write(&state_path, serde_json::to_vec_pretty(&persisted).unwrap()) + .expect("expire the durable receipt-limit deadline"); + + let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + for _ in 0..4 { + emitted.extend( + poll_and_ack(&mut recovered) + .expect("expired receipt-limit fallback must remain durably pollable"), + ); + if emitted + .iter() + .any(|event| event.event_type == "turn.interrupted") + { + break; + } + } + let interrupted = emitted + .iter() + .find(|event| event.event_type == "turn.interrupted") + .expect("the bounded accepted path must synthesize an interrupted turn"); + assert_eq!( + interrupted.payload["code"], + "semantic_tool_turn_receipt_limit_interrupt_deadline" + ); + assert_eq!(interrupted.payload["interruptAccepted"], true); + assert_eq!(interrupted.payload["providerTerminalObserved"], false); + assert!(!emitted + .iter() + .any(|event| event.event_type == "turn.failed")); + assert_eq!(call_count(&directory, "turn/interrupt"), 3); + + let persisted: Value = serde_json::from_slice( + &fs::read(directory.join("codex-provider-state.json")) + .expect("read bounded receipt-limit state"), + ) + .expect("parse bounded receipt-limit state"); + assert_eq!(persisted["lifecycle"], "provider_exited"); + assert!(persisted["activeProviderTurnId"].is_null()); + assert_eq!(persisted["receiptLimitInterruptPending"], false); + assert_eq!(persisted["receiptLimitInterruptAttempts"], 0); + assert!(persisted["receiptLimitInterruptDeadlineUnixMs"].is_null()); + + recovered + .shutdown() + .expect("bounded fallback stopped provider"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + #[test] fn unacknowledged_provider_events_survive_executor_restart() { let directory = temporary_directory("pending-event-recovery"); @@ -1672,6 +3917,70 @@ fn unacknowledged_provider_events_survive_executor_restart() { fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); } +#[test] +fn durable_backend_rejects_a_runtime_response_after_terminal_settlement() { + let directory = temporary_directory("durable-delayed-question-response"); + let config = provider_config(&directory, &["--question-before-failed-turn"]); + let mut executor = CodexCommandExecutor::new(&directory); + executor + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({"provider": config}), + )) + .expect("prepare provider"); + executor + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open provider session"); + executor + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Ask and then fail."}), + )) + .expect("start provider turn"); + + let mut request_id = None; + let mut terminal_seen = false; + for _ in 0..16 { + for event in poll_and_ack(&mut executor).expect("poll question and terminal") { + if event.event_type == "runtime_request.created" { + request_id = event + .payload + .pointer("/request/requestId") + .and_then(Value::as_str) + .map(str::to_owned); + } + terminal_seen |= event.event_type == "turn.failed"; + } + if request_id.is_some() && terminal_seen { + break; + } + } + let request_id = request_id.expect("observe the durable runtime request id"); + assert!(terminal_seen); + let error = executor + .execute(&command( + "resolve", + 4, + "request.resolve", + json!({ + "requestId": request_id, + "response": { + "schema": "paperclip.question_response.v1", + "answers": {"environment": {"selectedOptionIds": ["option-1"]}} + } + }), + )) + .expect_err("terminal runtime requests must fail closed"); + assert!(error.to_string().contains("outside an active turn")); + + executor.shutdown().expect("stop provider process"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + #[test] fn structured_question_round_trips_through_the_normalized_backend() { let directory = temporary_directory("questions"); @@ -1700,6 +4009,7 @@ fn structured_question_round_trips_through_the_normalized_backend() { assert_eq!(started.events[0].0, "turn.accepted"); let mut question_set = None; + let mut request_id = None; let mut provider_started_events = 0; for _ in 0..16 { for event in poll_and_ack(&mut executor).expect("poll question") { @@ -1710,6 +4020,11 @@ fn structured_question_round_trips_through_the_normalized_backend() { "paperclip.runtime_request.v2" ); question_set = event.payload.pointer("/request/input").cloned(); + request_id = event + .payload + .pointer("/request/requestId") + .and_then(Value::as_str) + .map(str::to_owned); } } if question_set.is_some() { @@ -1717,6 +4032,7 @@ fn structured_question_round_trips_through_the_normalized_backend() { } } let question_set = question_set.expect("normalized question set is emitted"); + let request_id = request_id.expect("normalized request id is emitted"); assert_eq!(provider_started_events, 1); assert_eq!(question_set["schema"], "paperclip.question_set.v1"); assert_eq!( @@ -1730,7 +4046,7 @@ fn structured_question_round_trips_through_the_normalized_backend() { 4, "request.resolve", json!({ - "requestId": "runtime-request-1", + "requestId": request_id, "response": { "schema": "paperclip.question_response.v1", "answers": {"environment": {"selectedOptionIds": ["option-1"]}} diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/provider_bridge.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/provider_bridge.rs index 17303b3bf9..2504f383ed 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/provider_bridge.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/provider_bridge.rs @@ -73,6 +73,13 @@ fn rejects_unknown_tools_and_conflicting_duplicate_results() { }; bridge.apply_result(result.clone()).unwrap(); bridge.apply_result(result).unwrap(); + assert!(bridge + .replay_result("call-1", "get_task_context", &json!({})) + .unwrap() + .is_some()); + assert!(bridge + .replay_result("call-1", "get_task_context", &json!({"changed": true})) + .is_err()); assert!(bridge .apply_result(ToolResult { call_id: "call-1".to_owned(), @@ -108,6 +115,29 @@ fn catalog_digest_matches_the_typescript_canonical_json_contract() { ); } +#[test] +fn catalog_digest_normalizes_json_numbers_like_javascript() { + let operation = AuthorizedTool { + operation_id: "get_task_context".into(), + version: 1, + description: "Read the active task context.".into(), + input_schema: json!({ + "type": "object", + "properties": { + "limit": { "type": "number", "default": 1.0 }, + "epsilon": { "type": "number", "default": 1e-6 }, + }, + }), + response_schema: json!({ "type": "object" }), + }; + let digest = authorized_tool_catalog_digest(&[operation]).unwrap(); + + assert_eq!( + digest, + "sha256:1c93693d9b5b48b46c83cd1c11d1ea329774f1b9b0ae741197cb2b8e992c4b8d" + ); +} + #[test] fn validates_the_operation_value_inside_a_semantic_dispatch_envelope() { let mut set = tools("sha256:catalog-a"); @@ -120,7 +150,7 @@ fn validates_the_operation_value_inside_a_semantic_dispatch_envelope() { let mut bridge = ProviderToolBridge::default(); set.catalog_digest = digest('a'); set.catalog_digest = authorized_tool_catalog_digest(&set.operations).unwrap(); - bridge.prepare(set).unwrap(); + bridge.prepare(set.clone()).unwrap(); bridge .begin_call("call-1".into(), "get_task_context".into(), json!({})) .unwrap(); @@ -139,6 +169,25 @@ fn validates_the_operation_value_inside_a_semantic_dispatch_envelope() { }) .unwrap(); assert_eq!(bridge.pending_calls().count(), 0); + + let mut second = ProviderToolBridge::default(); + second.prepare(set).unwrap(); + second + .begin_call("call-2".into(), "get_task_context".into(), json!({})) + .unwrap(); + second + .apply_result(ToolResult { + call_id: "call-2".into(), + operation_id: "get_task_context".into(), + result: json!({ + "ok": true, + "operationId": "get_task_context", + "callId": "call-2", + "value": { "value": "accepted" } + }), + is_error: false, + }) + .unwrap(); } #[test] @@ -159,6 +208,24 @@ fn rejects_noncanonical_digests_and_oversized_contract_values() { json!({ "value": "x".repeat(1024 * 1024) }), ) .is_err()); + + let retained = json!({"value": "x".repeat(700 * 1024)}); + for index in 0..5 { + bridge + .begin_call( + format!("call-{index}"), + "get_task_context".into(), + retained.clone(), + ) + .unwrap(); + } + assert!(bridge + .begin_call( + "call-over-aggregate-limit".into(), + "get_task_context".into(), + retained, + ) + .is_err()); } #[test] @@ -203,12 +270,214 @@ fn recovery_preserves_completed_call_replay_identities() { let encoded = serde_json::to_string(&bridge).unwrap(); let mut recovered: ProviderToolBridge = serde_json::from_str(&encoded).unwrap(); + recovered.validate_recovered().unwrap(); recovered.attach_existing_run().unwrap(); assert!(recovered .begin_call("call-1".into(), "get_task_context".into(), json!({})) .is_err()); } +#[test] +fn recovered_bridge_rejects_tampered_authorization_state() { + let mut bridge = ProviderToolBridge::default(); + bridge.prepare(tools("computed")).unwrap(); + let mut encoded = serde_json::to_value(&bridge).unwrap(); + encoded["authorized"]["get_task_context"]["description"] = json!("Tampered"); + let recovered: ProviderToolBridge = serde_json::from_value(encoded).unwrap(); + assert!(recovered.validate_recovered().is_err()); +} + +#[test] +fn cancellation_completes_pending_calls_and_rejects_late_results() { + let mut bridge = ProviderToolBridge::default(); + bridge.prepare(tools("computed")).unwrap(); + bridge + .begin_call("call-1".into(), "get_task_context".into(), json!({})) + .unwrap(); + let cancelled = bridge + .cancel_pending_calls("provider_turn_stopped") + .unwrap(); + assert_eq!(cancelled.len(), 1); + assert!(cancelled[0].is_error); + assert_eq!(bridge.pending_calls().count(), 0); + assert!(bridge + .apply_result(ToolResult { + call_id: "call-1".into(), + operation_id: "get_task_context".into(), + result: json!({"ok": true}), + is_error: false, + }) + .is_err()); +} + +#[test] +fn turn_settlement_releases_value_capacity_without_reusing_call_ids() { + let mut bridge = ProviderToolBridge::default(); + bridge.prepare(tools("computed")).unwrap(); + bridge + .begin_call("call-1".into(), "get_task_context".into(), json!({})) + .unwrap(); + bridge + .apply_result(ToolResult { + call_id: "call-1".into(), + operation_id: "get_task_context".into(), + result: json!({"ok": true}), + is_error: false, + }) + .unwrap(); + + assert!(bridge + .settle_turn("provider_turn_terminated") + .unwrap() + .is_empty()); + assert!(bridge + .begin_call("call-1".into(), "get_task_context".into(), json!({})) + .is_err()); + bridge + .begin_call("call-2".into(), "get_task_context".into(), json!({})) + .expect("a new turn can use a fresh call id after releasing exact values"); +} + +#[test] +fn completed_receipts_are_exact_until_the_controlled_turn_limit() { + let mut bridge = ProviderToolBridge::default(); + bridge.prepare(tools("computed")).unwrap(); + for index in 0..4_096 { + let call_id = format!("call-{index}"); + bridge + .begin_call(call_id.clone(), "get_task_context".into(), json!({})) + .expect("a completed call must release concurrent capacity"); + bridge + .apply_result(ToolResult { + call_id, + operation_id: "get_task_context".into(), + result: json!({"ok": true}), + is_error: false, + }) + .unwrap(); + } + + let error = bridge + .begin_call( + "call-after-limit".into(), + "get_task_context".into(), + json!({}), + ) + .expect_err("the bounded exact receipt ledger must stop the active turn"); + assert!(error.is_active_turn_receipt_limit()); + assert!(bridge + .replay_result("call-0", "get_task_context", &json!({})) + .unwrap() + .is_some()); + assert!(bridge + .replay_result("call-4095", "get_task_context", &json!({})) + .unwrap() + .is_some()); + + let recovered: ProviderToolBridge = + serde_json::from_str(&serde_json::to_string(&bridge).unwrap()).unwrap(); + recovered.validate_recovered().unwrap(); +} + +#[test] +fn turn_settlement_cannot_be_blocked_by_completed_value_pressure() { + let mut bridge = ProviderToolBridge::default(); + bridge.prepare(tools("computed")).unwrap(); + let large = json!({"value": "x".repeat(700 * 1024)}); + for index in 0..5 { + bridge + .begin_call( + format!("call-{index}"), + "get_task_context".into(), + large.clone(), + ) + .unwrap(); + } + for index in 0..4 { + bridge + .apply_result(ToolResult { + call_id: format!("call-{index}"), + operation_id: "get_task_context".into(), + result: large.clone(), + is_error: false, + }) + .unwrap(); + } + + let settled = bridge.settle_turn("provider_turn_terminated").unwrap(); + assert_eq!(settled.len(), 1); + assert!(settled[0].is_error); + assert!(bridge + .begin_call("call-0".into(), "get_task_context".into(), json!({})) + .is_err()); + bridge + .begin_call( + "call-after-settlement".into(), + "get_task_context".into(), + json!({}), + ) + .expect("settlement releases prior turn value retention"); +} + +#[test] +fn recovered_turn_preserves_exact_results_at_the_value_boundary() { + let mut bridge = ProviderToolBridge::default(); + bridge.prepare(tools("computed")).unwrap(); + let large = json!({"value": "x".repeat(700 * 1024)}); + for index in 0..5 { + bridge + .begin_call( + format!("call-{index}"), + "get_task_context".into(), + large.clone(), + ) + .unwrap(); + } + for index in 0..5 { + bridge + .apply_result(ToolResult { + call_id: format!("call-{index}"), + operation_id: "get_task_context".into(), + result: large.clone(), + is_error: false, + }) + .unwrap(); + } + + let error = bridge + .begin_call("call-next".into(), "get_task_context".into(), large.clone()) + .expect_err("exact replay values must not be discarded for later work"); + assert!(error.is_active_turn_receipt_limit()); + assert_eq!( + bridge + .replay_result("call-0", "get_task_context", &large) + .unwrap() + .unwrap() + .result, + large, + ); + bridge + .apply_result(ToolResult { + call_id: "call-0".into(), + operation_id: "get_task_context".into(), + result: large.clone(), + is_error: false, + }) + .expect("a matching exact result receipt remains idempotent"); + assert!(bridge + .apply_result(ToolResult { + call_id: "call-0".into(), + operation_id: "get_task_context".into(), + result: json!({"value": "changed"}), + is_error: false, + }) + .is_err()); + + let recovered: ProviderToolBridge = + serde_json::from_str(&serde_json::to_string(&bridge).unwrap()).unwrap(); + recovered.validate_recovered().unwrap(); +} + #[test] fn recovery_preserves_pending_calls_for_the_existing_run() { let mut bridge = ProviderToolBridge::default(); @@ -252,7 +521,7 @@ fn recovery_rejects_nonempty_state_without_a_catalog_digest() { let error = recovered .attach_existing_run() .expect_err("nonempty recovered state must remain bound to a catalog digest"); - assert!(error.to_string().contains("omit the catalog digest")); + assert!(error.to_string().contains("omitted its catalog identity")); } #[test] @@ -269,8 +538,8 @@ fn recovery_rejects_tampered_authorization_catalog_bindings() { let mut recovered: ProviderToolBridge = serde_json::from_value(changed_contract).unwrap(); let error = recovered .attach_existing_run() - .expect_err("recovery must recompute the catalog digest"); - assert!(error.to_string().contains("catalog digest")); + .expect_err("recovery must reconstruct the authorized catalog projection"); + assert!(error.to_string().contains("changed its authorized catalog")); let mut changed_map_key = encoded; let authorized = changed_map_key["authorized"].as_object_mut().unwrap(); @@ -280,7 +549,7 @@ fn recovery_rejects_tampered_authorization_catalog_bindings() { let error = recovered .attach_existing_run() .expect_err("recovery must bind map keys to declared operation identities"); - assert!(error.to_string().contains("identities are inconsistent")); + assert!(error.to_string().contains("changed its authorized catalog")); } #[test] @@ -326,18 +595,21 @@ fn recovery_rejects_tampered_retained_result_contracts() { let completed = serde_json::to_value(&bridge).unwrap(); let mut unauthorized = completed.clone(); - unauthorized["completed"]["call-1"]["operationId"] = json!("delete_company"); - let mut recovered: ProviderToolBridge = serde_json::from_value(unauthorized).unwrap(); - assert!(recovered.attach_existing_run().is_err()); + unauthorized["completed"]["call-1"]["result"]["operationId"] = json!("delete_company"); + let error = serde_json::from_value::(unauthorized) + .expect_err("durable decoding must reject mismatched call and result identities"); + assert!(error + .to_string() + .contains("retained provider tool receipt identity is inconsistent")); let mut invalid_output = completed; - invalid_output["completed"]["call-1"]["result"] = json!(["not", "an", "object"]); + invalid_output["completed"]["call-1"]["result"]["result"] = json!(["not", "an", "object"]); let mut recovered: ProviderToolBridge = serde_json::from_value(invalid_output).unwrap(); assert!(recovered.attach_existing_run().is_err()); - bridge.settle_turn().unwrap(); + bridge.settle_turn("provider_turn_terminated").unwrap(); let mut invalid_settled_output = serde_json::to_value(&bridge).unwrap(); - invalid_settled_output["settledResults"]["call-1"]["result"] = json!("invalid"); + invalid_settled_output["settledResults"]["call-1"]["result"]["result"] = json!("invalid"); let mut recovered: ProviderToolBridge = serde_json::from_value(invalid_settled_output).unwrap(); assert!(recovered.attach_existing_run().is_err()); } @@ -389,7 +661,7 @@ fn settles_completed_receipts_before_the_next_turn() { assert!(bridge .begin_call("call-next".into(), "get_task_context".into(), json!({})) .is_err()); - bridge.settle_turn().unwrap(); + bridge.settle_turn("provider_turn_terminated").unwrap(); assert!(bridge .begin_call("call-next".into(), "get_task_context".into(), json!({})) .is_ok()); @@ -410,7 +682,7 @@ fn settlement_preserves_call_ids_for_the_durable_run() { is_error: false, }) .unwrap(); - bridge.settle_turn().unwrap(); + bridge.settle_turn("provider_turn_terminated").unwrap(); let replay = ToolResult { call_id: "call-1".into(), @@ -456,7 +728,7 @@ fn settlement_preserves_call_ids_for_the_durable_run() { } #[test] -fn reserves_identity_capacity_before_accepting_a_call() { +fn exact_identity_overflow_saturates_the_durable_run() { let mut bridge = ProviderToolBridge::default(); bridge.prepare(tools("computed")).unwrap(); @@ -472,27 +744,147 @@ fn reserves_identity_capacity_before_accepting_a_call() { bridge .begin_call("last-call".into(), "get_task_context".into(), json!({})) .unwrap(); - bridge - .apply_result(ToolResult { - call_id: "last-call".into(), - operation_id: "get_task_context".into(), - result: json!({"ok": true}), - is_error: false, - }) - .unwrap(); - bridge.settle_turn().unwrap(); - assert!(bridge + let overflow = bridge + .begin_call("overflow".into(), "get_task_context".into(), json!({})) + .expect_err("the pending call reserves the final exact identity slot"); + assert!(overflow.is_active_turn_receipt_limit()); + + let encoded = serde_json::to_string(&bridge).unwrap(); + let mut recovered: ProviderToolBridge = serde_json::from_str(&encoded).unwrap(); + recovered.attach_existing_run().unwrap(); + assert!(recovered.durable_run_receipt_limit_reached()); + recovered + .settle_turn("provider_turn_terminated") + .expect("the controlled turn stop retains replay protection"); + assert!(recovered.durable_run_receipt_limit_reached()); + assert!(recovered.has_completed_call("settled-0")); + assert!(recovered.has_completed_call("last-call")); + let stopped_turn_receipt = recovered + .replay_result("last-call", "get_task_context", &json!({})) + .unwrap() + .expect("the call admitted before exhaustion retains an exact terminal receipt"); + assert!(stopped_turn_receipt.is_error); + assert_eq!( + stopped_turn_receipt.result["error"]["code"], + "provider_turn_terminated" + ); + assert!(recovered + .begin_call("settled-0".into(), "get_task_context".into(), json!({})) + .is_err()); + assert!(recovered .begin_call("overflow".into(), "get_task_context".into(), json!({})) .is_err()); - assert!(bridge.settle_turn().is_ok()); + recovered.prepare_turn().unwrap(); + assert!(recovered + .replay_result("last-call", "get_task_context", &json!({})) + .unwrap() + .is_none()); + assert!(recovered.has_completed_call("last-call")); + assert!(recovered.has_completed_call("settled-0")); + assert!(recovered.durable_run_receipt_limit_reached()); + let saturation = recovered + .begin_call("next-call".into(), "get_task_context".into(), json!({})) + .expect_err("a turn boundary must not reopen the saturated durable run"); + assert!(saturation.is_active_turn_receipt_limit()); + + let encoded = serde_json::to_string(&recovered).unwrap(); + let mut recovered: ProviderToolBridge = serde_json::from_str(&encoded).unwrap(); + recovered.attach_existing_run().unwrap(); + assert!(recovered.has_completed_call("settled-0")); + assert!(recovered.has_completed_call("last-call")); + assert!(recovered.durable_run_receipt_limit_reached()); + + recovered.attach_run(tools("computed")).unwrap(); + assert!(!recovered.durable_run_receipt_limit_reached()); + recovered + .begin_call("overflow".into(), "get_task_context".into(), json!({})) + .expect("a new durable run receives a fresh tool-call identity ledger"); +} + +#[test] +fn settled_result_byte_exhaustion_recovers_after_turn_cleanup() { + let mut bridge = ProviderToolBridge::default(); + bridge.prepare(tools("computed")).unwrap(); + let large_result = json!({"value": "x".repeat(750 * 1024)}); + + for index in 0..10 { + let call_id = format!("large-settled-{index}"); + bridge + .begin_call(call_id.clone(), "get_task_context".into(), json!({})) + .unwrap(); + bridge + .apply_result(ToolResult { + call_id, + operation_id: "get_task_context".into(), + result: large_result.clone(), + is_error: false, + }) + .unwrap(); + } + bridge.settle_turn("provider_turn_terminated").unwrap(); + + let error = bridge + .begin_call( + "over-byte-limit".into(), + "get_task_context".into(), + json!({}), + ) + .expect_err("settled byte exhaustion must stop the durable run"); + assert!(error.is_active_turn_receipt_limit()); + + let encoded = serde_json::to_string(&bridge).unwrap(); + let mut recovered: ProviderToolBridge = serde_json::from_str(&encoded).unwrap(); + recovered.attach_existing_run().unwrap(); + assert!(recovered.durable_run_receipt_limit_reached()); + assert_eq!( + recovered + .replay_result("large-settled-0", "get_task_context", &json!({})) + .unwrap() + .unwrap() + .result, + large_result + ); + assert!(recovered + .begin_call( + "over-byte-limit".into(), + "get_task_context".into(), + json!({}) + ) + .is_err()); + recovered.prepare_turn().unwrap(); + assert!(recovered + .replay_result("large-settled-0", "get_task_context", &json!({})) + .unwrap() + .is_none()); + assert!(recovered.has_completed_call("large-settled-0")); + assert!(!recovered.durable_run_receipt_limit_reached()); + recovered + .begin_call( + "after-turn-boundary".into(), + "get_task_context".into(), + json!({}), + ) + .expect("releasing bulky results clears transient byte pressure"); + recovered + .settle_turn("provider_turn_terminated") + .expect("the admitted next-turn call remains settleable"); + + recovered.attach_run(tools("computed")).unwrap(); + recovered + .begin_call( + "over-byte-limit".into(), + "get_task_context".into(), + json!({}), + ) + .expect("a new durable run resets the settled result budget"); } #[test] fn reserves_settled_result_bytes_before_accepting_a_call() { let mut bridge = ProviderToolBridge::default(); bridge.prepare(tools("computed")).unwrap(); - let large_result = json!({"value": "x".repeat(900 * 1024)}); + let large_result = json!({"value": "x".repeat(700 * 1024)}); let mut completed = 0; for index in 0..20 { @@ -523,7 +915,7 @@ fn reserves_settled_result_bytes_before_accepting_a_call() { ) .is_err()); bridge - .settle_turn() + .settle_turn("provider_turn_terminated") .expect("settlement cannot strand results whose bytes were reserved at admission"); assert_eq!( bridge @@ -534,7 +926,7 @@ fn reserves_settled_result_bytes_before_accepting_a_call() { is_error: false, }) .unwrap(), - json!({"value": "x".repeat(900 * 1024)}) + json!({"value": "x".repeat(700 * 1024)}) ); } @@ -544,15 +936,22 @@ fn recovery_rejects_an_oversized_settled_result_envelope() { bridge.prepare(tools("computed")).unwrap(); let mut encoded = serde_json::to_value(&bridge).unwrap(); let settled = encoded["settledResults"].as_object_mut().unwrap(); - for index in 0..10 { + for index in 0..12 { let call_id = format!("recovered-large-{index}"); settled.insert( call_id.clone(), json!({ - "callId": call_id, - "operationId": "get_task_context", - "result": {"value": "x".repeat(900 * 1024)}, - "isError": false + "call": { + "callId": call_id, + "operationId": "get_task_context", + "input": {} + }, + "result": { + "callId": call_id, + "operationId": "get_task_context", + "result": {"value": "x".repeat(700 * 1024)}, + "isError": false + } }), ); } @@ -568,18 +967,28 @@ fn recovery_rejects_state_without_room_for_a_pending_result() { bridge.prepare(tools("computed")).unwrap(); let mut encoded = serde_json::to_value(&bridge).unwrap(); let settled = encoded["settledResults"].as_object_mut().unwrap(); - for index in 0..8 { + let mut settled_ids = Vec::new(); + for index in 0..11 { let call_id = format!("recovered-large-{index}"); + settled_ids.push(json!(call_id)); settled.insert( call_id.clone(), json!({ - "callId": call_id, - "operationId": "get_task_context", - "result": {"value": "x".repeat(900 * 1024)}, - "isError": false + "call": { + "callId": call_id, + "operationId": "get_task_context", + "input": {} + }, + "result": { + "callId": call_id, + "operationId": "get_task_context", + "result": {"value": "x".repeat(700 * 1024)}, + "isError": false + } }), ); } + encoded["settledCallIds"] = serde_json::Value::Array(settled_ids); encoded["pending"]["pending-call"] = json!({ "callId": "pending-call", "operationId": "get_task_context", @@ -594,13 +1003,62 @@ fn recovery_rejects_state_without_room_for_a_pending_result() { } #[test] -fn refuses_to_settle_receipts_while_calls_are_pending() { +fn settles_pending_receipts_with_explicit_terminal_results() { let mut bridge = ProviderToolBridge::default(); bridge.prepare(tools("computed")).unwrap(); bridge .begin_call("call-1".into(), "get_task_context".into(), json!({})) .unwrap(); - assert!(bridge.settle_turn().is_err()); - assert_eq!(bridge.pending_calls().count(), 1); + let settled = bridge.settle_turn("provider_turn_terminated").unwrap(); + assert_eq!(settled.len(), 1); + assert_eq!(settled[0].call_id, "call-1"); + assert!(settled[0].is_error); + assert_eq!(bridge.pending_calls().count(), 0); +} + +#[test] +fn full_identity_ledger_fails_closed_across_turn_and_recovery() { + let mut bridge = ProviderToolBridge::default(); + bridge.prepare(tools("computed")).unwrap(); + let mut encoded = serde_json::to_value(&bridge).unwrap(); + encoded["settledCallIds"] = serde_json::Value::Array( + (0..65_536) + .map(|index| json!(format!("settled-{index:05}"))) + .collect(), + ); + let mut recovered: ProviderToolBridge = serde_json::from_value(encoded).unwrap(); + recovered.validate_recovered().unwrap(); + let saturation = recovered + .begin_call("current-call".into(), "get_task_context".into(), json!({})) + .expect_err("a full exact ledger must stop fresh work"); + assert!(saturation.is_active_turn_receipt_limit()); + assert!(recovered.durable_run_receipt_limit_reached()); + assert!(recovered.has_completed_call("settled-65535")); + assert!(recovered.has_completed_call("settled-00000")); + assert!(!recovered.has_completed_call("current-call")); + assert!(recovered + .begin_call("settled-00000".into(), "get_task_context".into(), json!({})) + .is_err()); + recovered.prepare_turn().unwrap(); + let fresh = recovered + .begin_call("fresh-call".into(), "get_task_context".into(), json!({})) + .expect_err("a turn boundary must preserve durable-run saturation"); + assert!(fresh.is_active_turn_receipt_limit()); + + let round_trip = serde_json::to_value(&recovered).unwrap(); + assert_eq!( + round_trip["settledCallIds"].as_array().unwrap().len(), + 65_536 + ); + let mut recovered_again: ProviderToolBridge = serde_json::from_value(round_trip).unwrap(); + recovered_again.attach_existing_run().unwrap(); + assert!(recovered_again.durable_run_receipt_limit_reached()); + assert!(recovered_again.has_completed_call("settled-00000")); + + recovered_again.attach_run(tools("computed")).unwrap(); + assert!(!recovered_again.durable_run_receipt_limit_reached()); + recovered_again + .begin_call("current-call".into(), "get_task_context".into(), json!({})) + .expect("only a new durable run resets replay authority"); }