diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs index 1095e28495..81763e1043 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs @@ -120,6 +120,66 @@ fn emit_ambiguous_turn_evidence( } } +fn emit_ambiguous_turn_item() -> io::Result<()> { + send(json!({ + "method": "item/completed", + "params": {"item": { + "id": "replacement-message-before-terminal", + "type": "agentMessage", + "status": "completed", + "text": "Replacement output before terminal authority." + }} + })) +} + +fn send_question(state: &FakeState) -> io::Result<()> { + let turn_id = state.active_turn_id.as_deref().unwrap_or("provider-turn-1"); + send(json!({ + "id": "runtime-request-1", + "method": "item/tool/requestUserInput", + "params": { + "threadId": state.thread_id, + "turnId": turn_id, + "itemId": "question-item-1", + "isBlocking": true, + "title": "Deployment input", + "questions": [{ + "id": "environment", + "header": "Environment", + "question": "Where should we deploy?", + "options": [ + {"label": "Staging", "description": "Deploy safely."}, + {"label": "Production", "description": "Deploy directly."} + ] + }] + } + })) +} + +fn send_runtime_request_flood(state: &FakeState, interrupt_count: u64) -> io::Result<()> { + let turn_id = state.active_turn_id.as_deref().unwrap_or("provider-turn-1"); + for index in 0..160_u64 { + send(json!({ + "id": format!("runtime-flood-{interrupt_count}-{index}"), + "method": "item/tool/requestUserInput", + "params": { + "threadId": state.thread_id, + "turnId": turn_id, + "itemId": format!("question-item-{interrupt_count}-{index}"), + "isBlocking": true, + "title": "Bounded cleanup input", + "questions": [{ + "id": "environment", + "header": "Environment", + "question": "Where should we deploy?", + "options": [{"label": "Staging", "description": "Deploy safely."}], + }], + }, + }))?; + } + Ok(()) +} + fn run() -> Result<(), Box> { let args = std::env::args().skip(1).collect::>(); let state_path = @@ -136,6 +196,19 @@ fn run() -> Result<(), Box> { let exit_after_tool_call_completion = args .iter() .any(|value| value == "--exit-after-tool-call-completion"); + let emit_tool_call_on_resume = args + .iter() + .any(|value| value == "--emit-tool-call-on-resume"); + let resume_unowned_turn_when_marked = args + .iter() + .any(|value| value == "--resume-unowned-turn-when-marked"); + let replay_completed_tool_call_count = argument(&args, "--replay-completed-tool-call-count") + .map(|value| value.parse::()) + .transpose()? + .unwrap_or_default(); + let finish_turn_with_pending_tool = args + .iter() + .any(|value| value == "--finish-turn-with-pending-tool"); let require_dynamic_tool = args.iter().any(|value| value == "--require-dynamic-tool"); let hold_turn = args.iter().any(|value| value == "--hold-turn"); let exit_after_turn_start = args.iter().any(|value| value == "--exit-after-turn-start"); @@ -163,6 +236,9 @@ fn run() -> Result<(), Box> { let missing_id_second_turn_start = args .iter() .any(|value| value == "--missing-id-second-turn-start"); + let missing_id_live_turn_start = args + .iter() + .any(|value| value == "--missing-id-live-turn-start"); let fail_after_accepting_second_turn_before_response = args .iter() .any(|value| value == "--fail-after-accepting-second-turn-before-response"); @@ -184,32 +260,93 @@ fn run() -> Result<(), Box> { let conflicting_ambiguous_second_turn = args .iter() .any(|value| value == "--conflicting-ambiguous-second-turn"); + let ambiguous_older_reused_turn = args + .iter() + .any(|value| value == "--ambiguous-older-reused-turn"); let omit_ambiguous_turn_started = args .iter() .any(|value| value == "--omit-ambiguous-turn-started"); let fail_after_thread_read = args.iter().any(|value| value == "--fail-after-thread-read"); + let fail_first_interrupt = args.iter().any(|value| value == "--fail-first-interrupt"); + let accept_interrupt_without_terminal_once = args + .iter() + .any(|value| value == "--accept-interrupt-without-terminal-once"); + let accept_interrupt_without_terminal = args + .iter() + .any(|value| value == "--accept-interrupt-without-terminal"); + let flood_runtime_requests_on_interrupt = args + .iter() + .any(|value| value == "--flood-runtime-requests-on-interrupt"); + let interrupt_terminal_delay_ms = argument(&args, "--interrupt-terminal-delay-ms") + .map(|value| value.parse::()) + .transpose()?; let exit_after_thread_read = args.iter().any(|value| value == "--exit-after-thread-read"); let fail_after_turn_completion_delay_ms = argument(&args, "--fail-after-turn-completion-delay-ms") .map(|value| value.parse::()) .transpose()?; + let delayed_tool_after_failed_turn = args + .iter() + .any(|value| value == "--delayed-tool-after-failed-turn"); + let delayed_tool_after_next_turn_start = args + .iter() + .any(|value| value == "--delayed-tool-after-next-turn-start"); + let delayed_tool_after_third_turn_start = args + .iter() + .any(|value| value == "--delayed-tool-after-third-turn-start"); + let delayed_tool_after_second_turn_completion = args + .iter() + .any(|value| value == "--delayed-tool-after-second-turn-completion"); + let tool_after_reused_turn_start = args + .iter() + .any(|value| value == "--tool-after-reused-turn-start"); + let tool_after_older_reused_turn_start = args + .iter() + .any(|value| value == "--tool-after-older-reused-turn-start"); + let question_before_failed_turn = args + .iter() + .any(|value| value == "--question-before-failed-turn"); + let reuse_question_id = args.iter().any(|value| value == "--reuse-question-id"); let pre_response_notification = args .iter() .any(|value| value == "--notification-before-response"); let mut state = load_state(&state_path); let mut turn_start_count = 0_u64; + let mut interrupt_count = 0_u64; + let mut delayed_interrupt_terminal_scheduled = false; + let mut answered_questions = 0u8; + let mut replayed_completed_tool_calls = 0_u64; for line in io::stdin().lock().lines() { let message: Value = serde_json::from_str(&line?)?; + if message.get("method").is_none() + && message + .get("id") + .and_then(Value::as_str) + .is_some_and(|id| id.starts_with("runtime-flood-")) + { + let outcome = if message.get("error").is_some() { + "runtime-response:rejected" + } else { + "runtime-response:cancelled" + }; + log_call(call_log.as_deref(), outcome)?; + continue; + } if message.get("method").is_none() && message.get("id") == Some(&json!("runtime-request-1")) { + if reuse_question_id && answered_questions == 0 { + answered_questions = 1; + send_question(&state)?; + continue; + } finish_turn(&state_path, &mut state, "completed")?; continue; } if message.get("method").is_none() && message.get("id") == Some(&json!("tool-request-1")) { if message.pointer("/result/success") == Some(&json!(false)) { log_call(call_log.as_deref(), "tool-response:failure")?; - if state.active_turn_id.is_some() { + if state.active_turn_id.is_some() && !hold_turn { finish_turn(&state_path, &mut state, "failed")?; } continue; @@ -225,7 +362,9 @@ fn run() -> Result<(), Box> { if result != json!({"ok": true, "task": {"id": "task-1"}}) { return Err("semantic tool response changed the operation result".into()); } - if replay_completed_tool_call { + log_call(call_log.as_deref(), &format!("tool-response:{text}"))?; + if replay_completed_tool_call && replayed_completed_tool_calls == 0 { + replayed_completed_tool_calls += 1; send(json!({ "id": "tool-request-replay", "method": "item/tool/call", @@ -237,9 +376,22 @@ fn run() -> Result<(), Box> { "arguments": {} } }))?; - continue; + } else if replayed_completed_tool_calls < replay_completed_tool_call_count { + replayed_completed_tool_calls += 1; + send(json!({ + "id": "tool-request-1", + "method": "item/tool/call", + "params": { + "threadId": state.thread_id, + "turnId": state.active_turn_id, + "callId": "semantic-call-1", + "tool": "get_task_context", + "arguments": {} + } + }))?; + } else if !hold_turn { + finish_turn(&state_path, &mut state, "completed")?; } - finish_turn(&state_path, &mut state, "completed")?; continue; } let Some(method) = message.get("method").and_then(Value::as_str) else { @@ -275,10 +427,31 @@ fn run() -> Result<(), Box> { if require_dynamic_tool && !has_task_context_tool(&message) { return Err("thread/resume omitted the authorized dynamic tool".into()); } + let unowned_turn_marker = state_path.with_file_name("resume-unowned-turn"); + if resume_unowned_turn_when_marked && unowned_turn_marker.exists() { + state.active_turn_id = Some("provider-turn-unowned".to_owned()); + save_state(&state_path, &state)?; + fs::remove_file(unowned_turn_marker)?; + } send(json!({ "id": id, "result": {"thread": {"id": state.thread_id, "sessionId": "codex-account-session"}} }))?; + if emit_tool_call_on_resume { + if let Some(turn_id) = state.active_turn_id.as_deref() { + send(json!({ + "id": "tool-request-1", + "method": "item/tool/call", + "params": { + "threadId": state.thread_id, + "turnId": turn_id, + "callId": "semantic-call-1", + "tool": "get_task_context", + "arguments": {} + } + }))?; + } + } } "thread/read" => { let turns = state @@ -319,17 +492,31 @@ fn run() -> Result<(), Box> { && (complete_ambiguous_second_turn || complete_ambiguous_second_turn_before_response || conflicting_ambiguous_second_turn); - let provider_turn_id = if emits_ambiguous_turn_evidence + let provider_turn_id = if turn_start_count == 3 && ambiguous_older_reused_turn { + "provider-turn-1".to_owned() + } else if emits_ambiguous_turn_evidence || (turn_start_count == 2 && (retain_ambiguous_second_turn_active || hold_ambiguous_second_turn_after_item)) { - "provider-turn-2" + "provider-turn-2".to_owned() + } else if tool_after_reused_turn_start + || (tool_after_older_reused_turn_start && turn_start_count == 3) + { + "provider-turn-1".to_owned() } else { - "provider-turn-1" + format!("provider-turn-{turn_start_count}") }; - state.active_turn_id = Some(provider_turn_id.to_owned()); + state.active_turn_id = Some(provider_turn_id.clone()); save_state(&state_path, &state)?; + if ambiguous_older_reused_turn && turn_start_count == 3 { + send(json!({ + "method": "turn/started", + "params": {"turn": {"id": provider_turn_id}} + }))?; + send(json!({"id": id, "error": {}}))?; + continue; + } if complete_ambiguous_second_turn_before_response && turn_start_count == 2 { emit_ambiguous_turn_evidence( &state_path, @@ -355,6 +542,9 @@ fn run() -> Result<(), Box> { return Ok(()); } if malformed_error_second_turn_start && turn_start_count == 2 { + if hold_ambiguous_second_turn_after_item { + emit_ambiguous_turn_item()?; + } send(json!({"id": id, "error": {}}))?; if emits_ambiguous_turn_evidence && !complete_ambiguous_second_turn_before_response @@ -374,15 +564,7 @@ fn run() -> Result<(), Box> { "result": {"turn": {"status": "inProgress"}} }))?; if hold_ambiguous_second_turn_after_item { - send(json!({ - "method": "item/completed", - "params": {"item": { - "id": "replacement-message-before-terminal", - "type": "agentMessage", - "status": "completed", - "text": "Replacement output before terminal authority." - }} - }))?; + emit_ambiguous_turn_item()?; continue; } if emits_ambiguous_turn_evidence @@ -397,16 +579,103 @@ fn run() -> Result<(), Box> { } return Err("configured failure after missing turn identity".into()); } + if missing_id_live_turn_start { + send(json!({ + "id": id, + "result": {"turn": {"status": "inProgress"}} + }))?; + continue; + } send(json!({ "id": id, - "result": {"turn": {"id": "provider-turn-1", "status": "inProgress"}} + "result": {"turn": {"id": provider_turn_id, "status": "inProgress"}} }))?; send(json!({ "method": "turn/started", - "params": {"turn": {"id": "provider-turn-1"}} + "params": {"turn": {"id": provider_turn_id}} }))?; if fail_after_second_turn_start && turn_start_count == 2 { return Err("configured failure after second turn start".into()); + } else if question_before_failed_turn { + send_question(&state)?; + send(json!({ + "method": "turn/completed", + "params": { + "threadId": state.thread_id, + "turn": {"id": provider_turn_id, "status": "failed"} + } + }))?; + state.active_turn_id = None; + save_state(&state_path, &state)?; + } else if delayed_tool_after_failed_turn { + send(json!({ + "method": "turn/failed", + "params": { + "threadId": state.thread_id, + "turn": {"id": provider_turn_id, "status": "failed"} + } + }))?; + state.active_turn_id = None; + save_state(&state_path, &state)?; + send(json!({ + "id": "tool-request-delayed", + "method": "item/tool/call", + "params": { + "threadId": state.thread_id, + "turnId": provider_turn_id, + "callId": "semantic-call-delayed", + "tool": "get_task_context", + "arguments": {} + } + }))?; + } else if delayed_tool_after_next_turn_start && turn_start_count == 2 { + send(json!({ + "id": "tool-request-delayed", + "method": "item/tool/call", + "params": { + "threadId": state.thread_id, + "turnId": "provider-turn-1", + "callId": "semantic-call-delayed", + "tool": "get_task_context", + "arguments": {} + } + }))?; + } else if delayed_tool_after_third_turn_start && turn_start_count == 3 { + send(json!({ + "id": "tool-request-two-turns-delayed", + "method": "item/tool/call", + "params": { + "threadId": state.thread_id, + "turnId": "provider-turn-1", + "callId": "semantic-call-two-turns-delayed", + "tool": "get_task_context", + "arguments": {} + } + }))?; + } else if tool_after_reused_turn_start && turn_start_count == 2 { + send(json!({ + "id": "tool-request-reused-turn", + "method": "item/tool/call", + "params": { + "threadId": state.thread_id, + "turnId": "provider-turn-1", + "callId": "semantic-call-reused-turn", + "tool": "get_task_context", + "arguments": {} + } + }))?; + } else if tool_after_older_reused_turn_start && turn_start_count == 3 { + send(json!({ + "id": "tool-request-older-reused-turn", + "method": "item/tool/call", + "params": { + "threadId": state.thread_id, + "turnId": "provider-turn-1", + "callId": "semantic-call-older-reused-turn", + "tool": "get_task_context", + "arguments": {} + } + }))?; } else if exit_after_turn_start { return Ok(()); } else if emit_tool_call { @@ -415,41 +684,35 @@ fn run() -> Result<(), Box> { "method": "item/tool/call", "params": { "threadId": state.thread_id, - "turnId": "provider-turn-1", + "turnId": provider_turn_id, "callId": "semantic-call-1", "tool": "get_task_context", "arguments": {} } }))?; - if complete_after_tool_call { + if complete_after_tool_call || finish_turn_with_pending_tool { finish_turn(&state_path, &mut state, "completed")?; if exit_after_tool_call_completion { return Ok(()); } } } else if emit_question { - send(json!({ - "id": "runtime-request-1", - "method": "item/tool/requestUserInput", - "params": { - "threadId": state.thread_id, - "turnId": "provider-turn-1", - "itemId": "question-item-1", - "isBlocking": true, - "title": "Deployment input", - "questions": [{ - "id": "environment", - "header": "Environment", - "question": "Where should we deploy?", - "options": [ - {"label": "Staging", "description": "Deploy safely."}, - {"label": "Production", "description": "Deploy directly."} - ] - }] - } - }))?; + send_question(&state)?; } else if !hold_turn { finish_turn(&state_path, &mut state, "completed")?; + if delayed_tool_after_second_turn_completion && turn_start_count == 2 { + send(json!({ + "id": "tool-request-idle-two-turns-delayed", + "method": "item/tool/call", + "params": { + "threadId": state.thread_id, + "turnId": "provider-turn-1", + "callId": "semantic-call-idle-two-turns-delayed", + "tool": "get_task_context", + "arguments": {} + } + }))?; + } if emit_post_completion_warning { send(json!({ "method": "warning", @@ -477,8 +740,44 @@ fn run() -> Result<(), Box> { } "turn/steer" => send(json!({"id": id, "result": {"accepted": true}}))?, "turn/interrupt" => { - send(json!({"id": id, "result": {"accepted": true}}))?; - finish_turn(&state_path, &mut state, "interrupted")?; + interrupt_count += 1; + if fail_first_interrupt && interrupt_count == 1 { + send(json!({ + "id": id, + "error": {"code": -32001, "message": "configured interrupt failure"} + }))?; + } else { + send(json!({"id": id, "result": {"accepted": true}}))?; + if flood_runtime_requests_on_interrupt { + send_runtime_request_flood(&state, interrupt_count)?; + } + if !accept_interrupt_without_terminal + && !(accept_interrupt_without_terminal_once + && interrupt_count == if fail_first_interrupt { 2 } else { 1 }) + { + if let Some(delay_ms) = interrupt_terminal_delay_ms { + if !delayed_interrupt_terminal_scheduled { + delayed_interrupt_terminal_scheduled = true; + let delayed_state_path = state_path.clone(); + let mut delayed_state = state.clone(); + thread::spawn(move || { + thread::sleep(Duration::from_millis(delay_ms)); + if let Err(error) = finish_turn( + &delayed_state_path, + &mut delayed_state, + "interrupted", + ) { + eprintln!( + "failed to emit delayed interrupt terminal: {error}" + ); + } + }); + } + } else { + finish_turn(&state_path, &mut state, "interrupted")?; + } + } + } } _ if id.is_some() => send(json!({ "id": id, diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs index 4764ed1e88..62aed56ea1 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs @@ -50,28 +50,26 @@ fn run_durable(args: &[String]) -> Result<(), LocalRunnerError> { optional_u64(args, name).map(|value| Duration::from_millis(value.unwrap_or(default))) }; let state_dir = PathBuf::from(value(args, "--state-dir")?); - run_durable_runner( - DurableRunnerConfig { - connect_url: value(args, "--connect-url")?, - state_dir: state_dir.clone(), - runner_instance_id: value(args, "--runner-id")?, - environment_lease_id: value(args, "--environment-lease-id")?, - run_id: value(args, "--run-id")?, - normalized_session_id: value(args, "--session-id")?, - turn_id: value(args, "--turn-id")?, - item_id: value(args, "--item-id")?, - runner_version: value(args, "--runner-version")?, - runner_digest: value(args, "--runner-digest")?, - max_outbox_bytes: usize_value(args, "--max-outbox-bytes", 16 * 1024 * 1024)?, - p0_reserve_bytes: usize_value(args, "--p0-reserve-bytes", 1024 * 1024)?, - max_frame_bytes: usize_value(args, "--max-frame-bytes", 1024 * 1024)?, - reconnect_delay: duration("--reconnect-delay-ms", 250)?, - max_runtime: duration("--max-runtime-ms", 60 * 60 * 1000)?, - }, - ticket, - CodexCommandExecutor::new(state_dir), - ) - .map_err(|error| LocalRunnerError::invalid(error.to_string())) + let config = DurableRunnerConfig { + connect_url: value(args, "--connect-url")?, + state_dir: state_dir.clone(), + runner_instance_id: value(args, "--runner-id")?, + environment_lease_id: value(args, "--environment-lease-id")?, + run_id: value(args, "--run-id")?, + normalized_session_id: value(args, "--session-id")?, + turn_id: value(args, "--turn-id")?, + item_id: value(args, "--item-id")?, + runner_version: value(args, "--runner-version")?, + runner_digest: value(args, "--runner-digest")?, + max_outbox_bytes: usize_value(args, "--max-outbox-bytes", 16 * 1024 * 1024)?, + p0_reserve_bytes: usize_value(args, "--p0-reserve-bytes", 1024 * 1024)?, + max_frame_bytes: usize_value(args, "--max-frame-bytes", 1024 * 1024)?, + reconnect_delay: duration("--reconnect-delay-ms", 250)?, + max_runtime: duration("--max-runtime-ms", 60 * 60 * 1000)?, + }; + let executor = CodexCommandExecutor::with_runner_config(state_dir, &config); + run_durable_runner(config, ticket, executor) + .map_err(|error| LocalRunnerError::invalid(error.to_string())) } fn run() -> Result<(), LocalRunnerError> { diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs index 322236013d..249c13f0ae 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs @@ -4,18 +4,24 @@ use std::time::Duration; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; use crate::durable::redact_text; use crate::local_runner::LocalRunnerError; use crate::process_supervisor::SupervisedProcess; -use crate::provider_bridge::{AuthorizedTool, ToolResult}; +use crate::provider_bridge::{AuthorizedTool, DurableReplayFilter, ToolResult}; +use crate::provider_events::normalized_codex_terminal_event_type; pub const CODEX_APP_SERVER_MAX_FRAME_BYTES: usize = 4 * 1024 * 1024; const MAX_BUFFERED_MESSAGES: usize = 1_024; +const MAX_BUFFERED_MESSAGE_BYTES: usize = 16 * 1024 * 1024; const MAX_INSTRUCTIONS_BYTES: usize = 1024 * 1024; const MAX_PENDING_TOOL_REQUESTS: usize = 4_096; const MAX_PENDING_TOOL_REQUEST_BYTES: usize = 16 * 1024 * 1024; const MAX_COMPLETED_TOOL_CALL_IDS: usize = 4_096; +const MAX_PENDING_RUNTIME_REQUESTS: usize = 128; +const MAX_PENDING_RUNTIME_REQUEST_BYTES: usize = 4 * 1024 * 1024; +pub(crate) const MAX_SETTLED_PROVIDER_TURN_IDS: usize = 4_096; type QuestionOptionLabels = BTreeMap>; type QuestionSetMapping = (String, Value, QuestionOptionLabels); @@ -135,11 +141,71 @@ struct CompletedTurnAuthority { provider_turn_id: String, } +#[derive(Default)] +struct SettledProviderTurnIds { + ids: BTreeSet, + filter: DurableReplayFilter, +} + +impl SettledProviderTurnIds { + fn insert(&mut self, provider_turn_id: String) -> bool { + if self.contains(&provider_turn_id) { + return true; + } + if self.ids.len() >= MAX_SETTLED_PROVIDER_TURN_IDS { + return false; + } + self.ids.insert(provider_turn_id) + } + + fn contains(&self, provider_turn_id: &str) -> bool { + self.ids.contains(provider_turn_id) + } + + fn at_capacity(&self) -> bool { + self.ids.len() >= MAX_SETTLED_PROVIDER_TURN_IDS || !self.filter.is_empty() + } + + fn restore(&mut self, provider_turn_id: String) -> Result<(), LocalRunnerError> { + if !self.insert(provider_turn_id) { + return Err(LocalRunnerError::invalid( + "Codex restored provider turn identity epoch exceeded its exact capacity", + )); + } + Ok(()) + } + + fn restore_all( + &mut self, + provider_turn_ids: impl IntoIterator, + replay_filter: DurableReplayFilter, + ) -> Result<(), LocalRunnerError> { + replay_filter + .validate() + .map_err(|error| LocalRunnerError::invalid(error.to_string()))?; + self.filter = replay_filter; + for provider_turn_id in provider_turn_ids { + if !self.insert(provider_turn_id) { + return Err(LocalRunnerError::invalid( + "Codex restored provider turn identity epoch exceeded its exact capacity", + )); + } + } + Ok(()) + } +} + enum ProviderRequestError { Rejected(LocalRunnerError), Ambiguous(LocalRunnerError), } +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum RejectedAcceptedTurn { + ReusedIdentity(String), + InvalidIdentity, +} + impl ProviderRequestError { fn into_inner(self) -> LocalRunnerError { match self { @@ -159,10 +225,12 @@ struct PendingToolRequest { #[derive(Clone, Debug, PartialEq)] struct PendingRuntimeRequest { rpc_id: Value, + turn_id: String, method: String, params: Value, question_set: Value, option_labels: QuestionOptionLabels, + retained_bytes: usize, } struct BufferedProviderMessage { @@ -178,22 +246,32 @@ enum AmbiguousTurnMessage { pub struct CodexProvider { process: SupervisedProcess, + config: CodexProviderConfig, + authorized_tools: Vec, next_request_id: u64, thread_id: String, provider_session_id: Option, active_provider_turn_id: Option, pending_messages: VecDeque, deferred_ambiguous_messages: VecDeque, + pending_message_bytes: usize, authorized_tool_ids: BTreeSet, pending_tool_requests: BTreeMap, completed_tool_call_ids: BTreeSet, + durable_tool_call_replays: bool, pending_tool_request_bytes: usize, pending_runtime_requests: BTreeMap, + pending_runtime_request_bytes: usize, + runtime_request_scope: [u8; 16], + next_runtime_request_sequence: u64, expected_shutdown: bool, process_generation: u64, completed_turn_authority: Option, completion_reconciliation_pending: bool, ambiguous_turn_start_pending: bool, + settled_provider_turn_ids: SettledProviderTurnIds, + rejected_accepted_turn: Option, + quarantined: bool, } impl CodexProvider { @@ -212,20 +290,7 @@ impl CodexProvider { Self::start_with_tools_for_generation(config, authorized_tools, resume_thread_id, 1) } - pub(crate) fn start_for_generation( - config: &CodexProviderConfig, - resume_thread_id: Option<&str>, - process_generation: u64, - ) -> Result { - Self::start_with_tools_for_generation( - config, - std::iter::empty(), - resume_thread_id, - process_generation, - ) - } - - fn start_with_tools_for_generation( + pub(crate) fn start_with_tools_for_generation( config: &CodexProviderConfig, authorized_tools: impl IntoIterator, resume_thread_id: Option<&str>, @@ -237,7 +302,9 @@ impl CodexProvider { "Codex process generation must be positive", )); } - let (dynamic_tools, authorized_tool_ids) = codex_dynamic_tools(authorized_tools)?; + let authorized_tools = authorized_tools.into_iter().collect::>(); + let (dynamic_tools, authorized_tool_ids) = + codex_dynamic_tools(authorized_tools.iter().cloned())?; let mut provider = Self { process: SupervisedProcess::spawn( &config.command, @@ -245,22 +312,32 @@ impl CodexProvider { Duration::from_secs(2), CODEX_APP_SERVER_MAX_FRAME_BYTES, )?, + config: config.clone(), + authorized_tools, next_request_id: 1, thread_id: String::new(), provider_session_id: None, active_provider_turn_id: None, pending_messages: VecDeque::new(), deferred_ambiguous_messages: VecDeque::new(), + pending_message_bytes: 0, authorized_tool_ids, pending_tool_requests: BTreeMap::new(), completed_tool_call_ids: BTreeSet::new(), + durable_tool_call_replays: false, pending_tool_request_bytes: 0, pending_runtime_requests: BTreeMap::new(), + pending_runtime_request_bytes: 0, + runtime_request_scope: new_runtime_request_scope()?, + next_runtime_request_sequence: 1, expected_shutdown: false, process_generation, completed_turn_authority: None, completion_reconciliation_pending: false, ambiguous_turn_start_pending: false, + settled_provider_turn_ids: SettledProviderTurnIds::default(), + rejected_accepted_turn: None, + quarantined: false, }; let initialized = provider.request( "initialize", @@ -320,7 +397,7 @@ impl CodexProvider { if resume_thread_id.is_some() { let snapshot = provider.read_thread()?; provider.active_provider_turn_id = latest_active_turn_id(&snapshot) - .map(|turn_id| bounded_identifier(Some(&turn_id), "Codex turn id")) + .map(|provider_turn_id| bounded_provider_turn_id(Some(&provider_turn_id))) .transpose()?; } Ok(provider) @@ -330,6 +407,10 @@ impl CodexProvider { self.process.id() } + pub(crate) fn process_generation(&self) -> u64 { + self.process_generation + } + pub fn thread_id(&self) -> &str { &self.thread_id } @@ -346,12 +427,19 @@ impl CodexProvider { self.ambiguous_turn_start_pending } + pub(crate) fn enable_durable_tool_call_replays(&mut self) { + // The durable backend validates the call id, operation, and input + // against its persisted receipt before returning a stored result. + // Direct provider consumers retain the stricter one-shot behavior. + self.durable_tool_call_replays = true; + } + pub(crate) fn restore_completed_turn_authority( &mut self, authoritative: bool, process_generation: Option, provider_turn_id: Option<&str>, - ) { + ) -> Result<(), LocalRunnerError> { self.completed_turn_authority = authoritative.then(|| CompletedTurnAuthority { // Legacy state did not record the generation. Generation zero is // deliberately older than every supervised process generation. @@ -360,6 +448,10 @@ impl CodexProvider { .unwrap_or("durable-completed-turn") .to_owned(), }); + if let Some(authority) = self.completed_turn_authority.as_ref() { + self.settled_provider_turn_ids + .restore(authority.provider_turn_id.clone())?; + } // Resuming a completed durable thread and reading its provider state // is recovery, not new turn work. Keep the prior terminal authoritative // until start_turn explicitly revokes it. @@ -369,6 +461,16 @@ impl CodexProvider { // of them supersedes a completed result. Only accepting a replacement // turn identity revokes this authority. self.completion_reconciliation_pending = false; + Ok(()) + } + + pub(crate) fn restore_settled_turn_identities( + &mut self, + provider_turn_ids: impl IntoIterator, + replay_filter: DurableReplayFilter, + ) -> Result<(), LocalRunnerError> { + self.settled_provider_turn_ids + .restore_all(provider_turn_ids, replay_filter) } pub(crate) fn completed_turn_authority(&self) -> Option<(u64, &str)> { @@ -380,7 +482,87 @@ impl CodexProvider { }) } + pub(crate) fn take_rejected_accepted_turn(&mut self) -> Option { + self.rejected_accepted_turn.take() + } + + pub(crate) fn restart_idle_identity_epoch(&mut self) -> Result<(), LocalRunnerError> { + if self.active_provider_turn_id.is_some() || self.ambiguous_turn_start_pending { + return Err(LocalRunnerError::invalid( + "Codex provider identity epoch cannot rotate while work is active", + )); + } + + let next_generation = self.process_generation.checked_add(1).ok_or_else(|| { + LocalRunnerError::invalid("Codex process generation exhausted during epoch rollover") + })?; + let config = self.config.clone(); + let authorized_tools = self.authorized_tools.clone(); + let thread_id = self.thread_id.clone(); + let completed_turn_authority = self.completed_turn_authority.clone(); + let completion_reconciliation_pending = self.completion_reconciliation_pending; + let durable_tool_call_replays = self.durable_tool_call_replays; + + // Exact turn identities may be forgotten only after the provider + // process that could emit them is gone. Resume the same thread in a + // fresh process generation, then preserve prior completion authority + // until a replacement turn identity is actually accepted. + self.shutdown()?; + let mut replacement = Self::start_with_tools_for_generation( + &config, + authorized_tools, + Some(&thread_id), + next_generation, + )?; + replacement.durable_tool_call_replays = durable_tool_call_replays; + if replacement.active_provider_turn_id.is_some() { + // A terminal can race the provider's own durable idle-state write. + // This process has work Paperclip never dispatched in the new + // epoch, so revoke its request authority and reap it. Retain the + // exact ledger for diagnostics, but never expose the unexpected + // turn as ordinary active work or admit a replacement. + replacement.settled_provider_turn_ids = + std::mem::take(&mut self.settled_provider_turn_ids); + replacement.active_provider_turn_id = None; + replacement.ambiguous_turn_start_pending = true; + replacement.rejected_accepted_turn = Some(RejectedAcceptedTurn::InvalidIdentity); + replacement.quarantined = true; + replacement.pending_messages.clear(); + replacement.deferred_ambiguous_messages.clear(); + replacement.pending_message_bytes = 0; + let _ = replacement.cancel_pending_requests(); + let _ = replacement.process.terminate_group(); + replacement.expected_shutdown = false; + *self = replacement; + return Err(LocalRunnerError::invalid( + "Codex provider epoch rollover resumed unowned active work; the provider was terminated", + )); + } + if let Some(authority) = completed_turn_authority.as_ref() { + replacement.restore_completed_turn_authority( + true, + Some(authority.process_generation), + Some(&authority.provider_turn_id), + )?; + } + replacement.completion_reconciliation_pending = completion_reconciliation_pending; + *self = replacement; + Ok(()) + } + + fn rollover_settled_turn_epoch_if_needed(&mut self) -> Result<(), LocalRunnerError> { + if !self.settled_provider_turn_ids.at_capacity() { + return Ok(()); + } + self.restart_idle_identity_epoch() + } + pub fn start_turn(&mut self, message: &str, cwd: &str) -> Result { + if self.quarantined { + return Err(LocalRunnerError::invalid( + "Codex provider is quarantined after unsafe recovered work", + )); + } if self.active_provider_turn_id.is_some() { return Err(LocalRunnerError::invalid( "Codex already has an active provider turn", @@ -396,6 +578,7 @@ impl CodexProvider { "Codex turn text is empty or exceeds the 1 MiB limit", )); } + self.rollover_settled_turn_epoch_if_needed()?; // Preserve the prior durable result until a replacement turn identity // is accepted. A rejected, ambiguous, or transport-failed attempt does // not prove that replacement work superseded the completed turn. @@ -403,6 +586,7 @@ impl CodexProvider { self.completion_reconciliation_pending = false; let prior_buffered_message_count = self.pending_messages.len(); self.ambiguous_turn_start_pending = true; + let runtime_request_scope = new_runtime_request_scope()?; let result = match self.request_classified( "turn/start", json!({ @@ -433,16 +617,32 @@ impl CodexProvider { } Err(ProviderRequestError::Ambiguous(error)) => return Err(error), }; - let provider_turn_id = bounded_identifier( - result - .pointer("/turn/id") - .or_else(|| result.get("turnId")) - .and_then(Value::as_str), - "Codex turn id", - )?; + let provider_turn_id = result + .pointer("/turn/id") + .or_else(|| result.get("turnId")) + .and_then(Value::as_str); + let provider_turn_id = match bounded_provider_turn_id(provider_turn_id) { + Ok(provider_turn_id) => provider_turn_id, + Err(error) => { + // A successful turn/start response means the provider may + // already be executing the work. Without a bounded identity, + // runnerd cannot durably bind, interrupt, or reconcile it. + // Terminate the process and let the durable backend close the + // run before returning the protocol error. + self.rejected_accepted_turn = Some(RejectedAcceptedTurn::InvalidIdentity); + self.expected_shutdown = true; + self.completed_turn_authority = None; + let _ = self.cancel_pending_requests(); + let _ = self.process.terminate_group(); + return Err(error); + } + }; + if self.settled_provider_turn_ids.contains(&provider_turn_id) { + return Err(self.reject_accepted_reused_turn_identity(provider_turn_id)); + } // Only a validated provider turn identity proves that replacement // work exists and supersedes the prior completed result. - self.accept_replacement_turn(provider_turn_id); + self.accept_replacement_turn(provider_turn_id, runtime_request_scope); Ok(result) } @@ -471,17 +671,12 @@ impl CodexProvider { )) })?; - if self - .completed_turn_authority - .as_ref() - .is_some_and(|authority| authority.provider_turn_id == provider_turn_id) - { - return Err(LocalRunnerError::invalid(format!( - "Codex {method} notification reused the previously completed turn id while resolving an ambiguous turn start" - ))); + if self.settled_provider_turn_ids.contains(&provider_turn_id) { + return Err(self.reject_accepted_reused_turn_identity(provider_turn_id)); } - self.accept_replacement_turn(provider_turn_id.clone()); + let runtime_request_scope = new_runtime_request_scope()?; + self.accept_replacement_turn(provider_turn_id.clone(), runtime_request_scope); if method == "turn/started" && message.get("id").is_none() { Ok(AmbiguousTurnMessage::ReconciledWithStart) } else { @@ -489,13 +684,39 @@ impl CodexProvider { } } - fn accept_replacement_turn(&mut self, provider_turn_id: String) { + fn accept_replacement_turn( + &mut self, + provider_turn_id: String, + runtime_request_scope: [u8; 16], + ) { self.ambiguous_turn_start_pending = false; self.expected_shutdown = false; self.completed_turn_authority = None; self.completion_reconciliation_pending = false; self.completed_tool_call_ids.clear(); + // Retain the prior settled identity while the next turn runs. Besides + // recognizing delayed prior-turn requests, this fails closed if a + // provider ambiguously reuses the same turn id for fresh work. self.active_provider_turn_id = Some(provider_turn_id); + self.runtime_request_scope = runtime_request_scope; + } + + fn reject_accepted_reused_turn_identity( + &mut self, + provider_turn_id: String, + ) -> LocalRunnerError { + // Both a successful response and identity-bearing evidence after an + // ambiguous response prove the provider accepted work. A settled + // identity cannot durably own that work, so terminate its process + // before returning instead of leaving an untracked turn alive. + self.rejected_accepted_turn = Some(RejectedAcceptedTurn::ReusedIdentity(provider_turn_id)); + self.expected_shutdown = true; + self.completed_turn_authority = None; + let _ = self.cancel_pending_requests(); + let _ = self.process.terminate_group(); + LocalRunnerError::invalid( + "Codex reused a settled provider turn identity after accepting work; the provider was terminated", + ) } pub fn steer_turn(&mut self, message: &str) -> Result { @@ -554,15 +775,84 @@ impl CodexProvider { .ok_or_else(|| { LocalRunnerError::invalid("runtime response has no pending Codex request") })?; + if self.active_provider_turn_id.as_deref() != Some(pending.turn_id.as_str()) { + return Err(LocalRunnerError::invalid( + "runtime response belongs to another Codex turn", + )); + } let result = codex_question_response(&pending, response)?; self.process .send(&json!({"id": pending.rpc_id, "result": result}))?; - self.pending_runtime_requests.remove(request_id); + if let Some(completed) = self.pending_runtime_requests.remove(request_id) { + self.pending_runtime_request_bytes = self + .pending_runtime_request_bytes + .saturating_sub(completed.retained_bytes); + } Ok(()) } + fn reject_post_terminal_request( + &mut self, + rpc_id: Value, + method: &str, + ) -> Result, LocalRunnerError> { + let message = format!( + "ignored delayed {} request after the Codex turn terminated", + bounded_method(method) + ); + let response = if method == "item/tool/call" { + json!({ + "id": rpc_id, + "result": codex_tool_failure("the Codex turn has already terminated"), + }) + } else { + json!({ + "id": rpc_id, + "error": {"code": -32000, "message": "the Codex turn has already terminated"}, + }) + }; + // The terminal notification is already authoritative and may be + // waiting in the durable outbox. A courtesy rejection must not turn a + // provider that has closed stdin into a fatal polling error. + let _ = self.process.send(&response); + Ok(Some(CodexProviderEvent::Notification { + method: "warning".to_owned(), + params: json!({"message": message, "providerMethod": bounded_method(method)}), + })) + } + pub fn poll(&mut self) -> Result, LocalRunnerError> { + if self.quarantined { + // Never interpret provider-originated requests after fail-closed + // quarantine. Drain output only so process termination cannot + // deadlock on a full pipe, then surface an unequivocal failure. + if self + .process + .receive_stdout_line(Duration::from_millis(1))? + .is_some() + { + return Ok(None); + } + return Ok(self + .process + .try_wait()? + .map(|exit| CodexProviderEvent::Exited { + exit_code: exit.exit_code, + success: false, + completed_turn_authoritative: false, + completed_turn_observed_by_process: false, + completion_reconciles_exit: false, + process_generation: self.process_generation, + completed_turn_process_generation: None, + })); + } let buffered = self.pending_messages.pop_front(); + if let Some(buffered) = buffered.as_ref() { + self.pending_message_bytes = self.pending_message_bytes.saturating_sub(json_size( + &buffered.value, + "buffered Codex provider message", + )?); + } let message = if let Some(buffered) = buffered { buffered.value } else { @@ -625,11 +915,24 @@ impl CodexProvider { match self.classify_ambiguous_turn_message(&message)? { AmbiguousTurnMessage::Ready => {} AmbiguousTurnMessage::Deferred => { - if self.deferred_ambiguous_messages.len() >= MAX_BUFFERED_MESSAGES { + if self + .pending_messages + .len() + .saturating_add(self.deferred_ambiguous_messages.len()) + >= MAX_BUFFERED_MESSAGES + { return Err(LocalRunnerError::invalid( "Codex emitted too many messages before resolving an ambiguous turn start", )); } + let retained_bytes = json_size(&message, "buffered Codex provider message")?; + self.pending_message_bytes = + retain_buffered_message_bytes(self.pending_message_bytes, retained_bytes) + .ok_or_else(|| { + LocalRunnerError::invalid( + "Codex buffered messages exceed the 16 MiB aggregate limit", + ) + })?; self.deferred_ambiguous_messages .push_back(BufferedProviderMessage { value: message }); return Ok(None); @@ -641,6 +944,14 @@ impl CodexProvider { } AmbiguousTurnMessage::ReconciledNeedsStart { provider_turn_id } => { let mut replay = std::mem::take(&mut self.deferred_ambiguous_messages); + let retained_bytes = json_size(&message, "buffered Codex provider message")?; + self.pending_message_bytes = + retain_buffered_message_bytes(self.pending_message_bytes, retained_bytes) + .ok_or_else(|| { + LocalRunnerError::invalid( + "Codex buffered messages exceed the 16 MiB aggregate limit", + ) + })?; replay.push_back(BufferedProviderMessage { value: message }); replay.append(&mut self.pending_messages); self.pending_messages = replay; @@ -666,6 +977,13 @@ impl CodexProvider { "Codex tool call named another thread", )); } + if request_targets_non_active_turn( + self.active_provider_turn_id.as_deref(), + &self.settled_provider_turn_ids, + ¶ms, + ) { + return self.reject_post_terminal_request(rpc_id, method); + } let active_turn_id = self.active_provider_turn_id.as_deref().ok_or_else(|| { LocalRunnerError::invalid("Codex tool call arrived outside an active turn") })?; @@ -719,7 +1037,8 @@ impl CodexProvider { input: input.clone(), retained_bytes, }; - if self.completed_tool_call_ids.contains(&call_id) { + let completed_replay = self.completed_tool_call_ids.contains(&call_id); + if completed_replay && !self.durable_tool_call_replays { return Err(LocalRunnerError::invalid( "Codex reused a completed tool call id", )); @@ -746,7 +1065,9 @@ impl CodexProvider { "Codex emitted too many pending tool calls", )); } - if self.completed_tool_call_ids.len() >= MAX_COMPLETED_TOOL_CALL_IDS { + if !completed_replay + && self.completed_tool_call_ids.len() >= MAX_COMPLETED_TOOL_CALL_IDS + { return Err(LocalRunnerError::invalid( "Codex emitted too many completed tool calls in one turn", )); @@ -770,36 +1091,93 @@ impl CodexProvider { "Codex runtime request named another thread", )); } - let active_turn_id = self.active_provider_turn_id.as_deref().ok_or_else(|| { + if request_targets_non_active_turn( + self.active_provider_turn_id.as_deref(), + &self.settled_provider_turn_ids, + ¶ms, + ) { + return self.reject_post_terminal_request(rpc_id, method); + } + let active_turn_id = self.active_provider_turn_id.clone().ok_or_else(|| { LocalRunnerError::invalid( "Codex runtime request arrived outside an active turn", ) })?; - if params.get("turnId").and_then(Value::as_str) != Some(active_turn_id) { + if params.get("turnId").and_then(Value::as_str) != Some(active_turn_id.as_str()) { return Err(LocalRunnerError::invalid( "Codex runtime request named another turn", )); } - let (request_id, question_set, option_labels) = + let (provider_request_id, question_set, option_labels) = codex_question_set(&rpc_id, ¶ms)?; + let retained_bytes = pending_runtime_request_size( + &rpc_id, + &active_turn_id, + method, + ¶ms, + &question_set, + &option_labels, + )?; let pending = PendingRuntimeRequest { - rpc_id, + rpc_id: rpc_id.clone(), + turn_id: active_turn_id.clone(), method: method.to_owned(), params, question_set: question_set.clone(), option_labels, + retained_bytes, }; - if let Some(existing) = self.pending_runtime_requests.get(&request_id) { + if let Some(existing) = self + .pending_runtime_requests + .values() + .find(|existing| existing.rpc_id == rpc_id) + { if existing != &pending { return Err(LocalRunnerError::invalid( "Codex reused a runtime request id with different input", )); } return Ok(None); - } else { - self.pending_runtime_requests - .insert(request_id.clone(), pending); } + let retained_request_bytes = retain_pending_runtime_request_bytes( + self.pending_runtime_request_bytes, + retained_bytes, + ); + if self.pending_runtime_requests.len() >= MAX_PENDING_RUNTIME_REQUESTS + || retained_request_bytes.is_none() + { + self.process.send(&json!({ + "id": rpc_id, + "error": { + "code": -32000, + "message": "Paperclip rejected this runtime request because the pending input capacity was reached", + }, + }))?; + return Ok(Some(CodexProviderEvent::Notification { + method: "warning".to_owned(), + params: json!({ + "message": "rejected a Codex runtime request at the bounded pending-input limit", + "providerMethod": "item/tool/requestUserInput", + }), + })); + } + let request_sequence = self.next_runtime_request_sequence; + self.next_runtime_request_sequence = self + .next_runtime_request_sequence + .checked_add(1) + .ok_or_else(|| { + LocalRunnerError::invalid("Codex runtime request sequence overflowed") + })?; + let request_id = scoped_runtime_request_id( + &self.runtime_request_scope, + &active_turn_id, + &provider_request_id, + request_sequence, + ); + self.pending_runtime_requests + .insert(request_id.clone(), pending); + self.pending_runtime_request_bytes = + retained_request_bytes.expect("bounded runtime request bytes checked above"); return Ok(Some(CodexProviderEvent::RuntimeRequest { request_id, question_set, @@ -817,24 +1195,60 @@ impl CodexProvider { if let Some(method) = message.get("method").and_then(Value::as_str) { let params = message.get("params").cloned().unwrap_or(Value::Null); + let terminal_event_type = normalized_codex_terminal_event_type(method, ¶ms); + let notification_turn_id = params + .get("turnId") + .or_else(|| params.pointer("/turn/id")) + .and_then(Value::as_str); + if terminal_event_type.is_some() + && notification_turn_id.is_some() + && notification_turn_id != self.active_provider_turn_id.as_deref() + && notification_turn_id + .is_some_and(|turn_id| self.settled_provider_turn_ids.contains(turn_id)) + { + return Ok(Some(CodexProviderEvent::Notification { + method: "warning".to_owned(), + params: json!({ + "message": "ignored a terminal notification for a non-active Codex turn", + "providerMethod": bounded_method(method), + }), + })); + } validate_notification_binding( &self.thread_id, self.active_provider_turn_id.as_deref(), ¶ms, )?; - if method == "turn/completed" { - let provider_turn_id = self.active_provider_turn_id.clone().ok_or_else(|| { - LocalRunnerError::invalid( - "Codex completion arrived outside an active provider turn", - ) - })?; + if let Some(terminal_event_type) = terminal_event_type { + if self.active_provider_turn_id.is_none() { + return Err(LocalRunnerError::invalid( + "Codex terminal arrived outside an active provider turn", + )); + } + let provider_turn_id = self + .active_provider_turn_id + .clone() + .expect("active provider turn checked above"); + let completed_turn_authority = if terminal_event_type == "turn.completed" { + Some(CompletedTurnAuthority { + process_generation: self.process_generation, + provider_turn_id: provider_turn_id.clone(), + }) + } else { + None + }; + if !self + .settled_provider_turn_ids + .insert(provider_turn_id.clone()) + { + return Err(LocalRunnerError::invalid( + "Codex provider turn identity epoch reached its exact capacity", + )); + } self.active_provider_turn_id = None; self.expected_shutdown = true; - self.completed_turn_authority = Some(CompletedTurnAuthority { - process_generation: self.process_generation, - provider_turn_id, - }); - self.completion_reconciliation_pending = true; + self.completed_turn_authority = completed_turn_authority; + self.completion_reconciliation_pending = terminal_event_type == "turn.completed"; // The provider terminal is authoritative once received. Clear // local request ownership and attempt courtesy responses, but // a provider that already closed stdin must not turn the @@ -898,6 +1312,7 @@ impl CodexProvider { fn cancel_pending_requests(&mut self) -> Result<(), LocalRunnerError> { let pending_runtime = std::mem::take(&mut self.pending_runtime_requests); let pending = std::mem::take(&mut self.pending_tool_requests); + self.pending_runtime_request_bytes = 0; self.pending_tool_request_bytes = 0; let mut first_error = None; for request in pending_runtime.into_values() { @@ -967,17 +1382,44 @@ impl CodexProvider { } return Ok(message.get("result").cloned().unwrap_or(Value::Null)); } - if self.pending_messages.len() >= MAX_BUFFERED_MESSAGES { + if self + .pending_messages + .len() + .saturating_add(self.deferred_ambiguous_messages.len()) + >= MAX_BUFFERED_MESSAGES + { return Err(ProviderRequestError::Ambiguous(LocalRunnerError::invalid( "Codex emitted too many messages before a request response", ))); } + let retained_bytes = json_size(&message, "buffered Codex provider message") + .map_err(ProviderRequestError::Ambiguous)?; + let next_retained_bytes = + retain_buffered_message_bytes(self.pending_message_bytes, retained_bytes) + .ok_or_else(|| { + ProviderRequestError::Ambiguous(LocalRunnerError::invalid( + "Codex buffered messages exceed the 16 MiB aggregate limit", + )) + })?; self.pending_messages .push_back(BufferedProviderMessage { value: message }); + self.pending_message_bytes = next_retained_bytes; } } } +fn json_size(value: &Value, label: &str) -> Result { + serde_json::to_vec(value) + .map(|bytes| bytes.len()) + .map_err(|error| LocalRunnerError::invalid(format!("{label} is not serializable: {error}"))) +} + +fn retain_buffered_message_bytes(current: usize, incoming: usize) -> Option { + current + .checked_add(incoming) + .filter(|total| *total <= MAX_BUFFERED_MESSAGE_BYTES) +} + fn pending_tool_request_size( parts: impl IntoIterator, ) -> Result { @@ -1002,6 +1444,29 @@ fn retain_pending_tool_request_bytes( }) } +fn pending_runtime_request_size( + rpc_id: &Value, + turn_id: &str, + method: &str, + params: &Value, + question_set: &Value, + option_labels: &QuestionOptionLabels, +) -> Result { + serde_json::to_vec(&(rpc_id, turn_id, method, params, question_set, option_labels)) + .map(|encoded| encoded.len()) + .map_err(|error| { + LocalRunnerError::invalid(format!( + "Codex pending runtime request is not serializable: {error}" + )) + }) +} + +fn retain_pending_runtime_request_bytes(current: usize, incoming: usize) -> Option { + current + .checked_add(incoming) + .filter(|total| *total <= MAX_PENDING_RUNTIME_REQUEST_BYTES) +} + fn codex_dynamic_tools( authorized_tools: impl IntoIterator, ) -> Result<(Vec, BTreeSet), LocalRunnerError> { @@ -1079,12 +1544,31 @@ fn codex_dynamic_tools( fn bounded_identifier(value: Option<&str>, label: &str) -> Result { let value = value.ok_or_else(|| LocalRunnerError::invalid(format!("{label} is required")))?; - if value.is_empty() || value.len() > 160 || value.chars().any(char::is_control) { + let mut characters = value.chars(); + let valid_first = characters + .next() + .is_some_and(|character| character.is_ascii_alphanumeric()); + let valid_rest = characters.all(|character| { + character.is_ascii_alphanumeric() || matches!(character, '_' | '-' | '.' | ':') + }); + if value.len() > 160 || !valid_first || !valid_rest { return Err(LocalRunnerError::invalid(format!("{label} is invalid"))); } Ok(value.to_owned()) } +fn bounded_provider_turn_id(value: Option<&str>) -> Result { + let value = value + .filter(|value| !value.is_empty()) + .ok_or_else(|| LocalRunnerError::invalid("Codex turn/start omitted turn.id"))?; + if value.len() > 240 || value.chars().any(char::is_control) { + return Err(LocalRunnerError::invalid( + "Codex turn/start returned an invalid turn.id", + )); + } + Ok(value.to_owned()) +} + fn codex_tool_failure(message: &str) -> Value { json!({ "success": false, @@ -1160,6 +1644,17 @@ fn notification_turn_id(params: &Value) -> Option<&str> { .filter(|value| !value.is_empty()) } +fn request_targets_non_active_turn( + active_turn_id: Option<&str>, + settled_turn_ids: &SettledProviderTurnIds, + params: &Value, +) -> bool { + let requested_turn_id = params.get("turnId").and_then(Value::as_str); + requested_turn_id.is_some_and(|requested| { + settled_turn_ids.contains(requested) || active_turn_id != Some(requested) + }) +} + fn latest_active_turn_id(snapshot: &Value) -> Option { snapshot .pointer("/thread/turns") @@ -1296,6 +1791,33 @@ fn codex_question_set( )) } +fn new_runtime_request_scope() -> Result<[u8; 16], LocalRunnerError> { + let mut scope = [0u8; 16]; + getrandom::fill(&mut scope).map_err(|error| { + LocalRunnerError::invalid(format!( + "failed to mint Codex runtime request scope: {error}" + )) + })?; + Ok(scope) +} + +fn scoped_runtime_request_id( + scope: &[u8; 16], + turn_id: &str, + provider_request_id: &str, + request_sequence: u64, +) -> String { + let mut digest = Sha256::new(); + digest.update(scope); + digest.update([0]); + digest.update(turn_id.as_bytes()); + digest.update([0]); + digest.update(provider_request_id.as_bytes()); + digest.update([0]); + digest.update(request_sequence.to_be_bytes()); + format!("runtime-request-{:x}", digest.finalize()) +} + fn codex_question_response( pending: &PendingRuntimeRequest, response: &Value, @@ -1436,10 +1958,12 @@ mod tests { assert_eq!(question_set["schema"], "paperclip.question_set.v1"); let pending = PendingRuntimeRequest { rpc_id: json!(41), + turn_id: "turn-1".to_owned(), method: "item/tool/requestUserInput".to_owned(), params: Value::Null, question_set, option_labels: labels, + retained_bytes: 0, }; let native = codex_question_response( &pending, @@ -1461,6 +1985,23 @@ mod tests { }), ) .is_err()); + let scope = [7u8; 16]; + assert_eq!( + scoped_runtime_request_id(&scope, "turn-1", "41", 1), + scoped_runtime_request_id(&scope, "turn-1", "41", 1), + ); + assert_ne!( + scoped_runtime_request_id(&scope, "turn-1", "41", 1), + scoped_runtime_request_id(&scope, "turn-2", "41", 1), + ); + assert_ne!( + scoped_runtime_request_id(&scope, "turn-1", "41", 1), + scoped_runtime_request_id(&[8u8; 16], "turn-1", "41", 1), + ); + assert_ne!( + scoped_runtime_request_id(&scope, "turn-1", "41", 1), + scoped_runtime_request_id(&scope, "turn-1", "41", 2), + ); assert!(codex_question_response( &pending, &json!({ @@ -1503,6 +2044,87 @@ mod tests { .is_ok()); } + #[test] + fn rejects_requests_bound_to_any_non_active_turn_nonfatally() { + let mut turn_one_settled = SettledProviderTurnIds::default(); + turn_one_settled.insert("turn-1".to_owned()); + let mut turn_two_settled = SettledProviderTurnIds::default(); + turn_two_settled.insert("turn-2".to_owned()); + let no_settled_turns = SettledProviderTurnIds::default(); + assert!(request_targets_non_active_turn( + Some("turn-2"), + &turn_one_settled, + &json!({"turnId": "turn-1"}), + )); + assert!(request_targets_non_active_turn( + Some("turn-2"), + &turn_one_settled, + &json!({"turnId": "turn-0"}), + )); + assert!(request_targets_non_active_turn( + None, + &turn_two_settled, + &json!({"turnId": "turn-1"}), + )); + assert!(request_targets_non_active_turn( + Some("turn-1"), + &turn_one_settled, + &json!({"turnId": "turn-1"}), + )); + assert!(!request_targets_non_active_turn( + Some("turn-2"), + &turn_one_settled, + &json!({"turnId": "turn-2"}), + )); + assert!(!request_targets_non_active_turn( + None, + &no_settled_turns, + &json!({}), + )); + } + + #[test] + fn settled_provider_turn_history_never_evicts_exact_identities() { + let mut settled = SettledProviderTurnIds::default(); + for index in 0..MAX_SETTLED_PROVIDER_TURN_IDS { + assert!(settled.insert(format!("turn-{index}"))); + } + + assert_eq!(settled.ids.len(), MAX_SETTLED_PROVIDER_TURN_IDS); + assert!(settled.contains("turn-0")); + assert!(settled.contains("turn-1")); + assert!(settled.at_capacity()); + assert!(!settled.insert(format!("turn-{MAX_SETTLED_PROVIDER_TURN_IDS}"))); + assert!(!settled.contains(&format!("turn-{MAX_SETTLED_PROVIDER_TURN_IDS}"))); + assert_eq!(settled.ids.len(), MAX_SETTLED_PROVIDER_TURN_IDS); + assert!(settled.contains("turn-0")); + assert!(settled.filter.is_empty()); + } + + #[test] + fn restored_provider_turn_identity_is_retained_in_release_builds() { + let mut settled = SettledProviderTurnIds::default(); + + settled.restore("turn-restored".to_owned()).unwrap(); + + assert!(settled.contains("turn-restored")); + } + + #[test] + fn restores_the_complete_durable_provider_turn_ledger() { + let mut settled = SettledProviderTurnIds::default(); + + settled + .restore_all( + ["turn-older".to_owned(), "turn-latest".to_owned()], + DurableReplayFilter::default(), + ) + .unwrap(); + + assert!(settled.contains("turn-older")); + assert!(settled.contains("turn-latest")); + } + #[test] fn bounds_all_retained_pending_tool_request_data_in_aggregate() { let request_bytes = pending_tool_request_size([1, 2, 3, 4]).unwrap(); @@ -1516,4 +2138,30 @@ mod tests { assert!(retain_pending_tool_request_bytes(usize::MAX, 1).is_err()); assert!(pending_tool_request_size([usize::MAX, 1]).is_err()); } + + #[test] + fn bounds_all_retained_runtime_request_data_in_aggregate() { + assert_eq!( + retain_pending_runtime_request_bytes(MAX_PENDING_RUNTIME_REQUEST_BYTES - 10, 10,), + Some(MAX_PENDING_RUNTIME_REQUEST_BYTES) + ); + assert_eq!( + retain_pending_runtime_request_bytes(MAX_PENDING_RUNTIME_REQUEST_BYTES, 1), + None + ); + assert_eq!(retain_pending_runtime_request_bytes(usize::MAX, 1), None); + } + + #[test] + fn bounds_messages_buffered_while_waiting_for_a_response() { + assert_eq!( + retain_buffered_message_bytes(MAX_BUFFERED_MESSAGE_BYTES - 10, 10), + Some(MAX_BUFFERED_MESSAGE_BYTES) + ); + assert_eq!( + retain_buffered_message_bytes(MAX_BUFFERED_MESSAGE_BYTES, 1), + None + ); + assert_eq!(retain_buffered_message_bytes(usize::MAX, 1), None); + } } diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs index ef3ec61456..3bbf2f9311 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs @@ -9,12 +9,14 @@ use std::time::Duration; pub use runner::{run_durable_runner, CommandExecution, CommandExecutor, PolledEvent}; pub(crate) use state::{ - create_private_temporary_file, open_private_regular_file, redact_text, verify_private_directory, + create_private_temporary_file, open_private_regular_file, redact_text, sanitize_value, + verify_private_directory, }; pub use state::{ Command, CommandDisposition, DurableState, DurableStateStore, EventPriority, StoredCommandResult, StoredOutboxEvent, }; +pub(crate) use transport::current_unix_ms; pub const PROTOCOL: &str = "paperclip.runner"; pub const PROTOCOL_VERSION: u64 = 1; diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/state.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/state.rs index 50c6df265c..0e27f28f3c 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/state.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/state.rs @@ -1078,7 +1078,23 @@ fn sensitive_key(key: &str) -> bool { .any(|needle| normalized.contains(needle)) } -fn sanitize_value(value: &Value) -> Value { +fn protocol_authorization_boundary(key: &str, value: &Value) -> bool { + key.eq_ignore_ascii_case("authorizationBoundary") + && value.as_str().is_some_and(|boundary| { + matches!( + boundary, + "company" + | "actor" + | "active_task" + | "grant" + | "governed_action" + | "lock" + | "revision" + ) + }) +} + +pub(crate) fn sanitize_value(value: &Value) -> Value { match value { Value::Object(object) => Value::Object( object @@ -1086,7 +1102,9 @@ fn sanitize_value(value: &Value) -> Value { .map(|(key, value)| { ( key.clone(), - if sensitive_key(key) { + if protocol_authorization_boundary(key, value) { + value.clone() + } else if sensitive_key(key) { Value::String("[REDACTED]".to_owned()) } else { sanitize_value(value) @@ -1323,6 +1341,21 @@ mod tests { ); } + #[test] + fn protocol_authorization_boundary_is_not_redacted_as_a_credential() { + let sanitized = sanitize_value(&json!({ + "authorizationBoundary": "active_task", + "nested": {"authorizationBoundary": "Bearer secret-value"}, + "authorization": "Bearer secret-value", + })); + assert_eq!(sanitized["authorizationBoundary"], json!("active_task")); + assert_eq!(sanitized["authorization"], json!("[REDACTED]")); + assert_eq!( + sanitized["nested"]["authorizationBoundary"], + json!("[REDACTED]") + ); + } + #[test] fn outbox_reserves_capacity_for_p0_and_bounds_frames() { let mut bounds_config = config(PathBuf::from("unused")); diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs index 860362da03..5c037db769 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs @@ -11,17 +11,69 @@ use std::os::unix::fs::PermissionsExt; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; -use crate::codex_provider::{CodexProvider, CodexProviderConfig, CodexProviderEvent}; -use crate::durable::{ - create_private_temporary_file, open_private_regular_file, verify_private_directory, Command, - CommandExecution, CommandExecutor, DurableRunnerError, EventPriority, PolledEvent, +use crate::codex_provider::{ + CodexProvider, CodexProviderConfig, CodexProviderEvent, RejectedAcceptedTurn, + MAX_SETTLED_PROVIDER_TURN_IDS, +}; +use crate::durable::{ + create_private_temporary_file, current_unix_ms, open_private_regular_file, sanitize_value, + verify_private_directory, Command, CommandExecution, CommandExecutor, DurableRunnerConfig, + DurableRunnerError, EventPriority, PolledEvent, +}; +use crate::provider_bridge::{ + authorized_tool_catalog_digest, semantic_value_digest, AuthorizedToolSet, DurableReplayFilter, + PendingToolCall, ProviderBridgeError, ProviderToolBridge, ToolResult, MAX_PENDING_CALLS, + TOOL_SET_SCHEMA, +}; +use crate::provider_events::{ + normalize_codex_notification, normalized_codex_terminal_event_type, NormalizedProviderEvent, }; -use crate::provider_events::{normalize_codex_notification, NormalizedProviderEvent}; const PROVIDER_STATE_SCHEMA: &str = "paperclip.runner.codex-provider-state.v1"; const PROVIDER_STATE_FILE: &str = "codex-provider-state.json"; -const MAX_PROVIDER_STATE_BYTES: u64 = 2 * 1024 * 1024; +const MAX_PROVIDER_STATE_BYTES: u64 = 16 * 1024 * 1024; const MAX_EVENTS_PER_POLL: usize = 128; +// One accepted semantic call can produce an input and a result event. Normal +// traffic cannot consume the additional capacity required to diagnose a +// receipt-limit stop, settle every retained call, and record the provider plus +// run terminal events. +const MAX_REGULAR_QUEUED_PROVIDER_EVENTS: usize = 2 * MAX_PENDING_CALLS + 3; +const MAX_RECEIPT_LIMIT_TERMINAL_RESERVE: usize = MAX_PENDING_CALLS + 4; +// During a receipt-limit stop, provider polling continues even when older +// events remain unacknowledged so an already-buffered authoritative terminal +// wins over the deadline fallback. Reserve one complete poll of cleanup events +// in addition to the semantic-result and terminal envelopes. +const MAX_TERMINAL_SETTLEMENT_EVENTS: usize = MAX_PENDING_CALLS + MAX_EVENTS_PER_POLL + 4; +const MAX_QUEUED_PROVIDER_EVENTS: usize = + MAX_REGULAR_QUEUED_PROVIDER_EVENTS + MAX_TERMINAL_SETTLEMENT_EVENTS; +const MAX_RECEIPT_LIMIT_INTERRUPT_ATTEMPTS: u8 = 3; +const RECEIPT_LIMIT_INTERRUPT_TERMINAL_DEADLINE_MS: u64 = 2_000; +const RECEIPT_LIMIT_ACCEPTED_TERMINAL_DEADLINE_MS: u64 = 30_000; + +fn receipt_limit_deadline_after(timeout_ms: u64) -> Result { + current_unix_ms()?.checked_add(timeout_ms).ok_or_else(|| { + DurableRunnerError::invalid("Codex receipt-limit interruption deadline overflowed") + }) +} + +#[derive(Clone, Debug)] +struct ProviderEventIdentity { + run_id: String, + normalized_session_id: String, + turn_id: String, + item_id: String, +} + +impl ProviderEventIdentity { + fn from_config(config: &DurableRunnerConfig) -> Self { + Self { + run_id: config.run_id.clone(), + normalized_session_id: config.normalized_session_id.clone(), + turn_id: config.turn_id.clone(), + item_id: config.item_id.clone(), + } + } +} #[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] #[serde(rename_all = "camelCase")] @@ -70,6 +122,106 @@ fn completion_contract( Ok(Some(binding)) } +fn authorized_tool_set(payload: &Value) -> Result { + if let Some(value) = payload.get("authorizedTools") { + return serde_json::from_value(value.clone()).map_err(|error| { + DurableRunnerError::invalid(format!("run.prepare authorizedTools is invalid: {error}")) + }); + } + let operations = Vec::new(); + let catalog_digest = authorized_tool_catalog_digest(&operations).map_err(|error| { + DurableRunnerError::invalid(format!("empty authorized tool set is invalid: {error}")) + })?; + Ok(AuthorizedToolSet { + schema: TOOL_SET_SCHEMA.to_owned(), + schema_version: 1, + catalog_digest, + operations, + }) +} + +fn semantic_correlation(identity: &ProviderEventIdentity) -> Value { + json!({ + "runId": identity.run_id, + "normalizedSessionId": identity.normalized_session_id, + "turnId": identity.turn_id, + "itemId": identity.item_id, + }) +} + +fn semantic_input_event( + identity: &ProviderEventIdentity, + call: &PendingToolCall, +) -> NormalizedProviderEvent { + let safe_input = sanitize_value(&call.input); + NormalizedProviderEvent { + event_type: "semantic_tool.input".to_owned(), + priority: EventPriority::P0, + payload: json!({ + "semantic_tool": { + "schema": "paperclip.prp.semantic_tool.v1", + "schemaVersion": 1, + "phase": "input", + "operationId": call.operation_id, + "callId": call.call_id, + "correlation": semantic_correlation(identity), + "idempotencyKey": Value::Null, + "content": { + "digest": semantic_value_digest(&safe_input), + "redactionDisposition": "digest_only", + "references": [], + }, + "input": safe_input, + }, + }), + } +} + +fn semantic_result_event( + identity: &ProviderEventIdentity, + result: &ToolResult, +) -> NormalizedProviderEvent { + let safe_result = sanitize_value(&result.result); + let envelope = safe_result + .get("resultReceipt") + .filter(|receipt| { + receipt.get("schema").and_then(Value::as_str) + == Some("paperclip.prp.semantic_tool.v1") + && receipt.get("phase").and_then(Value::as_str) == Some("result") + && receipt.get("operationId").and_then(Value::as_str) + == Some(result.operation_id.as_str()) + && receipt.get("callId").and_then(Value::as_str) == Some(result.call_id.as_str()) + && receipt.get("correlation") == Some(&semantic_correlation(identity)) + }) + .cloned() + .unwrap_or_else(|| { + json!({ + "schema": "paperclip.prp.semantic_tool.v1", + "schemaVersion": 1, + "phase": "result", + "operationId": result.operation_id, + "callId": result.call_id, + "correlation": semantic_correlation(identity), + "idempotencyKey": Value::Null, + "content": { + "digest": semantic_value_digest(&safe_result), + "redactionDisposition": "digest_only", + "references": [], + }, + "outcome": if result.is_error { "failed" } else { "succeeded" }, + "code": if result.is_error { "semantic_tool_failed" } else { "semantic_tool_succeeded" }, + "retryable": false, + "authorizationBoundary": "active_task", + "operationReceiptId": format!("operation_{}", result.call_id), + }) + }); + NormalizedProviderEvent { + event_type: "semantic_tool.result".to_owned(), + priority: EventPriority::P0, + payload: json!({"semantic_tool": envelope}), + } +} + fn terminal_events(state: &CodexProviderState, event_type: &str) -> Vec { let Some(contract) = state.completion_contract.as_ref() else { return Vec::new(); @@ -158,6 +310,8 @@ struct CodexProviderState { #[serde(default)] completion_contract: Option, #[serde(default)] + tool_bridge: ProviderToolBridge, + #[serde(default)] thread_id: Option, #[serde(default)] provider_session_id: Option, @@ -173,17 +327,69 @@ struct CodexProviderState { completed_turn_process_generation: Option, #[serde(default)] completed_provider_turn_id: Option, + // Unlike the live provider process, the durable run survives restarts. + // Terminal identities stay exact for one process-generation epoch. At the + // bound, an idle process is reaped before this epoch is rotated. + #[serde(default)] + settled_provider_turn_ids: std::collections::BTreeSet, + #[serde(default)] + settled_provider_turn_filter: DurableReplayFilter, + #[serde(default)] + receipt_limit_diagnostic_emitted: bool, + #[serde(default)] + receipt_limit_interrupt_pending: bool, + #[serde(default)] + receipt_limit_interrupt_accepted: bool, + #[serde(default)] + receipt_limit_interrupt_attempts: u8, + #[serde(default)] + receipt_limit_interrupt_deadline_unix_ms: Option, last_agent_message: Option, #[serde(default)] pending_events: VecDeque, + #[serde(default)] + queued_events: VecDeque, #[serde(default = "initial_provider_event_seq")] next_provider_event_seq: u64, } +#[derive(Debug, PartialEq)] +enum ToolCallAdmission { + CompletedReplay(ToolResult), + PendingReplay, + Pending(PendingToolCall), +} + +fn settled_provider_turn_contains( + identities: &std::collections::BTreeSet, + _filter: &DurableReplayFilter, + provider_turn_id: &str, +) -> bool { + identities.contains(provider_turn_id) +} + +fn remember_settled_provider_turn( + identities: &mut std::collections::BTreeSet, + _filter: &mut DurableReplayFilter, + provider_turn_id: String, +) -> Result<(), DurableRunnerError> { + if identities.contains(&provider_turn_id) { + return Ok(()); + } + if identities.len() >= MAX_SETTLED_PROVIDER_TURN_IDS { + return Err(DurableRunnerError::invalid( + "Codex provider turn identity epoch reached its exact capacity", + )); + } + identities.insert(provider_turn_id); + Ok(()) +} + impl CodexProviderState { fn new( config: CodexProviderConfig, completion_contract: Option, + tool_bridge: ProviderToolBridge, ) -> Self { let thread_id = config.provider_session_id.clone(); Self { @@ -191,6 +397,7 @@ impl CodexProviderState { lifecycle: "prepared".to_owned(), config, completion_contract, + tool_bridge, thread_id, provider_session_id: None, active_provider_turn_id: None, @@ -199,8 +406,16 @@ impl CodexProviderState { provider_process_generation: 0, completed_turn_process_generation: None, completed_provider_turn_id: None, + settled_provider_turn_ids: std::collections::BTreeSet::new(), + settled_provider_turn_filter: DurableReplayFilter::default(), + receipt_limit_diagnostic_emitted: false, + receipt_limit_interrupt_pending: false, + receipt_limit_interrupt_accepted: false, + receipt_limit_interrupt_attempts: 0, + receipt_limit_interrupt_deadline_unix_ms: None, last_agent_message: None, pending_events: VecDeque::new(), + queued_events: VecDeque::new(), next_provider_event_seq: initial_provider_event_seq(), } } @@ -209,6 +424,9 @@ impl CodexProviderState { self.config .validate() .map_err(|error| DurableRunnerError::invalid(error.to_string()))?; + self.tool_bridge.validate_recovered().map_err(|error| { + DurableRunnerError::invalid(format!("Codex semantic tool state is invalid: {error}")) + })?; let mut pending_event_ids = HashSet::new(); if self.schema != PROVIDER_STATE_SCHEMA || !matches!( @@ -262,24 +480,59 @@ impl CodexProviderState { || (!self.completed_turn_authoritative && (self.completed_turn_process_generation.is_some() || self.completed_provider_turn_id.is_some())) + || self.settled_provider_turn_ids.len() > MAX_SETTLED_PROVIDER_TURN_IDS + || self.settled_provider_turn_filter.validate().is_err() + || self + .settled_provider_turn_ids + .iter() + .any(|provider_turn_id| { + provider_turn_id.is_empty() + || provider_turn_id.len() > 240 + || provider_turn_id.chars().any(char::is_control) + }) + || self + .active_provider_turn_id + .as_ref() + .is_some_and(|provider_turn_id| { + settled_provider_turn_contains( + &self.settled_provider_turn_ids, + &self.settled_provider_turn_filter, + provider_turn_id, + ) + }) || self .completed_turn_process_generation .is_some_and(|generation| generation > self.provider_process_generation) + || (self.receipt_limit_diagnostic_emitted && self.active_provider_turn_id.is_none()) + || (self.receipt_limit_interrupt_pending + && (!self.receipt_limit_diagnostic_emitted + || self.active_provider_turn_id.is_none())) + || (self.receipt_limit_interrupt_accepted && !self.receipt_limit_interrupt_pending) + || self.receipt_limit_interrupt_attempts > MAX_RECEIPT_LIMIT_INTERRUPT_ATTEMPTS + || (!self.receipt_limit_interrupt_pending && self.receipt_limit_interrupt_attempts != 0) + || self + .receipt_limit_interrupt_deadline_unix_ms + .is_some_and(|deadline| deadline == 0 || !self.receipt_limit_interrupt_pending) || (matches!( self.lifecycle.as_str(), "prepared" | "session_open" | "closed" ) && self.active_provider_turn_id.is_some()) || self.next_provider_event_seq == 0 || self.pending_events.len() > MAX_EVENTS_PER_POLL + 3 - || self.pending_events.iter().any(|event| { - provider_event_sequence(&event.executor_event_id) - .is_none_or(|sequence| sequence >= self.next_provider_event_seq) - || !pending_event_ids.insert(event.executor_event_id.as_str()) - || event.event_type.is_empty() - || event.event_type.len() > 160 - || event.event_type.chars().any(char::is_control) - || !event.payload.is_object() - }) + || self.queued_events.len() > MAX_QUEUED_PROVIDER_EVENTS + || self + .pending_events + .iter() + .chain(self.queued_events.iter()) + .any(|event| { + provider_event_sequence(&event.executor_event_id) + .is_none_or(|sequence| sequence >= self.next_provider_event_seq) + || !pending_event_ids.insert(event.executor_event_id.as_str()) + || event.event_type.is_empty() + || event.event_type.len() > 160 + || event.event_type.chars().any(char::is_control) + || !event.payload.is_object() + }) { return Err(DurableRunnerError::invalid( "Codex provider state is malformed or inconsistent", @@ -288,20 +541,131 @@ impl CodexProviderState { Ok(()) } - fn push_event(&mut self, event: NormalizedProviderEvent) -> Result<(), DurableRunnerError> { + fn push_event_with_limit( + &mut self, + event: NormalizedProviderEvent, + max_queued_events: usize, + ) -> Result<(), DurableRunnerError> { + let queue_event = + !self.queued_events.is_empty() || self.pending_events.len() >= MAX_EVENTS_PER_POLL; + if queue_event && self.queued_events.len() >= max_queued_events { + return Err(DurableRunnerError::invalid( + "Codex provider event backlog exceeds its durable limit", + )); + } let sequence = self.next_provider_event_seq; self.next_provider_event_seq = sequence .checked_add(1) .ok_or_else(|| DurableRunnerError::invalid("provider event sequence exhausted"))?; - self.pending_events.push_back(PolledEvent { + let event = PolledEvent { executor_event_id: provider_event_id(sequence), event_type: event.event_type, priority: event.priority, payload: event.payload, - }); + }; + if queue_event { + self.queued_events.push_back(event); + } else { + self.pending_events.push_back(event); + } Ok(()) } + fn begin_receipt_limit_stop( + &mut self, + call_id: String, + operation_id: String, + deadline_unix_ms: u64, + ) -> Result { + if self.receipt_limit_diagnostic_emitted { + return Ok(self.receipt_limit_interrupt_pending); + } + self.push_terminal_event(NormalizedProviderEvent { + event_type: "harness.diagnostic".to_owned(), + priority: EventPriority::P0, + payload: json!({ + "provider": "codex", + "code": "semantic_tool_turn_receipt_limit", + "operationId": operation_id, + "callId": call_id, + "message": "The active provider turn reached its durable semantic-tool receipt limit and was interrupted", + "paperclipExecuted": false, + }), + })?; + self.receipt_limit_diagnostic_emitted = true; + self.receipt_limit_interrupt_pending = true; + self.receipt_limit_interrupt_attempts = 0; + self.receipt_limit_interrupt_deadline_unix_ms = Some(deadline_unix_ms); + Ok(true) + } + + fn record_receipt_limit_interrupt_attempt(&mut self) -> Result<(), DurableRunnerError> { + if self.receipt_limit_interrupt_attempts >= MAX_RECEIPT_LIMIT_INTERRUPT_ATTEMPTS { + return Err(DurableRunnerError::invalid( + "Codex receipt-limit interruption exceeded its durable retry bound", + )); + } + self.receipt_limit_interrupt_attempts += 1; + Ok(()) + } + + fn mark_receipt_limit_interrupt_accepted(&mut self, accepted_deadline_unix_ms: u64) { + // Provider restoration can reconcile the active turn as already + // settled while an interruption command is in flight. In that case + // `interrupt_turn` returns `already_settled` and recovery has already + // cleared the durable retry marker. Do not recreate an accepted state + // without a pending interruption or active turn. + if !self.receipt_limit_interrupt_pending || self.active_provider_turn_id.is_none() { + return; + } + if !self.receipt_limit_interrupt_accepted { + self.receipt_limit_interrupt_deadline_unix_ms = Some( + self.receipt_limit_interrupt_deadline_unix_ms + .unwrap_or_default() + .max(accepted_deadline_unix_ms), + ); + } + self.receipt_limit_interrupt_accepted = true; + } + + fn push_event(&mut self, event: NormalizedProviderEvent) -> Result<(), DurableRunnerError> { + self.push_event_with_limit(event, MAX_REGULAR_QUEUED_PROVIDER_EVENTS) + } + + fn push_terminal_event( + &mut self, + event: NormalizedProviderEvent, + ) -> Result<(), DurableRunnerError> { + self.push_event_with_limit(event, MAX_QUEUED_PROVIDER_EVENTS) + } + + fn push_receipt_limit_cleanup_event( + &mut self, + event: NormalizedProviderEvent, + ) -> Result<(), DurableRunnerError> { + let queue_event = + !self.queued_events.is_empty() || self.pending_events.len() >= MAX_EVENTS_PER_POLL; + if queue_event + && self.queued_events.len() + >= MAX_QUEUED_PROVIDER_EVENTS - MAX_RECEIPT_LIMIT_TERMINAL_RESERVE + { + // Continue draining the provider so an authoritative terminal can + // still be observed, but never let cleanup chatter consume the + // semantic-result and terminal-event reserve. + return Ok(()); + } + self.push_terminal_event(event) + } + + fn refill_pending_events(&mut self) { + while self.pending_events.len() < MAX_EVENTS_PER_POLL { + let Some(event) = self.queued_events.pop_front() else { + break; + }; + self.pending_events.push_back(event); + } + } + fn extend_events( &mut self, events: impl IntoIterator, @@ -312,6 +676,43 @@ impl CodexProviderState { Ok(()) } + fn admit_tool_call( + &mut self, + call_id: &str, + operation_id: &str, + input: &Value, + ) -> Result { + if let Some(result) = self + .tool_bridge + .replay_result(call_id, operation_id, input)? + { + // An exact completed replay is a transport retry. Its input and + // result receipts are already durable, so recording another event + // would make an otherwise idempotent replay consume bounded event + // capacity and could prevent returning the stored result. + return Ok(ToolCallAdmission::CompletedReplay(result)); + } + + let pending_replay = self + .tool_bridge + .pending_calls() + .any(|pending| pending.call_id == call_id); + let call = self.tool_bridge.begin_call( + call_id.to_owned(), + operation_id.to_owned(), + input.clone(), + )?; + if pending_replay { + // The first input receipt is already durable. An exact pending + // replay is only the provider re-establishing its request after a + // process restart; appending another event would make the retry + // consume bounded backlog capacity without adding information. + Ok(ToolCallAdmission::PendingReplay) + } else { + Ok(ToolCallAdmission::Pending(call)) + } + } + fn reconcile_active_provider_turn(&mut self, active_provider_turn_id: Option) { self.active_provider_turn_id = active_provider_turn_id; if self.active_provider_turn_id.is_some() { @@ -330,12 +731,53 @@ impl CodexProviderState { "session_open".to_owned() }; } + + fn settle_active_provider_turn_identity(&mut self) -> Result<(), DurableRunnerError> { + let provider_turn_id = self.active_provider_turn_id.clone().ok_or_else(|| { + DurableRunnerError::invalid("Codex terminal omitted its active provider turn identity") + })?; + remember_settled_provider_turn( + &mut self.settled_provider_turn_ids, + &mut self.settled_provider_turn_filter, + provider_turn_id, + )?; + Ok(()) + } + + fn recovered_settled_provider_turn_ids( + &self, + ) -> Result<(std::collections::BTreeSet, DurableReplayFilter), DurableRunnerError> { + let mut settled_provider_turn_ids = self.settled_provider_turn_ids.clone(); + let mut settled_provider_turn_filter = self.settled_provider_turn_filter.clone(); + // State written before the durable set was introduced retained only + // the latest completed identity. Fold that legacy authority into the + // new ledger before the provider is allowed to accept replacement work. + if let Some(provider_turn_id) = self.completed_provider_turn_id.clone() { + remember_settled_provider_turn( + &mut settled_provider_turn_ids, + &mut settled_provider_turn_filter, + provider_turn_id, + )?; + } + Ok((settled_provider_turn_ids, settled_provider_turn_filter)) + } + + fn extend_terminal_events( + &mut self, + events: impl IntoIterator, + ) -> Result<(), DurableRunnerError> { + for event in events { + self.push_terminal_event(event)?; + } + Ok(()) + } } pub struct CodexCommandExecutor { state_dir: PathBuf, state: Option, provider: Option, + event_identity: Option, restore_checked: bool, } @@ -345,10 +787,17 @@ impl CodexCommandExecutor { state_dir: state_dir.into(), state: None, provider: None, + event_identity: None, restore_checked: false, } } + pub fn with_runner_config(state_dir: impl Into, config: &DurableRunnerConfig) -> Self { + let mut executor = Self::new(state_dir); + executor.event_identity = Some(ProviderEventIdentity::from_config(config)); + executor + } + fn state_path(&self) -> PathBuf { self.state_dir.join(PROVIDER_STATE_FILE) } @@ -380,9 +829,14 @@ impl CodexCommandExecutor { file.read_to_end(&mut input).map_err(|error| { DurableRunnerError::invalid(format!("failed to read Codex provider state: {error}")) })?; - let state: CodexProviderState = serde_json::from_slice(&input).map_err(|error| { + let mut state: CodexProviderState = serde_json::from_slice(&input).map_err(|error| { DurableRunnerError::invalid(format!("Codex provider state is malformed: {error}")) })?; + state.tool_bridge.attach_existing_run().map_err(|error| { + DurableRunnerError::invalid(format!( + "Codex semantic tool state could not be reattached: {error}" + )) + })?; state.validate()?; self.state = Some(state); self.restore_provider_if_needed() @@ -413,15 +867,139 @@ impl CodexCommandExecutor { let completed_turn_process_generation = state.completed_turn_process_generation; let completed_provider_turn_id = state.completed_provider_turn_id.clone(); let ambiguous_turn_start_pending = state.ambiguous_turn_start_pending; - let mut provider = CodexProvider::start_for_generation( + let tool_replay_history_blocks_admission = + state.tool_bridge.replay_history_blocks_admission(); + let tool_receipt_epoch_has_active_receipts = state.tool_bridge.has_active_receipts(); + let (settled_provider_turn_ids, settled_provider_turn_filter) = + state.recovered_settled_provider_turn_ids()?; + let provider_epoch_requires_rollover = settled_provider_turn_ids.len() + >= MAX_SETTLED_PROVIDER_TURN_IDS + || !settled_provider_turn_filter.is_empty(); + let mut provider = CodexProvider::start_with_tools_for_generation( &state.config, + state.tool_bridge.authorized_tools().cloned(), Some(&thread_id), process_generation, ) .map_err(|error| { DurableRunnerError::invalid(format!("failed to resume Codex provider: {error}")) })?; + provider.enable_durable_tool_call_replays(); + provider + .restore_settled_turn_identities( + settled_provider_turn_ids.iter().cloned(), + settled_provider_turn_filter.clone(), + ) + .map_err(|error| { + DurableRunnerError::invalid(format!( + "failed to restore Codex provider turn identities: {error}" + )) + })?; let recovered_active_turn_id = provider.active_provider_turn_id().map(str::to_owned); + let legacy_epoch_is_ambiguous = (provider_epoch_requires_rollover + && (ambiguous_turn_start_pending || recovered_active_turn_id.is_some())) + || (tool_replay_history_blocks_admission + && (ambiguous_turn_start_pending + || recovered_active_turn_id.is_some() + || (previous_active_turn_id.is_some() + && tool_receipt_epoch_has_active_receipts))); + if legacy_epoch_is_ambiguous { + // A saturated legacy epoch cannot prove that recovered work can + // be identified and settled exactly. Reap the resumed process + // generation and close the run instead of risking duplicate work. + let provider_reported_active = recovered_active_turn_id.is_some(); + let provider_shutdown_failed = provider.shutdown().is_err(); + drop(provider); + let state = self + .state + .as_mut() + .expect("Codex state remains available during legacy recovery"); + state.provider_process_generation = process_generation; + state.settled_provider_turn_ids = settled_provider_turn_ids; + state.settled_provider_turn_filter = settled_provider_turn_filter; + state.active_provider_turn_id = None; + state.ambiguous_turn_start_pending = false; + state.completed_turn_authoritative = false; + state.completed_turn_process_generation = None; + state.completed_provider_turn_id = None; + state.receipt_limit_diagnostic_emitted = false; + state.receipt_limit_interrupt_pending = false; + state.receipt_limit_interrupt_accepted = false; + state.receipt_limit_interrupt_attempts = 0; + state.receipt_limit_interrupt_deadline_unix_ms = None; + state.last_agent_message = None; + state.lifecycle = "closed".to_owned(); + let _ = state.push_terminal_event(NormalizedProviderEvent { + event_type: "harness.diagnostic".to_owned(), + priority: EventPriority::P0, + payload: json!({ + "provider": "codex", + "code": "legacy_provider_turn_epoch_ambiguous", + "message": "Codex recovery could not safely identify and settle active work from a saturated legacy replay epoch; Paperclip terminated the provider and closed the durable run", + "paperclipAccepted": false, + "providerReportedActive": provider_reported_active, + "ambiguousStartPending": ambiguous_turn_start_pending, + "providerShutdownFailed": provider_shutdown_failed, + }), + }); + self.save_state()?; + return Ok(()); + } + if let Some(reused_provider_turn_id) = recovered_active_turn_id + .as_ref() + .filter(|provider_turn_id| { + settled_provider_turn_contains( + &settled_provider_turn_ids, + &settled_provider_turn_filter, + provider_turn_id, + ) + }) + .cloned() + { + // The durable terminal ledger is authoritative. A resumed provider + // that reports one of those identities as active is contradictory + // and may still be mutating the workspace. Terminate that process + // generation and persist the run closed before exposing recovery + // to the controller; otherwise this path would reopen settled work. + let provider_shutdown_failed = provider.shutdown().is_err(); + let state = self + .state + .as_mut() + .expect("Codex state remains available during recovery"); + state.provider_process_generation = process_generation; + state.settled_provider_turn_ids = settled_provider_turn_ids; + state.settled_provider_turn_filter = settled_provider_turn_filter; + state.active_provider_turn_id = None; + state.ambiguous_turn_start_pending = false; + state.completed_turn_authoritative = false; + state.completed_turn_process_generation = None; + state.completed_provider_turn_id = None; + state.receipt_limit_diagnostic_emitted = false; + state.receipt_limit_interrupt_pending = false; + state.receipt_limit_interrupt_accepted = false; + state.receipt_limit_interrupt_attempts = 0; + state.receipt_limit_interrupt_deadline_unix_ms = None; + state.last_agent_message = None; + state.lifecycle = "closed".to_owned(); + // Closing the provider is the safety boundary. Preserve that + // durable transition even when an already-full event backlog has + // no room for an additional diagnostic. + let _ = state.push_terminal_event(NormalizedProviderEvent { + event_type: "harness.diagnostic".to_owned(), + priority: EventPriority::P0, + payload: json!({ + "provider": "codex", + "code": "provider_turn_identity_reused", + "providerTurnId": reused_provider_turn_id, + "message": "Codex recovery reported a previously settled turn identity as active; Paperclip terminated the provider and closed the durable run", + "paperclipAccepted": false, + "providerReportedActive": true, + "providerShutdownFailed": provider_shutdown_failed, + }), + }); + self.save_state()?; + return Ok(()); + } if ambiguous_turn_start_pending { let recovered_turn_id = recovered_active_turn_id.as_deref().ok_or_else(|| { DurableRunnerError::invalid( @@ -434,37 +1012,99 @@ impl CodexCommandExecutor { )); } } - provider.restore_completed_turn_authority( - completed_turn_authoritative - && recovered_active_turn_id.is_none() - && !ambiguous_turn_start_pending, - completed_turn_process_generation, - completed_provider_turn_id.as_deref(), - ); + provider + .restore_completed_turn_authority( + completed_turn_authoritative + && recovered_active_turn_id.is_none() + && !ambiguous_turn_start_pending, + completed_turn_process_generation, + completed_provider_turn_id.as_deref(), + ) + .map_err(|error| { + DurableRunnerError::invalid(format!( + "failed to restore Codex completion authority: {error}" + )) + })?; self.provider = Some(provider); - self.state - .as_mut() - .expect("Codex state remains available during recovery") - .provider_process_generation = process_generation; - if provider_had_exited - || ambiguous_turn_start_pending - || recovered_active_turn_id != previous_active_turn_id { let state = self .state .as_mut() .expect("Codex state remains available during recovery"); + state.provider_process_generation = process_generation; + state.settled_provider_turn_ids = settled_provider_turn_ids; + state.settled_provider_turn_filter = settled_provider_turn_filter; + } + if provider_had_exited + || ambiguous_turn_start_pending + || recovered_active_turn_id != previous_active_turn_id + { + let recovered_turn_ended = + previous_active_turn_id.is_some() && recovered_active_turn_id.is_none(); + let identity = self.event_identity.clone(); + let state = self + .state + .as_mut() + .expect("Codex state remains available during recovery"); + if recovered_turn_ended { + if !provider_epoch_requires_rollover { + state.settle_active_provider_turn_identity()?; + } + let settled = state + .tool_bridge + .settle_turn("provider_turn_terminated") + .map_err(|error| { + DurableRunnerError::invalid(format!( + "failed to settle semantic tools during recovery: {error}" + )) + })?; + if !settled.is_empty() { + let identity = identity.as_ref().ok_or_else(|| { + DurableRunnerError::invalid( + "Codex semantic tool events require the durable runner identity", + ) + })?; + for result in settled { + state.push_terminal_event(semantic_result_event(identity, &result))?; + } + } + state.receipt_limit_diagnostic_emitted = false; + state.receipt_limit_interrupt_pending = false; + state.receipt_limit_interrupt_accepted = false; + state.receipt_limit_interrupt_attempts = 0; + state.receipt_limit_interrupt_deadline_unix_ms = None; + } state.reconcile_active_provider_turn(recovered_active_turn_id.clone()); - state.push_event(NormalizedProviderEvent { + let reconciled = NormalizedProviderEvent { event_type: "session.reconciled".to_owned(), priority: EventPriority::P0, payload: json!({ "provider": "codex", "providerSessionId": thread_id, - "previousProviderTurnId": previous_active_turn_id, - "activeProviderTurnId": recovered_active_turn_id, + "previousProviderTurnId": previous_active_turn_id.clone(), + "activeProviderTurnId": recovered_active_turn_id.clone(), }), - })?; + }; + if recovered_turn_ended { + state.push_terminal_event(reconciled)?; + // A turn that disappeared while runnerd was offline has no + // trustworthy success notification to replay. Terminate it + // conservatively so the controller cannot wait forever or + // mistake an unknown outcome for success. + state.push_terminal_event(NormalizedProviderEvent { + event_type: "turn.failed".to_owned(), + priority: EventPriority::P0, + payload: json!({ + "provider": "codex", + "providerTurnId": previous_active_turn_id, + "status": "failed", + "providerTerminalObserved": false, + }), + })?; + state.extend_terminal_events(terminal_events(state, "turn.failed"))?; + } else { + state.push_event(reconciled)?; + } } self.save_state()?; Ok(()) @@ -502,7 +1142,7 @@ impl CodexCommandExecutor { })?; if bytes.len() as u64 > MAX_PROVIDER_STATE_BYTES { return Err(DurableRunnerError::invalid( - "Codex provider state exceeds the 2 MiB limit", + "Codex provider state exceeds the 16 MiB limit", )); } let (temporary, mut file) = create_private_temporary_file(&path)?; @@ -542,7 +1182,8 @@ impl CodexCommandExecutor { .validate() .map_err(|error| DurableRunnerError::invalid(error.to_string()))?; let completion_contract = completion_contract(payload)?; - if let Some(state) = &self.state { + let tool_set = authorized_tool_set(payload)?; + if let Some(state) = self.state.as_mut() { if state.config != config || state.completion_contract != completion_contract { return Err(DurableRunnerError::invalid( "Codex provider or completion contract changed across the durable run", @@ -553,8 +1194,33 @@ impl CodexCommandExecutor { "Codex provider session is already closed", )); } + if state.tool_bridge.has_catalog() { + state + .tool_bridge + .verify_tool_set(&tool_set) + .map_err(|error| { + DurableRunnerError::invalid(format!( + "run.prepare tool contract changed: {error}" + )) + })?; + } else { + state.tool_bridge.prepare(tool_set).map_err(|error| { + DurableRunnerError::invalid(format!( + "run.prepare tool contract rejected: {error}" + )) + })?; + self.save_state()?; + } } else { - self.state = Some(CodexProviderState::new(config, completion_contract)); + let mut tool_bridge = ProviderToolBridge::default(); + tool_bridge.prepare(tool_set).map_err(|error| { + DurableRunnerError::invalid(format!("run.prepare tool contract rejected: {error}")) + })?; + self.state = Some(CodexProviderState::new( + config, + completion_contract, + tool_bridge, + )); self.save_state()?; } Ok(CommandExecution::result(json!({ @@ -579,24 +1245,50 @@ impl CodexCommandExecutor { .provider_process_generation .checked_add(1) .ok_or_else(|| DurableRunnerError::invalid("Codex process generation exhausted"))?; - let mut provider = CodexProvider::start_for_generation( + let (settled_provider_turn_ids, settled_provider_turn_filter) = + state.recovered_settled_provider_turn_ids()?; + let mut provider = CodexProvider::start_with_tools_for_generation( &state.config, + state.tool_bridge.authorized_tools().cloned(), state.thread_id.as_deref(), process_generation, ) .map_err(|error| { DurableRunnerError::invalid(format!("failed to start Codex provider: {error}")) })?; - provider.restore_completed_turn_authority( - state.completed_turn_authoritative && provider.active_provider_turn_id().is_none(), - state.completed_turn_process_generation, - state.completed_provider_turn_id.as_deref(), - ); + provider.enable_durable_tool_call_replays(); + provider + .restore_settled_turn_identities( + settled_provider_turn_ids.iter().cloned(), + settled_provider_turn_filter.clone(), + ) + .map_err(|error| { + DurableRunnerError::invalid(format!( + "failed to restore Codex provider turn identities: {error}" + )) + })?; + provider + .restore_completed_turn_authority( + state.completed_turn_authoritative + && provider.active_provider_turn_id().is_none(), + state.completed_turn_process_generation, + state.completed_provider_turn_id.as_deref(), + ) + .map_err(|error| { + DurableRunnerError::invalid(format!( + "failed to restore Codex completion authority: {error}" + )) + })?; self.provider = Some(provider); - self.state - .as_mut() - .expect("Codex state remains available after provider start") - .provider_process_generation = process_generation; + { + let state = self + .state + .as_mut() + .expect("Codex state remains available after provider start"); + state.provider_process_generation = process_generation; + state.settled_provider_turn_ids = settled_provider_turn_ids; + state.settled_provider_turn_filter = settled_provider_turn_filter; + } self.save_state()?; } self.provider @@ -604,6 +1296,21 @@ impl CodexCommandExecutor { .ok_or_else(|| DurableRunnerError::invalid("Codex provider is unavailable")) } + fn verify_attached_tools(&self, payload: &Value) -> Result<(), DurableRunnerError> { + if payload.get("authorizedTools").is_none() { + return Ok(()); + } + let tool_set = authorized_tool_set(payload)?; + self.state + .as_ref() + .ok_or_else(|| DurableRunnerError::invalid("Codex provider has not been prepared"))? + .tool_bridge + .verify_tool_set(&tool_set) + .map_err(|error| { + DurableRunnerError::invalid(format!("run.attach tool contract changed: {error}")) + }) + } + fn open_session(&mut self) -> Result { self.restore_provider_if_needed()?; if self @@ -636,6 +1343,11 @@ impl CodexCommandExecutor { state.thread_id = Some(thread_id.clone()); state.provider_session_id = provider_session_id.clone(); state.active_provider_turn_id = None; + state.receipt_limit_diagnostic_emitted = false; + state.receipt_limit_interrupt_pending = false; + state.receipt_limit_interrupt_accepted = false; + state.receipt_limit_interrupt_attempts = 0; + state.receipt_limit_interrupt_deadline_unix_ms = None; state.lifecycle = "session_open".to_owned(); state.config.provider_version.clone() }; @@ -667,8 +1379,156 @@ impl CodexCommandExecutor { }) } + fn close_after_rejected_provider_acceptance( + &mut self, + rejected_accepted_turn: &RejectedAcceptedTurn, + ) -> Result<(), DurableRunnerError> { + let provider_process_generation = self + .provider + .as_ref() + .map(CodexProvider::process_generation); + // The provider has already been terminated. Drop its quarantined + // handle before persisting the closure so recovery can never resume + // work that Codex accepted without Paperclip accepting its identity. + self.provider = None; + let state = self + .state + .as_mut() + .expect("Codex state remains available after rejected provider acceptance"); + if let Some(provider_process_generation) = provider_process_generation { + state.provider_process_generation = provider_process_generation; + } + state.active_provider_turn_id = None; + state.ambiguous_turn_start_pending = false; + state.completed_turn_authoritative = false; + state.completed_turn_process_generation = None; + state.completed_provider_turn_id = None; + state.receipt_limit_diagnostic_emitted = false; + state.receipt_limit_interrupt_pending = false; + state.receipt_limit_interrupt_accepted = false; + state.receipt_limit_interrupt_attempts = 0; + state.receipt_limit_interrupt_deadline_unix_ms = None; + state.last_agent_message = None; + state.lifecycle = "closed".to_owned(); + // Closure is the safety boundary. Preserve it even if a saturated + // event queue cannot retain this additional diagnostic. + let _ = state.push_terminal_event(NormalizedProviderEvent { + event_type: "harness.diagnostic".to_owned(), + priority: EventPriority::P0, + payload: json!({ + "provider": "codex", + "code": match rejected_accepted_turn { + RejectedAcceptedTurn::ReusedIdentity(_) => "provider_turn_identity_reused", + RejectedAcceptedTurn::InvalidIdentity => "provider_turn_identity_invalid", + }, + "providerTurnId": match rejected_accepted_turn { + RejectedAcceptedTurn::ReusedIdentity(provider_turn_id) => json!(provider_turn_id), + RejectedAcceptedTurn::InvalidIdentity => Value::Null, + }, + "message": match rejected_accepted_turn { + RejectedAcceptedTurn::ReusedIdentity(_) => "Codex accepted work with a previously settled turn identity; Paperclip terminated the provider and closed the durable run", + RejectedAcceptedTurn::InvalidIdentity => "Codex accepted work without a valid bounded turn identity; Paperclip terminated the provider and closed the durable run", + }, + "paperclipAccepted": false, + "providerAccepted": true, + }), + }); + self.save_state() + } + + fn rollover_provider_identity_epochs_if_needed(&mut self) -> Result<(), DurableRunnerError> { + let tool_rollover_required = self + .state + .as_ref() + .is_some_and(|state| state.tool_bridge.replay_history_blocks_admission()); + let rollover_required = self.state.as_ref().is_some_and(|state| { + state.settled_provider_turn_ids.len() >= MAX_SETTLED_PROVIDER_TURN_IDS + || !state.settled_provider_turn_filter.is_empty() + || tool_rollover_required + }); + if !rollover_required { + return Ok(()); + } + let rollover_is_safe = self.state.as_ref().is_some_and(|state| { + state.active_provider_turn_id.is_none() + && !state.ambiguous_turn_start_pending + && state.lifecycle == "session_open" + }); + if !rollover_is_safe { + return Err(DurableRunnerError::invalid( + "Codex provider identity epoch cannot rotate while work is active", + )); + } + + let (restart_result, process_generation, rejected_accepted_turn) = { + let provider = self.provider.as_mut().ok_or_else(|| { + DurableRunnerError::invalid( + "Codex provider identity epoch cannot rotate without an attached process", + ) + })?; + let restart_result = provider.restart_idle_identity_epoch(); + ( + restart_result, + provider.process_generation(), + provider.take_rejected_accepted_turn(), + ) + }; + if let Err(error) = restart_result { + if let Some(rejected_accepted_turn) = rejected_accepted_turn { + let failure_kind = match &rejected_accepted_turn { + RejectedAcceptedTurn::ReusedIdentity(_) => "accepted identity reuse", + RejectedAcceptedTurn::InvalidIdentity => "an invalid accepted identity", + }; + self.close_after_rejected_provider_acceptance(&rejected_accepted_turn)?; + return Err(DurableRunnerError::invalid(format!( + "Codex identity epoch rollover failed closed after {failure_kind}: {error}" + ))); + } + return Err(DurableRunnerError::invalid(format!( + "failed to rotate the completed Codex identity epoch: {error}" + ))); + }; + let state = self + .state + .as_mut() + .expect("Codex state remains available during identity epoch rollover"); + state.provider_process_generation = process_generation; + state.settled_provider_turn_ids.clear(); + if let Some(completed_provider_turn_id) = state.completed_provider_turn_id.clone() { + // The replacement process restored this still-authoritative + // terminal into its fresh epoch. Mirror that one tombstone in the + // durable ledger until accepting replacement work revokes the + // completion authority. + state + .settled_provider_turn_ids + .insert(completed_provider_turn_id); + } + state.settled_provider_turn_filter = DurableReplayFilter::default(); + if tool_rollover_required { + state + .tool_bridge + .rollover_replay_epoch_after_provider_restart() + .map_err(|error| { + DurableRunnerError::invalid(format!( + "failed to rotate Codex semantic tool replay authority: {error}" + )) + })?; + } + self.save_state()?; + Ok(()) + } + fn start_turn(&mut self, payload: &Value) -> Result { self.restore_provider_if_needed()?; + if self + .state + .as_ref() + .is_some_and(|state| state.lifecycle == "closed") + { + return Err(DurableRunnerError::invalid( + "Codex provider session is closed", + )); + } if self .state .as_ref() @@ -678,6 +1538,35 @@ impl CodexCommandExecutor { "Codex already has an active provider turn", )); } + if self + .state + .as_ref() + .is_some_and(|state| state.ambiguous_turn_start_pending) + { + return Err(DurableRunnerError::invalid( + "Codex has an unresolved ambiguous provider turn start", + )); + } + if self + .state + .as_ref() + .is_some_and(|state| state.queued_events.len() >= MAX_REGULAR_QUEUED_PROVIDER_EVENTS) + { + return Err(DurableRunnerError::invalid( + "cannot start a new Codex turn until terminal events are acknowledged", + )); + } + self.state + .as_mut() + .expect("Codex state remains available before turn receipt preparation") + .tool_bridge + .prepare_turn() + .map_err(|error| { + DurableRunnerError::invalid(format!( + "Codex semantic tool receipts could not prepare the next turn: {error}" + )) + })?; + self.rollover_provider_identity_epochs_if_needed()?; let text = payload .get("text") .and_then(Value::as_str) @@ -698,16 +1587,36 @@ impl CodexCommandExecutor { state.ambiguous_turn_start_pending = true; } self.save_state()?; - let (start_result, completion_authority_retained, ambiguous_turn_start_pending) = { + let ( + start_result, + completion_authority_retained, + ambiguous_turn_start_pending, + rejected_accepted_turn, + ) = { let provider = self.ensure_provider()?; let result = provider.start_turn(text, &cwd); ( result, provider.completed_turn_authority().is_some(), provider.ambiguous_turn_start_pending(), + provider.take_rejected_accepted_turn(), ) }; if let Err(error) = start_result { + if let Some(rejected_accepted_turn) = rejected_accepted_turn { + // Codex accepted this work before disclosing a usable durable + // identity. The provider has already been terminated; close + // this run before returning so recovery cannot resume the + // untracked turn from the provider's thread snapshot. + self.close_after_rejected_provider_acceptance(&rejected_accepted_turn)?; + let failure_kind = match &rejected_accepted_turn { + RejectedAcceptedTurn::ReusedIdentity(_) => "accepted identity reuse", + RejectedAcceptedTurn::InvalidIdentity => "an invalid accepted identity", + }; + return Err(DurableRunnerError::invalid(format!( + "Codex turn/start failed closed after {failure_kind}: {error}" + ))); + } let state = self .state .as_mut() @@ -748,6 +1657,11 @@ impl CodexCommandExecutor { state.completed_turn_authoritative = false; state.completed_turn_process_generation = None; state.completed_provider_turn_id = None; + state.receipt_limit_diagnostic_emitted = false; + state.receipt_limit_interrupt_pending = false; + state.receipt_limit_interrupt_accepted = false; + state.receipt_limit_interrupt_attempts = 0; + state.receipt_limit_interrupt_deadline_unix_ms = None; state.last_agent_message = None; state.lifecycle = "turn_active".to_owned(); self.save_state()?; @@ -773,6 +1687,30 @@ impl CodexCommandExecutor { "reason": reason, }))); } + let has_pending_tools = self + .state + .as_ref() + .is_some_and(|state| state.tool_bridge.pending_calls().next().is_some()); + if has_pending_tools { + let identity = self.event_identity()?; + let mut next_state = self + .state + .clone() + .expect("Codex state remains available during interruption"); + let cancelled = next_state + .tool_bridge + .cancel_pending_calls("provider_turn_stopped") + .map_err(|error| { + DurableRunnerError::invalid(format!( + "failed to cancel pending semantic tools: {error}" + )) + })?; + for result in cancelled { + next_state.push_terminal_event(semantic_result_event(&identity, &result))?; + } + self.persist_state(&next_state)?; + self.state = Some(next_state); + } self.ensure_provider()?.interrupt_turn().map_err(|error| { DurableRunnerError::invalid(format!("Codex turn interrupt failed: {error}")) })?; @@ -799,6 +1737,15 @@ impl CodexCommandExecutor { .get("requestId") .and_then(Value::as_str) .ok_or_else(|| DurableRunnerError::invalid("request.resolve requires requestId"))?; + if self + .state + .as_ref() + .is_none_or(|state| state.active_provider_turn_id.is_none()) + { + return Err(DurableRunnerError::invalid( + "cannot resolve a Codex runtime request outside an active turn", + )); + } let response = payload .get("response") .ok_or_else(|| DurableRunnerError::invalid("request.resolve requires response"))?; @@ -817,6 +1764,377 @@ impl CodexCommandExecutor { }) } + fn event_identity(&self) -> Result { + self.event_identity.clone().ok_or_else(|| { + DurableRunnerError::invalid( + "Codex semantic tool events require the durable runner identity", + ) + }) + } + + fn reject_tool_call( + &mut self, + call_id: String, + operation_id: String, + reason: String, + ) -> Result<(), DurableRunnerError> { + let state = self + .state + .as_mut() + .expect("Codex state remains available for a rejected tool call"); + let event = NormalizedProviderEvent { + event_type: "harness.diagnostic".to_owned(), + priority: EventPriority::P0, + payload: json!({ + "provider": "codex", + "code": "semantic_tool_denied", + "operationId": operation_id, + "callId": call_id, + "message": reason, + "paperclipExecuted": false, + }), + }; + if state.receipt_limit_interrupt_pending { + state.push_receipt_limit_cleanup_event(event)?; + } else { + state.push_event(event)?; + } + self.save_state()?; + let rejection = ToolResult { + call_id, + operation_id, + result: json!({ + "error": { + "code": "invalid_tool_call", + "message": "Paperclip rejected this semantic tool call", + "retryable": false, + }, + }), + is_error: true, + }; + self.provider + .as_mut() + .expect("provider remains present while rejecting its tool call") + .deliver_tool_result(&rejection) + .map_err(|delivery_error| { + DurableRunnerError::invalid(format!( + "failed to return the semantic tool rejection: {delivery_error}" + )) + }) + } + + fn stop_turn_at_tool_receipt_limit( + &mut self, + call_id: String, + operation_id: String, + ) -> Result<(), DurableRunnerError> { + let deadline_unix_ms = + receipt_limit_deadline_after(RECEIPT_LIMIT_INTERRUPT_TERMINAL_DEADLINE_MS)?; + let state = self + .state + .as_mut() + .expect("Codex state remains available at its tool receipt limit"); + let interrupt_pending = state.begin_receipt_limit_stop( + call_id.clone(), + operation_id.clone(), + deadline_unix_ms, + )?; + let first_interrupt_attempt = + interrupt_pending && state.receipt_limit_interrupt_attempts == 0; + if interrupt_pending { + self.save_state()?; + } + // The durable diagnostic owns the turn-level failure, while every + // buffered JSON-RPC call still receives an explicit provider error. + // Deliver the rejection before requesting interruption: Codex may + // close the transport as part of the interrupt, and a failed courtesy + // RPC must never abort runnerd's terminal polling loop. + let rejection = ToolResult { + call_id, + operation_id, + result: json!({ + "error": { + "code": "semantic_tool_turn_receipt_limit", + "message": "Paperclip stopped this turn at its durable semantic-tool receipt limit", + "retryable": false, + }, + }), + is_error: true, + }; + if let Some(provider) = self.provider.as_mut() { + let _ = provider.deliver_tool_result(&rejection); + } + if first_interrupt_attempt { + // Keep the durable retry marker until a terminal notification is + // observed. Provider acceptance acknowledges only this RPC; it + // does not prove that the turn stopped. Later buffered calls may + // therefore retry the idempotent interruption instead of leaving + // a still-active receipt-exhausted turn permanently unstopped. + self.state + .as_mut() + .expect("Codex state remains available before receipt-limit interruption") + .record_receipt_limit_interrupt_attempt()?; + self.save_state()?; + match self.interrupt_turn("semantic_tool_turn_receipt_limit") { + Ok(_) => { + let accepted_deadline_unix_ms = + receipt_limit_deadline_after(RECEIPT_LIMIT_ACCEPTED_TERMINAL_DEADLINE_MS)?; + self.state + .as_mut() + .expect("Codex state remains available after receipt-limit interruption") + .mark_receipt_limit_interrupt_accepted(accepted_deadline_unix_ms); + self.save_state()?; + } + // The first interruption attempt is also best-effort. Its + // durable retry marker is already saved, and propagating the + // transport error here would terminate runnerd before it can + // poll the provider's terminal notification. + Err(_) => {} + } + } + Ok(()) + } + + fn settle_receipt_limit_interrupt_after_deadline(&mut self) -> Result<(), DurableRunnerError> { + let interrupt_accepted = self + .state + .as_ref() + .is_some_and(|state| state.receipt_limit_interrupt_accepted); + let provider_shutdown_failed = self + .provider + .as_mut() + .is_some_and(|provider| provider.shutdown().is_err()); + self.provider = None; + let identity = self.event_identity()?; + let state = self + .state + .as_mut() + .expect("Codex state remains available at its receipt-limit retry bound"); + state.settle_active_provider_turn_identity()?; + let settled = state + .tool_bridge + .settle_turn("semantic_tool_turn_receipt_limit") + .map_err(|error| { + DurableRunnerError::invalid(format!( + "failed to settle semantic tools at the receipt-limit retry bound: {error}" + )) + })?; + for result in settled { + state.push_terminal_event(semantic_result_event(&identity, &result))?; + } + state.active_provider_turn_id = None; + state.completed_turn_authoritative = false; + state.receipt_limit_diagnostic_emitted = false; + state.receipt_limit_interrupt_pending = false; + state.receipt_limit_interrupt_accepted = false; + state.receipt_limit_interrupt_attempts = 0; + state.receipt_limit_interrupt_deadline_unix_ms = None; + state.lifecycle = "provider_exited".to_owned(); + let terminal_event_type = if interrupt_accepted { + "turn.interrupted" + } else { + "turn.failed" + }; + state.push_terminal_event(NormalizedProviderEvent { + event_type: terminal_event_type.to_owned(), + priority: EventPriority::P0, + payload: json!({ + "provider": "codex", + "code": if interrupt_accepted { + "semantic_tool_turn_receipt_limit_interrupt_deadline" + } else { + "semantic_tool_turn_receipt_limit_interrupt_unconfirmed" + }, + "message": if interrupt_accepted { + "Codex accepted the receipt-limit interruption but did not emit its terminal before the bounded shutdown deadline" + } else { + "Codex did not confirm terminal state after the bounded receipt-limit interruption attempts" + }, + "interruptAccepted": interrupt_accepted, + "providerTerminalObserved": false, + "providerShutdownFailed": provider_shutdown_failed, + }), + })?; + let terminal = terminal_events(state, terminal_event_type); + state.extend_terminal_events(terminal)?; + self.save_state() + } + + fn retry_receipt_limit_interrupt(&mut self) -> Result<(), DurableRunnerError> { + let (should_retry, attempts, persisted_deadline) = + self.state.as_ref().map_or((false, 0, None), |state| { + ( + state.receipt_limit_interrupt_pending + && state.active_provider_turn_id.is_some(), + state.receipt_limit_interrupt_attempts, + state.receipt_limit_interrupt_deadline_unix_ms, + ) + }); + if !should_retry { + return Ok(()); + } + let now_unix_ms = current_unix_ms()?; + let deadline_unix_ms = match persisted_deadline { + Some(deadline) => deadline, + None => { + // Older durable state did not record this additive field. Give + // an already-pending interruption one complete bounded window + // after recovery rather than falling back on poll count alone. + let timeout_ms = if self + .state + .as_ref() + .is_some_and(|state| state.receipt_limit_interrupt_accepted) + { + RECEIPT_LIMIT_ACCEPTED_TERMINAL_DEADLINE_MS + } else { + RECEIPT_LIMIT_INTERRUPT_TERMINAL_DEADLINE_MS + }; + let deadline = receipt_limit_deadline_after(timeout_ms)?; + self.state + .as_mut() + .expect("Codex state remains available while adding its receipt-limit deadline") + .receipt_limit_interrupt_deadline_unix_ms = Some(deadline); + self.save_state()?; + deadline + } + }; + // Attempts bound network traffic, while the durable wall-clock deadline + // gives an accepted asynchronous interruption time to deliver its + // authoritative terminal. The provider is always polled once more below + // before an elapsed deadline is converted into the conservative fallback. + if attempts >= MAX_RECEIPT_LIMIT_INTERRUPT_ATTEMPTS || now_unix_ms >= deadline_unix_ms { + return Ok(()); + } + // Polling is the autonomous recovery path until a terminal notification + // clears the durable marker. RPC acceptance alone does not establish + // that Codex stopped the turn, so accepted-but-unsettled interruptions + // remain idempotently retryable across polls and process restarts. + self.state + .as_mut() + .expect("Codex state remains available before receipt-limit retry") + .record_receipt_limit_interrupt_attempt()?; + self.save_state()?; + if self + .interrupt_turn("semantic_tool_turn_receipt_limit_retry") + .is_ok() + && self.state.as_ref().is_some_and(|state| { + state.receipt_limit_interrupt_pending && state.active_provider_turn_id.is_some() + }) + { + let accepted_deadline_unix_ms = + receipt_limit_deadline_after(RECEIPT_LIMIT_ACCEPTED_TERMINAL_DEADLINE_MS)?; + self.state + .as_mut() + .expect("Codex state remains available after receipt-limit retry") + .mark_receipt_limit_interrupt_accepted(accepted_deadline_unix_ms); + self.save_state()?; + } + Ok(()) + } + + fn settle_receipt_limit_interrupt_if_deadline_elapsed( + &mut self, + ) -> Result<(), DurableRunnerError> { + let deadline = self.state.as_ref().and_then(|state| { + (state.receipt_limit_interrupt_pending && state.active_provider_turn_id.is_some()) + .then_some(state.receipt_limit_interrupt_deadline_unix_ms) + .flatten() + }); + let Some(deadline) = deadline else { + return Ok(()); + }; + if current_unix_ms()? >= deadline { + self.settle_receipt_limit_interrupt_after_deadline()?; + } + Ok(()) + } + + fn handle_tool_call( + &mut self, + call_id: String, + operation_id: String, + input: Value, + ) -> Result<(), DurableRunnerError> { + let identity = self.event_identity()?; + let admission = self + .state + .as_mut() + .ok_or_else(|| DurableRunnerError::invalid("Codex provider is not prepared"))? + .admit_tool_call(&call_id, &operation_id, &input); + match admission { + Ok(ToolCallAdmission::CompletedReplay(result)) => { + self.provider + .as_mut() + .expect("provider remains present while handling its tool call") + .deliver_tool_result(&result) + .map_err(|error| { + DurableRunnerError::invalid(format!( + "failed to replay a durable semantic tool result: {error}" + )) + })?; + return Ok(()); + } + Ok(ToolCallAdmission::PendingReplay) => return Ok(()), + Err(error) => { + if error.is_active_turn_receipt_limit() { + return self.stop_turn_at_tool_receipt_limit(call_id, operation_id); + } + return self.reject_tool_call(call_id, operation_id, error.to_string()); + } + Ok(ToolCallAdmission::Pending(call)) => { + self.state + .as_mut() + .expect("Codex state remains available while accepting a tool call") + .push_event(semantic_input_event(&identity, &call))?; + self.save_state() + } + } + } + + fn deliver_semantic_result( + &mut self, + payload: &Value, + ) -> Result { + let result: ToolResult = serde_json::from_value(payload.clone()).map_err(|error| { + DurableRunnerError::invalid(format!("semantic tool result is invalid: {error}")) + })?; + let identity = self.event_identity()?; + let was_completed = self + .state + .as_ref() + .is_some_and(|state| state.tool_bridge.has_completed_call(&result.call_id)); + let mut next_state = self + .state + .clone() + .ok_or_else(|| DurableRunnerError::invalid("Codex provider is not prepared"))?; + next_state + .tool_bridge + .apply_result(result.clone()) + .map_err(|error| { + DurableRunnerError::invalid(format!("semantic tool result was rejected: {error}")) + })?; + if was_completed { + return Ok(CommandExecution::result(json!({ + "status": "duplicate", + "callId": result.call_id, + }))); + } + next_state.push_event(semantic_result_event(&identity, &result))?; + self.persist_state(&next_state)?; + self.state = Some(next_state); + self.ensure_provider()? + .deliver_tool_result(&result) + .map_err(|error| { + DurableRunnerError::invalid(format!( + "failed to return semantic tool result to Codex: {error}" + )) + })?; + Ok(CommandExecution::result(json!({ + "status": "delivered", + "callId": result.call_id, + }))) + } + fn close_session(&mut self) -> Result { if let Some(provider) = self.provider.as_mut() { provider.shutdown().map_err(|error| { @@ -830,6 +2148,11 @@ impl CodexCommandExecutor { .ok_or_else(|| DurableRunnerError::invalid("Codex provider is not prepared"))?; state.active_provider_turn_id = None; state.ambiguous_turn_start_pending = false; + state.receipt_limit_diagnostic_emitted = false; + state.receipt_limit_interrupt_pending = false; + state.receipt_limit_interrupt_accepted = false; + state.receipt_limit_interrupt_attempts = 0; + state.receipt_limit_interrupt_deadline_unix_ms = None; state.lifecycle = "closed".to_owned(); let thread_id = state.thread_id.clone(); self.save_state()?; @@ -865,15 +2188,24 @@ impl CodexCommandExecutor { // an ambiguous-start failure cannot degrade into an empty successful // poll on the same executor. self.restore_provider_if_needed()?; - if self - .state - .as_ref() - .is_some_and(|state| !state.pending_events.is_empty()) + // Receipt-limit interruption is autonomous recovery. It must advance + // even while older durable events await acknowledgement, otherwise a + // slow or disconnected controller can keep an exhausted provider turn + // alive forever. Terminal settlement uses the reserved event capacity. + self.retry_receipt_limit_interrupt()?; + let receipt_limit_terminal_poll = self.state.as_ref().is_some_and(|state| { + state.receipt_limit_interrupt_pending && state.active_provider_turn_id.is_some() + }); + if !receipt_limit_terminal_poll + && self + .state + .as_ref() + .is_some_and(|state| !state.pending_events.is_empty()) { return Ok(()); } if self.provider.is_none() { - return Ok(()); + return self.settle_receipt_limit_interrupt_if_deadline_elapsed(); } for _ in 0..MAX_EVENTS_PER_POLL { let event = self @@ -889,11 +2221,9 @@ impl CodexCommandExecutor { CodexProviderEvent::ToolCall { call_id, operation_id, - .. + input, } => { - return Err(DurableRunnerError::invalid(format!( - "Codex emitted semantic tool call {call_id} for {operation_id} before the durable tool bridge was attached" - ))); + self.handle_tool_call(call_id, operation_id, input)?; } CodexProviderEvent::Notification { method, params } => { let active_provider_turn_id = if method == "turn/started" { @@ -904,14 +2234,17 @@ impl CodexCommandExecutor { } else { None }; - let completed_turn_authority = if method == "turn/completed" { - self.provider - .as_ref() - .and_then(CodexProvider::completed_turn_authority) - .map(|(generation, turn_id)| (generation, turn_id.to_owned())) - } else { - None - }; + let normalized_terminal_type = + normalized_codex_terminal_event_type(&method, ¶ms); + let completed_turn_authority = + if normalized_terminal_type == Some("turn.completed") { + self.provider + .as_ref() + .and_then(CodexProvider::completed_turn_authority) + .map(|(generation, turn_id)| (generation, turn_id.to_owned())) + } else { + None + }; let normalized = normalize_codex_notification(&method, ¶ms); let terminal_event_type = normalized .iter() @@ -926,6 +2259,7 @@ impl CodexCommandExecutor { | "turn.interrupted" ) }); + let identity = self.event_identity.clone(); let state = self .state .as_mut() @@ -948,23 +2282,63 @@ impl CodexCommandExecutor { })?; state.reconcile_active_provider_turn(Some(provider_turn_id)); } - if method == "turn/completed" { - let (process_generation, provider_turn_id) = completed_turn_authority - .ok_or_else(|| { + if terminal_event_type.is_some() { + state.settle_active_provider_turn_identity()?; + let settled = state + .tool_bridge + .settle_turn("provider_turn_terminated") + .map_err(|error| { + DurableRunnerError::invalid(format!( + "failed to settle semantic tools at turn termination: {error}" + )) + })?; + if !settled.is_empty() { + let identity = identity.as_ref().ok_or_else(|| { + DurableRunnerError::invalid( + "Codex semantic tool events require the durable runner identity", + ) + })?; + for result in settled { + state.push_terminal_event(semantic_result_event( + identity, &result, + ))?; + } + } + state.active_provider_turn_id = None; + if terminal_event_type.as_deref() == Some("turn.completed") { + let (process_generation, provider_turn_id) = completed_turn_authority + .ok_or_else(|| { DurableRunnerError::invalid( "Codex completion omitted process and turn authority", ) })?; - state.active_provider_turn_id = None; - state.completed_turn_authoritative = true; - state.completed_turn_process_generation = Some(process_generation); - state.completed_provider_turn_id = Some(provider_turn_id); + state.completed_turn_authoritative = true; + state.completed_turn_process_generation = Some(process_generation); + state.completed_provider_turn_id = Some(provider_turn_id); + } else { + state.completed_turn_authoritative = false; + state.completed_turn_process_generation = None; + state.completed_provider_turn_id = None; + } + state.receipt_limit_diagnostic_emitted = false; + state.receipt_limit_interrupt_pending = false; + state.receipt_limit_interrupt_accepted = false; + state.receipt_limit_interrupt_attempts = 0; + state.receipt_limit_interrupt_deadline_unix_ms = None; state.ambiguous_turn_start_pending = false; state.lifecycle = "session_open".to_owned(); } - state.extend_events(normalized)?; + if terminal_event_type.is_some() { + state.extend_terminal_events(normalized)?; + } else if receipt_limit_terminal_poll { + for event in normalized { + state.push_receipt_limit_cleanup_event(event)?; + } + } else { + state.extend_events(normalized)?; + } if let Some(event_type) = terminal_event_type { - state.extend_events(terminal_events(state, &event_type))?; + state.extend_terminal_events(terminal_events(state, &event_type))?; } self.save_state()?; } @@ -977,29 +2351,35 @@ impl CodexCommandExecutor { .or_else(|| question_set.pointer("/questions/0/prompt")) .and_then(Value::as_str) .unwrap_or("Codex needs your input"); - self.state + let state = self + .state .as_mut() - .expect("Codex state remains available while polling") - .push_event(NormalizedProviderEvent { - event_type: "runtime_request.created".to_owned(), - priority: EventPriority::P0, - payload: json!({ - "request": { - "schema": "paperclip.runtime_request.v2", - "requestKind": "runtime", - "requestId": request_id, - "type": "input", - "status": "pending", - "prompt": prompt, - "input": question_set, - "origin": { - "adapter": "codex-app-server", - "provider": "codex", - "method": "item/tool/requestUserInput", - }, + .expect("Codex state remains available while polling"); + let event = NormalizedProviderEvent { + event_type: "runtime_request.created".to_owned(), + priority: EventPriority::P0, + payload: json!({ + "request": { + "schema": "paperclip.runtime_request.v2", + "requestKind": "runtime", + "requestId": request_id, + "type": "input", + "status": "pending", + "prompt": prompt, + "input": question_set, + "origin": { + "adapter": "codex-app-server", + "provider": "codex", + "method": "item/tool/requestUserInput", }, - }), - })?; + }, + }), + }; + if receipt_limit_terminal_poll { + state.push_receipt_limit_cleanup_event(event)?; + } else { + state.push_event(event)?; + } self.save_state()?; } CodexProviderEvent::Exited { @@ -1022,7 +2402,7 @@ impl CodexCommandExecutor { // whether this separate session exit belongs to that // completion or is a later idle-provider failure. state.lifecycle = "provider_exited".to_owned(); - state.push_event(NormalizedProviderEvent { + state.push_terminal_event(NormalizedProviderEvent { // A completed turn remains authoritative, while the // reusable provider session independently becomes // unavailable. Avoid emitting session.failed for @@ -1053,7 +2433,11 @@ impl CodexCommandExecutor { } } } - Ok(()) + // Check the durable deadline only after a complete provider poll. A + // terminal that arrived after interruption acceptance but before this + // observation remains authoritative even when several fast controller + // polls have already exhausted the interruption-attempt budget. + self.settle_receipt_limit_interrupt_if_deadline_elapsed() } } @@ -1066,6 +2450,7 @@ impl CommandExecutor for CodexCommandExecutor { if self.state.is_none() && command.payload.get("provider").is_some() { self.prepare(&command.payload)?; } + self.verify_attached_tools(&command.payload)?; let mut execution = self.open_session()?; execution.events.push(( "run.attached".to_owned(), @@ -1081,6 +2466,7 @@ impl CommandExecutor for CodexCommandExecutor { self.interrupt_turn(&command.command_type) } "request.resolve" => self.resolve_request(&command.payload), + "semantic_tool.result" => self.deliver_semantic_result(&command.payload), "session.snapshot" => self.snapshot(), "session.close" | "session.destroy" => self.close_session(), "runner.drain" | "runner.suspend" | "runner.shutdown" => { @@ -1119,6 +2505,7 @@ impl CommandExecutor for CodexCommandExecutor { )); } next_state.pending_events.drain(..count); + next_state.refill_pending_events(); self.persist_state(&next_state)?; self.state = Some(next_state); Ok(()) @@ -1160,6 +2547,7 @@ mod tests { approval_policy: "never".to_owned(), }, completion_contract: None, + tool_bridge: ProviderToolBridge::default(), thread_id: Some("thread-1".to_owned()), provider_session_id: None, active_provider_turn_id: None, @@ -1168,8 +2556,16 @@ mod tests { provider_process_generation: 0, completed_turn_process_generation: None, completed_provider_turn_id: None, + settled_provider_turn_ids: std::collections::BTreeSet::new(), + settled_provider_turn_filter: DurableReplayFilter::default(), + receipt_limit_diagnostic_emitted: false, + receipt_limit_interrupt_pending: false, + receipt_limit_interrupt_accepted: false, + receipt_limit_interrupt_attempts: 0, + receipt_limit_interrupt_deadline_unix_ms: None, last_agent_message: None, pending_events: VecDeque::new(), + queued_events: VecDeque::new(), next_provider_event_seq: initial_provider_event_seq(), }; assert!(state.validate().is_err()); @@ -1194,6 +2590,7 @@ mod tests { approval_policy: "never".to_owned(), }, None, + ProviderToolBridge::default(), ); state.thread_id = Some("thread-1".to_owned()); state.lifecycle = "session_open".to_owned(); @@ -1236,6 +2633,7 @@ mod tests { revision: "1".to_owned(), criterion_ids: vec!["objective".to_owned()], }), + ProviderToolBridge::default(), ); state.last_agent_message = Some("Finished the requested work.".to_owned()); let events = terminal_events(&state, "turn.completed"); @@ -1244,4 +2642,752 @@ mod tests { assert_eq!(events[1].event_type, "run.terminal"); assert_eq!(events[1].payload["runTerminalState"], "succeeded"); } + + #[test] + fn semantic_input_digest_covers_the_transmitted_redacted_value() { + let identity = ProviderEventIdentity { + run_id: "run-1".to_owned(), + normalized_session_id: "session-1".to_owned(), + turn_id: "turn-1".to_owned(), + item_id: "item-1".to_owned(), + }; + let call = PendingToolCall { + call_id: "call-1".to_owned(), + operation_id: "get_task_context".to_owned(), + input: json!({"password": "do-not-persist", "safe": true}), + }; + let event = semantic_input_event(&identity, &call); + let transmitted = &event.payload["semantic_tool"]["input"]; + assert_eq!(transmitted["password"], "[REDACTED]"); + assert_eq!( + event.payload["semantic_tool"]["content"]["digest"], + semantic_value_digest(transmitted) + ); + } + + #[test] + fn receipt_limit_diagnostic_is_durable_and_turn_idempotent() { + let mut state = CodexProviderState::new( + CodexProviderConfig { + provider: "codex".to_owned(), + driver: "codex_app_server".to_owned(), + provider_version: "test".to_owned(), + command: PathBuf::from("codex"), + args: vec!["app-server".to_owned()], + cwd: std::env::current_dir() + .unwrap() + .to_string_lossy() + .into_owned(), + model: None, + provider_session_id: None, + instructions: String::new(), + approval_policy: "never".to_owned(), + }, + None, + ProviderToolBridge::default(), + ); + state.thread_id = Some("thread-1".to_owned()); + state.active_provider_turn_id = Some("turn-1".to_owned()); + state.lifecycle = "turn_active".to_owned(); + + assert!(state + .begin_receipt_limit_stop("call-first".to_owned(), "tool.first".to_owned(), 10_000) + .unwrap()); + state.mark_receipt_limit_interrupt_accepted(50_000); + assert!(state + .begin_receipt_limit_stop("call-second".to_owned(), "tool.second".to_owned(), 20_000) + .unwrap()); + assert_eq!(state.pending_events.len(), 1); + assert_eq!(state.pending_events[0].payload["callId"], "call-first"); + assert_eq!(state.receipt_limit_interrupt_deadline_unix_ms, Some(50_000)); + + let mut recovered: CodexProviderState = + serde_json::from_slice(&serde_json::to_vec(&state).unwrap()).unwrap(); + recovered.validate().unwrap(); + assert!(recovered.receipt_limit_interrupt_accepted); + assert_eq!( + recovered.receipt_limit_interrupt_deadline_unix_ms, + Some(50_000) + ); + assert!(recovered + .begin_receipt_limit_stop( + "call-after-restart".to_owned(), + "tool.third".to_owned(), + 30_000, + ) + .unwrap()); + assert_eq!(recovered.pending_events.len(), 1); + // Only a terminal notification clears the retry marker. Accepting an + // interrupt request does not prove that the provider stopped. + assert!(recovered + .begin_receipt_limit_stop( + "call-after-success".to_owned(), + "tool.fourth".to_owned(), + 40_000, + ) + .unwrap()); + assert_eq!(recovered.pending_events.len(), 1); + } + + #[test] + fn settled_receipt_limit_interrupt_cannot_be_marked_accepted() { + let mut state = CodexProviderState::new( + CodexProviderConfig { + provider: "codex".to_owned(), + driver: "codex_app_server".to_owned(), + provider_version: "test".to_owned(), + command: PathBuf::from("codex"), + args: vec!["app-server".to_owned()], + cwd: std::env::current_dir() + .unwrap() + .to_string_lossy() + .into_owned(), + model: None, + provider_session_id: None, + instructions: String::new(), + approval_policy: "never".to_owned(), + }, + None, + ProviderToolBridge::default(), + ); + state.thread_id = Some("thread-1".to_owned()); + state.active_provider_turn_id = Some("turn-1".to_owned()); + state.lifecycle = "turn_active".to_owned(); + state + .begin_receipt_limit_stop("call-1".to_owned(), "tool.one".to_owned(), 10_000) + .unwrap(); + state.record_receipt_limit_interrupt_attempt().unwrap(); + + // Recovery observed that the turn ended before the interrupt RPC was + // issued and cleared the receipt-limit interruption state. + state.settle_active_provider_turn_identity().unwrap(); + state.active_provider_turn_id = None; + state.lifecycle = "session_open".to_owned(); + state.receipt_limit_diagnostic_emitted = false; + state.receipt_limit_interrupt_pending = false; + state.receipt_limit_interrupt_accepted = false; + state.receipt_limit_interrupt_attempts = 0; + state.receipt_limit_interrupt_deadline_unix_ms = None; + + state.mark_receipt_limit_interrupt_accepted(50_000); + + assert!(!state.receipt_limit_interrupt_accepted); + assert!(state.receipt_limit_interrupt_deadline_unix_ms.is_none()); + state.validate().unwrap(); + } + + #[test] + fn regular_backlog_preserves_receipt_limit_and_terminal_settlement_capacity() { + let mut state = CodexProviderState::new( + CodexProviderConfig { + provider: "codex".to_owned(), + driver: "codex_app_server".to_owned(), + provider_version: "test".to_owned(), + command: PathBuf::from("codex"), + args: vec!["app-server".to_owned()], + cwd: std::env::current_dir() + .unwrap() + .to_string_lossy() + .into_owned(), + model: None, + provider_session_id: None, + instructions: String::new(), + approval_policy: "never".to_owned(), + }, + None, + ProviderToolBridge::default(), + ); + let identity = ProviderEventIdentity { + run_id: "run-1".to_owned(), + normalized_session_id: "session-1".to_owned(), + turn_id: "turn-1".to_owned(), + item_id: "item-1".to_owned(), + }; + let ordinary_event = || NormalizedProviderEvent { + event_type: "harness.diagnostic".to_owned(), + priority: EventPriority::P1, + payload: json!({"code": "ordinary_backlog"}), + }; + for _ in 0..(MAX_EVENTS_PER_POLL + MAX_REGULAR_QUEUED_PROVIDER_EVENTS) { + state.push_event(ordinary_event()).unwrap(); + } + assert_eq!(state.pending_events.len(), MAX_EVENTS_PER_POLL); + assert_eq!( + state.queued_events.len(), + MAX_REGULAR_QUEUED_PROVIDER_EVENTS + ); + assert!(state.push_event(ordinary_event()).is_err()); + + for _ in 0..MAX_EVENTS_PER_POLL { + state + .push_receipt_limit_cleanup_event(ordinary_event()) + .unwrap(); + } + let cleanup_boundary = state.queued_events.len(); + state + .push_receipt_limit_cleanup_event(ordinary_event()) + .expect("cleanup overflow is dropped while preserving terminal capacity"); + assert_eq!(state.queued_events.len(), cleanup_boundary); + + for index in 0..MAX_PENDING_CALLS { + state + .push_terminal_event(semantic_result_event( + &identity, + &ToolResult { + call_id: format!("call-{index}"), + operation_id: "get_task_context".to_owned(), + result: json!({"error": {"code": "provider_turn_terminated"}}), + is_error: true, + }, + )) + .unwrap(); + } + for event_type in [ + "harness.diagnostic", + "turn.completed", + "run.result.proposed", + "run.terminal", + ] { + state + .push_terminal_event(NormalizedProviderEvent { + event_type: event_type.to_owned(), + priority: EventPriority::P0, + payload: json!({"terminal": true}), + }) + .unwrap(); + } + + assert_eq!(state.pending_events.len(), MAX_EVENTS_PER_POLL); + assert_eq!(state.queued_events.len(), MAX_QUEUED_PROVIDER_EVENTS); + state.validate().unwrap(); + assert!(state.push_terminal_event(ordinary_event()).is_err()); + assert!(serde_json::to_vec(&state).unwrap().len() as u64 <= MAX_PROVIDER_STATE_BYTES); + } + + #[test] + fn completed_replays_are_read_only_at_the_regular_event_boundary() { + let operation = crate::provider_bridge::AuthorizedTool { + operation_id: "get_task_context".to_owned(), + version: 1, + description: "Read the active task context.".to_owned(), + input_schema: json!({"type": "object"}), + response_schema: json!({"type": "object"}), + }; + let mut bridge = ProviderToolBridge::default(); + bridge + .prepare(AuthorizedToolSet { + schema: TOOL_SET_SCHEMA.to_owned(), + schema_version: 1, + catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation)) + .unwrap(), + operations: vec![operation], + }) + .unwrap(); + bridge + .begin_call( + "call-replayed".to_owned(), + "get_task_context".to_owned(), + json!({}), + ) + .unwrap(); + let replayed_result = ToolResult { + call_id: "call-replayed".to_owned(), + operation_id: "get_task_context".to_owned(), + result: json!({"ok": true}), + is_error: false, + }; + bridge.apply_result(replayed_result.clone()).unwrap(); + + let mut state = CodexProviderState::new( + CodexProviderConfig { + provider: "codex".to_owned(), + driver: "codex_app_server".to_owned(), + provider_version: "test".to_owned(), + command: PathBuf::from("codex"), + args: vec!["app-server".to_owned()], + cwd: std::env::current_dir() + .unwrap() + .to_string_lossy() + .into_owned(), + model: None, + provider_session_id: None, + instructions: String::new(), + approval_policy: "never".to_owned(), + }, + None, + bridge, + ); + let identity = ProviderEventIdentity { + run_id: "run-1".to_owned(), + normalized_session_id: "session-1".to_owned(), + turn_id: "turn-1".to_owned(), + item_id: "item-1".to_owned(), + }; + + // Keep the pending window occupied, then model the input and result + // receipts retained by the maximum 4,096 completed calls. Only three + // regular queued-event slots remain at this boundary. + for index in 0..MAX_EVENTS_PER_POLL { + state + .push_event(NormalizedProviderEvent { + event_type: "provider.notice.recorded".to_owned(), + priority: EventPriority::P1, + payload: json!({"index": index}), + }) + .unwrap(); + } + for index in 0..MAX_PENDING_CALLS { + let call = PendingToolCall { + call_id: format!("call-{index}"), + operation_id: "get_task_context".to_owned(), + input: json!({}), + }; + state + .push_event(semantic_input_event(&identity, &call)) + .unwrap(); + state + .push_event(semantic_result_event( + &identity, + &ToolResult { + call_id: call.call_id, + operation_id: call.operation_id, + result: json!({"ok": true}), + is_error: false, + }, + )) + .unwrap(); + } + assert_eq!( + MAX_REGULAR_QUEUED_PROVIDER_EVENTS - state.queued_events.len(), + 3 + ); + let pending_len = state.pending_events.len(); + let queued_len = state.queued_events.len(); + let next_sequence = state.next_provider_event_seq; + + for _ in 0..4 { + assert_eq!( + state + .admit_tool_call("call-replayed", "get_task_context", &json!({})) + .unwrap(), + ToolCallAdmission::CompletedReplay(replayed_result.clone()) + ); + } + assert_eq!(state.pending_events.len(), pending_len); + assert_eq!(state.queued_events.len(), queued_len); + assert_eq!(state.next_provider_event_seq, next_sequence); + state.validate().unwrap(); + } + + #[test] + fn pending_replays_are_read_only_at_the_regular_event_boundary() { + let operation = crate::provider_bridge::AuthorizedTool { + operation_id: "get_task_context".to_owned(), + version: 1, + description: "Read the active task context.".to_owned(), + input_schema: json!({"type": "object"}), + response_schema: json!({"type": "object"}), + }; + let mut bridge = ProviderToolBridge::default(); + bridge + .prepare(AuthorizedToolSet { + schema: TOOL_SET_SCHEMA.to_owned(), + schema_version: 1, + catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation)) + .unwrap(), + operations: vec![operation], + }) + .unwrap(); + bridge + .begin_call( + "call-pending".to_owned(), + "get_task_context".to_owned(), + json!({}), + ) + .unwrap(); + + let mut state = CodexProviderState::new( + CodexProviderConfig { + provider: "codex".to_owned(), + driver: "codex_app_server".to_owned(), + provider_version: "test".to_owned(), + command: PathBuf::from("codex"), + args: vec!["app-server".to_owned()], + cwd: std::env::current_dir() + .unwrap() + .to_string_lossy() + .into_owned(), + model: None, + provider_session_id: None, + instructions: String::new(), + approval_policy: "never".to_owned(), + }, + None, + bridge, + ); + for index in 0..(MAX_EVENTS_PER_POLL + MAX_REGULAR_QUEUED_PROVIDER_EVENTS) { + state + .push_event(NormalizedProviderEvent { + event_type: "provider.notice.recorded".to_owned(), + priority: EventPriority::P1, + payload: json!({"index": index}), + }) + .unwrap(); + } + assert_eq!(state.pending_events.len(), MAX_EVENTS_PER_POLL); + assert_eq!( + state.queued_events.len(), + MAX_REGULAR_QUEUED_PROVIDER_EVENTS + ); + let next_sequence = state.next_provider_event_seq; + + for _ in 0..4 { + assert_eq!( + state + .admit_tool_call("call-pending", "get_task_context", &json!({})) + .unwrap(), + ToolCallAdmission::PendingReplay + ); + } + assert_eq!(state.pending_events.len(), MAX_EVENTS_PER_POLL); + assert_eq!( + state.queued_events.len(), + MAX_REGULAR_QUEUED_PROVIDER_EVENTS + ); + assert_eq!(state.next_provider_event_seq, next_sequence); + state.validate().unwrap(); + } + + #[test] + fn exact_regular_backlog_capacity_rejects_turn_admission() { + let mut state = CodexProviderState::new( + CodexProviderConfig { + provider: "codex".to_owned(), + driver: "codex_app_server".to_owned(), + provider_version: "test".to_owned(), + command: PathBuf::from("codex"), + args: vec!["app-server".to_owned()], + cwd: std::env::current_dir() + .unwrap() + .to_string_lossy() + .into_owned(), + model: None, + provider_session_id: None, + instructions: String::new(), + approval_policy: "never".to_owned(), + }, + None, + ProviderToolBridge::default(), + ); + state.lifecycle = "prepared".to_owned(); + for _ in 0..(MAX_EVENTS_PER_POLL + MAX_REGULAR_QUEUED_PROVIDER_EVENTS) { + state + .push_event(NormalizedProviderEvent { + event_type: "harness.diagnostic".to_owned(), + priority: EventPriority::P1, + payload: json!({"code": "admission_boundary"}), + }) + .unwrap(); + } + assert_eq!( + state.queued_events.len(), + MAX_REGULAR_QUEUED_PROVIDER_EVENTS, + ); + let mut executor = CodexCommandExecutor::new(PathBuf::from("unused-test-state")); + executor.state = Some(state); + executor.restore_checked = true; + + let error = executor + .start_turn(&json!({"text": "must not reach the provider"})) + .expect_err("the exact regular backlog limit must reject admission"); + assert!(error + .to_string() + .contains("until terminal events are acknowledged")); + assert!(executor.provider.is_none()); + } + + #[test] + fn transient_receipt_limit_clears_before_a_later_turn() { + let operation = crate::provider_bridge::AuthorizedTool { + operation_id: "get_task_context".to_owned(), + version: 1, + description: "Read the active task context.".to_owned(), + input_schema: json!({"type": "object"}), + response_schema: json!({"type": "object"}), + }; + let mut bridge = ProviderToolBridge::default(); + bridge + .prepare(AuthorizedToolSet { + schema: TOOL_SET_SCHEMA.to_owned(), + schema_version: 1, + catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation)) + .unwrap(), + operations: vec![operation], + }) + .unwrap(); + let mut encoded = serde_json::to_value(&bridge).unwrap(); + encoded["durableRunReceiptLimitReached"] = Value::Bool(true); + let mut bridge: ProviderToolBridge = serde_json::from_value(encoded).unwrap(); + bridge.attach_existing_run().unwrap(); + + assert!(bridge.durable_run_receipt_limit_reached()); + bridge.prepare_turn().unwrap(); + assert!(!bridge.durable_run_receipt_limit_reached()); + } + + #[test] + fn restore_reattaches_the_durable_tool_result_byte_counter() { + let directory = std::env::temp_dir().join(format!( + "paperclip-provider-tool-byte-restore-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&directory); + fs::create_dir_all(&directory).unwrap(); + let operation = crate::provider_bridge::AuthorizedTool { + operation_id: "get_task_context".to_owned(), + version: 1, + description: "Read the active task context.".to_owned(), + input_schema: json!({"type": "object"}), + response_schema: json!({"type": "object"}), + }; + let mut bridge = ProviderToolBridge::default(); + bridge + .prepare(AuthorizedToolSet { + schema: TOOL_SET_SCHEMA.to_owned(), + schema_version: 1, + catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation)) + .unwrap(), + operations: vec![operation], + }) + .unwrap(); + bridge + .begin_call( + "call-1".to_owned(), + "get_task_context".to_owned(), + json!({}), + ) + .unwrap(); + bridge + .apply_result(ToolResult { + call_id: "call-1".to_owned(), + operation_id: "get_task_context".to_owned(), + result: json!({"ok": true}), + is_error: false, + }) + .unwrap(); + bridge.settle_turn("provider_turn_terminated").unwrap(); + assert!(bridge.retained_result_bytes_for_test() > 0); + + let state = CodexProviderState::new( + CodexProviderConfig { + provider: "codex".to_owned(), + driver: "codex_app_server".to_owned(), + provider_version: "test".to_owned(), + command: PathBuf::from("codex"), + args: vec!["app-server".to_owned()], + cwd: std::env::current_dir() + .unwrap() + .to_string_lossy() + .into_owned(), + model: None, + provider_session_id: None, + instructions: String::new(), + approval_policy: "never".to_owned(), + }, + None, + bridge, + ); + let writer = CodexCommandExecutor::new(&directory); + writer.persist_state(&state).unwrap(); + + let mut recovered = CodexCommandExecutor::new(&directory); + recovered.restore().unwrap(); + assert!( + recovered + .state + .as_ref() + .unwrap() + .tool_bridge + .retained_result_bytes_for_test() + > 0 + ); + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn durable_provider_turn_ledger_backfills_legacy_completion_authority() { + let mut state = CodexProviderState::new( + CodexProviderConfig { + provider: "codex".to_owned(), + driver: "codex_app_server".to_owned(), + provider_version: "test".to_owned(), + command: PathBuf::from("codex"), + args: vec!["app-server".to_owned()], + cwd: std::env::current_dir() + .unwrap() + .to_string_lossy() + .into_owned(), + model: None, + provider_session_id: None, + instructions: String::new(), + approval_policy: "never".to_owned(), + }, + None, + ProviderToolBridge::default(), + ); + state.completed_turn_authoritative = true; + state.completed_turn_process_generation = Some(1); + state.completed_provider_turn_id = Some("provider-turn-legacy".to_owned()); + state.provider_process_generation = 1; + + let (recovered, recovered_filter) = state.recovered_settled_provider_turn_ids().unwrap(); + + assert!(settled_provider_turn_contains( + &recovered, + &recovered_filter, + "provider-turn-legacy" + )); + } + + #[test] + fn durable_provider_turn_ledger_never_evicts_within_an_epoch() { + let mut state = CodexProviderState::new( + CodexProviderConfig { + provider: "codex".to_owned(), + driver: "codex_app_server".to_owned(), + provider_version: "test".to_owned(), + command: PathBuf::from("codex"), + args: vec!["app-server".to_owned()], + cwd: std::env::current_dir() + .unwrap() + .to_string_lossy() + .into_owned(), + model: None, + provider_session_id: None, + instructions: String::new(), + approval_policy: "never".to_owned(), + }, + None, + ProviderToolBridge::default(), + ); + state.thread_id = Some("thread-1".to_owned()); + state.lifecycle = "turn_active".to_owned(); + for index in 0..MAX_SETTLED_PROVIDER_TURN_IDS - 1 { + state + .settled_provider_turn_ids + .insert(format!("provider-turn-{index:04}")); + } + state.active_provider_turn_id = Some("provider-turn-final".to_owned()); + + state.settle_active_provider_turn_identity().unwrap(); + state.active_provider_turn_id = None; + state.lifecycle = "session_open".to_owned(); + + assert_eq!( + state.settled_provider_turn_ids.len(), + MAX_SETTLED_PROVIDER_TURN_IDS + ); + assert!(state.settled_provider_turn_filter.is_empty()); + assert!(state + .settled_provider_turn_ids + .contains("provider-turn-final")); + assert!(state + .settled_provider_turn_ids + .contains("provider-turn-0000")); + + state.lifecycle = "turn_active".to_owned(); + state.active_provider_turn_id = Some("provider-turn-overflow".to_owned()); + assert!(state.settle_active_provider_turn_identity().is_err()); + assert!(!state + .settled_provider_turn_ids + .contains("provider-turn-overflow")); + state.active_provider_turn_id = None; + state.lifecycle = "session_open".to_owned(); + state.validate().unwrap(); + let recovered: CodexProviderState = + serde_json::from_str(&serde_json::to_string(&state).unwrap()).unwrap(); + assert_eq!( + recovered.settled_provider_turn_ids, + state.settled_provider_turn_ids + ); + assert_eq!( + recovered.settled_provider_turn_filter, + state.settled_provider_turn_filter + ); + recovered.validate().unwrap(); + } + + #[test] + fn receipt_limit_deadline_settlement_preserves_unacknowledged_events() { + let directory = std::env::temp_dir().join(format!( + "paperclip-provider-receipt-deadline-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&directory); + fs::create_dir_all(&directory).unwrap(); + let mut state = CodexProviderState::new( + CodexProviderConfig { + provider: "codex".to_owned(), + driver: "codex_app_server".to_owned(), + provider_version: "test".to_owned(), + command: PathBuf::from("codex"), + args: vec!["app-server".to_owned()], + cwd: std::env::current_dir() + .unwrap() + .to_string_lossy() + .into_owned(), + model: None, + provider_session_id: None, + instructions: String::new(), + approval_policy: "never".to_owned(), + }, + None, + ProviderToolBridge::default(), + ); + state.thread_id = Some("thread-1".to_owned()); + state.active_provider_turn_id = Some("turn-1".to_owned()); + state.lifecycle = "turn_active".to_owned(); + state.receipt_limit_diagnostic_emitted = true; + state.receipt_limit_interrupt_pending = true; + state.receipt_limit_interrupt_attempts = MAX_RECEIPT_LIMIT_INTERRUPT_ATTEMPTS; + state.receipt_limit_interrupt_deadline_unix_ms = Some(1); + state + .push_event(NormalizedProviderEvent { + event_type: "provider.notice.recorded".to_owned(), + priority: EventPriority::P1, + payload: json!({"message": "awaiting acknowledgement"}), + }) + .unwrap(); + let mut executor = CodexCommandExecutor::new(&directory); + executor.state = Some(state); + executor.event_identity = Some(ProviderEventIdentity { + run_id: "run-1".to_owned(), + normalized_session_id: "session-1".to_owned(), + turn_id: "turn-1".to_owned(), + item_id: "item-1".to_owned(), + }); + executor.restore_checked = true; + + // Exercise receipt-limit recovery directly. `poll_provider` also + // restores a missing provider by design, while this unit test + // intentionally injects state without constructing a provider. + executor.retry_receipt_limit_interrupt().unwrap(); + executor + .settle_receipt_limit_interrupt_if_deadline_elapsed() + .unwrap(); + + let state = executor.state.as_ref().unwrap(); + assert_eq!(state.lifecycle, "provider_exited"); + assert!(state.active_provider_turn_id.is_none()); + assert!(!state.receipt_limit_interrupt_pending); + assert!(state.pending_events.iter().any(|event| { + event.event_type == "provider.notice.recorded" + && event.payload == json!({"message": "awaiting acknowledgement"}) + })); + assert!(state + .pending_events + .iter() + .any(|event| event.event_type == "turn.failed")); + fs::remove_dir_all(directory).unwrap(); + } } diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_bridge.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_bridge.rs index f5ef365baf..62213d3378 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_bridge.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_bridge.rs @@ -1,9 +1,9 @@ -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::error::Error; use std::fmt::{self, Display, Formatter}; use serde::de::{self, MapAccess, Visitor}; -use serde::{Deserialize, Deserializer, Serialize}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde_json::Value; use sha2::{Digest, Sha256}; @@ -12,18 +12,32 @@ pub const TOOL_CALL_SCHEMA: &str = "paperclip.prp.semantic_tool.v1"; pub const TOOL_RESULT_COMMAND: &str = "semantic_tool.result"; const MAX_AUTHORIZED_TOOLS: usize = 256; const MAX_DESCRIPTION_BYTES: usize = 16 * 1024; -const MAX_SCHEMA_BYTES: usize = 1024 * 1024; -const MAX_TOOL_SET_BYTES: usize = 4 * 1024 * 1024; -const MAX_TOOL_VALUE_BYTES: usize = 1024 * 1024; -// Settled results are authoritative replay receipts and live for the durable -// run. Bound their complete encoded map, while reserving enough room for every -// active call to later produce a maximum-sized result. The 1 KiB allowance -// covers the map key, bounded call/operation identities, JSON field names, and -// escaping around a 1 MiB result value. +const MAX_SCHEMA_BYTES: usize = 512 * 1024; +// Leave room for the authenticated PRP command or event envelope inside the +// default 1 MiB transport frame. +const MAX_TOOL_SET_BYTES: usize = 768 * 1024; +const MAX_TOOL_VALUE_BYTES: usize = 768 * 1024; +const MAX_ACCEPTED_TOOL_VALUE_BYTES: usize = 4 * 1024 * 1024; +const MAX_RETAINED_TOOL_VALUE_BYTES: usize = 8 * 1024 * 1024; +// Exact settled receipts are authoritative for the durable run. Bound their +// complete encoded representation independently of the raw-value budget so a +// long sequence of tiny results cannot accumulate unbounded identity and JSON +// envelope overhead. const MAX_SETTLED_RESULT_BYTES: usize = 8 * 1024 * 1024; -const MAX_SETTLED_RESULT_ENTRY_BYTES: usize = MAX_TOOL_VALUE_BYTES + 1024; -const MAX_RETAINED_CALLS: usize = 4_096; +pub(crate) const MAX_PENDING_CALLS: usize = 4_096; +// Active-turn inputs and results are separately bounded. Settled identities +// remain exact for a complete provider-process epoch so a delayed request +// cannot replay an old call ID after crossing a turn boundary. At the bound, +// the backend must reap the idle process before it rotates this ledger. +const MAX_DURABLE_CALL_RECEIPTS: usize = 4_096; const MAX_SETTLED_CALL_IDS: usize = 65_536; +// Retain the legacy serialized filter shape for recovery compatibility. New +// state never inserts probabilistic identities. A recovered non-empty filter +// makes the durable run fail closed until attach_run because its identities +// cannot safely be distinguished from fresh calls. +const REPLAY_FILTER_WORDS: usize = 32_768; +const ACTIVE_TURN_RECEIPT_LIMIT_MESSAGE: &str = + "durable provider tool receipt limit reached for the active turn"; #[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] #[serde(rename_all = "camelCase")] @@ -62,26 +76,145 @@ pub struct ToolResult { pub is_error: bool, } +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +#[serde(rename_all = "camelCase")] +struct CompletedToolCall { + call: PendingToolCall, + result: ToolResult, +} + +#[derive(Clone, Debug, Default, PartialEq)] +struct SettledCallIds { + order: VecDeque, + members: BTreeSet, +} + +// Preserve the existing on-disk array shape so durable state written before +// this bounded ordering was introduced remains readable. New state records IDs +// from oldest to newest, which makes pruning deterministic across recovery. +impl Serialize for SettledCallIds { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + self.order.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for SettledCallIds { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let values = VecDeque::::deserialize(deserializer)?; + let mut settled = Self::default(); + for value in values { + if settled.members.insert(value.clone()) { + settled.order.push_back(value); + } + } + Ok(settled) + } +} + +impl SettledCallIds { + fn clear(&mut self) { + self.order.clear(); + self.members.clear(); + } + + fn contains(&self, call_id: &str) -> bool { + self.members.contains(call_id) + } + + fn is_empty(&self) -> bool { + self.order.is_empty() + } + + fn len(&self) -> usize { + self.order.len() + } + + fn iter(&self) -> impl Iterator { + self.order.iter() + } + + fn extend_recent( + &mut self, + call_ids: impl IntoIterator, + limit: usize, + ) -> Vec { + for call_id in call_ids { + if self.members.insert(call_id.clone()) { + self.order.push_back(call_id); + } + } + let mut evicted = Vec::new(); + while self.order.len() > limit { + if let Some(call_id) = self.order.pop_front() { + self.members.remove(&call_id); + evicted.push(call_id); + } + } + evicted + } +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)] +pub(crate) struct DurableReplayFilter { + words: Vec, +} + +impl DurableReplayFilter { + pub(crate) fn is_empty(&self) -> bool { + self.words.is_empty() + } + + pub(crate) fn validate(&self) -> Result<(), ProviderBridgeError> { + if self.words.is_empty() || self.words.len() == REPLAY_FILTER_WORDS { + Ok(()) + } else { + Err(ProviderBridgeError::invalid( + "durable replay filter has an invalid size", + )) + } + } +} + #[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)] #[serde(rename_all = "camelCase")] pub struct ProviderToolBridge { authorized: BTreeMap, + #[serde(default)] + catalog_operations: Vec, catalog_digest: Option, pending: BTreeMap, #[serde(deserialize_with = "deserialize_retained_results")] - completed: BTreeMap, + completed: BTreeMap, + // Keep authoritative result bodies through the provider's replay window; + // prepare_turn releases them after settlement. #[serde(default, deserialize_with = "deserialize_retained_results")] - settled_results: BTreeMap, + settled_results: BTreeMap, // Derived from completed + settled results. It is intentionally omitted // from durable JSON and recomputed by attach_existing_run so old state and // tampered counters cannot bypass the byte envelope. #[serde(skip)] retained_result_bytes: usize, - // Compatibility tombstones for state written before settled results were - // retained. They still fail closed on call-id reuse, but cannot replay a - // value that the older state format discarded. + // Exact replay tombstones span every provider turn in the durable run. #[serde(default)] - settled_call_ids: BTreeSet, + settled_call_ids: SettledCallIds, + #[serde(default)] + settled_call_filter: DurableReplayFilter, + // Resource exhaustion stops the active turn. prepare_turn releases bulky + // result payloads and recomputes this marker, but replay ambiguity remains + // fail-closed until attach_run or a verified idle provider-process restart + // establishes fresh replay authority. + #[serde( + default, + alias = "settledHistoryResetPending", + skip_serializing_if = "is_false" + )] + durable_run_receipt_limit_reached: bool, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -91,6 +224,14 @@ impl ProviderBridgeError { fn invalid(message: impl Into) -> Self { Self(message.into()) } + + fn active_turn_receipt_limit() -> Self { + Self::invalid(ACTIVE_TURN_RECEIPT_LIMIT_MESSAGE) + } + + pub fn is_active_turn_receipt_limit(&self) -> bool { + self.0 == ACTIVE_TURN_RECEIPT_LIMIT_MESSAGE + } } impl Display for ProviderBridgeError { @@ -101,6 +242,10 @@ impl Display for ProviderBridgeError { impl Error for ProviderBridgeError {} +fn is_false(value: &bool) -> bool { + !*value +} + impl ProviderToolBridge { pub fn prepare(&mut self, tool_set: AuthorizedToolSet) -> Result<(), ProviderBridgeError> { self.prepare_internal(tool_set, false) @@ -117,6 +262,8 @@ impl ProviderToolBridge { self.settled_results.clear(); self.retained_result_bytes = 0; self.settled_call_ids.clear(); + self.settled_call_filter = DurableReplayFilter::default(); + self.durable_run_receipt_limit_reached = false; Ok(()) } @@ -125,92 +272,9 @@ impl ProviderToolBridge { // preserve them so an interrupted dispatcher can resume or replay the // authoritative result. `attach_run` remains the boundary that rejects // carrying pending calls into a different run. - if self - .authorized - .iter() - .any(|(operation_id, tool)| operation_id != &tool.operation_id) - { - return Err(ProviderBridgeError::invalid( - "recovered authorized tool identities are inconsistent", - )); - } - let Some(catalog_digest) = self.catalog_digest.clone() else { - if self.authorized.is_empty() - && self.pending.is_empty() - && self.completed.is_empty() - && self.settled_results.is_empty() - && self.settled_call_ids.is_empty() - { - self.retained_result_bytes = 0; - return Ok(()); - } - return Err(ProviderBridgeError::invalid( - "recovered authorized tools omit the catalog digest", - )); - }; - let recovered_tool_set = AuthorizedToolSet { - schema: TOOL_SET_SCHEMA.to_owned(), - schema_version: 1, - catalog_digest, - operations: self.authorized.values().cloned().collect(), - }; - validate_authorized_tool_set(&recovered_tool_set).map_err(|error| { - ProviderBridgeError::invalid(format!( - "recovered authorized tool catalog is invalid: {error}" - )) - })?; - if self - .settled_call_ids - .len() - .checked_add(self.settled_results.len()) - .and_then(|total| total.checked_add(self.pending.len())) - .and_then(|total| total.checked_add(self.completed.len())) - .is_none_or(|total| total > MAX_SETTLED_CALL_IDS) - || self.pending.len().saturating_add(self.completed.len()) > MAX_RETAINED_CALLS - || self - .settled_call_ids - .iter() - .any(|call_id| !is_stable_call_id(call_id)) - || self.settled_results.iter().any(|(call_id, result)| { - !is_stable_call_id(call_id) - || call_id != &result.call_id - || self.settled_call_ids.contains(call_id) - || validate_retained_result(result).is_err() - || validate_tool_result_contract(&self.authorized, result).is_err() - }) - || self.settled_call_ids.iter().any(|call_id| { - self.pending.contains_key(call_id) || self.completed.contains_key(call_id) - }) - || self.settled_results.keys().any(|call_id| { - self.pending.contains_key(call_id) || self.completed.contains_key(call_id) - }) - || self.pending.iter().any(|(call_id, call)| { - !is_stable_call_id(call_id) - || call_id != &call.call_id - || self.completed.contains_key(call_id) - || self.settled_call_ids.contains(call_id) - || self.settled_results.contains_key(call_id) - || validate_pending_tool_call(&self.authorized, call).is_err() - }) - || self.completed.iter().any(|(call_id, result)| { - !is_stable_call_id(call_id) - || call_id != &result.call_id - || self.pending.contains_key(call_id) - || validate_retained_result(result).is_err() - || validate_tool_result_contract(&self.authorized, result).is_err() - }) - { - return Err(ProviderBridgeError::invalid( - "recovered provider tool call state is invalid", - )); - } + self.validate_recovered()?; self.retained_result_bytes = retained_result_bytes(self.settled_results.iter().chain(self.completed.iter()))?; - self.ensure_settled_result_capacity(0).map_err(|_| { - ProviderBridgeError::invalid( - "recovered provider tool results exceed the durable byte limit", - ) - })?; Ok(()) } @@ -235,6 +299,7 @@ impl ProviderToolBridge { } } self.catalog_digest = Some(tool_set.catalog_digest); + self.catalog_operations = tool_set.operations.clone(); self.authorized = tool_set .operations .into_iter() @@ -244,7 +309,245 @@ impl ProviderToolBridge { } pub fn authorized_tools(&self) -> impl Iterator { - self.authorized.values() + self.catalog_operations.iter() + } + + pub fn validate_recovered(&self) -> Result<(), ProviderBridgeError> { + let Some(catalog_digest) = self.catalog_digest.clone() else { + return if self.authorized.is_empty() + && self.catalog_operations.is_empty() + && self.pending.is_empty() + && self.completed.is_empty() + && self.settled_results.is_empty() + && self.settled_call_ids.is_empty() + && self.settled_call_filter.is_empty() + && !self.durable_run_receipt_limit_reached + { + Ok(()) + } else { + Err(ProviderBridgeError::invalid( + "recovered provider tool bridge omitted its catalog identity", + )) + }; + }; + let mut expected = ProviderToolBridge::default(); + expected.prepare(AuthorizedToolSet { + schema: TOOL_SET_SCHEMA.to_owned(), + schema_version: 1, + catalog_digest, + operations: self.catalog_operations.clone(), + })?; + if self.authorized != expected.authorized { + return Err(ProviderBridgeError::invalid( + "recovered provider tool bridge changed its authorized catalog", + )); + } + let retained_receipts = self.pending.len().saturating_add(self.completed.len()); + if self.pending.len() > MAX_PENDING_CALLS || retained_receipts > MAX_DURABLE_CALL_RECEIPTS { + return Err(ProviderBridgeError::invalid( + "recovered provider tool bridge exceeds its call limit", + )); + } + self.validate_retained_value_bytes()?; + self.settled_call_filter.validate()?; + if self.settled_call_ids.len() > MAX_SETTLED_CALL_IDS { + return Err(ProviderBridgeError::invalid( + "recovered provider tool bridge exceeds its settled call identity limit", + )); + } + if !self.settled_identity_capacity_allows(0) { + return Err(ProviderBridgeError::invalid( + "recovered provider tool bridge cannot settle its active call identities", + )); + } + for call_id in self.settled_call_ids.iter() { + validate_stable_id(call_id, "settled tool call id")?; + if self.pending.contains_key(call_id) || self.completed.contains_key(call_id) { + return Err(ProviderBridgeError::invalid( + "recovered settled provider tool call identity is inconsistent", + )); + } + } + let mut pending_validator = expected.clone(); + for (call_id, call) in &self.pending { + if call_id != &call.call_id + || self.completed.contains_key(call_id) + || self.settled_results.contains_key(call_id) + || self.has_settled_call_id(call_id) + { + return Err(ProviderBridgeError::invalid( + "recovered provider tool call identity is inconsistent", + )); + } + pending_validator.begin_call( + call.call_id.clone(), + call.operation_id.clone(), + call.input.clone(), + )?; + } + for (call_id, completed) in &self.completed { + if call_id != &completed.call.call_id + || call_id != &completed.result.call_id + || completed.call.operation_id != completed.result.operation_id + || self.settled_results.contains_key(call_id) + || self.has_settled_call_id(call_id) + { + return Err(ProviderBridgeError::invalid( + "recovered completed tool call identity is inconsistent", + )); + } + let mut completed_validator = expected.clone(); + completed_validator.begin_call( + completed.call.call_id.clone(), + completed.call.operation_id.clone(), + completed.call.input.clone(), + )?; + completed_validator.apply_result(completed.result.clone())?; + } + for (call_id, settled) in &self.settled_results { + if call_id != &settled.call.call_id + || call_id != &settled.result.call_id + || settled.call.operation_id != settled.result.operation_id + || !self.settled_call_ids.contains(call_id) + || self.pending.contains_key(call_id) + || self.completed.contains_key(call_id) + { + return Err(ProviderBridgeError::invalid( + "recovered settled tool result identity is inconsistent", + )); + } + let mut settled_validator = expected.clone(); + settled_validator.begin_call( + settled.call.call_id.clone(), + settled.call.operation_id.clone(), + settled.call.input.clone(), + )?; + settled_validator.apply_result(settled.result.clone())?; + } + let retained_bytes = + retained_result_bytes(self.settled_results.iter().chain(self.completed.iter()))?; + ensure_settled_result_capacity(retained_bytes, self.pending.values()).map_err(|_| { + ProviderBridgeError::invalid( + "recovered provider tool results exceed the durable byte limit", + ) + })?; + Ok(()) + } + + pub fn verify_tool_set(&self, tool_set: &AuthorizedToolSet) -> Result<(), ProviderBridgeError> { + let mut expected = ProviderToolBridge::default(); + expected.prepare(tool_set.clone())?; + if self.catalog_digest != expected.catalog_digest + || self.catalog_operations != expected.catalog_operations + || self.authorized != expected.authorized + { + return Err(ProviderBridgeError::invalid( + "authorized tool set changed across a durable session", + )); + } + Ok(()) + } + + pub fn has_catalog(&self) -> bool { + self.catalog_digest.is_some() + } + + pub fn durable_run_receipt_limit_reached(&self) -> bool { + self.durable_run_receipt_limit_reached + } + + fn has_settled_call_id(&self, call_id: &str) -> bool { + self.settled_call_ids.contains(call_id) + } + + pub(crate) fn replay_history_blocks_admission(&self) -> bool { + self.settled_call_ids.len() >= MAX_SETTLED_CALL_IDS || !self.settled_call_filter.is_empty() + } + + pub(crate) fn has_active_receipts(&self) -> bool { + !self.pending.is_empty() || !self.completed.is_empty() + } + + /// Forget an exact replay epoch only after its provider process has been + /// reaped and an idle replacement generation has been established. The + /// process boundary is what prevents an old provider request from being + /// admitted after its tombstone is released; callers must not use this as + /// ordinary capacity eviction. + pub(crate) fn rollover_replay_epoch_after_provider_restart( + &mut self, + ) -> Result<(), ProviderBridgeError> { + if !self.replay_history_blocks_admission() { + return Err(ProviderBridgeError::invalid( + "cannot rotate provider tool replay authority before its exact capacity boundary", + )); + } + if !self.pending.is_empty() + || !self.completed.is_empty() + || !self.settled_results.is_empty() + || self.retained_result_bytes != 0 + { + return Err(ProviderBridgeError::invalid( + "cannot rotate provider tool replay authority while receipts are retained", + )); + } + self.settled_call_ids.clear(); + self.settled_call_filter = DurableReplayFilter::default(); + self.durable_run_receipt_limit_reached = false; + Ok(()) + } + + fn settled_identity_capacity_allows(&self, additional_calls: usize) -> bool { + self.settled_call_ids + .len() + .checked_add(self.total_call_receipts()) + .and_then(|total| total.checked_add(additional_calls)) + .is_some_and(|total| total <= MAX_SETTLED_CALL_IDS) + } + + pub fn prepare_turn(&mut self) -> Result<(), ProviderBridgeError> { + if !self.pending.is_empty() || !self.completed.is_empty() { + return Err(ProviderBridgeError::invalid( + "cannot prepare the next provider turn while tool calls are active", + )); + } + // The authoritative result bodies are needed only while the provider + // can replay the just-settled turn. Release that bulky data at the + // verified turn boundary, but retain every call-ID tombstone and any + // recovered legacy filter until the owning provider process is reaped. + self.settled_results.clear(); + self.retained_result_bytes = 0; + self.durable_run_receipt_limit_reached = self.replay_history_blocks_admission(); + Ok(()) + } + + #[cfg(test)] + pub(crate) fn retained_result_bytes_for_test(&self) -> usize { + self.retained_result_bytes + } + + pub fn replay_result( + &self, + call_id: &str, + operation_id: &str, + input: &Value, + ) -> Result, ProviderBridgeError> { + if let Some(completed) = self + .completed + .get(call_id) + .or_else(|| self.settled_results.get(call_id)) + { + if completed.call.operation_id != operation_id || &completed.call.input != input { + return Err(ProviderBridgeError::invalid( + "provider replayed a completed tool call with different input", + )); + } + return Ok(Some(completed.result.clone())); + } + Ok(None) + } + + pub fn has_completed_call(&self, call_id: &str) -> bool { + self.completed.contains_key(call_id) || self.has_settled_call_id(call_id) } pub fn begin_call( @@ -253,6 +556,24 @@ impl ProviderToolBridge { operation_id: String, input: Value, ) -> Result { + validate_stable_id(&call_id, "tool call id")?; + validate_operation_id(&operation_id)?; + let authorized = self.authorized.get(&operation_id).ok_or_else(|| { + ProviderBridgeError::invalid(format!( + "provider requested unauthorized tool {operation_id}" + )) + })?; + let validator = jsonschema::validator_for(&authorized.input_schema).map_err(|_| { + ProviderBridgeError::invalid(format!( + "tool {operation_id} has an invalid durable input JSON Schema" + )) + })?; + if !validator.is_valid(&input) { + return Err(ProviderBridgeError::invalid(format!( + "provider arguments for {operation_id} failed JSON Schema validation" + ))); + } + bounded_json(&input, MAX_TOOL_VALUE_BYTES, "provider tool input")?; let call = PendingToolCall { call_id: call_id.clone(), operation_id, @@ -270,55 +591,93 @@ impl ProviderToolBridge { } if self.completed.contains_key(&call_id) || self.settled_results.contains_key(&call_id) - || self.settled_call_ids.contains(&call_id) + || self.has_settled_call_id(&call_id) { return Err(ProviderBridgeError::invalid( "provider reused a completed tool call id", )); } - if self.pending.len().saturating_add(self.completed.len()) >= MAX_RETAINED_CALLS { + // A durable receipt ledger may be exactly full or legacy state may + // contain only a probabilistic summary of evicted identities. Neither + // state can admit more work safely in this provider-process epoch. The + // backend reaps an idle process before establishing a fresh epoch. + if self.replay_history_blocks_admission() { + self.durable_run_receipt_limit_reached = true; + return Err(ProviderBridgeError::active_turn_receipt_limit()); + } + if self.durable_run_receipt_limit_reached { + return Err(ProviderBridgeError::active_turn_receipt_limit()); + } + // Pending and completed calls become exact settled identities when the + // turn terminates. Reserve that identity capacity before dispatch so + // every admitted call can be settled without stranding active state. + if !self.settled_identity_capacity_allows(1) { + self.durable_run_receipt_limit_reached = true; + return Err(ProviderBridgeError::active_turn_receipt_limit()); + } + if self.pending.len() >= MAX_PENDING_CALLS { return Err(ProviderBridgeError::invalid( - "provider tool receipt limit reached for the active turn", + "concurrent provider tool call limit reached", )); } - // Reserve durable identity space before accepting work. Settlement can - // then never fail merely because earlier turns filled the ledger and - // leave completed receipts stranded in the active-turn budget. - if self - .settled_call_ids - .len() - .saturating_add(self.settled_results.len()) - .saturating_add(self.pending.len()) - .saturating_add(self.completed.len()) - >= MAX_SETTLED_CALL_IDS + if self.total_call_receipts() >= MAX_DURABLE_CALL_RECEIPTS { + return Err(ProviderBridgeError::active_turn_receipt_limit()); + } + let input_bytes = json_size(&call.input, "provider tool input")?; + let pending_bytes = self.pending_value_bytes()?; + if pending_bytes + .checked_add(input_bytes) + .is_none_or(|total| total > MAX_ACCEPTED_TOOL_VALUE_BYTES) { return Err(ProviderBridgeError::invalid( - "durable provider tool call identity limit reached", + "retained provider tool values exceed the 4 MiB acceptance limit", )); } - // Reserve the worst-case encoded result before accepting the call. - // This makes apply_result and settlement infallible with respect to - // durable result capacity: accepted work can always retain its exact - // authoritative replay value. - self.ensure_settled_result_capacity(1)?; + let result_reserve = self + .pending + .len() + .saturating_add(1) + .checked_mul(MAX_TOOL_VALUE_BYTES) + .ok_or_else(|| ProviderBridgeError::invalid("provider tool result reserve overflow"))?; + let retained_value_bytes = self.retained_value_bytes()?; + if retained_value_bytes + .checked_add(input_bytes) + .and_then(|total| total.checked_add(MAX_TOOL_VALUE_BYTES)) + .is_none_or(|total| total > MAX_RETAINED_TOOL_VALUE_BYTES) + { + self.durable_run_receipt_limit_reached = true; + return Err(ProviderBridgeError::active_turn_receipt_limit()); + } + if retained_value_bytes + .checked_add(input_bytes) + .and_then(|total| total.checked_add(result_reserve)) + .is_none_or(|total| total > MAX_RETAINED_TOOL_VALUE_BYTES) + { + return Err(ProviderBridgeError::active_turn_receipt_limit()); + } + // Reserve the complete encoded receipt, including this exact input and + // a maximum-sized result, before accepting work. An admitted call can + // therefore always retain its authoritative result at settlement. + if ensure_settled_result_capacity(self.retained_result_bytes, std::iter::once(&call)) + .is_err() + { + self.durable_run_receipt_limit_reached = true; + return Err(ProviderBridgeError::active_turn_receipt_limit()); + } + if self.ensure_settled_result_capacity(Some(&call)).is_err() { + return Err(ProviderBridgeError::active_turn_receipt_limit()); + } self.pending.insert(call_id, call.clone()); Ok(call) } pub fn apply_result(&mut self, result: ToolResult) -> Result { - if result.call_id.is_empty() - || result.call_id.len() > 160 - || result.call_id.chars().any(char::is_control) - { - return Err(ProviderBridgeError::invalid( - "tool result call id is invalid", - )); - } + validate_stable_id(&result.call_id, "tool result call id")?; validate_operation_id(&result.operation_id)?; bounded_json(&result.result, MAX_TOOL_VALUE_BYTES, "provider tool result")?; if let Some(existing) = self.completed.get(&result.call_id) { - return if existing == &result { - Ok(existing.result.clone()) + return if existing.result == result { + Ok(existing.result.result.clone()) } else { Err(ProviderBridgeError::invalid( "conflicting duplicate tool result", @@ -326,20 +685,20 @@ impl ProviderToolBridge { }; } if let Some(existing) = self.settled_results.get(&result.call_id) { - return if existing == &result { - Ok(existing.result.clone()) + return if existing.result == result { + Ok(existing.result.result.clone()) } else { Err(ProviderBridgeError::invalid( "conflicting duplicate settled tool result", )) }; } - if self.settled_call_ids.contains(&result.call_id) { + if self.has_settled_call_id(&result.call_id) { return Err(ProviderBridgeError::invalid( "legacy settled tool result cannot be replayed", )); } - let pending = self.pending.get(&result.call_id).ok_or_else(|| { + let pending = self.pending.get(&result.call_id).cloned().ok_or_else(|| { ProviderBridgeError::invalid("tool result does not match a pending provider call") })?; if pending.operation_id != result.operation_id { @@ -347,89 +706,218 @@ impl ProviderToolBridge { "tool result operation does not match its call", )); } - validate_tool_result_contract(&self.authorized, &result)?; - let result_bytes = retained_result_entry_bytes(&result.call_id, &result)?; - let next_retained_bytes = self - .retained_result_bytes + let authorized = self.authorized.get(&result.operation_id).ok_or_else(|| { + ProviderBridgeError::invalid("tool result operation is no longer authorized") + })?; + let validator = jsonschema::validator_for(&authorized.response_schema).map_err(|_| { + ProviderBridgeError::invalid(format!( + "tool {} has an invalid durable response JSON Schema", + result.operation_id + )) + })?; + let response = semantic_response_value(&result)?; + if !result.is_error { + // Paperclip semantic dispatchers return an authoritative envelope; + // provider contracts describe the operation-specific value inside + // `result`. Direct values remain valid for compatibility with v1 + // peers that do not wrap their semantic result. + if let Some(response) = response { + if !validator.is_valid(response) { + return Err(ProviderBridgeError::invalid(format!( + "tool result for {} failed JSON Schema validation", + result.operation_id + ))); + } + } + } + let result_bytes = json_size(&result.result, "provider tool result")?; + let retained_bytes = self.retained_value_bytes()?; + if retained_bytes .checked_add(result_bytes) - .ok_or_else(|| ProviderBridgeError::invalid("durable provider result size overflow"))?; - let remaining_pending_reserve = self - .pending - .len() - .saturating_sub(1) - .checked_mul(MAX_SETTLED_RESULT_ENTRY_BYTES) - .ok_or_else(|| ProviderBridgeError::invalid("durable provider result size overflow"))?; - if next_retained_bytes - .checked_add(remaining_pending_reserve) - .is_none_or(|bytes| bytes > MAX_SETTLED_RESULT_BYTES) + .is_none_or(|total| total > MAX_RETAINED_TOOL_VALUE_BYTES) { return Err(ProviderBridgeError::invalid( - "durable provider tool result byte limit reached", + "retained provider tool values exceed the 8 MiB aggregate limit", )); } + let completed = CompletedToolCall { + call: pending, + result: result.clone(), + }; + let receipt_bytes = retained_result_entry_bytes(&result.call_id, &completed)?; + let next_retained_bytes = self + .retained_result_bytes + .checked_add(receipt_bytes) + .ok_or_else(|| ProviderBridgeError::invalid("durable provider result size overflow"))?; + ensure_settled_result_capacity( + next_retained_bytes, + self.pending + .iter() + .filter(|(call_id, _)| *call_id != &result.call_id) + .map(|(_, call)| call), + )?; self.pending.remove(&result.call_id); self.retained_result_bytes = next_retained_bytes; - self.completed - .insert(result.call_id.clone(), result.clone()); + self.completed.insert(result.call_id.clone(), completed); Ok(result.result) } - pub fn settle_turn(&mut self) -> Result<(), ProviderBridgeError> { - if !self.pending.is_empty() { - return Err(ProviderBridgeError::invalid( - "cannot settle provider tool receipts while calls are pending", - )); - } - self.retained_result_bytes = - retained_result_bytes(self.settled_results.iter().chain(self.completed.iter()))?; - self.ensure_settled_result_capacity(0)?; - // The identity capacity was reserved in `begin_call`, so moving the - // authoritative receipts cannot fail a valid admitted turn. The check - // above rejects only recovered state that bypassed attach validation. - self.settled_results.append(&mut self.completed); - Ok(()) - } - pub fn pending_calls(&self) -> impl Iterator { self.pending.values() } - fn ensure_settled_result_capacity( - &self, - additional_pending: usize, - ) -> Result<(), ProviderBridgeError> { - let pending_count = self + pub fn cancel_pending_calls( + &mut self, + code: &str, + ) -> Result, ProviderBridgeError> { + let mut next = self.clone(); + let results = next.cancel_pending_calls_internal(code)?; + *self = next; + Ok(results) + } + + pub fn settle_turn(&mut self, code: &str) -> Result, ProviderBridgeError> { + validate_stable_id(code, "tool cancellation code")?; + let results = self .pending - .len() - .checked_add(additional_pending) - .ok_or_else(|| { - ProviderBridgeError::invalid("durable provider result count overflow") - })?; - let pending_reserve = pending_count - .checked_mul(MAX_SETTLED_RESULT_ENTRY_BYTES) - .ok_or_else(|| ProviderBridgeError::invalid("durable provider result size overflow"))?; + .values() + .map(|call| cancelled_tool_result(call, code)) + .collect::>(); + let mut settled_entries = self.completed.clone(); + for (call_id, call) in &self.pending { + settled_entries.insert( + call_id.clone(), + CompletedToolCall { + call: call.clone(), + result: cancelled_tool_result(call, code), + }, + ); + } + let next_retained_bytes = + retained_result_bytes(self.settled_results.iter().chain(settled_entries.iter()))?; + ensure_settled_result_capacity(next_retained_bytes, std::iter::empty())?; + + // Byte capacity was reserved at admission. Keep every identity exact + // for the lifetime of this provider-process epoch; prepare_turn + // releases only the bulky result bodies. + let new_identity_count = settled_entries + .keys() + .filter(|call_id| !self.settled_call_ids.contains(call_id)) + .count(); if self - .retained_result_bytes - .checked_add(pending_reserve) - .is_none_or(|bytes| bytes > MAX_SETTLED_RESULT_BYTES) + .settled_call_ids + .len() + .checked_add(new_identity_count) + .is_none_or(|total| total > MAX_SETTLED_CALL_IDS) { + return Err(ProviderBridgeError::active_turn_receipt_limit()); + } + let evicted = self + .settled_call_ids + .extend_recent(settled_entries.keys().cloned(), MAX_SETTLED_CALL_IDS); + debug_assert!(evicted.is_empty()); + self.pending.clear(); + self.completed.clear(); + self.settled_results.append(&mut settled_entries); + self.retained_result_bytes = next_retained_bytes; + Ok(results) + } + + fn cancel_pending_calls_internal( + &mut self, + code: &str, + ) -> Result, ProviderBridgeError> { + validate_stable_id(code, "tool cancellation code")?; + let pending = self.pending.values().cloned().collect::>(); + let mut results = Vec::with_capacity(pending.len()); + for call in pending { + let result = cancelled_tool_result(&call, code); + self.apply_result(result.clone())?; + results.push(result); + } + Ok(results) + } + + fn retained_value_bytes(&self) -> Result { + let pending = self + .pending + .values() + .map(|call| json_size(&call.input, "retained provider tool input")); + let completed = self.completed.values().flat_map(|entry| { + [ + json_size(&entry.call.input, "retained provider tool input"), + json_size(&entry.result.result, "retained provider tool result"), + ] + }); + let settled = self.settled_results.values().flat_map(|entry| { + [ + json_size(&entry.call.input, "retained provider tool input"), + json_size(&entry.result.result, "retained provider tool result"), + ] + }); + pending + .chain(completed) + .chain(settled) + .try_fold(0usize, |total, bytes| { + total.checked_add(bytes?).ok_or_else(|| { + ProviderBridgeError::invalid("retained provider tool values overflow") + }) + }) + } + + fn pending_value_bytes(&self) -> Result { + self.pending.values().try_fold(0usize, |total, call| { + total + .checked_add(json_size(&call.input, "retained provider tool input")?) + .ok_or_else(|| { + ProviderBridgeError::invalid("retained provider tool values overflow") + }) + }) + } + + fn total_call_receipts(&self) -> usize { + self.pending.len().saturating_add(self.completed.len()) + } + + fn validate_retained_value_bytes(&self) -> Result<(), ProviderBridgeError> { + if self.retained_value_bytes()? > MAX_RETAINED_TOOL_VALUE_BYTES { return Err(ProviderBridgeError::invalid( - "durable provider tool result byte limit reached", + "retained provider tool values exceed the 8 MiB aggregate limit", )); } Ok(()) } + + fn ensure_settled_result_capacity( + &self, + additional_pending: Option<&PendingToolCall>, + ) -> Result<(), ProviderBridgeError> { + ensure_settled_result_capacity( + self.retained_result_bytes, + self.pending.values().chain(additional_pending.into_iter()), + ) + } } -fn validate_retained_result(result: &ToolResult) -> Result<(), ProviderBridgeError> { - if !is_stable_call_id(&result.call_id) { +fn validate_retained_result(entry: &CompletedToolCall) -> Result<(), ProviderBridgeError> { + validate_stable_id(&entry.call.call_id, "retained tool call id")?; + validate_stable_id(&entry.result.call_id, "retained tool result call id")?; + validate_operation_id(&entry.call.operation_id)?; + validate_operation_id(&entry.result.operation_id)?; + if entry.call.call_id != entry.result.call_id + || entry.call.operation_id != entry.result.operation_id + { return Err(ProviderBridgeError::invalid( - "retained tool result call id is invalid", + "retained provider tool receipt identity is inconsistent", )); } - validate_operation_id(&result.operation_id)?; bounded_json( - &result.result, + &entry.call.input, + MAX_TOOL_VALUE_BYTES, + "retained provider tool input", + )?; + bounded_json( + &entry.result.result, MAX_TOOL_VALUE_BYTES, "retained provider tool result", ) @@ -439,9 +927,7 @@ fn validate_pending_tool_call( authorized: &BTreeMap, call: &PendingToolCall, ) -> Result<(), ProviderBridgeError> { - if !is_stable_call_id(&call.call_id) { - return Err(ProviderBridgeError::invalid("tool call id is invalid")); - } + validate_stable_id(&call.call_id, "tool call id")?; validate_operation_id(&call.operation_id)?; let tool = authorized.get(&call.operation_id).ok_or_else(|| { ProviderBridgeError::invalid(format!( @@ -571,16 +1057,16 @@ fn validate_authorized_tool_set(tool_set: &AuthorizedToolSet) -> Result<(), Prov fn retained_result_entry_bytes( call_id: &str, - result: &ToolResult, + result: &CompletedToolCall, ) -> Result { // A two-item tuple has the same delimiter cost as a one-entry JSON map. // Summing tuples therefore equals one entry exactly and conservatively // overcounts a multi-entry map by one byte per additional receipt. - encoded_json_bytes(&(call_id, result), "retained provider tool result") + json_size(&(call_id, result), "retained provider tool result") } fn retained_result_bytes<'a>( - results: impl IntoIterator, + results: impl IntoIterator, ) -> Result { results .into_iter() @@ -593,16 +1079,53 @@ fn retained_result_bytes<'a>( }) } +fn pending_result_reserve_bytes(call: &PendingToolCall) -> Result { + let placeholder = CompletedToolCall { + call: call.clone(), + result: ToolResult { + call_id: call.call_id.clone(), + operation_id: call.operation_id.clone(), + result: Value::Null, + is_error: false, + }, + }; + retained_result_entry_bytes(&call.call_id, &placeholder)? + .checked_sub(json_size(&Value::Null, "provider tool result reserve")?) + .and_then(|bytes| bytes.checked_add(MAX_TOOL_VALUE_BYTES)) + .ok_or_else(|| ProviderBridgeError::invalid("durable provider result size overflow")) +} + +fn ensure_settled_result_capacity<'a>( + retained_bytes: usize, + pending: impl IntoIterator, +) -> Result<(), ProviderBridgeError> { + let reserved_bytes = pending + .into_iter() + .try_fold(retained_bytes, |total, call| { + total + .checked_add(pending_result_reserve_bytes(call)?) + .ok_or_else(|| { + ProviderBridgeError::invalid("durable provider result size overflow") + }) + })?; + if reserved_bytes > MAX_SETTLED_RESULT_BYTES { + return Err(ProviderBridgeError::invalid( + "durable provider tool result byte limit reached", + )); + } + Ok(()) +} + fn deserialize_retained_results<'de, D>( deserializer: D, -) -> Result, D::Error> +) -> Result, D::Error> where D: Deserializer<'de>, { struct RetainedResultsVisitor; impl<'de> Visitor<'de> for RetainedResultsVisitor { - type Value = BTreeMap; + type Value = BTreeMap; fn expecting(&self, formatter: &mut Formatter<'_>) -> fmt::Result { formatter.write_str("a bounded map of retained provider tool results") @@ -622,14 +1145,14 @@ where } let mut results = BTreeMap::new(); let mut retained_bytes = 0usize; - while let Some((call_id, result)) = map.next_entry::()? { + while let Some((call_id, result)) = map.next_entry::()? { if results.len() >= MAX_SETTLED_CALL_IDS { return Err(de::Error::custom( "retained provider tool result count exceeds the durable limit", )); } validate_retained_result(&result).map_err(de::Error::custom)?; - if call_id != result.call_id { + if call_id != result.call.call_id || call_id != result.result.call_id { return Err(de::Error::custom( "retained provider tool result identity is inconsistent", )); @@ -658,8 +1181,19 @@ where deserializer.deserialize_map(RetainedResultsVisitor) } -fn is_stable_call_id(value: &str) -> bool { - !value.is_empty() && value.len() <= 160 && !value.chars().any(char::is_control) +fn cancelled_tool_result(call: &PendingToolCall, code: &str) -> ToolResult { + ToolResult { + call_id: call.call_id.clone(), + operation_id: call.operation_id.clone(), + result: serde_json::json!({ + "error": { + "code": code, + "message": "The provider turn stopped before this semantic tool completed", + "retryable": false, + }, + }), + is_error: true, + } } pub fn authorized_tool_catalog_digest( @@ -678,11 +1212,16 @@ pub fn authorized_tool_catalog_digest( Ok(format!("sha256:{digest:x}")) } +pub fn semantic_value_digest(value: &Value) -> String { + let digest = Sha256::digest(canonical_json(value).as_bytes()); + format!("sha256:{digest:x}") +} + fn canonical_json(value: &Value) -> String { match value { Value::Null => "null".to_owned(), Value::Bool(value) => value.to_string(), - Value::Number(value) => value.to_string(), + Value::Number(value) => canonical_json_number(value), Value::String(value) => { serde_json::to_string(value).expect("serializing an in-memory JSON string cannot fail") } @@ -696,7 +1235,7 @@ fn canonical_json(value: &Value) -> String { ), Value::Object(object) => { let mut entries = object.iter().collect::>(); - entries.sort_by_key(|(key, _)| *key); + entries.sort_by(|(left, _), (right, _)| left.encode_utf16().cmp(right.encode_utf16())); format!( "{{{}}}", entries @@ -714,6 +1253,82 @@ fn canonical_json(value: &Value) -> String { } } +fn canonical_json_number(value: &serde_json::Number) -> String { + if value.is_i64() || value.is_u64() { + return value.to_string(); + } + let Some(float) = value.as_f64() else { + return value.to_string(); + }; + if float == 0.0 { + return "0".to_owned(); + } + + // serde_json preserves valid lexical spellings such as `1.0`, whereas + // JSON.stringify canonicalizes JavaScript numbers. Normalize the shortest + // serde representation to the ECMAScript decimal/exponent thresholds so + // the server and runner hash the same schema value. + let encoded = value.to_string().to_ascii_lowercase(); + let (negative, unsigned) = encoded + .strip_prefix('-') + .map_or((false, encoded.as_str()), |rest| (true, rest)); + let (coefficient, explicit_exponent) = unsigned + .split_once('e') + .map_or((unsigned, 0_i32), |(coefficient, exponent)| { + (coefficient, exponent.parse::().unwrap_or(0)) + }); + let fraction_digits = coefficient + .split_once('.') + .map_or(0_i32, |(_, fraction)| fraction.len() as i32); + let mut digits = coefficient + .bytes() + .filter(|byte| *byte != b'.') + .map(char::from) + .collect::(); + let mut decimal_position = digits.len() as i32 + explicit_exponent - fraction_digits; + + let leading_zeros = digits.bytes().take_while(|byte| *byte == b'0').count(); + digits.drain(..leading_zeros); + decimal_position -= leading_zeros as i32; + while digits.ends_with('0') { + digits.pop(); + } + if digits.is_empty() { + return "0".to_owned(); + } + + let body = if (1e-6..1e21).contains(&float.abs()) { + if decimal_position <= 0 { + format!("0.{}{}", "0".repeat((-decimal_position) as usize), digits) + } else if decimal_position >= digits.len() as i32 { + format!( + "{}{}", + digits, + "0".repeat((decimal_position - digits.len() as i32) as usize) + ) + } else { + let split = decimal_position as usize; + format!("{}.{}", &digits[..split], &digits[split..]) + } + } else { + let exponent = decimal_position - 1; + let coefficient = if digits.len() == 1 { + digits + } else { + format!("{}.{}", &digits[..1], &digits[1..]) + }; + format!( + "{coefficient}e{}{exponent}", + if exponent >= 0 { "+" } else { "" } + ) + }; + if negative { + format!("-{body}") + } else { + body + } +} + fn semantic_response_value(result: &ToolResult) -> Result, ProviderBridgeError> { let Some(envelope) = result.result.as_object() else { return Ok(Some(&result.result)); @@ -732,9 +1347,11 @@ fn semantic_response_value(result: &ToolResult) -> Result, Provid )); } if ok { - envelope.get("result").map(Some).ok_or_else(|| { - ProviderBridgeError::invalid("successful semantic result omitted result") - }) + envelope + .get("result") + .or_else(|| envelope.get("value")) + .map(Some) + .ok_or_else(|| ProviderBridgeError::invalid("successful semantic result omitted value")) } else if envelope.get("denial").is_some() || envelope.get("error").is_some() { Ok(None) } else { @@ -745,6 +1362,10 @@ fn semantic_response_value(result: &ToolResult) -> Result, Provid } fn validate_operation_id(value: &str) -> Result<(), ProviderBridgeError> { + validate_stable_id(value, "tool operation id") +} + +fn validate_stable_id(value: &str, label: &str) -> Result<(), ProviderBridgeError> { let mut chars = value.chars(); let first = chars .next() @@ -755,7 +1376,7 @@ fn validate_operation_id(value: &str) -> Result<(), ProviderBridgeError> { if first && rest && value.len() <= 160 { Ok(()) } else { - Err(ProviderBridgeError::invalid("tool operation id is invalid")) + Err(ProviderBridgeError::invalid(format!("{label} is invalid"))) } } @@ -774,7 +1395,7 @@ fn bounded_json( max_bytes: usize, label: &str, ) -> Result<(), ProviderBridgeError> { - let bytes = encoded_json_bytes(value, label)?; + let bytes = json_size(value, label)?; if bytes > max_bytes { return Err(ProviderBridgeError::invalid(format!( "{label} exceeds the {max_bytes} byte limit" @@ -783,7 +1404,7 @@ fn bounded_json( Ok(()) } -fn encoded_json_bytes(value: &impl Serialize, label: &str) -> Result { +fn json_size(value: &impl Serialize, label: &str) -> Result { serde_json::to_vec(value) .map(|bytes| bytes.len()) .map_err(|_| ProviderBridgeError::invalid(format!("{label} is not serializable"))) @@ -792,24 +1413,524 @@ fn encoded_json_bytes(value: &impl Serialize, label: &str) -> Result Option<&'static str> { + let status = match method { + "turn/failed" => "failed", + "turn/cancelled" => "cancelled", + "turn/interrupted" => "interrupted", + "turn/completed" => string( + params + .pointer("/turn/status") + .or_else(|| params.get("status")), + ), + _ => return None, + }; + Some(match status { + "failed" | "error" => "turn.failed", + "cancelled" | "canceled" => "turn.cancelled", + "interrupted" | "aborted" => "turn.interrupted", + _ => "turn.completed", + }) +} + fn bounded_text(value: &str, max_chars: usize) -> String { redact_text(value).chars().take(max_chars).collect() } @@ -118,18 +141,19 @@ pub fn normalize_codex_notification(method: &str, params: &Value) -> Vec { - let status = string( - params - .pointer("/turn/status") - .or_else(|| params.get("status")), - ); - let event_type = match status { - "failed" | "error" => "turn.failed", - "cancelled" | "canceled" => "turn.cancelled", - "interrupted" | "aborted" => "turn.interrupted", - _ => "turn.completed", + "turn/completed" | "turn/failed" | "turn/cancelled" | "turn/interrupted" => { + let status = match method { + "turn/failed" => "failed", + "turn/cancelled" => "cancelled", + "turn/interrupted" => "interrupted", + _ => string( + params + .pointer("/turn/status") + .or_else(|| params.get("status")), + ), }; + let event_type = normalized_codex_terminal_event_type(method, params) + .expect("matched Codex terminal method has a normalized terminal type"); push( &mut events, event_type, @@ -366,6 +390,16 @@ mod tests { ); assert_eq!(terminal[0].event_type, "turn.failed"); assert_eq!(terminal[0].priority, EventPriority::P0); + for (method, expected) in [ + ("turn/failed", "turn.failed"), + ("turn/cancelled", "turn.cancelled"), + ("turn/interrupted", "turn.interrupted"), + ] { + let terminal = + normalize_codex_notification(method, &json!({"turnId": "provider-turn"})); + assert_eq!(terminal[0].event_type, expected); + assert_eq!(terminal[0].priority, EventPriority::P0); + } let usage = normalize_codex_notification( "thread/tokenUsage/updated", diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs index 08c3380360..4e8050f1b1 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs @@ -5,9 +5,14 @@ use std::sync::atomic::{AtomicU64, Ordering}; use paperclip_runner_core::codex_provider::{ CodexProvider, CodexProviderConfig, CodexProviderEvent, }; -use paperclip_runner_core::durable::{Command, CommandExecutor, DurableRunnerError, PolledEvent}; +use paperclip_runner_core::durable::{ + Command, CommandExecutor, DurableRunnerConfig, DurableRunnerError, PolledEvent, +}; use paperclip_runner_core::provider_backend::CodexCommandExecutor; -use paperclip_runner_core::provider_bridge::{AuthorizedTool, ToolResult}; +use paperclip_runner_core::provider_bridge::{ + authorized_tool_catalog_digest, AuthorizedTool, AuthorizedToolSet, ProviderToolBridge, + ToolResult, TOOL_SET_SCHEMA, +}; use paperclip_runner_core::provider_events::normalize_codex_notification; use serde_json::{json, Value}; @@ -62,6 +67,36 @@ fn task_context_tool() -> AuthorizedTool { } } +fn task_context_tool_set() -> AuthorizedToolSet { + let operations = vec![task_context_tool()]; + AuthorizedToolSet { + schema: TOOL_SET_SCHEMA.to_owned(), + schema_version: 1, + catalog_digest: authorized_tool_catalog_digest(&operations).unwrap(), + operations, + } +} + +fn durable_config(directory: &Path) -> DurableRunnerConfig { + DurableRunnerConfig { + connect_url: "ws://127.0.0.1:3000/runner".to_owned(), + state_dir: directory.to_path_buf(), + runner_instance_id: "runner-1".to_owned(), + environment_lease_id: "lease-1".to_owned(), + run_id: "run-1".to_owned(), + normalized_session_id: "session-1".to_owned(), + turn_id: "turn-1".to_owned(), + item_id: "item-1".to_owned(), + runner_version: "test-1".to_owned(), + runner_digest: format!("sha256:{}", "a".repeat(64)), + max_outbox_bytes: 16 * 1024 * 1024, + p0_reserve_bytes: 1024 * 1024, + max_frame_bytes: 1024 * 1024, + reconnect_delay: std::time::Duration::from_millis(1), + max_runtime: std::time::Duration::from_secs(5), + } +} + fn command(id: &str, sequence: u64, command_type: &str, payload: Value) -> Command { Command { schema: "paperclip.prp.command.v1".to_owned(), @@ -83,6 +118,14 @@ fn call_count(directory: &Path, method: &str) -> usize { .count() } +fn recorded_tool_responses(directory: &Path) -> Vec { + fs::read_to_string(directory.join("calls.log")) + .unwrap_or_default() + .lines() + .filter_map(|line| line.strip_prefix("tool-response:").map(str::to_owned)) + .collect() +} + fn poll_and_ack( executor: &mut CodexCommandExecutor, ) -> Result, DurableRunnerError> { @@ -91,6 +134,132 @@ fn poll_and_ack( Ok(events) } +fn wait_for_notification(provider: &mut CodexProvider, expected_method: &str) -> Value { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while std::time::Instant::now() < deadline { + match provider.poll().expect("poll provider notification") { + Some(CodexProviderEvent::Notification { method, params }) => { + if method == expected_method { + return params; + } + } + Some(_) => {} + None => std::thread::sleep(std::time::Duration::from_millis(1)), + } + } + panic!("did not observe Codex {expected_method} notification before the deadline"); +} + +fn wait_for_provider_error(provider: &mut CodexProvider) -> String { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while std::time::Instant::now() < deadline { + match provider.poll() { + Err(error) => return error.to_string(), + Ok(Some(_)) => {} + Ok(None) => std::thread::sleep(std::time::Duration::from_millis(1)), + } + } + panic!("did not observe the expected Codex provider error before the deadline"); +} + +fn wait_for_executor_event( + executor: &mut CodexCommandExecutor, + expected_event_type: &str, +) -> PolledEvent { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while std::time::Instant::now() < deadline { + let events = poll_and_ack(executor).expect("poll Codex executor event"); + if let Some(event) = events + .into_iter() + .find(|event| event.event_type == expected_event_type) + { + return event; + } + std::thread::sleep(std::time::Duration::from_millis(1)); + } + panic!("did not observe Codex {expected_event_type} event before the deadline"); +} + +fn wait_for_provider_exit(provider: &mut CodexProvider) { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while std::time::Instant::now() < deadline { + if matches!( + provider.poll().expect("poll terminated provider"), + Some(CodexProviderEvent::Exited { .. }) + ) { + return; + } + std::thread::sleep(std::time::Duration::from_millis(1)); + } + panic!("the provider accepted a reused turn identity but remained live"); +} + +fn wait_for_reused_identity_reap(provider: &mut CodexProvider) { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while std::time::Instant::now() < deadline { + match provider.poll() { + Err(error) => assert!( + error.to_string().contains("reused a settled"), + "unexpected error while reaping reused provider identity: {error}" + ), + Ok(Some(CodexProviderEvent::Exited { + completed_turn_authoritative, + .. + })) => { + assert!( + !completed_turn_authoritative, + "accepted identity reuse must revoke prior completion authority" + ); + return; + } + Ok(Some(_)) => {} + Ok(None) => std::thread::sleep(std::time::Duration::from_millis(1)), + } + } + panic!("the provider accepted a reused turn identity but was not reaped"); +} + +fn wait_for_fake_provider_idle(directory: &Path) { + let state_path = directory.join("fake-state.json"); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while std::time::Instant::now() < deadline { + let active_turn_id = fs::read(&state_path) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + .and_then(|state| state.get("activeTurnId").cloned()); + if active_turn_id == Some(Value::Null) { + return; + } + std::thread::sleep(std::time::Duration::from_millis(1)); + } + panic!("the fake provider did not persist its idle turn state before the deadline"); +} + +fn saturate_provider_tool_receipts(directory: &Path) { + let mut bridge = ProviderToolBridge::default(); + bridge.prepare(task_context_tool_set()).unwrap(); + for index in 0..4_096 { + let call_id = format!("retained-call-{index}"); + bridge + .begin_call(call_id.clone(), "get_task_context".into(), json!({})) + .unwrap(); + bridge + .apply_result(ToolResult { + call_id, + operation_id: "get_task_context".into(), + result: json!({"ok": true}), + is_error: false, + }) + .unwrap(); + } + let state_path = directory.join("codex-provider-state.json"); + let mut persisted: Value = + serde_json::from_slice(&fs::read(&state_path).expect("read provider state")).unwrap(); + persisted["toolBridge"] = serde_json::to_value(bridge).unwrap(); + fs::write(&state_path, serde_json::to_vec_pretty(&persisted).unwrap()) + .expect("write saturated provider state"); +} + #[test] fn codex_transport_buffers_notifications_while_waiting_for_responses() { let directory = temporary_directory("buffering"); @@ -505,6 +674,285 @@ fn clean_provider_exit_does_not_refail_a_completed_turn() { fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); } +#[test] +fn durable_backend_reaps_before_rotating_full_provider_identity_epochs() { + let directory = temporary_directory("provider-identity-epoch-rollover"); + let config = provider_config(&directory, &["--require-dynamic-tool", "--emit-tool-call"]); + let runner_config = durable_config(&directory); + let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + first + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({ + "provider": config, + "authorizedTools": task_context_tool_set(), + }), + )) + .expect("prepare Codex provider"); + first + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open Codex session"); + drop(first); + + let state_path = directory.join("codex-provider-state.json"); + let mut persisted: Value = serde_json::from_slice( + &fs::read(&state_path).expect("read provider state before epoch rollover"), + ) + .expect("parse provider state before epoch rollover"); + let prior_generation = persisted["providerProcessGeneration"] + .as_u64() + .expect("provider generation is persisted"); + persisted["settledProviderTurnIds"] = Value::Array( + (0..4_096) + .map(|index| Value::String(format!("provider-turn-{index}"))) + .collect(), + ); + persisted["completedTurnAuthoritative"] = Value::Bool(true); + persisted["completedTurnProcessGeneration"] = json!(prior_generation); + persisted["completedProviderTurnId"] = json!("provider-turn-4095"); + persisted["toolBridge"]["settledCallIds"] = Value::Array( + (0..65_536) + .map(|index| Value::String(format!("semantic-call-{index}"))) + .collect(), + ); + persisted["toolBridge"]["durableRunReceiptLimitReached"] = Value::Bool(true); + fs::write(&state_path, serde_json::to_vec_pretty(&persisted).unwrap()) + .expect("write full provider identity epochs"); + + let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + recovered + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Start after a verified provider epoch rollover."}), + )) + .expect("roll over the idle provider generation and start fresh work"); + + let rolled: Value = serde_json::from_slice( + &fs::read(&state_path).expect("read provider state after epoch rollover"), + ) + .expect("parse provider state after epoch rollover"); + assert!(rolled["providerProcessGeneration"].as_u64().unwrap() > prior_generation); + assert_eq!( + rolled["settledProviderTurnIds"], + json!(["provider-turn-4095"]) + ); + assert_eq!(rolled["settledProviderTurnFilter"], json!({"words": []})); + assert_eq!(rolled["toolBridge"]["settledCallIds"], json!([])); + assert_eq!( + rolled["toolBridge"]["settledCallFilter"], + json!({"words": []}) + ); + assert!(rolled["toolBridge"]["durableRunReceiptLimitReached"].is_null()); + + let mut event_types = Vec::new(); + let semantic_deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while std::time::Instant::now() < semantic_deadline { + event_types.extend( + poll_and_ack(&mut recovered) + .expect("poll semantic work in the fresh provider epoch") + .into_iter() + .map(|event| event.event_type), + ); + if event_types + .iter() + .any(|event| event == "semantic_tool.input") + { + break; + } + std::thread::sleep(std::time::Duration::from_millis(1)); + } + assert!(event_types + .iter() + .any(|event| event == "semantic_tool.input")); + + recovered.shutdown().expect("stop rolled provider process"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn durable_backend_closes_when_identity_rollover_resumes_unowned_work() { + let directory = temporary_directory("provider-identity-rollover-unowned-work"); + let config = provider_config(&directory, &["--resume-unowned-turn-when-marked"]); + let mut first = CodexCommandExecutor::new(&directory); + first + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({"provider": config}), + )) + .expect("prepare Codex provider"); + first + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open Codex session"); + drop(first); + + let state_path = directory.join("codex-provider-state.json"); + let mut persisted: Value = serde_json::from_slice( + &fs::read(&state_path).expect("read provider state before unsafe rollover"), + ) + .expect("parse provider state before unsafe rollover"); + persisted["settledProviderTurnIds"] = Value::Array( + (0..4_096) + .map(|index| Value::String(format!("provider-turn-{index}"))) + .collect(), + ); + fs::write(&state_path, serde_json::to_vec_pretty(&persisted).unwrap()) + .expect("write full provider identity epoch"); + + let mut recovered = CodexCommandExecutor::new(&directory); + poll_and_ack(&mut recovered).expect("attach an idle provider before rollover"); + wait_for_fake_provider_idle(&directory); + let attached: Value = + serde_json::from_slice(&fs::read(&state_path).expect("read attached provider state")) + .expect("parse attached provider state"); + let attached_generation = attached["providerProcessGeneration"] + .as_u64() + .expect("attached provider generation is persisted"); + + // Race the provider's idle snapshot with work Paperclip never dispatched. + // The replacement process observes this turn during thread/resume and must + // close the durable run instead of leaving a quarantined session open. + fs::write(directory.join("resume-unowned-turn"), b"armed") + .expect("arm unowned provider work before rollover"); + let resumes_before_rollover = call_count(&directory, "thread/resume"); + let error = recovered + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Never overlap the unowned provider turn."}), + )) + .expect_err("identity rollover fails closed after resuming unowned work"); + assert!(error + .to_string() + .contains("identity epoch rollover failed closed after an invalid accepted identity")); + assert_eq!( + call_count(&directory, "thread/resume"), + resumes_before_rollover + 1, + ); + + let closed: Value = + serde_json::from_slice(&fs::read(&state_path).expect("read fail-closed provider state")) + .expect("parse fail-closed provider state"); + assert_eq!(closed["lifecycle"], "closed"); + assert!(closed["activeProviderTurnId"].is_null()); + assert_eq!(closed["ambiguousTurnStartPending"], false); + assert_eq!(closed["completedTurnAuthoritative"], false); + assert!(closed["providerProcessGeneration"].as_u64().unwrap() > attached_generation); + + let events = poll_and_ack(&mut recovered).expect("read fail-closed rollover diagnostic"); + assert!(events.iter().any(|event| { + event.event_type == "harness.diagnostic" + && event.payload["code"] == "provider_turn_identity_invalid" + && event.payload["paperclipAccepted"] == false + && event.payload["providerAccepted"] == true + })); + + let resumes_before_retry = call_count(&directory, "thread/resume"); + assert!(recovered + .execute(&command( + "turn-retry", + 4, + "turn.start", + json!({"text": "Do not resume the quarantined provider."}), + )) + .unwrap_err() + .to_string() + .contains("provider session is closed")); + assert_eq!( + call_count(&directory, "thread/resume"), + resumes_before_retry, + "closed rollover state must never resume the unowned provider turn", + ); + + recovered.shutdown().expect("close fail-closed executor"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn direct_provider_reaps_before_rotating_a_full_turn_identity_epoch() { + let directory = temporary_directory("direct-turn-identity-epoch-rollover"); + let config = provider_config( + &directory, + &[ + "--require-dynamic-tool", + "--resume-unowned-turn-when-marked", + ], + ); + let mut provider = CodexProvider::start_with_tools(&config, [task_context_tool()], None) + .expect("start Codex provider"); + let initial_process_id = provider.process_id(); + + // Keep every identity exact until the process boundary makes it safe to + // forget them. The next turn must transparently resume in a new process + // generation instead of permanently rejecting this session. + for index in 0..4_096 { + provider + .start_turn(&format!("Complete provider turn {index}."), &config.cwd) + .expect("start provider turn before identity rollover"); + wait_for_notification(&mut provider, "turn/completed"); + } + + provider + .start_turn( + "Continue after the exact identity epoch fills.", + &config.cwd, + ) + .expect("roll over the provider process and start fresh work"); + assert_ne!(provider.process_id(), initial_process_id); + assert_eq!(call_count(&directory, "thread/resume"), 1); + wait_for_notification(&mut provider, "turn/completed"); + let rolled_process_id = provider.process_id(); + + // Fill the next process epoch and force its resume probe to observe an + // active turn. Rollover must retain that work instead of dispatching a + // concurrent replacement after forgetting the old exact identities. The + // prior generation's authoritative completion remains as one tombstone in + // this epoch, so 4,094 additional completions fill the remaining slots. + for index in 1..4_095 { + provider + .start_turn( + &format!("Complete rolled provider turn {index}."), + &config.cwd, + ) + .expect("start provider turn in the rolled identity epoch"); + wait_for_notification(&mut provider, "turn/completed"); + } + assert_eq!(provider.process_id(), rolled_process_id); + // The terminal notification is flushed before the fake provider persists + // its idle state. Wait for that write, then arm a one-shot resume race so + // only the replacement generation reports unowned active work. + wait_for_fake_provider_idle(&directory); + fs::write(directory.join("resume-unowned-turn"), b"armed") + .expect("arm unowned work for the replacement provider resume"); + let error = provider + .start_turn( + "Do not overlap the turn recovered during epoch rollover.", + &config.cwd, + ) + .expect_err("a resumed active turn is reaped before replacement work"); + assert!(error + .to_string() + .contains("resumed unowned active work; the provider was terminated")); + assert_eq!(provider.active_provider_turn_id(), None); + assert!(provider + .start_turn( + "Never admit replacement work after quarantine.", + &config.cwd, + ) + .unwrap_err() + .to_string() + .contains("quarantined after unsafe recovered work")); + wait_for_provider_exit(&mut provider); + + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + #[test] fn post_completion_observation_does_not_hide_same_or_resumed_process_failure() { let directory = temporary_directory("completion-then-nonzero-exit"); @@ -650,7 +1098,11 @@ fn rejected_replacement_turn_start_preserves_result_and_exit_authority() { .start_turn("Reject replacement work.", &config.cwd) .expect_err("the replacement turn/start returns a definite rejection"); let mut buffered_notification_seen = false; - let rejected_start_exit = (0..64).find_map(|_| { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + let rejected_start_exit = loop { + if std::time::Instant::now() >= deadline { + break None; + } match provider .poll() .expect("poll exit after rejected replacement start") @@ -661,21 +1113,23 @@ fn rejected_replacement_turn_start_preserves_result_and_exit_authority() { == Some("buffered before replacement rejection") => { buffered_notification_seen = true; - None } Some(CodexProviderEvent::Exited { success, completed_turn_authoritative, completion_reconciles_exit, .. - }) => Some(( - success, - completed_turn_authoritative, - completion_reconciles_exit, - )), - _ => None, + }) => { + break Some(( + success, + completed_turn_authoritative, + completion_reconciles_exit, + )); + } + _ => {} } - }); + std::thread::sleep(std::time::Duration::from_millis(1)); + }; assert!(buffered_notification_seen); assert_eq!(rejected_start_exit, Some((false, true, false))); @@ -727,7 +1181,11 @@ fn rejected_replacement_turn_start_does_not_hide_contradictory_turn_evidence() { "unexpected duplicate-start error: {duplicate_error}" ); let mut contradictory_turn_seen = false; - let rejected_start_exit = (0..64).find_map(|_| { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + let rejected_start_exit = loop { + if std::time::Instant::now() >= deadline { + break None; + } match provider .poll() .expect("poll exit after contradictory replacement rejection") @@ -738,21 +1196,23 @@ fn rejected_replacement_turn_start_does_not_hide_contradictory_turn_evidence() { == Some("provider-turn-contradiction") => { contradictory_turn_seen = true; - None } Some(CodexProviderEvent::Exited { success, completed_turn_authoritative, completion_reconciles_exit, .. - }) => Some(( - success, - completed_turn_authoritative, - completion_reconciles_exit, - )), - _ => None, + }) => { + break Some(( + success, + completed_turn_authoritative, + completion_reconciles_exit, + )); + } + _ => {} } - }); + std::thread::sleep(std::time::Duration::from_millis(1)); + }; assert!(contradictory_turn_seen); assert_eq!( provider.active_provider_turn_id(), @@ -771,7 +1231,6 @@ fn ambiguous_or_dead_replacement_start_preserves_result_not_exit_authority() { "--fail-after-accepting-second-turn-before-response", ), ("malformed-error", "--malformed-error-second-turn-start"), - ("missing-turn-id", "--missing-id-second-turn-start"), ] { let directory = temporary_directory(label); let config = provider_config(&directory, &[switch]); @@ -822,6 +1281,63 @@ fn ambiguous_or_dead_replacement_start_preserves_result_not_exit_authority() { } } +#[test] +fn durable_backend_closes_after_accepted_turn_omits_its_identity() { + let directory = temporary_directory("accepted-turn-missing-identity"); + let config = provider_config(&directory, &["--missing-id-live-turn-start"]); + let runner_config = durable_config(&directory); + let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + executor + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({"provider": config}), + )) + .expect("prepare Codex provider"); + executor + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open Codex session"); + + let error = executor + .execute(&command( + "turn-without-id", + 3, + "turn.start", + json!({"text": "Accept work but omit its durable identity."}), + )) + .expect_err("accepted work without a turn identity must fail closed"); + assert!(error.to_string().contains("omitted turn.id")); + + let persisted: Value = serde_json::from_slice( + &fs::read(directory.join("codex-provider-state.json")) + .expect("read fail-closed provider state"), + ) + .expect("parse fail-closed provider state"); + assert_eq!(persisted["lifecycle"], "closed"); + assert!(persisted["activeProviderTurnId"].is_null()); + + drop(executor); + let resumes_before_closed_restore = call_count(&directory, "thread/resume"); + let mut closed = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + let events = closed + .poll_events() + .expect("closed invalid-identity state remains readable without resuming Codex"); + assert!(events.iter().any(|event| { + event.event_type == "harness.diagnostic" + && event.payload["code"] == "provider_turn_identity_invalid" + && event.payload["providerTurnId"].is_null() + })); + assert_eq!( + call_count(&directory, "thread/resume"), + resumes_before_closed_restore, + "recovery must not resume provider work accepted without a durable identity" + ); + + closed.shutdown().expect("close fail-closed executor"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + #[test] fn ambiguous_replacement_turn_adopts_one_later_completion_identity() { for (label, switch, omit_started) in [ @@ -940,7 +1456,7 @@ fn ambiguous_replacement_turn_rejects_conflicting_later_identity() { let config = provider_config( &directory, &[ - "--missing-id-second-turn-start", + "--malformed-error-second-turn-start", "--conflicting-ambiguous-second-turn", ], ); @@ -948,41 +1464,23 @@ fn ambiguous_replacement_turn_rejects_conflicting_later_identity() { provider .start_turn("Complete the first turn.", &config.cwd) .expect("start first provider turn"); - let first_completed = (0..32).any(|_| { - matches!( - provider.poll().expect("poll first turn"), - Some(CodexProviderEvent::Notification { method, .. }) - if method == "turn/completed" - ) - }); - assert!( - first_completed, - "observe the authoritative first completion" - ); + wait_for_notification(&mut provider, "turn/completed"); provider .start_turn("Accept replacement work ambiguously.", &config.cwd) - .expect_err("the replacement response omits its turn identity"); - let replacement_started = provider - .poll() - .expect("poll replacement start") - .expect("replacement start is available"); - assert!(matches!( - replacement_started, - CodexProviderEvent::Notification { method, params } - if method == "turn/started" - && params.pointer("/turn/id").and_then(Value::as_str) - == Some("provider-turn-2") - )); + .expect_err("the replacement response is transport-ambiguous"); + let replacement_started = wait_for_notification(&mut provider, "turn/started"); + assert_eq!( + replacement_started + .pointer("/turn/id") + .and_then(Value::as_str), + Some("provider-turn-2") + ); assert_eq!(provider.active_provider_turn_id(), Some("provider-turn-2")); - let conflicting_completion = provider - .poll() - .expect_err("a second replacement identity must fail closed"); + let conflicting_completion = wait_for_provider_error(&mut provider); assert!( - conflicting_completion - .to_string() - .contains("another active turn"), + conflicting_completion.contains("another active turn"), "unexpected conflicting-identity error: {conflicting_completion}" ); assert_eq!(provider.active_provider_turn_id(), Some("provider-turn-2")); @@ -990,6 +1488,53 @@ fn ambiguous_replacement_turn_rejects_conflicting_later_identity() { fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); } +#[test] +fn ambiguous_replacement_turn_rejects_an_older_settled_identity() { + let directory = temporary_directory("ambiguous-older-settled-turn-identity"); + let config = provider_config(&directory, &["--ambiguous-older-reused-turn"]); + let mut provider = CodexProvider::start(&config, None).expect("start Codex provider"); + for message in ["Complete turn one.", "Complete turn two."] { + provider + .start_turn(message, &config.cwd) + .expect("start completed provider turn"); + wait_for_notification(&mut provider, "turn/completed"); + } + provider + .start_turn("Ambiguously reuse the first turn identity.", &config.cwd) + .expect_err("the replacement response is transport-ambiguous"); + let error = wait_for_provider_error(&mut provider); + assert!( + error.contains("reused a settled provider turn identity"), + "unexpected older-identity error: {error}" + ); + assert_eq!(provider.active_provider_turn_id(), None); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + let completed_turn_authoritative = (0..) + .take_while(|_| std::time::Instant::now() < deadline) + .find_map(|_| { + match provider + .poll() + .expect("poll provider after rejected identity") + { + Some(CodexProviderEvent::Exited { + completed_turn_authoritative, + .. + }) => Some(completed_turn_authoritative), + Some(_) => None, + None => { + std::thread::sleep(std::time::Duration::from_millis(1)); + None + } + } + }); + assert!( + completed_turn_authoritative == Some(false), + "accepted work with an older settled identity must terminate the provider and revoke completion authority" + ); + + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + #[test] fn clean_exit_after_ambiguous_replacement_start_fails_the_durable_session() { let directory = temporary_directory("durable-clean-exit-after-ambiguous-turn-start"); @@ -1161,7 +1706,7 @@ fn replacement_item_is_not_persisted_before_ambiguous_turn_identity() { let config = provider_config( &directory, &[ - "--missing-id-second-turn-start", + "--malformed-error-second-turn-start", "--hold-ambiguous-second-turn-after-item", ], ); @@ -1185,15 +1730,7 @@ fn replacement_item_is_not_persisted_before_ambiguous_turn_identity() { json!({"text": "Complete the first turn."}), )) .expect("start first provider turn"); - for _ in 0..32 { - if poll_and_ack(&mut executor) - .expect("poll first turn") - .iter() - .any(|event| event.event_type == "turn.completed") - { - break; - } - } + wait_for_executor_event(&mut executor, "turn.completed"); executor .execute(&command( @@ -1202,7 +1739,7 @@ fn replacement_item_is_not_persisted_before_ambiguous_turn_identity() { "turn.start", json!({"text": "Emit replacement output before terminal authority."}), )) - .expect_err("replacement response omits its identity"); + .expect_err("replacement response is transport-ambiguous"); assert!( poll_and_ack(&mut executor) .expect("defer identity-less replacement output") @@ -1252,7 +1789,7 @@ fn completed_ambiguous_replacement_fails_closed_after_process_loss() { let config = provider_config( &directory, &[ - "--missing-id-second-turn-start", + "--malformed-error-second-turn-start", "--complete-ambiguous-second-turn-before-response", "--omit-ambiguous-turn-started", ], @@ -1277,15 +1814,7 @@ fn completed_ambiguous_replacement_fails_closed_after_process_loss() { json!({"text": "Complete the first turn."}), )) .expect("start first provider turn"); - for _ in 0..32 { - if poll_and_ack(&mut executor) - .expect("poll first turn") - .iter() - .any(|event| event.event_type == "turn.completed") - { - break; - } - } + wait_for_executor_event(&mut executor, "turn.completed"); executor .execute(&command( @@ -1294,7 +1823,7 @@ fn completed_ambiguous_replacement_fails_closed_after_process_loss() { "turn.start", json!({"text": "Complete replacement work before returning an invalid response."}), )) - .expect_err("replacement response omits its identity"); + .expect_err("replacement response is transport-ambiguous"); executor.shutdown().expect("stop first provider process"); drop(executor); @@ -1335,7 +1864,7 @@ fn ambiguous_replacement_completion_replaces_durable_turn_authority() { let config = provider_config( &directory, &[ - "--missing-id-second-turn-start", + "--malformed-error-second-turn-start", "--complete-ambiguous-second-turn", ], ); @@ -1360,19 +1889,7 @@ fn ambiguous_replacement_completion_replaces_durable_turn_authority() { )) .expect("start first provider turn"); - let mut first_events = Vec::new(); - for _ in 0..32 { - first_events.extend( - poll_and_ack(&mut executor) - .expect("poll first turn") - .into_iter() - .map(|event| event.event_type), - ); - if first_events.iter().any(|event| event == "turn.completed") { - break; - } - } - assert!(first_events.iter().any(|event| event == "turn.completed")); + wait_for_executor_event(&mut executor, "turn.completed"); executor .execute(&command( @@ -1381,10 +1898,11 @@ fn ambiguous_replacement_completion_replaces_durable_turn_authority() { "turn.start", json!({"text": "Complete replacement work after the malformed response."}), )) - .expect_err("accepted replacement response omits its turn identity"); + .expect_err("accepted replacement response is transport-ambiguous"); let mut replacement_events = Vec::new(); - for _ in 0..64 { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while std::time::Instant::now() < deadline { replacement_events.extend( poll_and_ack(&mut executor) .expect("poll accepted replacement evidence") @@ -1397,6 +1915,7 @@ fn ambiguous_replacement_completion_replaces_durable_turn_authority() { { break; } + std::thread::sleep(std::time::Duration::from_millis(1)); } assert!(replacement_events .iter() @@ -1478,6 +1997,461 @@ fn codex_rejects_a_tool_call_that_was_not_advertised() { fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); } +#[test] +fn codex_rejects_delayed_calls_after_every_terminal_notification() { + let directory = temporary_directory("delayed-tool-after-failure"); + let config = provider_config(&directory, &["--delayed-tool-after-failed-turn"]); + let mut provider = CodexProvider::start_with_tools(&config, [task_context_tool()], None) + .expect("start Codex with an authorized tool"); + provider + .start_turn("Fail before invoking a tool.", &config.cwd) + .expect("start provider turn"); + + wait_for_notification(&mut provider, "turn/failed"); + assert_eq!(provider.active_provider_turn_id(), None); + let warning = wait_for_notification(&mut provider, "warning"); + assert!(warning["message"] + .as_str() + .is_some_and(|message| message.contains("after the Codex turn terminated"))); + + let _ = provider.shutdown(); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn codex_rejects_a_delayed_prior_turn_call_while_the_next_turn_is_active() { + let directory = temporary_directory("delayed-tool-after-next-turn-start"); + let config = provider_config(&directory, &["--delayed-tool-after-next-turn-start"]); + let mut provider = CodexProvider::start_with_tools(&config, [task_context_tool()], None) + .expect("start Codex with an authorized tool"); + provider + .start_turn("Complete the first turn.", &config.cwd) + .expect("start first provider turn"); + wait_for_notification(&mut provider, "turn/completed"); + + provider + .start_turn("Keep the second turn active.", &config.cwd) + .expect("start second provider turn"); + assert_eq!(provider.active_provider_turn_id(), Some("provider-turn-2")); + let warning = wait_for_notification(&mut provider, "warning"); + assert!(warning["message"] + .as_str() + .is_some_and(|message| message.contains("after the Codex turn terminated"))); + assert_eq!(provider.active_provider_turn_id(), Some("provider-turn-2")); + + let _ = provider.shutdown(); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn codex_rejects_a_two_turn_old_call_while_fresh_work_is_active() { + let directory = temporary_directory("delayed-tool-after-third-turn-start"); + let config = provider_config(&directory, &["--delayed-tool-after-third-turn-start"]); + let mut provider = CodexProvider::start_with_tools(&config, [task_context_tool()], None) + .expect("start Codex with an authorized tool"); + + for message in ["Complete turn one.", "Complete turn two."] { + provider + .start_turn(message, &config.cwd) + .expect("start completed provider turn"); + wait_for_notification(&mut provider, "turn/completed"); + } + provider + .start_turn("Keep turn three active.", &config.cwd) + .expect("start third provider turn"); + assert_eq!(provider.active_provider_turn_id(), Some("provider-turn-3")); + + let warning = wait_for_notification(&mut provider, "warning"); + assert_eq!(warning["providerMethod"], "item/tool/call"); + assert_eq!(provider.active_provider_turn_id(), Some("provider-turn-3")); + + let _ = provider.shutdown(); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn codex_fails_closed_when_a_provider_reuses_a_settled_turn_id() { + let directory = temporary_directory("tool-after-reused-turn-start"); + let config = provider_config(&directory, &["--tool-after-reused-turn-start"]); + let mut provider = CodexProvider::start_with_tools(&config, [task_context_tool()], None) + .expect("start Codex with an authorized tool"); + provider + .start_turn("Complete the first turn.", &config.cwd) + .expect("start first provider turn"); + wait_for_notification(&mut provider, "turn/completed"); + + let error = provider + .start_turn("Reuse the settled provider identity.", &config.cwd) + .expect_err("reject a provider turn with a reused identity"); + assert!(error.to_string().contains("reused a settled")); + assert_eq!(provider.active_provider_turn_id(), None); + wait_for_reused_identity_reap(&mut provider); + + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn codex_fails_closed_when_a_provider_reuses_an_older_settled_turn_id() { + let directory = temporary_directory("tool-after-older-reused-turn-start"); + let config = provider_config(&directory, &["--tool-after-older-reused-turn-start"]); + let mut provider = CodexProvider::start_with_tools(&config, [task_context_tool()], None) + .expect("start Codex with an authorized tool"); + for message in ["Complete turn one.", "Complete turn two."] { + provider + .start_turn(message, &config.cwd) + .expect("start completed provider turn"); + wait_for_notification(&mut provider, "turn/completed"); + } + + let error = provider + .start_turn("Reuse the older settled provider identity.", &config.cwd) + .expect_err("reject a provider turn with an older reused identity"); + assert!(error.to_string().contains("reused a settled")); + assert_eq!(provider.active_provider_turn_id(), None); + wait_for_reused_identity_reap(&mut provider); + + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn durable_backend_rejects_an_older_provider_turn_id_after_restart() { + let directory = temporary_directory("durable-older-reused-turn-after-restart"); + let config = provider_config(&directory, &[]); + let runner_config = durable_config(&directory); + let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + first + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({"provider": config}), + )) + .expect("prepare Codex provider"); + first + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open Codex session"); + + for (sequence, message) in [(3, "Complete turn one."), (4, "Complete turn two.")] { + first + .execute(&command( + &format!("turn-{sequence}"), + sequence, + "turn.start", + json!({"text": message}), + )) + .expect("start completed provider turn"); + let mut completed = false; + for _ in 0..32 { + completed |= poll_and_ack(&mut first) + .expect("poll completed provider turn") + .iter() + .any(|event| event.event_type == "turn.completed"); + if completed { + break; + } + } + assert!(completed); + } + first.shutdown().expect("stop first provider process"); + drop(first); + + // The fake provider's process-local counter restarts at provider-turn-1. + // Durable state must still remember that older identity, not only the most + // recent provider-turn-2 completion authority. + let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + let error = recovered + .execute(&command( + "turn-reused", + 5, + "turn.start", + json!({"text": "Do not accept an older provider turn identity."}), + )) + .expect_err("reject a provider turn identity retained before restart"); + assert!(error.to_string().contains("reused a settled")); + + let persisted: Value = serde_json::from_slice( + &fs::read(directory.join("codex-provider-state.json")) + .expect("read fail-closed provider state"), + ) + .expect("parse fail-closed provider state"); + assert_eq!(persisted["lifecycle"], "closed"); + assert!(persisted["activeProviderTurnId"].is_null()); + + drop(recovered); + let resumes_before_closed_restore = call_count(&directory, "thread/resume"); + let mut closed = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + let events = closed + .poll_events() + .expect("closed reused-identity state remains readable without resuming Codex"); + assert!(events.iter().any(|event| { + event.event_type == "harness.diagnostic" + && event.payload["code"] == "provider_turn_identity_reused" + })); + assert_eq!( + call_count(&directory, "thread/resume"), + resumes_before_closed_restore, + "recovery must not resume provider work accepted under a reused identity" + ); + + closed.shutdown().expect("close fail-closed executor"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn durable_recovery_closes_a_provider_that_reopens_a_settled_turn() { + let directory = temporary_directory("durable-settled-turn-active-on-recovery"); + let config = provider_config(&directory, &[]); + let runner_config = durable_config(&directory); + let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + first + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({"provider": config}), + )) + .expect("prepare Codex provider"); + first + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open Codex session"); + first + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Complete the durable turn."}), + )) + .expect("start provider turn"); + let mut completed = false; + for _ in 0..32 { + completed |= poll_and_ack(&mut first) + .expect("poll completed provider turn") + .iter() + .any(|event| event.event_type == "turn.completed"); + if completed { + break; + } + } + assert!(completed); + first.shutdown().expect("stop first provider process"); + drop(first); + + // Contradict the durable terminal ledger with a resumed provider snapshot + // that reports the exact settled identity as active again. + fs::write( + directory.join("fake-state.json"), + serde_json::to_vec_pretty(&json!({ + "threadId": "codex-thread-1", + "activeTurnId": "provider-turn-1", + })) + .unwrap(), + ) + .expect("write contradictory fake provider state"); + + let resumes_before_recovery = call_count(&directory, "thread/resume"); + let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + let events = recovered + .poll_events() + .expect("fail-closed recovery remains observable"); + assert!(events.iter().any(|event| { + event.event_type == "harness.diagnostic" + && event.payload["code"] == "provider_turn_identity_reused" + && event.payload["providerTurnId"] == "provider-turn-1" + })); + assert_eq!( + call_count(&directory, "thread/resume"), + resumes_before_recovery + 1, + ); + let persisted: Value = serde_json::from_slice( + &fs::read(directory.join("codex-provider-state.json")) + .expect("read fail-closed provider state"), + ) + .expect("parse fail-closed provider state"); + assert_eq!(persisted["lifecycle"], "closed"); + assert!(persisted["activeProviderTurnId"].is_null()); + assert_eq!(persisted["completedTurnAuthoritative"], false); + + drop(recovered); + let resumes_before_closed_restore = call_count(&directory, "thread/resume"); + let mut closed = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + closed + .poll_events() + .expect("closed recovery state remains readable"); + assert_eq!( + call_count(&directory, "thread/resume"), + resumes_before_closed_restore, + "closed recovery must not resume the contradictory provider turn again", + ); + + closed.shutdown().expect("close fail-closed executor"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn codex_rejects_a_two_turn_old_call_while_idle() { + let directory = temporary_directory("delayed-tool-after-second-turn-completion"); + let config = provider_config(&directory, &["--delayed-tool-after-second-turn-completion"]); + let mut provider = CodexProvider::start_with_tools(&config, [task_context_tool()], None) + .expect("start Codex with an authorized tool"); + + for message in ["Complete turn one.", "Complete turn two."] { + provider + .start_turn(message, &config.cwd) + .expect("start completed provider turn"); + wait_for_notification(&mut provider, "turn/completed"); + } + assert_eq!(provider.active_provider_turn_id(), None); + + let warning = wait_for_notification(&mut provider, "warning"); + assert_eq!(warning["providerMethod"], "item/tool/call"); + assert_eq!(provider.active_provider_turn_id(), None); + + let _ = provider.shutdown(); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn durable_backend_delivers_a_terminal_before_rejecting_a_delayed_tool_call() { + let directory = temporary_directory("durable-delayed-tool-after-terminal"); + let config = provider_config(&directory, &["--delayed-tool-after-failed-turn"]); + let runner_config = durable_config(&directory); + let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + executor + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({ + "provider": config, + "authorizedTools": task_context_tool_set(), + "completionContract": { + "revision": "sha256:delayed-tool-contract", + "criterionIds": ["criterion_delayed_tool"] + }, + }), + )) + .unwrap(); + executor + .execute(&command("open", 2, "session.open", json!({}))) + .unwrap(); + executor + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Fail before invoking a delayed tool."}), + )) + .unwrap(); + + let mut observed = Vec::new(); + for _ in 0..32 { + observed.extend(poll_and_ack(&mut executor).expect("poll durable provider events")); + if observed + .iter() + .any(|event| event.event_type == "provider.notice.recorded") + { + break; + } + } + let terminal = observed + .iter() + .position(|event| event.event_type == "run.terminal") + .expect("the durable terminal is delivered"); + let rejection = observed + .iter() + .position(|event| event.event_type == "provider.notice.recorded") + .expect("the delayed request is rejected non-fatally"); + assert!(terminal < rejection); + + executor.shutdown().unwrap(); + fs::remove_dir_all(directory).unwrap(); +} + +#[test] +fn codex_rejects_a_runtime_response_after_its_turn_terminates() { + let directory = temporary_directory("delayed-question-response"); + let config = provider_config(&directory, &["--question-before-failed-turn"]); + let mut provider = CodexProvider::start(&config, None).expect("start Codex provider"); + provider + .start_turn("Ask and then fail.", &config.cwd) + .expect("start provider turn"); + + let mut request_id = None; + let mut terminal_seen = false; + for _ in 0..16 { + match provider.poll().expect("poll question and terminal") { + Some(CodexProviderEvent::RuntimeRequest { + request_id: observed, + .. + }) => request_id = Some(observed), + Some(CodexProviderEvent::Notification { method, .. }) if method == "turn/completed" => { + terminal_seen = true; + } + _ => {} + } + if request_id.is_some() && terminal_seen { + break; + } + } + let request_id = request_id.expect("observe the runtime request before termination"); + assert!(terminal_seen); + let error = provider + .resolve_runtime_request( + &request_id, + &json!({ + "schema": "paperclip.question_response.v1", + "answers": {"environment": {"selectedOptionIds": ["option-1"]}} + }), + ) + .expect_err("terminal requests must not remain resolvable"); + assert!(error.to_string().contains("no pending Codex request")); + + let _ = provider.shutdown(); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn reused_provider_question_ids_get_unique_controller_identities() { + let directory = temporary_directory("reused-question-id"); + let config = provider_config(&directory, &["--emit-question", "--reuse-question-id"]); + let mut provider = CodexProvider::start(&config, None).expect("start Codex provider"); + provider + .start_turn("Ask twice with one provider request id.", &config.cwd) + .expect("start provider turn"); + + let first_request_id = (0..16) + .find_map(|_| match provider.poll().expect("poll first question") { + Some(CodexProviderEvent::RuntimeRequest { request_id, .. }) => Some(request_id), + _ => None, + }) + .expect("observe first runtime request"); + let response = json!({ + "schema": "paperclip.question_response.v1", + "answers": {"environment": {"selectedOptionIds": ["option-1"]}} + }); + provider + .resolve_runtime_request(&first_request_id, &response) + .expect("resolve first runtime request"); + let second_request_id = (0..16) + .find_map(|_| match provider.poll().expect("poll second question") { + Some(CodexProviderEvent::RuntimeRequest { request_id, .. }) => Some(request_id), + _ => None, + }) + .expect("observe second runtime request"); + + assert_ne!(first_request_id, second_request_id); + let stale = provider + .resolve_runtime_request(&first_request_id, &response) + .expect_err("the first controller identity cannot resolve the second question"); + assert!(stale.to_string().contains("no pending Codex request")); + provider + .resolve_runtime_request(&second_request_id, &response) + .expect("resolve second runtime request"); + + let _ = provider.shutdown(); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + #[test] fn codex_resume_advertises_the_same_authorized_tools() { let directory = temporary_directory("dynamic-tool-resume"); @@ -1490,6 +2464,535 @@ fn codex_resume_advertises_the_same_authorized_tools() { fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); } +#[test] +fn durable_backend_routes_a_semantic_tool_result_back_to_codex() { + let directory = temporary_directory("durable-dynamic-tool"); + let config = provider_config(&directory, &["--require-dynamic-tool", "--emit-tool-call"]); + let runner_config = durable_config(&directory); + let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + executor + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({ + "provider": config, + "authorizedTools": task_context_tool_set(), + }), + )) + .expect("prepare the durable Codex tool set"); + executor + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open the Codex session"); + executor + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Inspect the durable fake task."}), + )) + .expect("start the Codex turn"); + + let mut semantic_input = None; + for _ in 0..32 { + let events = poll_and_ack(&mut executor).expect("poll semantic input"); + semantic_input = events + .iter() + .find(|event| event.event_type == "semantic_tool.input") + .cloned() + .or(semantic_input); + if semantic_input.is_some() { + break; + } + } + let semantic_input = semantic_input.expect("durable semantic input is emitted"); + assert_eq!( + semantic_input.payload["semantic_tool"]["correlation"]["runId"], + "run-1" + ); + assert_eq!( + semantic_input.payload["semantic_tool"]["operationId"], + "get_task_context" + ); + + let delivered = executor + .execute(&command( + "tool-result", + 4, + "semantic_tool.result", + json!({ + "callId": "semantic-call-1", + "operationId": "get_task_context", + "result": {"ok": true, "task": {"id": "task-1"}}, + "isError": false, + }), + )) + .expect("deliver the durable semantic result"); + assert_eq!(delivered.result["status"], "delivered"); + + let mut result_seen = false; + let mut terminal_seen = false; + for _ in 0..32 { + let events = poll_and_ack(&mut executor).expect("poll result and completion"); + result_seen |= events + .iter() + .any(|event| event.event_type == "semantic_tool.result"); + terminal_seen |= events + .iter() + .any(|event| event.event_type == "turn.completed"); + if result_seen && terminal_seen { + break; + } + } + assert!(result_seen); + assert!(terminal_seen); + executor.shutdown().expect("stop provider"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn durable_backend_replays_completed_results_without_mutating_the_event_queue() { + let directory = temporary_directory("durable-completed-tool-replay"); + let config = provider_config( + &directory, + &[ + "--require-dynamic-tool", + "--emit-tool-call", + "--hold-turn", + "--replay-completed-tool-call-count", + "4", + ], + ); + let runner_config = durable_config(&directory); + let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + executor + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({ + "provider": config, + "authorizedTools": task_context_tool_set(), + }), + )) + .expect("prepare the durable Codex tool set"); + executor + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open the Codex session"); + executor + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Replay the completed fake tool call."}), + )) + .expect("start the Codex turn"); + + let mut input_seen = false; + for _ in 0..32 { + input_seen |= poll_and_ack(&mut executor) + .expect("poll semantic input") + .iter() + .any(|event| event.event_type == "semantic_tool.input"); + if input_seen { + break; + } + } + assert!(input_seen, "durable semantic input is emitted"); + + executor + .execute(&command( + "tool-result", + 4, + "semantic_tool.result", + json!({ + "callId": "semantic-call-1", + "operationId": "get_task_context", + "result": {"ok": true, "task": {"id": "task-1"}}, + "isError": false, + }), + )) + .expect("deliver the original durable semantic result"); + let result_events = poll_and_ack(&mut executor).expect("acknowledge semantic result"); + assert_eq!( + result_events + .iter() + .filter(|event| event.event_type == "semantic_tool.result") + .count(), + 1 + ); + + let state_path = directory.join("codex-provider-state.json"); + let state_before_replays = fs::read(&state_path).expect("read state before exact replays"); + let expected_response = r#"{"ok":true,"task":{"id":"task-1"}}"#; + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while recorded_tool_responses(&directory).len() < 5 && std::time::Instant::now() < deadline { + let events = poll_and_ack(&mut executor).expect("service completed tool replay"); + assert!( + events.iter().all(|event| { + event.event_type != "semantic_tool.input" + && event.event_type != "semantic_tool.reconciled" + && event.event_type != "semantic_tool.result" + }), + "an exact completed replay must not add semantic reconciliation events" + ); + std::thread::sleep(std::time::Duration::from_millis(1)); + } + + let responses = recorded_tool_responses(&directory); + assert_eq!( + responses.len(), + 5, + "the original result and four replays return" + ); + assert!(responses + .iter() + .all(|response| response == expected_response)); + assert_eq!( + fs::read(&state_path).expect("read state after exact replays"), + state_before_replays, + "exact completed replays must not consume durable event capacity" + ); + + executor.shutdown().expect("stop provider"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn durable_backend_replays_pending_tool_calls_without_mutating_the_event_queue() { + let directory = temporary_directory("durable-tool-recovery"); + let config = provider_config( + &directory, + &[ + "--require-dynamic-tool", + "--emit-tool-call", + "--emit-tool-call-on-resume", + ], + ); + let runner_config = durable_config(&directory); + let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + first + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({ + "provider": config, + "authorizedTools": task_context_tool_set(), + }), + )) + .expect("prepare the recoverable tool bridge"); + first + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open the first provider"); + first + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Hold the tool call across recovery."}), + )) + .expect("start the first turn"); + let mut input_seen = false; + for _ in 0..32 { + input_seen |= poll_and_ack(&mut first) + .expect("poll first tool input") + .iter() + .any(|event| event.event_type == "semantic_tool.input"); + if input_seen { + break; + } + } + assert!(input_seen); + drop(first); + + let state_path = directory.join("codex-provider-state.json"); + let mut recovered = None; + for replay in 0..4 { + let before: Value = serde_json::from_slice( + &fs::read(&state_path).expect("read state before provider recovery"), + ) + .expect("parse state before provider recovery"); + let resume_count = call_count(&directory, "thread/resume"); + let mut next = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + for _ in 0..32 { + let events = poll_and_ack(&mut next).expect("poll exact pending replay"); + assert!(events.iter().all(|event| { + event.event_type != "semantic_tool.input" + && event.event_type != "semantic_tool.reconciled" + && event.event_type != "semantic_tool.result" + })); + if call_count(&directory, "thread/resume") > resume_count { + break; + } + } + assert!( + call_count(&directory, "thread/resume") > resume_count, + "recovery {replay} resumes the active provider turn" + ); + for _ in 0..4 { + let events = poll_and_ack(&mut next).expect("poll exact pending replay"); + assert!(events.iter().all(|event| { + event.event_type != "semantic_tool.input" + && event.event_type != "semantic_tool.reconciled" + && event.event_type != "semantic_tool.result" + })); + } + let after: Value = serde_json::from_slice( + &fs::read(&state_path).expect("read state after pending replay"), + ) + .expect("parse state after pending replay"); + assert_eq!( + after["pendingEvents"], before["pendingEvents"], + "exact pending replay {replay} must not append pending events" + ); + assert_eq!( + after["queuedEvents"], before["queuedEvents"], + "exact pending replay {replay} must not append queued events" + ); + assert_eq!( + after["nextProviderEventSeq"], before["nextProviderEventSeq"], + "exact pending replay {replay} must not consume durable event capacity" + ); + recovered = Some(next); + if replay < 3 { + drop(recovered.take()); + } + } + let mut recovered = recovered.expect("retain the final recovered provider"); + recovered + .execute(&command( + "tool-result", + 4, + "semantic_tool.result", + json!({ + "callId": "semantic-call-1", + "operationId": "get_task_context", + "result": {"ok": true, "task": {"id": "task-1"}}, + "isError": false, + }), + )) + .expect("complete the replayed tool call"); + let events = poll_and_ack(&mut recovered).expect("poll the completed pending call"); + assert_eq!( + events + .iter() + .filter(|event| event.event_type == "semantic_tool.result") + .count(), + 1 + ); + recovered.shutdown().expect("stop recovered provider"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn durable_backend_settles_pending_tools_when_recovery_finds_the_turn_ended() { + let directory = temporary_directory("durable-tool-ended-offline"); + let config = provider_config(&directory, &["--require-dynamic-tool", "--emit-tool-call"]); + let runner_config = durable_config(&directory); + let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + first + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({ + "provider": config, + "authorizedTools": task_context_tool_set(), + "completionContract": { + "revision": "sha256:offline-recovery-contract", + "criterionIds": ["criterion_offline_recovery"] + }, + }), + )) + .expect("prepare the recoverable tool bridge"); + first + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open the first provider"); + first + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "End while the runner is offline."}), + )) + .expect("start the first turn"); + let mut input_seen = false; + for _ in 0..32 { + input_seen |= poll_and_ack(&mut first) + .expect("poll first tool input") + .iter() + .any(|event| event.event_type == "semantic_tool.input"); + if input_seen { + break; + } + } + assert!(input_seen); + drop(first); + + fs::write( + directory.join("fake-state.json"), + serde_json::to_vec_pretty(&json!({ + "threadId": "codex-thread-1", + "activeTurnId": null, + })) + .unwrap(), + ) + .expect("record that the provider turn ended while offline"); + + let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + recovered + .execute(&command("snapshot", 4, "session.snapshot", json!({}))) + .expect("restore the provider session"); + let mut observed = Vec::new(); + for _ in 0..32 { + observed.extend( + poll_and_ack(&mut recovered) + .expect("poll recovered settlement") + .into_iter() + .map(|event| event.event_type), + ); + if observed.iter().any(|event| event == "run.terminal") { + break; + } + } + let semantic_result = observed + .iter() + .position(|event| event == "semantic_tool.result") + .expect("recovery settles the pending semantic tool"); + let reconciled = observed + .iter() + .position(|event| event == "session.reconciled") + .expect("recovery emits a reconciliation event"); + let terminal = observed + .iter() + .position(|event| event == "run.terminal") + .expect("offline turn recovery terminates the run"); + assert!(semantic_result < reconciled); + assert!(reconciled < terminal); + assert!(recovered + .execute(&command( + "late-result", + 5, + "semantic_tool.result", + json!({ + "callId": "semantic-call-1", + "operationId": "get_task_context", + "result": {"ok": true}, + "isError": false, + }), + )) + .is_err()); + + recovered.shutdown().expect("stop recovered provider"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn durable_backend_rejects_tool_catalog_drift_during_attach() { + let directory = temporary_directory("durable-tool-attach-drift"); + let config = provider_config(&directory, &[]); + let runner_config = durable_config(&directory); + let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + executor + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({ + "provider": config, + "authorizedTools": task_context_tool_set(), + }), + )) + .expect("prepare the durable tool catalog"); + + let mut changed = task_context_tool_set(); + changed.operations[0].description = "Changed after recovery.".to_owned(); + changed.catalog_digest = authorized_tool_catalog_digest(&changed.operations).unwrap(); + let error = executor + .execute(&command( + "attach", + 2, + "run.attach", + json!({"authorizedTools": changed}), + )) + .expect_err("attach must reject tool catalog drift"); + assert!(error.to_string().contains("tool contract changed")); + + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn durable_backend_settles_tools_before_a_natural_terminal_event() { + let directory = temporary_directory("durable-tool-terminal"); + let config = provider_config( + &directory, + &[ + "--require-dynamic-tool", + "--emit-tool-call", + "--finish-turn-with-pending-tool", + ], + ); + let runner_config = durable_config(&directory); + let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + executor + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({ + "provider": config, + "authorizedTools": task_context_tool_set(), + }), + )) + .unwrap(); + executor + .execute(&command("open", 2, "session.open", json!({}))) + .unwrap(); + executor + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Terminate with a pending tool."}), + )) + .unwrap(); + + let mut observed = Vec::new(); + for _ in 0..32 { + let events = poll_and_ack(&mut executor).unwrap(); + observed.extend(events.into_iter().map(|event| event.event_type)); + if observed.iter().any(|event| event == "turn.completed") { + break; + } + } + let semantic_result = observed + .iter() + .position(|event| event == "semantic_tool.result") + .expect("terminal settlement emits a failed semantic result"); + let terminal = observed + .iter() + .position(|event| event == "turn.completed") + .expect("provider terminal event is emitted"); + assert!(semantic_result < terminal); + assert!(executor + .execute(&command( + "late-result", + 4, + "semantic_tool.result", + json!({ + "callId": "semantic-call-1", + "operationId": "get_task_context", + "result": {"ok": true}, + "isError": false, + }), + )) + .is_err()); + + executor.shutdown().unwrap(); + fs::remove_dir_all(directory).unwrap(); +} + #[test] fn durable_backend_resumes_the_active_thread_without_restarting_the_turn() { let directory = temporary_directory("resume"); @@ -1548,6 +3051,256 @@ fn durable_backend_resumes_the_active_thread_without_restarting_the_turn() { fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); } +#[test] +fn legacy_full_active_epoch_is_closed_on_recovery() { + let directory = temporary_directory("legacy-full-active-recovery"); + let config = provider_config(&directory, &["--hold-turn"]); + let mut first = CodexCommandExecutor::new(&directory); + first + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({"provider": config}), + )) + .expect("prepare Codex provider"); + first + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open Codex session"); + first + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Hold the legacy full-epoch turn."}), + )) + .expect("start held provider turn"); + first.shutdown().expect("stop first provider process"); + drop(first); + + let state_path = directory.join("codex-provider-state.json"); + let mut persisted: Value = + serde_json::from_slice(&fs::read(&state_path).expect("read active provider state")) + .expect("parse active provider state"); + let prior_generation = persisted["providerProcessGeneration"] + .as_u64() + .expect("provider generation is persisted"); + persisted["settledProviderTurnIds"] = Value::Array( + (0..4_096) + .map(|index| Value::String(format!("legacy-provider-turn-{index}"))) + .collect(), + ); + fs::write(&state_path, serde_json::to_vec_pretty(&persisted).unwrap()) + .expect("write legacy full active state"); + + let mut recovered = CodexCommandExecutor::new(&directory); + let events = recovered + .poll_events() + .expect("legacy full-epoch recovery remains observable"); + assert!(events.iter().any(|event| { + event.event_type == "harness.diagnostic" + && event.payload["code"] == "legacy_provider_turn_epoch_ambiguous" + })); + let closed: Value = + serde_json::from_slice(&fs::read(&state_path).expect("read closed legacy full state")) + .expect("parse closed legacy full state"); + assert_eq!(closed["lifecycle"], "closed"); + assert!(closed["activeProviderTurnId"].is_null()); + assert_eq!(closed["ambiguousTurnStartPending"], false); + assert_eq!(closed["completedTurnAuthoritative"], false); + assert!(closed["settledProviderTurnFilter"]["words"] + .as_array() + .expect("legacy filter shape remains valid") + .is_empty()); + assert_eq!( + closed["settledProviderTurnIds"].as_array().unwrap().len(), + 4_096 + ); + assert!(closed["providerProcessGeneration"].as_u64().unwrap() > prior_generation); + assert!(recovered + .execute(&command( + "replacement-turn", + 4, + "turn.start", + json!({"text": "Do not reopen closed legacy work."}), + )) + .expect_err("closed legacy full-epoch state rejects replacement work") + .to_string() + .contains("closed")); + assert_eq!(call_count(&directory, "turn/start"), 1); + + recovered.shutdown().expect("close recovered executor"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn legacy_filtered_ambiguous_epoch_is_closed_on_recovery() { + let directory = temporary_directory("legacy-filtered-ambiguous-recovery"); + let config = provider_config(&directory, &[]); + let mut first = CodexCommandExecutor::new(&directory); + first + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({"provider": config}), + )) + .expect("prepare Codex provider"); + first + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open Codex session"); + first.shutdown().expect("stop first provider process"); + drop(first); + + let state_path = directory.join("codex-provider-state.json"); + let mut persisted: Value = + serde_json::from_slice(&fs::read(&state_path).expect("read idle provider state")) + .expect("parse idle provider state"); + let mut legacy_words = vec![0_u64; 32_768]; + legacy_words[0] = 1; + persisted["settledProviderTurnFilter"] = json!({"words": legacy_words}); + persisted["ambiguousTurnStartPending"] = Value::Bool(true); + fs::write(&state_path, serde_json::to_vec_pretty(&persisted).unwrap()) + .expect("write legacy-filtered ambiguous state"); + + let mut recovered = CodexCommandExecutor::new(&directory); + let events = recovered + .poll_events() + .expect("legacy ambiguous recovery remains observable"); + let diagnostic = events + .iter() + .find(|event| { + event.event_type == "harness.diagnostic" + && event.payload["code"] == "legacy_provider_turn_epoch_ambiguous" + }) + .expect("legacy ambiguous recovery emits a terminal diagnostic"); + assert_eq!(diagnostic.payload["ambiguousStartPending"], true); + assert_eq!(diagnostic.payload["providerReportedActive"], false); + let closed: Value = + serde_json::from_slice(&fs::read(&state_path).expect("read closed legacy ambiguous state")) + .expect("parse closed legacy ambiguous state"); + assert_eq!(closed["lifecycle"], "closed"); + assert!(closed["activeProviderTurnId"].is_null()); + assert_eq!(closed["ambiguousTurnStartPending"], false); + assert!(!closed["settledProviderTurnFilter"]["words"] + .as_array() + .expect("legacy filter remains durable evidence") + .is_empty()); + assert_eq!(call_count(&directory, "turn/start"), 0); + + recovered.shutdown().expect("close recovered executor"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn legacy_full_ended_epoch_reconciles_before_idle_rollover() { + let directory = temporary_directory("legacy-full-ended-recovery"); + let config = provider_config(&directory, &["--hold-turn"]); + let mut first = CodexCommandExecutor::new(&directory); + first + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({"provider": config}), + )) + .expect("prepare Codex provider"); + first + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open Codex session"); + first + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Let this legacy turn end while runnerd is away."}), + )) + .expect("start held provider turn"); + first.shutdown().expect("stop first provider process"); + drop(first); + + let state_path = directory.join("codex-provider-state.json"); + let mut persisted: Value = + serde_json::from_slice(&fs::read(&state_path).expect("read active provider state")) + .expect("parse active provider state"); + persisted["settledProviderTurnIds"] = Value::Array( + (0..4_096) + .map(|index| Value::String(format!("legacy-provider-turn-{index}"))) + .collect(), + ); + fs::write(&state_path, serde_json::to_vec_pretty(&persisted).unwrap()) + .expect("write full legacy provider epoch"); + fs::write( + directory.join("fake-state.json"), + serde_json::to_vec_pretty(&json!({"threadId": "codex-thread-1"})).unwrap(), + ) + .expect("mark the provider turn ended while runnerd was away"); + + let mut recovered = CodexCommandExecutor::new(&directory); + let events = recovered + .poll_events() + .expect("reconcile an ended turn from a full legacy epoch"); + assert!(events.iter().any(|event| event.event_type == "turn.failed")); + let reconciled: Value = + serde_json::from_slice(&fs::read(&state_path).expect("read reconciled legacy state")) + .expect("parse reconciled legacy state"); + assert_eq!(reconciled["lifecycle"], "session_open"); + assert!(reconciled["activeProviderTurnId"].is_null()); + assert_eq!( + reconciled["settledProviderTurnIds"] + .as_array() + .unwrap() + .len(), + 4_096 + ); + + recovered + .execute(&command( + "replacement-turn", + 4, + "turn.start", + json!({"text": "Start only after the idle epoch rolls over."}), + )) + .expect("roll over the idle full epoch and start replacement work"); + let rolled: Value = + serde_json::from_slice(&fs::read(&state_path).expect("read rolled legacy state")) + .expect("parse rolled legacy state"); + assert!(rolled["settledProviderTurnIds"] + .as_array() + .expect("rolled exact identities remain an array") + .is_empty()); + assert!(rolled["settledProviderTurnFilter"]["words"] + .as_array() + .expect("rolled filter remains valid") + .is_empty()); + assert_eq!(call_count(&directory, "turn/start"), 2); + + recovered.shutdown().expect("stop rolled provider process"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn codex_resume_rejects_an_oversized_active_turn_identity() { + let directory = temporary_directory("resume-oversized-active-turn-id"); + let config = provider_config(&directory, &[]); + fs::write( + directory.join("fake-state.json"), + serde_json::to_vec_pretty(&json!({ + "threadId": "codex-thread-1", + "activeTurnId": "x".repeat(241), + })) + .unwrap(), + ) + .expect("write fake provider state with an oversized active identity"); + + let error = CodexProvider::start(&config, Some("codex-thread-1")) + .err() + .expect("reject an oversized recovered provider turn identity"); + assert!(error.to_string().contains("invalid turn.id")); + + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + #[test] fn provider_exit_preserves_and_reconciles_the_active_turn() { let directory = temporary_directory("exit-active-turn"); @@ -1615,6 +3368,498 @@ fn provider_exit_preserves_and_reconciles_the_active_turn() { fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); } +#[test] +fn receipt_limit_rejects_the_call_and_keeps_polling_when_interrupt_fails() { + let directory = temporary_directory("receipt-limit-interrupt-failure"); + let config = provider_config( + &directory, + &[ + "--require-dynamic-tool", + "--hold-turn", + "--emit-tool-call-on-resume", + "--fail-first-interrupt", + "--accept-interrupt-without-terminal-once", + ], + ); + let runner_config = durable_config(&directory); + let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + first + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({ + "provider": config, + "authorizedTools": task_context_tool_set(), + }), + )) + .expect("prepare Codex provider"); + first + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open Codex session"); + first + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Hold this saturated turn for recovery."}), + )) + .expect("start held provider turn"); + drop(first); + + saturate_provider_tool_receipts(&directory); + + let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + let events = poll_and_ack(&mut recovered) + .expect("a failed first interrupt must not terminate durable provider polling"); + assert!(events.iter().any(|event| { + event.event_type == "harness.diagnostic" + && event.payload["code"] == "semantic_tool_turn_receipt_limit" + })); + assert_eq!(call_count(&directory, "tool-response:failure"), 1); + assert_eq!(call_count(&directory, "turn/interrupt"), 1); + + let mut settled = Vec::new(); + for _ in 0..4 { + settled.extend( + poll_and_ack(&mut recovered) + .expect("polling must autonomously retry the durable receipt-limit interrupt"), + ); + if settled + .iter() + .any(|event| event.event_type == "turn.interrupted") + { + break; + } + } + assert!( + settled + .iter() + .any(|event| event.event_type == "turn.interrupted"), + "the retry must settle the receipt-exhausted turn" + ); + assert_eq!(call_count(&directory, "turn/interrupt"), 3); + + recovered.shutdown().expect("stop recovered provider"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn receipt_limit_retry_preserves_a_turn_settled_during_provider_recovery() { + let directory = temporary_directory("receipt-limit-recovered-settlement"); + let config = provider_config( + &directory, + &[ + "--require-dynamic-tool", + "--hold-turn", + "--emit-tool-call-on-resume", + "--fail-first-interrupt", + ], + ); + let runner_config = durable_config(&directory); + let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + first + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({ + "provider": config, + "authorizedTools": task_context_tool_set(), + }), + )) + .expect("prepare Codex provider"); + first + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open Codex session"); + first + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Recover a provider-settled receipt-limited turn."}), + )) + .expect("start held provider turn"); + drop(first); + + saturate_provider_tool_receipts(&directory); + + let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + let events = poll_and_ack(&mut recovered) + .expect("the first failed interruption must leave a durable retry"); + assert!(events.iter().any(|event| { + event.event_type == "harness.diagnostic" + && event.payload["code"] == "semantic_tool_turn_receipt_limit" + })); + assert_eq!(call_count(&directory, "turn/interrupt"), 1); + + // Lose the live transport while retaining its durable interruption retry, + // then model the provider settling the turn before runnerd reconnects. + recovered + .shutdown() + .expect("stop the provider before retry recovery"); + let fake_state_path = directory.join("fake-state.json"); + let mut fake_state: Value = + serde_json::from_slice(&fs::read(&fake_state_path).expect("read fake provider state")) + .expect("parse fake provider state"); + fake_state["activeTurnId"] = Value::Null; + fs::write( + &fake_state_path, + serde_json::to_vec_pretty(&fake_state).unwrap(), + ) + .expect("settle the fake provider turn before recovery"); + + let events = poll_and_ack(&mut recovered) + .expect("an already-settled retry must preserve the recovered terminal state"); + assert!(events.iter().any(|event| { + event.event_type == "session.reconciled" && event.payload["activeProviderTurnId"].is_null() + })); + assert_eq!( + call_count(&directory, "turn/interrupt"), + 1, + "an already-settled recovered turn must not receive another interrupt RPC" + ); + + let persisted: Value = serde_json::from_slice( + &fs::read(directory.join("codex-provider-state.json")) + .expect("read reconciled provider state"), + ) + .expect("parse reconciled provider state"); + assert!(persisted["activeProviderTurnId"].is_null()); + assert_eq!(persisted["receiptLimitInterruptPending"], false); + assert_eq!(persisted["receiptLimitInterruptAccepted"], false); + assert_eq!(persisted["receiptLimitInterruptAttempts"], 0); + assert!(persisted["receiptLimitInterruptDeadlineUnixMs"].is_null()); + + recovered.shutdown().expect("stop recovered provider"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn receipt_limit_accepts_a_terminal_after_the_initial_interrupt_deadline() { + let directory = temporary_directory("receipt-limit-delayed-terminal"); + let config = provider_config( + &directory, + &[ + "--require-dynamic-tool", + "--hold-turn", + "--emit-tool-call-on-resume", + "--interrupt-terminal-delay-ms", + "2100", + ], + ); + let runner_config = durable_config(&directory); + let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + first + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({ + "provider": config, + "authorizedTools": task_context_tool_set(), + }), + )) + .expect("prepare Codex provider"); + first + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open Codex session"); + first + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Accept a delayed authoritative interruption."}), + )) + .expect("start held provider turn"); + drop(first); + saturate_provider_tool_receipts(&directory); + + let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + let mut emitted = Vec::new(); + for _ in 0..4_096 { + emitted.extend( + poll_and_ack(&mut recovered) + .expect("a delayed accepted interrupt must remain durably pollable"), + ); + if emitted + .iter() + .any(|event| event.event_type == "turn.interrupted") + { + break; + } + } + assert!( + emitted + .iter() + .any(|event| event.event_type == "turn.interrupted"), + "the delayed provider terminal must remain authoritative after two seconds" + ); + assert!(!emitted.iter().any(|event| { + event.event_type == "turn.interrupted" + && event.payload["code"] == "semantic_tool_turn_receipt_limit_interrupt_deadline" + })); + assert!( + !emitted.iter().any(|event| { + event.event_type == "turn.failed" + && event.payload["code"] == "semantic_tool_turn_receipt_limit_interrupt_unconfirmed" + }), + "fast provider polls must not replace a delayed terminal with fallback failure" + ); + + recovered.shutdown().expect("stop recovered provider"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn receipt_limit_polls_an_authoritative_terminal_with_unacknowledged_events() { + let directory = temporary_directory("receipt-limit-terminal-with-unacked-events"); + let config = provider_config( + &directory, + &[ + "--require-dynamic-tool", + "--hold-turn", + "--emit-tool-call-on-resume", + "--accept-interrupt-without-terminal-once", + ], + ); + let runner_config = durable_config(&directory); + let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + first + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({ + "provider": config, + "authorizedTools": task_context_tool_set(), + }), + )) + .expect("prepare Codex provider"); + first + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open Codex session"); + first + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Retain the receipt-limit diagnostic until terminal polling."}), + )) + .expect("start held provider turn"); + drop(first); + saturate_provider_tool_receipts(&directory); + + let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + let pending = recovered + .poll_events() + .expect("begin the durable receipt-limit stop"); + assert!(pending.iter().any(|event| { + event.event_type == "harness.diagnostic" + && event.payload["code"] == "semantic_tool_turn_receipt_limit" + })); + assert!(!pending + .iter() + .any(|event| event.event_type == "turn.interrupted")); + + let terminal = recovered + .poll_events() + .expect("poll the provider terminal before old events are acknowledged"); + assert!(terminal.iter().any(|event| { + event.event_type == "turn.interrupted" && event.payload.get("code").is_none() + })); + assert!(!terminal.iter().any(|event| { + event.event_type == "turn.interrupted" + && event.payload["code"] == "semantic_tool_turn_receipt_limit_interrupt_deadline" + })); + recovered + .acknowledge_events(terminal.len()) + .expect("acknowledge the diagnostic and authoritative terminal together"); + + recovered.shutdown().expect("stop recovered provider"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn receipt_limit_polling_bounds_and_rejects_runtime_request_floods() { + let directory = temporary_directory("receipt-limit-runtime-request-flood"); + let config = provider_config( + &directory, + &[ + "--require-dynamic-tool", + "--hold-turn", + "--emit-tool-call-on-resume", + "--accept-interrupt-without-terminal", + "--flood-runtime-requests-on-interrupt", + ], + ); + let runner_config = durable_config(&directory); + let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + first + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({ + "provider": config, + "authorizedTools": task_context_tool_set(), + }), + )) + .expect("prepare Codex provider"); + first + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open Codex session"); + first + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Bound runtime requests while stopping this turn."}), + )) + .expect("start held provider turn"); + drop(first); + saturate_provider_tool_receipts(&directory); + + let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + let mut rejected_at_capacity = false; + while !rejected_at_capacity && std::time::Instant::now() < deadline { + let events = recovered + .poll_events() + .expect("runtime-request cleanup remains bounded across repeated polls"); + assert!(events.len() <= 128); + rejected_at_capacity = events.iter().any(|event| { + event.event_type == "provider.notice.recorded" + && event.payload["summary"] + == "rejected a Codex runtime request at the bounded pending-input limit" + }); + recovered + .acknowledge_events(events.len()) + .expect("advance the bounded durable event prefix"); + if events.is_empty() { + std::thread::sleep(std::time::Duration::from_millis(1)); + } + } + assert!( + rejected_at_capacity, + "production records that requests above the pending count/byte envelope were rejected" + ); + + recovered.shutdown().expect("stop recovered provider"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + +#[test] +fn receipt_limit_synthesizes_interrupted_after_an_accepted_terminal_deadline() { + let directory = temporary_directory("receipt-limit-missing-terminal"); + let config = provider_config( + &directory, + &[ + "--require-dynamic-tool", + "--hold-turn", + "--emit-tool-call-on-resume", + "--accept-interrupt-without-terminal", + ], + ); + let runner_config = durable_config(&directory); + let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + first + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({ + "provider": config, + "authorizedTools": task_context_tool_set(), + }), + )) + .expect("prepare Codex provider"); + first + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open Codex session"); + first + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Bound a missing interrupt terminal."}), + )) + .expect("start held provider turn"); + drop(first); + saturate_provider_tool_receipts(&directory); + + let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + let mut emitted = Vec::new(); + for _ in 0..8 { + emitted.extend( + poll_and_ack(&mut recovered) + .expect("receipt-limit fallback must remain durably pollable"), + ); + if call_count(&directory, "turn/interrupt") == 3 { + break; + } + } + assert!(!emitted + .iter() + .any(|event| event.event_type == "turn.failed")); + recovered + .shutdown() + .expect("pause the provider before expiring its durable deadline"); + drop(recovered); + + let state_path = directory.join("codex-provider-state.json"); + let mut persisted: Value = + serde_json::from_slice(&fs::read(&state_path).expect("read pending receipt-limit state")) + .expect("parse pending receipt-limit state"); + persisted["receiptLimitInterruptDeadlineUnixMs"] = json!(1); + fs::write(&state_path, serde_json::to_vec_pretty(&persisted).unwrap()) + .expect("expire the durable receipt-limit deadline"); + + let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + for _ in 0..4 { + emitted.extend( + poll_and_ack(&mut recovered) + .expect("expired receipt-limit fallback must remain durably pollable"), + ); + if emitted + .iter() + .any(|event| event.event_type == "turn.interrupted") + { + break; + } + } + let interrupted = emitted + .iter() + .find(|event| event.event_type == "turn.interrupted") + .expect("the bounded accepted path must synthesize an interrupted turn"); + assert_eq!( + interrupted.payload["code"], + "semantic_tool_turn_receipt_limit_interrupt_deadline" + ); + assert_eq!(interrupted.payload["interruptAccepted"], true); + assert_eq!(interrupted.payload["providerTerminalObserved"], false); + assert!(!emitted + .iter() + .any(|event| event.event_type == "turn.failed")); + assert_eq!(call_count(&directory, "turn/interrupt"), 3); + + let persisted: Value = serde_json::from_slice( + &fs::read(directory.join("codex-provider-state.json")) + .expect("read bounded receipt-limit state"), + ) + .expect("parse bounded receipt-limit state"); + assert_eq!(persisted["lifecycle"], "provider_exited"); + assert!(persisted["activeProviderTurnId"].is_null()); + assert_eq!(persisted["receiptLimitInterruptPending"], false); + assert_eq!(persisted["receiptLimitInterruptAttempts"], 0); + assert!(persisted["receiptLimitInterruptDeadlineUnixMs"].is_null()); + + recovered + .shutdown() + .expect("bounded fallback stopped provider"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + #[test] fn unacknowledged_provider_events_survive_executor_restart() { let directory = temporary_directory("pending-event-recovery"); @@ -1672,6 +3917,70 @@ fn unacknowledged_provider_events_survive_executor_restart() { fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); } +#[test] +fn durable_backend_rejects_a_runtime_response_after_terminal_settlement() { + let directory = temporary_directory("durable-delayed-question-response"); + let config = provider_config(&directory, &["--question-before-failed-turn"]); + let mut executor = CodexCommandExecutor::new(&directory); + executor + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({"provider": config}), + )) + .expect("prepare provider"); + executor + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open provider session"); + executor + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Ask and then fail."}), + )) + .expect("start provider turn"); + + let mut request_id = None; + let mut terminal_seen = false; + for _ in 0..16 { + for event in poll_and_ack(&mut executor).expect("poll question and terminal") { + if event.event_type == "runtime_request.created" { + request_id = event + .payload + .pointer("/request/requestId") + .and_then(Value::as_str) + .map(str::to_owned); + } + terminal_seen |= event.event_type == "turn.failed"; + } + if request_id.is_some() && terminal_seen { + break; + } + } + let request_id = request_id.expect("observe the durable runtime request id"); + assert!(terminal_seen); + let error = executor + .execute(&command( + "resolve", + 4, + "request.resolve", + json!({ + "requestId": request_id, + "response": { + "schema": "paperclip.question_response.v1", + "answers": {"environment": {"selectedOptionIds": ["option-1"]}} + } + }), + )) + .expect_err("terminal runtime requests must fail closed"); + assert!(error.to_string().contains("outside an active turn")); + + executor.shutdown().expect("stop provider process"); + fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); +} + #[test] fn structured_question_round_trips_through_the_normalized_backend() { let directory = temporary_directory("questions"); @@ -1700,6 +4009,7 @@ fn structured_question_round_trips_through_the_normalized_backend() { assert_eq!(started.events[0].0, "turn.accepted"); let mut question_set = None; + let mut request_id = None; let mut provider_started_events = 0; for _ in 0..16 { for event in poll_and_ack(&mut executor).expect("poll question") { @@ -1710,6 +4020,11 @@ fn structured_question_round_trips_through_the_normalized_backend() { "paperclip.runtime_request.v2" ); question_set = event.payload.pointer("/request/input").cloned(); + request_id = event + .payload + .pointer("/request/requestId") + .and_then(Value::as_str) + .map(str::to_owned); } } if question_set.is_some() { @@ -1717,6 +4032,7 @@ fn structured_question_round_trips_through_the_normalized_backend() { } } let question_set = question_set.expect("normalized question set is emitted"); + let request_id = request_id.expect("normalized request id is emitted"); assert_eq!(provider_started_events, 1); assert_eq!(question_set["schema"], "paperclip.question_set.v1"); assert_eq!( @@ -1730,7 +4046,7 @@ fn structured_question_round_trips_through_the_normalized_backend() { 4, "request.resolve", json!({ - "requestId": "runtime-request-1", + "requestId": request_id, "response": { "schema": "paperclip.question_response.v1", "answers": {"environment": {"selectedOptionIds": ["option-1"]}} diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/provider_bridge.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/provider_bridge.rs index 17303b3bf9..2504f383ed 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/provider_bridge.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/provider_bridge.rs @@ -73,6 +73,13 @@ fn rejects_unknown_tools_and_conflicting_duplicate_results() { }; bridge.apply_result(result.clone()).unwrap(); bridge.apply_result(result).unwrap(); + assert!(bridge + .replay_result("call-1", "get_task_context", &json!({})) + .unwrap() + .is_some()); + assert!(bridge + .replay_result("call-1", "get_task_context", &json!({"changed": true})) + .is_err()); assert!(bridge .apply_result(ToolResult { call_id: "call-1".to_owned(), @@ -108,6 +115,29 @@ fn catalog_digest_matches_the_typescript_canonical_json_contract() { ); } +#[test] +fn catalog_digest_normalizes_json_numbers_like_javascript() { + let operation = AuthorizedTool { + operation_id: "get_task_context".into(), + version: 1, + description: "Read the active task context.".into(), + input_schema: json!({ + "type": "object", + "properties": { + "limit": { "type": "number", "default": 1.0 }, + "epsilon": { "type": "number", "default": 1e-6 }, + }, + }), + response_schema: json!({ "type": "object" }), + }; + let digest = authorized_tool_catalog_digest(&[operation]).unwrap(); + + assert_eq!( + digest, + "sha256:1c93693d9b5b48b46c83cd1c11d1ea329774f1b9b0ae741197cb2b8e992c4b8d" + ); +} + #[test] fn validates_the_operation_value_inside_a_semantic_dispatch_envelope() { let mut set = tools("sha256:catalog-a"); @@ -120,7 +150,7 @@ fn validates_the_operation_value_inside_a_semantic_dispatch_envelope() { let mut bridge = ProviderToolBridge::default(); set.catalog_digest = digest('a'); set.catalog_digest = authorized_tool_catalog_digest(&set.operations).unwrap(); - bridge.prepare(set).unwrap(); + bridge.prepare(set.clone()).unwrap(); bridge .begin_call("call-1".into(), "get_task_context".into(), json!({})) .unwrap(); @@ -139,6 +169,25 @@ fn validates_the_operation_value_inside_a_semantic_dispatch_envelope() { }) .unwrap(); assert_eq!(bridge.pending_calls().count(), 0); + + let mut second = ProviderToolBridge::default(); + second.prepare(set).unwrap(); + second + .begin_call("call-2".into(), "get_task_context".into(), json!({})) + .unwrap(); + second + .apply_result(ToolResult { + call_id: "call-2".into(), + operation_id: "get_task_context".into(), + result: json!({ + "ok": true, + "operationId": "get_task_context", + "callId": "call-2", + "value": { "value": "accepted" } + }), + is_error: false, + }) + .unwrap(); } #[test] @@ -159,6 +208,24 @@ fn rejects_noncanonical_digests_and_oversized_contract_values() { json!({ "value": "x".repeat(1024 * 1024) }), ) .is_err()); + + let retained = json!({"value": "x".repeat(700 * 1024)}); + for index in 0..5 { + bridge + .begin_call( + format!("call-{index}"), + "get_task_context".into(), + retained.clone(), + ) + .unwrap(); + } + assert!(bridge + .begin_call( + "call-over-aggregate-limit".into(), + "get_task_context".into(), + retained, + ) + .is_err()); } #[test] @@ -203,12 +270,214 @@ fn recovery_preserves_completed_call_replay_identities() { let encoded = serde_json::to_string(&bridge).unwrap(); let mut recovered: ProviderToolBridge = serde_json::from_str(&encoded).unwrap(); + recovered.validate_recovered().unwrap(); recovered.attach_existing_run().unwrap(); assert!(recovered .begin_call("call-1".into(), "get_task_context".into(), json!({})) .is_err()); } +#[test] +fn recovered_bridge_rejects_tampered_authorization_state() { + let mut bridge = ProviderToolBridge::default(); + bridge.prepare(tools("computed")).unwrap(); + let mut encoded = serde_json::to_value(&bridge).unwrap(); + encoded["authorized"]["get_task_context"]["description"] = json!("Tampered"); + let recovered: ProviderToolBridge = serde_json::from_value(encoded).unwrap(); + assert!(recovered.validate_recovered().is_err()); +} + +#[test] +fn cancellation_completes_pending_calls_and_rejects_late_results() { + let mut bridge = ProviderToolBridge::default(); + bridge.prepare(tools("computed")).unwrap(); + bridge + .begin_call("call-1".into(), "get_task_context".into(), json!({})) + .unwrap(); + let cancelled = bridge + .cancel_pending_calls("provider_turn_stopped") + .unwrap(); + assert_eq!(cancelled.len(), 1); + assert!(cancelled[0].is_error); + assert_eq!(bridge.pending_calls().count(), 0); + assert!(bridge + .apply_result(ToolResult { + call_id: "call-1".into(), + operation_id: "get_task_context".into(), + result: json!({"ok": true}), + is_error: false, + }) + .is_err()); +} + +#[test] +fn turn_settlement_releases_value_capacity_without_reusing_call_ids() { + let mut bridge = ProviderToolBridge::default(); + bridge.prepare(tools("computed")).unwrap(); + bridge + .begin_call("call-1".into(), "get_task_context".into(), json!({})) + .unwrap(); + bridge + .apply_result(ToolResult { + call_id: "call-1".into(), + operation_id: "get_task_context".into(), + result: json!({"ok": true}), + is_error: false, + }) + .unwrap(); + + assert!(bridge + .settle_turn("provider_turn_terminated") + .unwrap() + .is_empty()); + assert!(bridge + .begin_call("call-1".into(), "get_task_context".into(), json!({})) + .is_err()); + bridge + .begin_call("call-2".into(), "get_task_context".into(), json!({})) + .expect("a new turn can use a fresh call id after releasing exact values"); +} + +#[test] +fn completed_receipts_are_exact_until_the_controlled_turn_limit() { + let mut bridge = ProviderToolBridge::default(); + bridge.prepare(tools("computed")).unwrap(); + for index in 0..4_096 { + let call_id = format!("call-{index}"); + bridge + .begin_call(call_id.clone(), "get_task_context".into(), json!({})) + .expect("a completed call must release concurrent capacity"); + bridge + .apply_result(ToolResult { + call_id, + operation_id: "get_task_context".into(), + result: json!({"ok": true}), + is_error: false, + }) + .unwrap(); + } + + let error = bridge + .begin_call( + "call-after-limit".into(), + "get_task_context".into(), + json!({}), + ) + .expect_err("the bounded exact receipt ledger must stop the active turn"); + assert!(error.is_active_turn_receipt_limit()); + assert!(bridge + .replay_result("call-0", "get_task_context", &json!({})) + .unwrap() + .is_some()); + assert!(bridge + .replay_result("call-4095", "get_task_context", &json!({})) + .unwrap() + .is_some()); + + let recovered: ProviderToolBridge = + serde_json::from_str(&serde_json::to_string(&bridge).unwrap()).unwrap(); + recovered.validate_recovered().unwrap(); +} + +#[test] +fn turn_settlement_cannot_be_blocked_by_completed_value_pressure() { + let mut bridge = ProviderToolBridge::default(); + bridge.prepare(tools("computed")).unwrap(); + let large = json!({"value": "x".repeat(700 * 1024)}); + for index in 0..5 { + bridge + .begin_call( + format!("call-{index}"), + "get_task_context".into(), + large.clone(), + ) + .unwrap(); + } + for index in 0..4 { + bridge + .apply_result(ToolResult { + call_id: format!("call-{index}"), + operation_id: "get_task_context".into(), + result: large.clone(), + is_error: false, + }) + .unwrap(); + } + + let settled = bridge.settle_turn("provider_turn_terminated").unwrap(); + assert_eq!(settled.len(), 1); + assert!(settled[0].is_error); + assert!(bridge + .begin_call("call-0".into(), "get_task_context".into(), json!({})) + .is_err()); + bridge + .begin_call( + "call-after-settlement".into(), + "get_task_context".into(), + json!({}), + ) + .expect("settlement releases prior turn value retention"); +} + +#[test] +fn recovered_turn_preserves_exact_results_at_the_value_boundary() { + let mut bridge = ProviderToolBridge::default(); + bridge.prepare(tools("computed")).unwrap(); + let large = json!({"value": "x".repeat(700 * 1024)}); + for index in 0..5 { + bridge + .begin_call( + format!("call-{index}"), + "get_task_context".into(), + large.clone(), + ) + .unwrap(); + } + for index in 0..5 { + bridge + .apply_result(ToolResult { + call_id: format!("call-{index}"), + operation_id: "get_task_context".into(), + result: large.clone(), + is_error: false, + }) + .unwrap(); + } + + let error = bridge + .begin_call("call-next".into(), "get_task_context".into(), large.clone()) + .expect_err("exact replay values must not be discarded for later work"); + assert!(error.is_active_turn_receipt_limit()); + assert_eq!( + bridge + .replay_result("call-0", "get_task_context", &large) + .unwrap() + .unwrap() + .result, + large, + ); + bridge + .apply_result(ToolResult { + call_id: "call-0".into(), + operation_id: "get_task_context".into(), + result: large.clone(), + is_error: false, + }) + .expect("a matching exact result receipt remains idempotent"); + assert!(bridge + .apply_result(ToolResult { + call_id: "call-0".into(), + operation_id: "get_task_context".into(), + result: json!({"value": "changed"}), + is_error: false, + }) + .is_err()); + + let recovered: ProviderToolBridge = + serde_json::from_str(&serde_json::to_string(&bridge).unwrap()).unwrap(); + recovered.validate_recovered().unwrap(); +} + #[test] fn recovery_preserves_pending_calls_for_the_existing_run() { let mut bridge = ProviderToolBridge::default(); @@ -252,7 +521,7 @@ fn recovery_rejects_nonempty_state_without_a_catalog_digest() { let error = recovered .attach_existing_run() .expect_err("nonempty recovered state must remain bound to a catalog digest"); - assert!(error.to_string().contains("omit the catalog digest")); + assert!(error.to_string().contains("omitted its catalog identity")); } #[test] @@ -269,8 +538,8 @@ fn recovery_rejects_tampered_authorization_catalog_bindings() { let mut recovered: ProviderToolBridge = serde_json::from_value(changed_contract).unwrap(); let error = recovered .attach_existing_run() - .expect_err("recovery must recompute the catalog digest"); - assert!(error.to_string().contains("catalog digest")); + .expect_err("recovery must reconstruct the authorized catalog projection"); + assert!(error.to_string().contains("changed its authorized catalog")); let mut changed_map_key = encoded; let authorized = changed_map_key["authorized"].as_object_mut().unwrap(); @@ -280,7 +549,7 @@ fn recovery_rejects_tampered_authorization_catalog_bindings() { let error = recovered .attach_existing_run() .expect_err("recovery must bind map keys to declared operation identities"); - assert!(error.to_string().contains("identities are inconsistent")); + assert!(error.to_string().contains("changed its authorized catalog")); } #[test] @@ -326,18 +595,21 @@ fn recovery_rejects_tampered_retained_result_contracts() { let completed = serde_json::to_value(&bridge).unwrap(); let mut unauthorized = completed.clone(); - unauthorized["completed"]["call-1"]["operationId"] = json!("delete_company"); - let mut recovered: ProviderToolBridge = serde_json::from_value(unauthorized).unwrap(); - assert!(recovered.attach_existing_run().is_err()); + unauthorized["completed"]["call-1"]["result"]["operationId"] = json!("delete_company"); + let error = serde_json::from_value::(unauthorized) + .expect_err("durable decoding must reject mismatched call and result identities"); + assert!(error + .to_string() + .contains("retained provider tool receipt identity is inconsistent")); let mut invalid_output = completed; - invalid_output["completed"]["call-1"]["result"] = json!(["not", "an", "object"]); + invalid_output["completed"]["call-1"]["result"]["result"] = json!(["not", "an", "object"]); let mut recovered: ProviderToolBridge = serde_json::from_value(invalid_output).unwrap(); assert!(recovered.attach_existing_run().is_err()); - bridge.settle_turn().unwrap(); + bridge.settle_turn("provider_turn_terminated").unwrap(); let mut invalid_settled_output = serde_json::to_value(&bridge).unwrap(); - invalid_settled_output["settledResults"]["call-1"]["result"] = json!("invalid"); + invalid_settled_output["settledResults"]["call-1"]["result"]["result"] = json!("invalid"); let mut recovered: ProviderToolBridge = serde_json::from_value(invalid_settled_output).unwrap(); assert!(recovered.attach_existing_run().is_err()); } @@ -389,7 +661,7 @@ fn settles_completed_receipts_before_the_next_turn() { assert!(bridge .begin_call("call-next".into(), "get_task_context".into(), json!({})) .is_err()); - bridge.settle_turn().unwrap(); + bridge.settle_turn("provider_turn_terminated").unwrap(); assert!(bridge .begin_call("call-next".into(), "get_task_context".into(), json!({})) .is_ok()); @@ -410,7 +682,7 @@ fn settlement_preserves_call_ids_for_the_durable_run() { is_error: false, }) .unwrap(); - bridge.settle_turn().unwrap(); + bridge.settle_turn("provider_turn_terminated").unwrap(); let replay = ToolResult { call_id: "call-1".into(), @@ -456,7 +728,7 @@ fn settlement_preserves_call_ids_for_the_durable_run() { } #[test] -fn reserves_identity_capacity_before_accepting_a_call() { +fn exact_identity_overflow_saturates_the_durable_run() { let mut bridge = ProviderToolBridge::default(); bridge.prepare(tools("computed")).unwrap(); @@ -472,27 +744,147 @@ fn reserves_identity_capacity_before_accepting_a_call() { bridge .begin_call("last-call".into(), "get_task_context".into(), json!({})) .unwrap(); - bridge - .apply_result(ToolResult { - call_id: "last-call".into(), - operation_id: "get_task_context".into(), - result: json!({"ok": true}), - is_error: false, - }) - .unwrap(); - bridge.settle_turn().unwrap(); - assert!(bridge + let overflow = bridge + .begin_call("overflow".into(), "get_task_context".into(), json!({})) + .expect_err("the pending call reserves the final exact identity slot"); + assert!(overflow.is_active_turn_receipt_limit()); + + let encoded = serde_json::to_string(&bridge).unwrap(); + let mut recovered: ProviderToolBridge = serde_json::from_str(&encoded).unwrap(); + recovered.attach_existing_run().unwrap(); + assert!(recovered.durable_run_receipt_limit_reached()); + recovered + .settle_turn("provider_turn_terminated") + .expect("the controlled turn stop retains replay protection"); + assert!(recovered.durable_run_receipt_limit_reached()); + assert!(recovered.has_completed_call("settled-0")); + assert!(recovered.has_completed_call("last-call")); + let stopped_turn_receipt = recovered + .replay_result("last-call", "get_task_context", &json!({})) + .unwrap() + .expect("the call admitted before exhaustion retains an exact terminal receipt"); + assert!(stopped_turn_receipt.is_error); + assert_eq!( + stopped_turn_receipt.result["error"]["code"], + "provider_turn_terminated" + ); + assert!(recovered + .begin_call("settled-0".into(), "get_task_context".into(), json!({})) + .is_err()); + assert!(recovered .begin_call("overflow".into(), "get_task_context".into(), json!({})) .is_err()); - assert!(bridge.settle_turn().is_ok()); + recovered.prepare_turn().unwrap(); + assert!(recovered + .replay_result("last-call", "get_task_context", &json!({})) + .unwrap() + .is_none()); + assert!(recovered.has_completed_call("last-call")); + assert!(recovered.has_completed_call("settled-0")); + assert!(recovered.durable_run_receipt_limit_reached()); + let saturation = recovered + .begin_call("next-call".into(), "get_task_context".into(), json!({})) + .expect_err("a turn boundary must not reopen the saturated durable run"); + assert!(saturation.is_active_turn_receipt_limit()); + + let encoded = serde_json::to_string(&recovered).unwrap(); + let mut recovered: ProviderToolBridge = serde_json::from_str(&encoded).unwrap(); + recovered.attach_existing_run().unwrap(); + assert!(recovered.has_completed_call("settled-0")); + assert!(recovered.has_completed_call("last-call")); + assert!(recovered.durable_run_receipt_limit_reached()); + + recovered.attach_run(tools("computed")).unwrap(); + assert!(!recovered.durable_run_receipt_limit_reached()); + recovered + .begin_call("overflow".into(), "get_task_context".into(), json!({})) + .expect("a new durable run receives a fresh tool-call identity ledger"); +} + +#[test] +fn settled_result_byte_exhaustion_recovers_after_turn_cleanup() { + let mut bridge = ProviderToolBridge::default(); + bridge.prepare(tools("computed")).unwrap(); + let large_result = json!({"value": "x".repeat(750 * 1024)}); + + for index in 0..10 { + let call_id = format!("large-settled-{index}"); + bridge + .begin_call(call_id.clone(), "get_task_context".into(), json!({})) + .unwrap(); + bridge + .apply_result(ToolResult { + call_id, + operation_id: "get_task_context".into(), + result: large_result.clone(), + is_error: false, + }) + .unwrap(); + } + bridge.settle_turn("provider_turn_terminated").unwrap(); + + let error = bridge + .begin_call( + "over-byte-limit".into(), + "get_task_context".into(), + json!({}), + ) + .expect_err("settled byte exhaustion must stop the durable run"); + assert!(error.is_active_turn_receipt_limit()); + + let encoded = serde_json::to_string(&bridge).unwrap(); + let mut recovered: ProviderToolBridge = serde_json::from_str(&encoded).unwrap(); + recovered.attach_existing_run().unwrap(); + assert!(recovered.durable_run_receipt_limit_reached()); + assert_eq!( + recovered + .replay_result("large-settled-0", "get_task_context", &json!({})) + .unwrap() + .unwrap() + .result, + large_result + ); + assert!(recovered + .begin_call( + "over-byte-limit".into(), + "get_task_context".into(), + json!({}) + ) + .is_err()); + recovered.prepare_turn().unwrap(); + assert!(recovered + .replay_result("large-settled-0", "get_task_context", &json!({})) + .unwrap() + .is_none()); + assert!(recovered.has_completed_call("large-settled-0")); + assert!(!recovered.durable_run_receipt_limit_reached()); + recovered + .begin_call( + "after-turn-boundary".into(), + "get_task_context".into(), + json!({}), + ) + .expect("releasing bulky results clears transient byte pressure"); + recovered + .settle_turn("provider_turn_terminated") + .expect("the admitted next-turn call remains settleable"); + + recovered.attach_run(tools("computed")).unwrap(); + recovered + .begin_call( + "over-byte-limit".into(), + "get_task_context".into(), + json!({}), + ) + .expect("a new durable run resets the settled result budget"); } #[test] fn reserves_settled_result_bytes_before_accepting_a_call() { let mut bridge = ProviderToolBridge::default(); bridge.prepare(tools("computed")).unwrap(); - let large_result = json!({"value": "x".repeat(900 * 1024)}); + let large_result = json!({"value": "x".repeat(700 * 1024)}); let mut completed = 0; for index in 0..20 { @@ -523,7 +915,7 @@ fn reserves_settled_result_bytes_before_accepting_a_call() { ) .is_err()); bridge - .settle_turn() + .settle_turn("provider_turn_terminated") .expect("settlement cannot strand results whose bytes were reserved at admission"); assert_eq!( bridge @@ -534,7 +926,7 @@ fn reserves_settled_result_bytes_before_accepting_a_call() { is_error: false, }) .unwrap(), - json!({"value": "x".repeat(900 * 1024)}) + json!({"value": "x".repeat(700 * 1024)}) ); } @@ -544,15 +936,22 @@ fn recovery_rejects_an_oversized_settled_result_envelope() { bridge.prepare(tools("computed")).unwrap(); let mut encoded = serde_json::to_value(&bridge).unwrap(); let settled = encoded["settledResults"].as_object_mut().unwrap(); - for index in 0..10 { + for index in 0..12 { let call_id = format!("recovered-large-{index}"); settled.insert( call_id.clone(), json!({ - "callId": call_id, - "operationId": "get_task_context", - "result": {"value": "x".repeat(900 * 1024)}, - "isError": false + "call": { + "callId": call_id, + "operationId": "get_task_context", + "input": {} + }, + "result": { + "callId": call_id, + "operationId": "get_task_context", + "result": {"value": "x".repeat(700 * 1024)}, + "isError": false + } }), ); } @@ -568,18 +967,28 @@ fn recovery_rejects_state_without_room_for_a_pending_result() { bridge.prepare(tools("computed")).unwrap(); let mut encoded = serde_json::to_value(&bridge).unwrap(); let settled = encoded["settledResults"].as_object_mut().unwrap(); - for index in 0..8 { + let mut settled_ids = Vec::new(); + for index in 0..11 { let call_id = format!("recovered-large-{index}"); + settled_ids.push(json!(call_id)); settled.insert( call_id.clone(), json!({ - "callId": call_id, - "operationId": "get_task_context", - "result": {"value": "x".repeat(900 * 1024)}, - "isError": false + "call": { + "callId": call_id, + "operationId": "get_task_context", + "input": {} + }, + "result": { + "callId": call_id, + "operationId": "get_task_context", + "result": {"value": "x".repeat(700 * 1024)}, + "isError": false + } }), ); } + encoded["settledCallIds"] = serde_json::Value::Array(settled_ids); encoded["pending"]["pending-call"] = json!({ "callId": "pending-call", "operationId": "get_task_context", @@ -594,13 +1003,62 @@ fn recovery_rejects_state_without_room_for_a_pending_result() { } #[test] -fn refuses_to_settle_receipts_while_calls_are_pending() { +fn settles_pending_receipts_with_explicit_terminal_results() { let mut bridge = ProviderToolBridge::default(); bridge.prepare(tools("computed")).unwrap(); bridge .begin_call("call-1".into(), "get_task_context".into(), json!({})) .unwrap(); - assert!(bridge.settle_turn().is_err()); - assert_eq!(bridge.pending_calls().count(), 1); + let settled = bridge.settle_turn("provider_turn_terminated").unwrap(); + assert_eq!(settled.len(), 1); + assert_eq!(settled[0].call_id, "call-1"); + assert!(settled[0].is_error); + assert_eq!(bridge.pending_calls().count(), 0); +} + +#[test] +fn full_identity_ledger_fails_closed_across_turn_and_recovery() { + let mut bridge = ProviderToolBridge::default(); + bridge.prepare(tools("computed")).unwrap(); + let mut encoded = serde_json::to_value(&bridge).unwrap(); + encoded["settledCallIds"] = serde_json::Value::Array( + (0..65_536) + .map(|index| json!(format!("settled-{index:05}"))) + .collect(), + ); + let mut recovered: ProviderToolBridge = serde_json::from_value(encoded).unwrap(); + recovered.validate_recovered().unwrap(); + let saturation = recovered + .begin_call("current-call".into(), "get_task_context".into(), json!({})) + .expect_err("a full exact ledger must stop fresh work"); + assert!(saturation.is_active_turn_receipt_limit()); + assert!(recovered.durable_run_receipt_limit_reached()); + assert!(recovered.has_completed_call("settled-65535")); + assert!(recovered.has_completed_call("settled-00000")); + assert!(!recovered.has_completed_call("current-call")); + assert!(recovered + .begin_call("settled-00000".into(), "get_task_context".into(), json!({})) + .is_err()); + recovered.prepare_turn().unwrap(); + let fresh = recovered + .begin_call("fresh-call".into(), "get_task_context".into(), json!({})) + .expect_err("a turn boundary must preserve durable-run saturation"); + assert!(fresh.is_active_turn_receipt_limit()); + + let round_trip = serde_json::to_value(&recovered).unwrap(); + assert_eq!( + round_trip["settledCallIds"].as_array().unwrap().len(), + 65_536 + ); + let mut recovered_again: ProviderToolBridge = serde_json::from_value(round_trip).unwrap(); + recovered_again.attach_existing_run().unwrap(); + assert!(recovered_again.durable_run_receipt_limit_reached()); + assert!(recovered_again.has_completed_call("settled-00000")); + + recovered_again.attach_run(tools("computed")).unwrap(); + assert!(!recovered_again.durable_run_receipt_limit_reached()); + recovered_again + .begin_call("current-call".into(), "get_task_context".into(), json!({})) + .expect("only a new durable run resets replay authority"); }