From 6681104692d018b9ad76665a10509781deef3598 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Wed, 9 Sep 2026 08:53:07 -0500 Subject: [PATCH] fix(runner): preserve provider identity and terminal failures (#13074) ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - The runner translates provider events into durable task execution. > - Provider notifications can refer to another thread or a descendant. > - Root validation treated these notifications as fatal, and later layers could lose the original failure. > - This pull request classifies event identity and preserves structured terminal failures. > - The server can then choose safe recovery without weakening tool authority. ## Linked Issues or Issue Description Refs #13038. This change incorporates the shared protocol-integrity and bounded-cleanup prerequisites from that PR. It does not include the chat feature. Related failure classification work: #13028. **What happened?** An informational provider notification for another thread could terminate the root session. A failed stream could then become a missing-result error and lose its cause. **Expected behavior** Ignore unrelated informational notices with bounded diagnostics. Reject invalid authoritative events. Preserve the original failure code and recovery meaning through cleanup. **Steps to reproduce** Run a native Codex task. Deliver a notification for an unrelated thread, or close the stream after a structured failure. Inspect the root outcome and recorded failure. **Paperclip version or commit** Reproduced before e20010472. This branch includes the current session-goal contract from that commit. **Deployment mode** Built from source. Native runner with Codex. ## What Changed - Classify root, provider-confirmed descendant, stale, unrelated, and invalid provider events. - Keep tool requests bound to their original execution authority. - Preserve typed failures through transport, session, and durable control-plane cleanup. - Keep bounded cleanup failures separate from the primary execution failure. - Add compatible shared contracts for continuation context, execution status, and explicit reconciliation. The dependent PR adds their server and UI consumers. ## Verification - Runner TypeScript and ACPX suites: 1,733 passed, 7 skipped. Node contracts: 38 passed. - Real provider-process fixtures cover 300 descendant identities across restart, the 4,096-identity capacity boundary, and rejection of continuation after terminal acknowledgement and restart. - Repository build, typecheck, and full `pnpm test:run` passed on the rebased stack (18,448 tests passed, 49 skipped). The full Rust workspace passed with `--test-threads=1`; parallel execution exposed an existing fixture port-reservation race. All latest-head CI checks passed. One unchanged artifact-document concurrency test failed on the first CI run and passed on its single rerun. - Added notification, streaming failure, protocol integrity, cleanup quarantine, and durable failure tests. ## Risks Provider event classification must retain the new session-goal behavior on master. Descendant notifications must never gain root tool authority. Shared contract fields are additive. This PR does not migrate data or start replacement provider work. ## Model Used OpenAI GPT-6 through Codex. The exact deployment ID and context window were not exposed. Used reasoning, tool use, code execution, and browser automation. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- .../src/bin/fake-codex-app-server.rs | 40 +- .../crates/runner-core/src/codex_provider.rs | 403 ++++++++++- .../runner-core/src/provider_backend.rs | 139 +++- .../runner-core/tests/codex_provider.rs | 244 ++++++- .../backends/harness-driver-backend.test.ts | 116 ++++ .../src/backends/harness-driver-backend.ts | 62 +- .../src/contracts/native-session-backend.ts | 53 ++ .../durable-prp-control-plane.test.ts | 531 ++++++++++++++- .../durable-prp-control-plane.ts | 98 ++- .../codex/codex-app-server-driver-impl.ts | 5 + .../codex-app-server-driver.recovery.test.ts | 54 ++ .../drivers/codex/codex-harness-session.ts | 13 + .../codex/codex-notification-identity.test.ts | 61 ++ .../codex/codex-notification-identity.ts | 95 +++ .../codex/codex-protocol-integrity.test.ts | 641 ++++++++++++++++++ .../codex/codex-session-notifications.ts | 55 +- .../src/drivers/codex/codex-session-state.ts | 58 +- .../codex/codex-thread-normalization.ts | 1 + .../src/live/runnerd-codex-transport.test.ts | 259 +++++++ .../src/live/runnerd-codex-transport.ts | 91 ++- .../src/native-session-runtime.test.ts | 408 ++++++++++- .../src/native-session-runtime.ts | 137 +++- .../paperclip-runner/src/provider-events.ts | 58 +- packages/shared/src/index.ts | 4 + .../src/types/execution-continuation.ts | 52 ++ .../shared/src/types/execution-projection.ts | 58 ++ packages/shared/src/types/heartbeat.ts | 1 + packages/shared/src/types/issue.ts | 5 + packages/shared/src/validators/issue.ts | 6 + 29 files changed, 3605 insertions(+), 143 deletions(-) create mode 100644 packages/paperclip-runner/src/drivers/codex/codex-notification-identity.test.ts create mode 100644 packages/paperclip-runner/src/drivers/codex/codex-notification-identity.ts create mode 100644 packages/paperclip-runner/src/drivers/codex/codex-protocol-integrity.test.ts create mode 100644 packages/shared/src/types/execution-continuation.ts create mode 100644 packages/shared/src/types/execution-projection.ts diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs index ee0ca08b6e..b29643ce58 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs @@ -256,7 +256,7 @@ fn finish_turn(state_path: &Path, state: &mut FakeState, status: &str) -> io::Re .unwrap_or_else(|| "provider-turn-1".to_owned()); send(json!({ "method": "item/completed", - "params": {"item": { + "params": {"threadId": state.thread_id, "turnId": turn_id, "item": { "id": "message-1", "type": "agentMessage", "status": "completed", @@ -307,10 +307,10 @@ fn emit_ambiguous_turn_evidence( } } -fn emit_ambiguous_turn_item() -> io::Result<()> { +fn emit_ambiguous_turn_item(state: &FakeState) -> io::Result<()> { send(json!({ "method": "item/completed", - "params": {"item": { + "params": {"threadId": state.thread_id, "item": { "id": "replacement-message-before-terminal", "type": "agentMessage", "status": "completed", @@ -691,6 +691,9 @@ fn run() -> Result<(), Box> { .any(|value| value == "--question-before-failed-turn"); let fail_turn_immediately = args.iter().any(|value| value == "--fail-turn-immediately"); let reuse_question_id = args.iter().any(|value| value == "--reuse-question-id"); + let descendant_notifications = args + .iter() + .any(|value| value == "--descendant-notifications"); let pre_response_notification = args .iter() .any(|value| value == "--notification-before-response"); @@ -942,6 +945,12 @@ fn run() -> Result<(), Box> { "id": id, "result": {"thread": {"id": state.thread_id, "sessionId": "codex-account-session"}} }))?; + if descendant_notifications { + // Restoration must retain lineage without a replay of thread/started. + send(json!({"method": "turn/completed", "params": { + "threadId": "descendant-299", "turnId": "child-turn", "status": "completed" + }}))?; + } if emit_tool_call_on_resume { if let Some(turn_id) = state.active_turn_id.as_deref() { send(json!({ @@ -1175,7 +1184,7 @@ fn run() -> Result<(), Box> { } if malformed_error_second_turn_start && turn_start_count == 2 { if hold_ambiguous_second_turn_after_item { - emit_ambiguous_turn_item()?; + emit_ambiguous_turn_item(&state)?; } send(json!({"id": id, "error": {}}))?; if emits_ambiguous_turn_evidence @@ -1196,7 +1205,7 @@ fn run() -> Result<(), Box> { "result": {"turn": {"status": "inProgress"}} }))?; if hold_ambiguous_second_turn_after_item { - emit_ambiguous_turn_item()?; + emit_ambiguous_turn_item(&state)?; continue; } if emits_ambiguous_turn_evidence @@ -1226,6 +1235,23 @@ fn run() -> Result<(), Box> { "method": "turn/started", "params": {"turn": {"id": provider_turn_id}} }))?; + if descendant_notifications { + for index in 0..300 { + send(json!({"method": "thread/started", "params": {"thread": { + "id": format!("descendant-{index}"), + "source": {"subAgent": {"thread_spawn": {"parent_thread_id": state.thread_id}}} + }}}))?; + } + send(json!({"method": "turn/completed", "params": { + "threadId": "descendant-299", "turnId": "child-turn", "status": "completed" + }}))?; + } + if args.iter().any(|value| value == "--descendant-overflow") { + send(json!({"method": "thread/started", "params": {"thread": { + "id": "descendant-overflow", + "source": {"subAgent": {"thread_spawn": {"parent_thread_id": state.thread_id}}} + }}}))?; + } if fail_after_second_turn_start && turn_start_count == 2 { return Err("configured failure after second turn start".into()); } else if fail_turn_immediately { @@ -1392,11 +1418,11 @@ fn run() -> Result<(), Box> { }), json!({ "method": "rawResponseItem/completed", - "params": {"item": {"id": "raw-tail", "type": "reasoning"}} + "params": {"threadId": state.thread_id, "turnId": provider_turn_id, "item": {"id": "raw-tail", "type": "reasoning"}} }), json!({ "method": "rawResponse/completed", - "params": {"response": {"id": "response-tail"}} + "params": {"threadId": state.thread_id, "turnId": provider_turn_id, "response": {"id": "response-tail"}} }), json!({ "method": "thread/goal/updated", diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs index e04269d641..1ea4c769a8 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs @@ -54,6 +54,17 @@ const MAX_PENDING_RUNTIME_REQUESTS: usize = 128; const MAX_PENDING_RUNTIME_REQUEST_BYTES: usize = 4 * 1024 * 1024; const OPENCODE_RUNTIME_REQUEST_METHOD: &str = "paperclip/runtimeRequest"; pub(crate) const MAX_SETTLED_PROVIDER_TURN_IDS: usize = 4_096; +pub(crate) const MAX_DESCENDANT_THREAD_IDS: usize = 4_096; + +fn remember_descendant_thread(ids: &mut BTreeSet, id: &str) -> Result { + if ids.contains(id) { + return Ok(false); + } + if ids.len() >= MAX_DESCENDANT_THREAD_IDS { + return Err("provider_descendant_capacity_exhausted"); + } + Ok(ids.insert(id.to_owned())) +} type QuestionOptionLabels = BTreeMap>; type QuestionSetMapping = (String, Value, QuestionOptionLabels); @@ -383,6 +394,19 @@ pub enum CodexProviderEvent { method: String, params: Value, }, + /// Provider-confirmed child progress has no root terminal or tool authority. + DescendantNotification { + method: String, + params: Value, + }, + /// An invalid authoritative event must retain its failure meaning across PRP. + ProtocolFailure { + diagnostic: Value, + }, + /// A bounded provider resource was exhausted; this is not identity corruption. + ResourceLimit { + diagnostic: Value, + }, RuntimeRequest { request_id: String, question_set: Value, @@ -537,6 +561,8 @@ pub struct CodexProvider { goal_allows_autonomous_turns: bool, ambiguous_turn_start_pending: bool, settled_provider_turn_ids: SettledProviderTurnIds, + descendant_thread_ids: BTreeSet, + notification_identity_diagnostics: usize, rejected_accepted_turn: Option, quarantined: bool, trace: Option, @@ -798,6 +824,8 @@ impl CodexProvider { goal_allows_autonomous_turns: false, ambiguous_turn_start_pending: false, settled_provider_turn_ids: SettledProviderTurnIds::default(), + descendant_thread_ids: BTreeSet::new(), + notification_identity_diagnostics: 0, rejected_accepted_turn: None, quarantined: false, trace: ProviderTraceSink::from_environment(), @@ -1060,6 +1088,20 @@ impl CodexProvider { ); } } + Some(CodexProviderEvent::ResourceLimit { .. }) => { + return Err(LocalRunnerError::invalid("Codex descendant capacity requires reconciliation before fresh-session continuation")); + } + Some(CodexProviderEvent::ProtocolFailure { .. }) => { + return Err(LocalRunnerError::invalid( + "Codex protocol integrity failed during warm attachment", + )); + } + Some(CodexProviderEvent::DescendantNotification { .. }) => { + // Child output cannot certify a quiescent root attachment. + return Err(LocalRunnerError::invalid( + "Codex descendant remains active during warm run attachment", + )); + } Some(CodexProviderEvent::ToolCall { .. }) | Some(CodexProviderEvent::RuntimeRequest { .. }) => { return Err(LocalRunnerError::invalid( @@ -1154,6 +1196,13 @@ impl CodexProvider { Ok(()) } + pub(crate) fn restore_descendant_thread_identities(&mut self, identities: &BTreeSet) { + // Exact provider-confirmed lineage lives as long as the root session. + // Evicting it would turn later child progress into a root integrity fault. + self.descendant_thread_ids + .extend(identities.iter().cloned()); + } + pub(crate) fn restore_settled_turn_identities( &mut self, provider_turn_ids: impl IntoIterator, @@ -1856,9 +1905,11 @@ impl CodexProvider { if method == "item/tool/call" { let params = message.get("params").cloned().unwrap_or(Value::Null); if params.get("threadId").and_then(Value::as_str) != Some(self.thread_id.as_str()) { - return Err(LocalRunnerError::invalid( - "Codex tool call named another thread", - )); + return Ok(Some(self.identity_failure( + method, + ¶ms, + "thread_binding_mismatch", + ))); } if request_targets_non_active_turn( self.active_provider_turn_id.as_deref(), @@ -1871,9 +1922,11 @@ impl CodexProvider { LocalRunnerError::invalid("Codex tool call arrived outside an active turn") })?; if params.get("turnId").and_then(Value::as_str) != Some(active_turn_id) { - return Err(LocalRunnerError::invalid( - "Codex tool call named another turn", - )); + return Ok(Some(self.identity_failure( + method, + ¶ms, + "turn_binding_mismatch", + ))); } let call_id = bounded_identifier( params.get("callId").and_then(Value::as_str), @@ -1976,9 +2029,11 @@ impl CodexProvider { && params.get("threadId").and_then(Value::as_str) != Some(self.thread_id.as_str()) { - return Err(LocalRunnerError::invalid( - "Codex runtime request named another thread", - )); + return Ok(Some(self.identity_failure( + method, + ¶ms, + "thread_binding_mismatch", + ))); } if request_targets_non_active_turn( self.active_provider_turn_id.as_deref(), @@ -2088,30 +2143,110 @@ impl CodexProvider { if let Some(method) = message.get("method").and_then(Value::as_str) { let params = message.get("params").cloned().unwrap_or(Value::Null); + let identity = match classify_notification_thread( + method, + &self.thread_id, + &self.descendant_thread_ids, + ¶ms, + ) { + Ok(identity) => identity, + Err(_) => { + return Ok(Some(self.identity_failure( + method, + ¶ms, + "thread_binding_mismatch", + ))) + } + }; + if identity == NotificationThread::Descendant { + let id = + notification_thread_id(¶ms).expect("classified descendant has an identity"); + let newly_known = match remember_descendant_thread( + &mut self.descendant_thread_ids, + id, + ) { + Ok(newly_known) => newly_known, + Err(code) => { + return Ok(Some(CodexProviderEvent::ResourceLimit { + diagnostic: json!({ + "code": code, "recoverable": false, "classification": "resource_capacity", + "message": "Codex reached the child-thread inventory limit. Reconcile child work before continuing in a fresh provider session.", + "method": bounded_method(method), "limit": MAX_DESCENDANT_THREAD_IDS, + "expectedThreadId": self.thread_id, "receivedThreadId": id, + }), + })) + } + }; + // Retain each discovered child's effect inventory independently of + // the informational diagnostic budget, then bound repeated progress. + if !newly_known && self.notification_identity_diagnostics >= 32 { + return Ok(None); + } + self.notification_identity_diagnostics += 1; + // Descendant terminals are progress only. They never settle root authority. + return Ok(Some(CodexProviderEvent::DescendantNotification { + method: method.to_owned(), + params, + })); + } + if identity == NotificationThread::UnrelatedInformation { + self.notification_identity_diagnostics += 1; + if self.notification_identity_diagnostics > 32 { + return Ok(None); + } + return Ok(Some(CodexProviderEvent::Notification { + method: "warning".to_owned(), + params: json!({ + "threadId": self.thread_id, + "message": "ignored unrelated provider information", + "providerMethod": bounded_method(method), + "classification": "unrelated_information", + "expectedThreadId": self.thread_id, + "receivedThreadId": notification_thread_id(¶ms).map(|id| id.chars().take(256).collect::()), + "expectedTurnId": self.active_provider_turn_id, + "receivedTurnId": notification_turn_id(¶ms).map(|id| id.chars().take(256).collect::()), + }), + })); + } let terminal_event_type = normalized_codex_terminal_event_type(method, ¶ms); let notification_turn_id = params .get("turnId") .or_else(|| params.pointer("/turn/id")) .and_then(Value::as_str); - if terminal_event_type.is_some() - && notification_turn_id.is_some() + if notification_turn_id.is_some() && notification_turn_id != self.active_provider_turn_id.as_deref() && notification_turn_id .is_some_and(|turn_id| self.settled_provider_turn_ids.contains(turn_id)) { + self.notification_identity_diagnostics += 1; + if self.notification_identity_diagnostics > 32 { + return Ok(None); + } return Ok(Some(CodexProviderEvent::Notification { method: "warning".to_owned(), params: json!({ - "message": "ignored a terminal notification for a non-active Codex turn", - "providerMethod": bounded_method(method), + "message": "ignored a notification for a settled Codex turn", + "providerMethod": bounded_method(method), "classification": "stale_settled_turn", + "expectedThreadId": self.thread_id, + "receivedThreadId": self.thread_id, + "expectedTurnId": self.active_provider_turn_id, + "receivedTurnId": notification_turn_id.map(|id| id.chars().take(256).collect::()), }), })); } - validate_notification_binding( + if validate_notification_binding( &self.thread_id, self.active_provider_turn_id.as_deref(), ¶ms, - )?; + ) + .is_err() + { + return Ok(Some(self.identity_failure( + method, + ¶ms, + "turn_binding_mismatch", + ))); + } if let Some(terminal_event_type) = terminal_event_type { if self.active_provider_turn_id.is_none() { return Err(LocalRunnerError::invalid( @@ -2200,10 +2335,25 @@ impl CodexProvider { Ok(()) } + fn identity_failure(&self, method: &str, params: &Value, code: &str) -> CodexProviderEvent { + CodexProviderEvent::ProtocolFailure { + diagnostic: json!({ + "code": code, "recoverable": false, + "message": "Codex rejected an event outside the active execution identity", + "classification": "invalid_authoritative", "method": bounded_method(method), + "expectedThreadId": self.thread_id.chars().take(256).collect::(), + "receivedThreadId": notification_thread_id(params).map(|id| id.chars().take(256).collect::()), + "expectedTurnId": self.active_provider_turn_id.as_ref().map(|id| id.chars().take(256).collect::()), + "receivedTurnId": notification_turn_id(params).map(|id| id.chars().take(256).collect::()), + }), + } + } + pub fn shutdown(&mut self) -> Result<(), LocalRunnerError> { self.expected_shutdown = true; - self.cancel_pending_requests()?; - let result = self.process.terminate_group().map(|_| ()); + // A failed courtesy response must never prevent process fencing. + let cancellation = self.cancel_pending_requests(); + let result = self.process.terminate_group().map(|_| ()).and(cancellation); if let Some(trace) = self.trace.as_mut() { trace.finish(); } @@ -2695,6 +2845,119 @@ fn contains_provider_work_binding(value: &Value) -> bool { } } +#[derive(Debug, PartialEq)] +enum NotificationThread { + Root, + Descendant, + UnrelatedInformation, +} + +fn notification_thread_id(params: &Value) -> Option<&str> { + params + .get("threadId") + .or_else(|| params.pointer("/thread/id")) + .or_else(|| params.pointer("/turn/threadId")) + .and_then(Value::as_str) +} + +fn classify_notification_thread( + method: &str, + root: &str, + descendants: &BTreeSet, + params: &Value, +) -> Result { + let identities: Vec<&Value> = [ + params.get("threadId"), + params.pointer("/thread/id"), + params.pointer("/turn/threadId"), + ] + .into_iter() + .flatten() + .filter(|v| !v.is_null()) + .collect(); + if identities.iter().any(|id| { + id.as_str() + .is_none_or(|value| value.is_empty() || value.len() > 240) + }) || identities.windows(2).any(|ids| ids[0] != ids[1]) + { + return Err(LocalRunnerError::invalid( + "Codex notification has malformed thread identity", + )); + } + let turn_ids: Vec<&Value> = [params.get("turnId"), params.pointer("/turn/id")] + .into_iter() + .flatten() + .filter(|value| !value.is_null()) + .collect(); + if turn_ids.iter().any(|id| { + id.as_str() + .is_none_or(|value| value.is_empty() || value.len() > 240) + }) || turn_ids.windows(2).any(|ids| ids[0] != ids[1]) + { + return Err(LocalRunnerError::invalid( + "Codex notification has malformed turn identity", + )); + } + let thread = notification_thread_id(params); + if thread.is_none() + && turn_ids.is_empty() + && !matches!( + method, + "warning" + | "configWarning" + | "guardianWarning" + | "deprecationNotice" + | "remoteControl/status/changed" + | "mcpServer/startupStatus/updated" + | "account/rateLimits/updated" + ) + { + return Err(LocalRunnerError::invalid( + "Codex authoritative notification omitted thread identity", + )); + } + // Unbound transport warnings belong to this provider connection. Preserve + // their diagnostic meaning; only a different named thread is unrelated. + if thread.is_none() || thread == Some(root) { + return Ok(NotificationThread::Root); + } + let parent = [ + "/thread/source/subAgent/thread_spawn/parent_thread_id", + "/thread/source/subAgent/threadSpawn/parentThreadId", + "/thread/source/subagent/thread_spawn/parent_thread_id", + ] + .iter() + .find_map(|path| params.pointer(path).and_then(Value::as_str)); + if thread.is_some() + && (thread.is_some_and(|id| descendants.contains(id)) + || (method == "thread/started" + && parent.is_some_and(|id| id == root || descendants.contains(id)))) + { + if method.starts_with("paperclip/") { + return Err(LocalRunnerError::invalid( + "Codex descendant cannot supply root execution authority", + )); + } + return Ok(NotificationThread::Descendant); + } + if matches!( + method, + "thread/started" + | "thread/status/changed" + | "thread/closed" + | "thread/tokenUsage/updated" + | "warning" + | "configWarning" + | "guardianWarning" + | "deprecationNotice" + ) { + return Ok(NotificationThread::UnrelatedInformation); + } + Err(LocalRunnerError::invalid( + "Codex authoritative notification named another thread", + )) +} + fn validate_notification_binding( thread_id: &str, active_turn_id: Option<&str>, @@ -3682,3 +3945,109 @@ mod tests { assert_eq!(retain_buffered_message_bytes(usize::MAX, 1), None); } } + +#[cfg(test)] +mod notification_identity_tests { + use super::*; + #[test] + fn bounds_lineage_without_eviction_or_misclassifying_capacity_as_integrity() { + let mut ids = BTreeSet::new(); + for index in 0..MAX_DESCENDANT_THREAD_IDS { + assert_eq!( + remember_descendant_thread(&mut ids, &format!("child-{index}")), + Ok(true) + ); + } + assert_eq!(remember_descendant_thread(&mut ids, "child-0"), Ok(false)); + assert_eq!( + remember_descendant_thread(&mut ids, "overflow"), + Err("provider_descendant_capacity_exhausted") + ); + assert_eq!(ids.len(), MAX_DESCENDANT_THREAD_IDS); + assert!(ids.contains("child-0")); + assert!(!ids.contains("overflow")); + assert_eq!( + classify_notification_thread( + "turn/completed", + "root", + &ids, + &json!({"threadId":"child-0", "turnId":"child-turn"}) + ) + .unwrap(), + NotificationThread::Descendant + ); + } + #[test] + fn rejects_missing_authority_and_malformed_turn_identities() { + for method in [ + "item/started", + "item/completed", + "item/agentMessage/delta", + "thread/goal/updated", + "unknown/authority", + ] { + assert!( + classify_notification_thread(method, "root", &BTreeSet::new(), &json!({})).is_err() + ); + } + for params in [ + json!({"status":"completed"}), + json!({"threadId":"root","turnId":7}), + json!({"threadId":"root","turnId":"a","turn":{"id":"b"}}), + ] { + assert!(classify_notification_thread( + "turn/completed", + "root", + &BTreeSet::new(), + ¶ms + ) + .is_err()); + } + assert_eq!( + classify_notification_thread( + "configWarning", + "root", + &BTreeSet::new(), + &json!({"message":"warning"}) + ) + .unwrap(), + NotificationThread::Root + ); + } + #[test] + fn classifies_provider_lineage_before_root_authority() { + let children = BTreeSet::from(["child".to_owned()]); + assert_eq!( + classify_notification_thread( + "thread/status/changed", + "root", + &children, + &json!({"threadId":"unrelated"}) + ) + .unwrap(), + NotificationThread::UnrelatedInformation + ); + assert_eq!( + classify_notification_thread( + "turn/completed", + "root", + &children, + &json!({"threadId":"child", "turnId":"child-turn"}) + ) + .unwrap(), + NotificationThread::Descendant + ); + assert_eq!(classify_notification_thread("thread/started", "root", &children, &json!({"thread":{"id":"grandchild","source":{"subAgent":{"thread_spawn":{"parent_thread_id":"child"}}}}})).unwrap(), NotificationThread::Descendant); + for (method, params) in [ + ("paperclip/runResult", json!({"threadId":"child"})), + ("turn/completed", json!({"threadId":"unrelated"})), + ("item/started", json!({"threadId":42})), + ( + "item/started", + json!({"threadId":"root", "thread":{"id":"other"}}), + ), + ] { + assert!(classify_notification_thread(method, "root", &children, ¶ms).is_err()); + } + } +} diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs index 6e06fdedfc..4ede9394e6 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs @@ -797,6 +797,8 @@ struct CodexProviderState { #[serde(default)] settled_provider_turn_ids: std::collections::BTreeSet, #[serde(default)] + descendant_thread_ids: std::collections::BTreeSet, + #[serde(default)] settled_provider_turn_filter: DurableReplayFilter, #[serde(default)] receipt_limit_diagnostic_emitted: bool, @@ -882,6 +884,7 @@ impl CodexProviderState { completed_turn_process_generation: None, completed_provider_turn_id: None, settled_provider_turn_ids: std::collections::BTreeSet::new(), + descendant_thread_ids: std::collections::BTreeSet::new(), settled_provider_turn_filter: DurableReplayFilter::default(), receipt_limit_diagnostic_emitted: false, receipt_limit_interrupt_pending: false, @@ -908,10 +911,20 @@ impl CodexProviderState { DurableRunnerError::invalid(format!("Codex semantic tool state is invalid: {error}")) })?; let mut pending_event_ids = HashSet::new(); - if self.schema != PROVIDER_STATE_SCHEMA + if self.descendant_thread_ids.len() > crate::codex_provider::MAX_DESCENDANT_THREAD_IDS + || self + .descendant_thread_ids + .iter() + .any(|id| id.is_empty() || id.len() > 240) + || self.schema != PROVIDER_STATE_SCHEMA || !matches!( self.lifecycle.as_str(), - "prepared" | "session_open" | "turn_active" | "provider_exited" | "closed" + "prepared" + | "session_open" + | "turn_active" + | "provider_exited" + | "reconciliation_required" + | "closed" ) || self .thread_id @@ -1468,6 +1481,7 @@ impl CodexCommandExecutor { "failed to resume {provider_name} provider: {error}" )) })?; + provider.restore_descendant_thread_identities(&state.descendant_thread_ids); provider.enable_durable_tool_call_replays(); provider .restore_settled_turn_identities( @@ -1883,6 +1897,15 @@ impl CodexCommandExecutor { fn ensure_provider(&mut self) -> Result<&mut CodexProvider, DurableRunnerError> { self.restore_provider_if_needed()?; + if self + .state + .as_ref() + .is_some_and(|state| state.lifecycle == "reconciliation_required") + { + return Err(DurableRunnerError::invalid( + "Codex provider session requires explicit reconciliation and a fresh session", + )); + } if self.provider.is_none() { let state = self.state.as_ref().ok_or_else(|| { DurableRunnerError::invalid("Codex provider has not been prepared") @@ -1914,6 +1937,7 @@ impl CodexCommandExecutor { .map_err(|error| { DurableRunnerError::invalid(format!("failed to start Codex provider: {error}")) })?; + provider.restore_descendant_thread_identities(&state.descendant_thread_ids); provider.enable_durable_tool_call_replays(); provider .restore_settled_turn_identities( @@ -3447,6 +3471,95 @@ impl CodexCommandExecutor { } => { self.handle_tool_call(call_id, operation_id, input)?; } + CodexProviderEvent::ProtocolFailure { diagnostic } + | CodexProviderEvent::ResourceLimit { diagnostic } => { + let resource_capacity = diagnostic["classification"] == "resource_capacity"; + let state = self + .state + .as_mut() + .expect("Codex state available while polling"); + // Neither an integrity failure nor a full lineage ledger can + // safely reopen this provider session, even after event ACK. + state.lifecycle = "reconciliation_required".to_owned(); + state.completed_turn_authoritative = false; + state.completed_turn_process_generation = None; + state.completed_provider_turn_id = None; + state.settle_active_provider_turn_identity()?; + state.active_provider_turn_id = None; + state.push_terminal_event(NormalizedProviderEvent { + event_type: "harness.diagnostic".to_owned(), + priority: EventPriority::P0, + payload: diagnostic.clone(), + })?; + state.push_terminal_event(NormalizedProviderEvent { + event_type: "turn.failed".to_owned(), + priority: EventPriority::P0, + payload: json!({ "provider": state.config.provider, "status": "failed", + "code": diagnostic["code"], "recoverable": false, + "message": diagnostic["message"], "error": diagnostic }), + })?; + state.extend_terminal_events(terminal_events(state, "turn.failed", None))?; + // Commit the authoritative failure before best-effort provider cleanup. + self.save_state()?; + if let Some(mut provider) = self.provider.take() { + if let Some(frame_id) = trace_frame_id { + provider.record_provider_trace_interpretation( + frame_id, + if resource_capacity { "codex.resource_capacity" } else { "codex.identity.invalid_authoritative" }, + "rejected", + Vec::new(), + if resource_capacity { "Provider resource capacity requires explicit reconciliation" } else { "Rejected provider authority outside the root execution identity" }, + ); + } + let _ = provider.shutdown(); + } + break; + } + CodexProviderEvent::DescendantNotification { method, params } => { + // Do not normalize a child's turn/completed as a root terminal. + // Retain bounded lineage evidence without credential-bearing payloads. + let child = params + .get("threadId") + .or_else(|| params.pointer("/thread/id")) + .and_then(Value::as_str) + .map(|id| id.chars().take(256).collect::()); + let root_thread = self + .provider + .as_ref() + .map(|provider| provider.thread_id().to_owned()); + let root_turn = self + .provider + .as_ref() + .and_then(CodexProvider::active_provider_turn_id) + .map(str::to_owned); + let child_turn = params + .get("turnId") + .or_else(|| params.pointer("/turn/id")) + .and_then(Value::as_str) + .map(|id| id.chars().take(256).collect::()); + let state = self + .state + .as_mut() + .expect("Codex state remains available while polling"); + if let Some(id) = child.as_ref() { + state.descendant_thread_ids.insert(id.clone()); + } + state.extend_events(vec![NormalizedProviderEvent { + event_type: "harness.diagnostic".to_owned(), + priority: EventPriority::P1, + payload: json!({ "code": "provider_notification_identity", "classification": "descendant", + "method": method.chars().take(128).collect::(), "receivedThreadId": child, "expectedThreadId": root_thread, + "receivedTurnId": child_turn, "expectedTurnId": root_turn }), + }])?; + self.save_state()?; + if let (Some(frame_id), Some(provider)) = + (trace_frame_id, self.provider.as_mut()) + { + provider.record_provider_trace_interpretation(frame_id, + "codex.identity.descendant", "mapped", Vec::new(), + "Provider-confirmed descendant progress has no root terminal or tool authority"); + } + } CodexProviderEvent::Notification { method, params } => { let active_provider_turn_id = if method == "turn/started" { self.provider @@ -3806,6 +3919,27 @@ impl CodexCommandExecutor { impl CommandExecutor for CodexCommandExecutor { fn execute(&mut self, command: &Command) -> Result { self.restore()?; + if self + .state + .as_ref() + .is_some_and(|state| state.lifecycle == "reconciliation_required") + && !matches!( + command.command_type.as_str(), + "session.snapshot" + | "session.close" + | "session.destroy" + | "runner.drain" + | "runner.suspend" + | "runner.shutdown" + | "turn.interrupt" + | "run.cancel" + | "turn.stop" + ) + { + return Err(DurableRunnerError::invalid( + "Codex provider session requires explicit reconciliation and a fresh session", + )); + } match command.command_type.as_str() { "run.prepare" => self.prepare(&command.payload), "run.attach" => { @@ -4291,6 +4425,7 @@ mod tests { completed_turn_process_generation: None, completed_provider_turn_id: None, settled_provider_turn_ids: std::collections::BTreeSet::new(), + descendant_thread_ids: std::collections::BTreeSet::new(), settled_provider_turn_filter: DurableReplayFilter::default(), receipt_limit_diagnostic_emitted: false, receipt_limit_interrupt_pending: false, diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs index 48a6230f0b..1f6574e3cd 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs @@ -1654,11 +1654,22 @@ fn ambiguous_replacement_turn_rejects_conflicting_later_identity() { ); assert_eq!(provider.active_provider_turn_id(), Some("provider-turn-2")); - let conflicting_completion = wait_for_provider_error(&mut provider); - assert!( - conflicting_completion.contains("another active turn"), - "unexpected conflicting-identity error: {conflicting_completion}" - ); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + let diagnostic = loop { + assert!( + std::time::Instant::now() < deadline, + "expected structured integrity failure" + ); + if let Some(CodexProviderEvent::ProtocolFailure { diagnostic }) = + provider.poll().expect("poll identity failure") + { + break diagnostic; + } + std::thread::sleep(std::time::Duration::from_millis(1)); + }; + assert_eq!(diagnostic["code"], "turn_binding_mismatch"); + assert_eq!(diagnostic["recoverable"], false); + assert_eq!(diagnostic["expectedTurnId"], "provider-turn-2"); assert_eq!(provider.active_provider_turn_id(), Some("provider-turn-2")); fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); @@ -4839,3 +4850,226 @@ fn codex_completion_emits_the_bound_result_before_the_terminal_event() { executor.shutdown().expect("stop provider process"); fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); } + +#[test] +fn durable_integrity_failure_preserves_code_and_stops_provider_authority() { + let directory = temporary_directory("durable-identity-failure"); + let config = provider_config( + &directory, + &[ + "--malformed-error-second-turn-start", + "--conflicting-ambiguous-second-turn", + ], + ); + let mut executor = CodexCommandExecutor::new(&directory); + executor + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({"provider": config}), + )) + .expect("prepare"); + executor + .execute(&command("open", 2, "session.open", json!({}))) + .expect("open"); + executor + .execute(&command( + "first", + 3, + "turn.start", + json!({"text": "Complete the first turn."}), + )) + .expect("first turn"); + wait_for_executor_event(&mut executor, "turn.completed"); + executor + .execute(&command( + "second", + 4, + "turn.start", + json!({"text": "Start replacement work."}), + )) + .expect_err("ambiguous response"); + let failed = wait_for_executor_event(&mut executor, "turn.failed"); + assert_eq!(failed.payload["code"], "turn_binding_mismatch"); + assert_eq!(failed.payload["recoverable"], false); + let persisted: Value = + serde_json::from_slice(&fs::read(directory.join("codex-provider-state.json")).unwrap()) + .unwrap(); + assert_eq!(persisted["lifecycle"], "reconciliation_required"); + assert_eq!(persisted["completedTurnAuthoritative"], false); + executor.shutdown().expect("cleanup"); + let mut restored = CodexCommandExecutor::new(&directory); + let error = restored + .execute(&command("retry", 5, "turn.start", json!({"text":"Retry"}))) + .expect_err("an integrity failure must also remain fenced after restart"); + assert!(error + .to_string() + .contains("requires explicit reconciliation")); + restored.shutdown().expect("restored cleanup"); + fs::remove_dir_all(directory).unwrap(); +} + +#[test] +fn durable_descendant_lineage_survives_capacity_and_provider_restoration() { + let directory = temporary_directory("descendant-restoration"); + let config = provider_config( + &directory, + &["--descendant-notifications", "--durable-turn-ids"], + ); + let runner_config = durable_config(&directory); + let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + first + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({ + "provider": config, "authorizedTools": task_context_tool_set(), + "completionContract": {"revision": "lineage-contract", "criterionIds": ["lineage"]} + }), + )) + .unwrap(); + first + .execute(&command("open", 2, "session.open", json!({}))) + .unwrap(); + first + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Read test context."}), + )) + .unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + let mut completed = false; + let mut children = std::collections::BTreeSet::new(); + while std::time::Instant::now() < deadline && !completed { + for event in poll_and_ack(&mut first).unwrap() { + assert_ne!(event.event_type, "turn.failed"); + if event.payload["classification"] == "descendant" { + children.insert( + event.payload["receivedThreadId"] + .as_str() + .unwrap() + .to_owned(), + ); + } + completed |= event.event_type == "run.terminal"; + } + std::thread::sleep(std::time::Duration::from_millis(1)); + } + assert!(completed); + assert_eq!(children.len(), 300); + first.shutdown().unwrap(); + drop(first); + + let mut restored = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + let mut child_seen = false; + while std::time::Instant::now() < deadline && !child_seen { + for event in poll_and_ack(&mut restored).unwrap() { + assert_ne!(event.event_type, "turn.failed"); + assert_ne!( + event.event_type, "run.terminal", + "child completion cannot complete the root" + ); + child_seen |= event.payload["classification"] == "descendant" + && event.payload["receivedThreadId"] == "descendant-299"; + } + std::thread::sleep(std::time::Duration::from_millis(1)); + } + assert!( + child_seen, + "restoration must recognize an existing child's terminal without rediscovery" + ); + restored.shutdown().unwrap(); + drop(restored); + + // Seed the bounded persisted inventory instead of performing thousands of + // redundant disk writes, then exercise the real overflow event and fencing. + let state_path = directory.join("codex-provider-state.json"); + let mut persisted: Value = serde_json::from_slice(&fs::read(&state_path).unwrap()).unwrap(); + persisted["descendantThreadIds"] = json!((0..4096) + .map(|index| format!("descendant-{index}")) + .collect::>()); + persisted["config"]["args"] + .as_array_mut() + .unwrap() + .push(json!("--descendant-overflow")); + fs::write(&state_path, serde_json::to_vec(&persisted).unwrap()).unwrap(); + let mut bounded = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + bounded + .execute(&command( + "bounded-turn", + 4, + "turn.start", + json!({"text":"Continue bounded child work."}), + )) + .unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + let mut capacity_failed = false; + while std::time::Instant::now() < deadline && !capacity_failed { + for event in poll_and_ack(&mut bounded).unwrap() { + if event.event_type == "turn.failed" { + assert_eq!( + event.payload["code"], + "provider_descendant_capacity_exhausted" + ); + assert_eq!( + event.payload["error"]["classification"], + "resource_capacity" + ); + capacity_failed = true; + } + } + std::thread::sleep(std::time::Duration::from_millis(1)); + } + assert!( + capacity_failed, + "capacity exhaustion must become a specific durable recovery reason" + ); + let persisted: Value = serde_json::from_slice(&fs::read(&state_path).unwrap()).unwrap(); + assert_eq!( + persisted["descendantThreadIds"].as_array().unwrap().len(), + 4096 + ); + assert_eq!(persisted["lifecycle"], "reconciliation_required"); + // Acknowledging all terminal events must not authorize another provider turn. + while !poll_and_ack(&mut bounded).unwrap().is_empty() {} + for kind in ["turn.start", "session.open", "run.attach"] { + let error = bounded + .execute(&command("blocked", 5, kind, json!({"text":"Retry"}))) + .expect_err("reconciliation cannot be bypassed in the current executor"); + assert!(error + .to_string() + .contains("requires explicit reconciliation")); + } + bounded.shutdown().unwrap(); + let mut restored = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + for kind in ["turn.start", "session.open", "run.attach"] { + let error = restored + .execute(&command( + "blocked-after-restart", + 6, + kind, + json!({"text":"Retry"}), + )) + .expect_err("restart must retain the reconciliation fence"); + assert!(error + .to_string() + .contains("requires explicit reconciliation")); + } + let persisted_after_restart: Value = + serde_json::from_slice(&fs::read(&state_path).unwrap()).unwrap(); + assert_eq!( + persisted_after_restart["providerProcessGeneration"], + persisted["providerProcessGeneration"] + ); + assert_eq!( + persisted_after_restart["lifecycle"], + "reconciliation_required" + ); + restored.shutdown().unwrap(); + fs::remove_dir_all(directory).unwrap(); +} diff --git a/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts b/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts index c94f4c3023..f742160320 100644 --- a/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts +++ b/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from "vitest"; import type { HarnessDriver, HarnessSession, PersistedHarnessSession } from "../contracts/harness-driver.js"; import type { PrpEvent, PrpStructuredRunResult, PrpTerminalState } from "../protocol/replay-contract.js"; +import { NativeSessionProtocolIntegrityError } from "../contracts/native-session-backend.js"; import { HarnessDriverBackend } from "./harness-driver-backend.js"; const result: PrpStructuredRunResult = { @@ -711,6 +712,121 @@ describe("HarnessDriverBackend", () => { ]); }); + it.each(["typed", "typed-snapshot", "lookalike", "generic"] as const)( + "only a typed integrity fault forbids pending-input fallback: %s", + async (kind) => { + const typed = kind === "typed" || kind === "typed-snapshot"; + const fault = typed + ? new NativeSessionProtocolIntegrityError( + "semantic_input_digest_mismatch", + ) + : kind === "lookalike" + ? Object.assign(new Error("native_event_replay_conflict"), { + code: "native_event_replay_conflict", + reason: "semantic_input_digest_mismatch", + }) + : new Error("provider transport lost"); + class PendingInputSession extends FakeHarnessSession { + override async *events() { + yield prpEvent(1, "runtime_request.created", { + request: { + schema: "paperclip.runtime_request.v2", + requestKind: "runtime", + requestId: "input-1", + type: "input", + status: "pending", + turnId: "turn-1", + itemId: "input-1", + input: { + schema: "paperclip.question_set.v1", + questions: [ + { + id: "color", + prompt: "Which color?", + required: true, + answerMode: "text", + }, + ], + }, + }, + }); + throw kind === "typed-snapshot" + ? new Error("ordinary stream failure") + : fault; + } + override async snapshot(): Promise { + if (kind === "typed-snapshot") throw fault; + return super.snapshot(); + } + } + const session = await new HarnessDriverBackend({ + ...driver, + openSession: async () => new PendingInputSession(), + }).openSession({ + identity: { + runId: "run-1", + sessionId: "session-1", + companyId: "company-1", + issueId: "issue-1", + agentId: "agent-1", + }, + workingDirectory: "/workspace", + }); + const events: PrpEvent[] = []; + const consumed = (async () => { + for await (const event of session.events()) events.push(event); + })(); + if (typed) { + await expect(consumed).rejects.toBe(fault); + expect(events.map((event) => event.eventType)).toEqual([ + "runtime_request.created", + ]); + await expect(session.result()).rejects.toBe(fault); + await expect(session.snapshot()).rejects.toBe(fault); + } else { + await consumed; + expect(events.map((event) => event.eventType)).toEqual([ + "runtime_request.created", + "runtime_request.expired", + "turn.interrupted", + ]); + } + await session.close({ reason: "fixture complete" }); + }, + ); + + it("never exposes an earlier terminal result after a later typed integrity fault", async () => { + const fault = new NativeSessionProtocolIntegrityError( + "semantic_input_digest_mismatch", + ); + class TerminalThenIntegrityFailure extends FakeHarnessSession { + override async *events() { + yield* super.events(); + throw fault; + } + } + const session = await new HarnessDriverBackend({ + ...driver, + openSession: async () => new TerminalThenIntegrityFailure(), + }).openSession({ + identity: { + runId: "run-1", + sessionId: "session-1", + companyId: "company-1", + issueId: "issue-1", + agentId: "agent-1", + }, + workingDirectory: "/workspace", + }); + const iterator = session.events()[Symbol.asyncIterator](); + await iterator.next(); + await iterator.next(); + await expect(iterator.next()).rejects.toBe(fault); + await expect(session.result()).rejects.toBe(fault); + await expect(session.snapshot()).rejects.toBe(fault); + await session.close({ reason: "fixture complete" }); + }); + it("emits one non-replayable input expiration and terminal wait after provider loss", async () => { const questionSet = { schema: "paperclip.question_set.v1" as const, diff --git a/packages/paperclip-runner/src/backends/harness-driver-backend.ts b/packages/paperclip-runner/src/backends/harness-driver-backend.ts index 3461f84a76..dac4ea378f 100644 --- a/packages/paperclip-runner/src/backends/harness-driver-backend.ts +++ b/packages/paperclip-runner/src/backends/harness-driver-backend.ts @@ -12,6 +12,7 @@ import type { OpenNativeSessionInput, PersistedNativeSession, } from "../contracts/native-session-backend.js"; +import { NativeSessionProtocolIntegrityError } from "../contracts/native-session-backend.js"; import type { PrpEvent, PrpTerminalState, @@ -385,6 +386,31 @@ class HarnessNativeSession implements NativeSession { readonly #session: HarnessSession; #terminal: PrpTerminalState | null = null; #explicitlyCancelled = false; + #protocolIntegrityFailure: NativeSessionProtocolIntegrityError | null = null; + + #assertProtocolIntegrity(): void { + if (this.#protocolIntegrityFailure !== null) + throw this.#protocolIntegrityFailure; + } + + #rethrowProtocolIntegrity(error: unknown): void { + if (!(error instanceof NativeSessionProtocolIntegrityError)) return; + this.#protocolIntegrityFailure ??= error; + this.#terminal = null; + throw this.#protocolIntegrityFailure; + } + + async #harnessSnapshot(): Promise { + this.#assertProtocolIntegrity(); + try { + const snapshot = await this.#session.snapshot(); + this.#assertProtocolIntegrity(); + return snapshot; + } catch (error) { + this.#rethrowProtocolIntegrity(error); + throw error; + } + } constructor( input: OpenNativeSessionInput, @@ -421,6 +447,7 @@ class HarnessNativeSession implements NativeSession { async attachRun(input: { identity: OpenNativeSessionInput["identity"]; }): Promise { + this.#assertProtocolIntegrity(); const currentIdentity = this.#input.identity; if ( input.identity.sessionId !== currentIdentity.sessionId || @@ -433,7 +460,13 @@ class HarnessNativeSession implements NativeSession { if (this.#session.attachRun === undefined) { throw new Error("native_session_multi_run_unavailable"); } - await this.#session.attachRun({ runId: input.identity.runId }); + try { + await this.#session.attachRun({ runId: input.identity.runId }); + this.#assertProtocolIntegrity(); + } catch (error) { + this.#rethrowProtocolIntegrity(error); + throw error; + } this.#input = { ...this.#input, identity: structuredClone(input.identity) }; this.#terminal = null; this.#explicitlyCancelled = false; @@ -445,6 +478,7 @@ class HarnessNativeSession implements NativeSession { } async *events(): AsyncIterable { + this.#assertProtocolIntegrity(); let sourceInstanceId: string | null = null; let lastSourceSequence = 0; let sawTerminal = false; @@ -497,7 +531,7 @@ class HarnessNativeSession implements NativeSession { ].includes(event.eventType) ) { sawTerminal = true; - const snapshot = await this.#session.snapshot(); + const snapshot = await this.#harnessSnapshot(); const disposition = snapshot.semanticResult?.result.reportedWorkDisposition ?? "yielded"; @@ -523,6 +557,7 @@ class HarnessNativeSession implements NativeSession { yield structuredClone(event); } } catch (error) { + this.#rethrowProtocolIntegrity(error); streamFailure = error; } @@ -531,7 +566,10 @@ class HarnessNativeSession implements NativeSession { // governed wait; the control plane can materialize the continuation without // ever trying to replay the dead provider request. if (!sawTerminal && !this.#explicitlyCancelled && sourceInstanceId) { - const snapshot = await this.#session.snapshot().catch(() => null); + const snapshot = await this.#harnessSnapshot().catch((error) => { + this.#rethrowProtocolIntegrity(error); + return null; + }); let governedWaitTurnId: string | undefined; for (const request of observedPendingInputs.values()) { const sourceSeq = @@ -604,8 +642,16 @@ class HarnessNativeSession implements NativeSession { if (streamFailure && !synthesizedDurableWait) throw streamFailure; } - startTurn(input: Parameters[0]) { - return this.#session.startTurn(input); + async startTurn(input: Parameters[0]) { + this.#assertProtocolIntegrity(); + try { + const started = await this.#session.startTurn(input); + this.#assertProtocolIntegrity(); + return started; + } catch (error) { + this.#rethrowProtocolIntegrity(error); + throw error; + } } steer(input: { @@ -681,8 +727,9 @@ class HarnessNativeSession implements NativeSession { } async result() { + this.#assertProtocolIntegrity(); if (this.#explicitlyCancelled) return null; - const snapshot = await this.#session.snapshot(); + const snapshot = await this.#harnessSnapshot(); if ( snapshot.semanticResult === undefined || snapshot.semanticResult === null @@ -700,7 +747,8 @@ class HarnessNativeSession implements NativeSession { } async snapshot(): Promise { - const snapshot = await this.#session.snapshot(); + this.#assertProtocolIntegrity(); + const snapshot = await this.#harnessSnapshot(); return { backendKind: "runner", driverKind: snapshot.driverKind, diff --git a/packages/paperclip-runner/src/contracts/native-session-backend.ts b/packages/paperclip-runner/src/contracts/native-session-backend.ts index 59c565959e..ed2219fbdf 100644 --- a/packages/paperclip-runner/src/contracts/native-session-backend.ts +++ b/packages/paperclip-runner/src/contracts/native-session-backend.ts @@ -90,6 +90,50 @@ export interface NativeSessionSnapshotOptions { signal: AbortSignal; } +/** The exact close owner has torn down its controller without proving suspension. */ +export class NativeSessionCloseUnrecoverableError extends Error { + readonly code = "native_session_close_unrecoverable"; + + constructor() { + super( + "provider_transport_failed: runner did not durably suspend before checkpoint", + ); + this.name = "NativeSessionCloseUnrecoverableError"; + } +} + +/** An authenticated, exactly bound runner event failed permanent integrity checks. */ +export class NativeSessionProtocolIntegrityError extends Error { + readonly code = "native_event_replay_conflict"; + readonly recovery = "operator_required"; + + constructor( + readonly reason: + | "semantic_input_digest_mismatch" + | "source_event_replay_conflict", + ) { + super( + reason === "semantic_input_digest_mismatch" + ? "native_event_replay_conflict: authenticated runner semantic input failed integrity validation; automatic recovery is stopped." + : "native_event_replay_conflict: authenticated runner event conflicts with committed history; automatic recovery is stopped.", + ); + this.name = "NativeSessionProtocolIntegrityError"; + } +} + +/** Admission is blocked by a retained owner that has no safe automatic close retry. */ +export class NativeSessionCleanupQuarantinedError extends Error { + readonly code = "native_session_cleanup_quarantined"; + readonly recovery = "operator_required"; + + constructor() { + super( + "native_session_cleanup_quarantined: prior session cleanup requires operator recovery; verify its retained process ownership and checkpoint before a controlled restart. Clearing a task session does not resolve this quarantine.", + ); + this.name = "NativeSessionCleanupQuarantinedError"; + } +} + export interface NativeSession { identity(): NativeRunIdentity; capabilities(): Promise; @@ -167,3 +211,12 @@ export interface NativeSessionBackend { options: NativeSessionRecoveryOptions, ): Promise; } + +/** A provider failed terminal is not a missing completion proposal. */ +export class NativeProviderTerminalFailure extends Error { + readonly code = "native_provider_terminal_failed"; + constructor(readonly providerCode: string, readonly recoverable: boolean, message = "Provider session ended with a failed terminal") { + super(message); + this.name = "NativeProviderTerminalFailure"; + } +} diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts index d5a5fed36b..cb08b4725a 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts @@ -15,8 +15,10 @@ import { connect, type Socket } from "node:net"; import { tmpdir } from "node:os"; import { resolve } from "node:path"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; +import { NativeSessionProtocolIntegrityError } from "../contracts/native-session-backend.js"; +import { createCapabilityRunnerdCodexTransport } from "../live/runnerd-codex-transport.js"; import { validatePrpEvent } from "../protocol/replay-contract.js"; import { digestPaperclipSemanticContent } from "../semantic-tools/receipts.js"; import { @@ -689,10 +691,13 @@ async function authenticate( controlPlane: DurablePrpControlPlane, token: string, selectedIdentity: DurableRecoveryIdentity = identity, + runnerDigest = expectedRunnerDigest, ): Promise { const { socket, reader } = await upgradeSocket(controlPlane.connectUrl); const material = credentialMaterial(token); - sendMaskedJson(socket, authHello(material.credentialId, selectedIdentity)); + const hello = authHello(material.credentialId, selectedIdentity); + (hello.payload as Record).runnerDigest = runnerDigest; + sendMaskedJson(socket, hello); const challenge = await reader.next(); if (challenge === null) return null; const challengePayload = challenge.payload as Record; @@ -877,7 +882,529 @@ function semanticInputEvent(sourceSeq = 1): Record { }; } +function corruptSemanticInputDigest( + event = semanticInputEvent(), +): Record { + const payload = (event.payload as Record).payload as Record< + string, + unknown + >; + const semantic = payload.semantic_tool as Record; + (semantic.content as Record).digest = + `sha256:${"0".repeat(64)}`; + return event; +} + describe.sequential("DurablePrpControlPlane", () => { + it.each([false, true])( + "promptly fails the real transport request and notification paths on authenticated bad semantic input (throwing observer: %s)", + async (throwingObserver) => { + const root = mkdtempSync( + resolve(tmpdir(), "paperclip-prp-transport-integrity-"), + ); + let authority: DurablePrpControlPlane | undefined; + let launched = false; + const diagnostics: string[] = []; + // The launcher below is synthetic; use the current executable only as + // its artifact identity, without depending on a staged Rust build. + const runnerBinary = process.execPath; + const runnerDigest = `sha256:${createHash("sha256").update(readFileSync(runnerBinary)).digest("hex")}`; + const handler = vi.fn(async () => ({ success: true, contentItems: [] })); + const bundle = createCapabilityRunnerdCodexTransport({ + stateDirectory: root, + prpIdentity: identity, + runnerBinary, + codexCommand: process.execPath, + codexArgs: [], + sourceCodexHome: null, + environment: {}, + runnerReconnectGraceMs: 900_000, + onDiagnostic: (message) => { + diagnostics.push(message); + if ( + throwingObserver && + message.includes("native_event_replay_conflict") + ) { + throw new Error("diagnostic observer failed"); + } + }, + controlPlaneRegistration: async (core) => { + authority = core; + await core.start(); + return { connectUrl: core.connectUrl, release: () => core.stop() }; + }, + // Only the runner process is synthetic. Authentication, encrypted frames, + // authority validation, transport latching, and both consumers are real. + runnerProcessLauncher: () => { + launched = true; + return { + child: { exitCode: null, kill: () => true }, + completion: new Promise(() => undefined), + }; + }, + }); + bundle.transport.setServerRequestHandler(handler); + const requestFailure = bundle.transport + .request("thread/start", { cwd: tmpdir() }) + .catch((error: unknown) => error); + const notificationFailure = (async () => { + for await (const _notification of bundle.transport.notifications()) { + // No provider content is expected before startup completes. + } + return null; + })().catch((error: unknown) => error); + try { + await vi.waitFor(() => expect(launched).toBe(true)); + const core = authority!; + const client = (await authenticate( + core, + core.issueBootstrapTicket(), + identity, + runnerDigest, + ))!; + const event = corruptSemanticInputDigest(); + const semantic = ( + (event.payload as Record).payload as Record< + string, + unknown + > + ).semantic_tool as Record; + semantic.input = { summary: "DO-NOT-LEAK-integrity-test" }; + sendSecure(client, event); + await expect(receiveSecure(client)).resolves.toBeNull(); + const requestError = await requestFailure; + expect(requestError).toBeInstanceOf( + NativeSessionProtocolIntegrityError, + ); + expect(await notificationFailure).toBe(requestError); + await expect(bundle.transport.request("initialize", {})).rejects.toBe( + requestError, + ); + expect(handler).not.toHaveBeenCalled(); + expect(core.store.state.ackedSourceSeq).toBe(0); + expect(core.store.state.committedEvents).toEqual([]); + expect(diagnostics.join("\n")).not.toContain("DO-NOT-LEAK"); + expect( + diagnostics.filter((message) => + message.includes("native_event_replay_conflict"), + ), + ).toHaveLength(1); + } finally { + await bundle.detachControllerForRestart(); + await authority?.stop(); + await Promise.allSettled([requestFailure, notificationFailure]); + rmSync(root, { recursive: true, force: true }); + } + }, + ); + + it("latches an authenticated semantic integrity fault without ACK or dispatch and still permits suspension", async () => { + const root = mkdtempSync(resolve(tmpdir(), "paperclip-prp-integrity-")); + const onProtocolIntegrityError = vi.fn(); + const onCommittedEvent = vi.fn(async () => undefined); + const onSemanticToolInput = vi.fn(async () => ({ result: { ok: true } })); + const core = new DurablePrpControlPlane({ + stateDirectory: root, + identity, + expectedRunnerVersion, + expectedRunnerDigest, + onProtocolIntegrityError, + onCommittedEvent, + onSemanticToolInput, + }); + try { + await core.start(); + const client = (await authenticate(core, core.issueBootstrapTicket()))!; + sendSecure(client, corruptSemanticInputDigest()); + await expect(receiveSecure(client)).resolves.toBeNull(); + expect(onProtocolIntegrityError).toHaveBeenCalledTimes(1); + const error = onProtocolIntegrityError.mock.calls[0]![0]; + expect(error).toBeInstanceOf(NativeSessionProtocolIntegrityError); + expect(error).toMatchObject({ + code: "native_event_replay_conflict", + reason: "semantic_input_digest_mismatch", + recovery: "operator_required", + }); + expect(core.store.state.ackedSourceSeq).toBe(0); + expect(core.store.state.committedEvents).toEqual([]); + expect(onCommittedEvent).not.toHaveBeenCalled(); + expect(onSemanticToolInput).not.toHaveBeenCalled(); + + // A reconnect cannot reclassify this owner as healthy, even if it now + // supplies valid bytes for the failed sequence. Its callback is one-shot. + const replay = (await authenticate(core, client.leaseToken!))!; + sendSecure(replay, semanticInputEvent()); + await expect(receiveSecure(replay)).resolves.toBeNull(); + expect(onProtocolIntegrityError).toHaveBeenCalledTimes(1); + expect(core.store.state.ackedSourceSeq).toBe(0); + expect(onCommittedEvent).not.toHaveBeenCalled(); + expect(onSemanticToolInput).not.toHaveBeenCalled(); + expect(() => + core.rotateRunIdentity({ ...identity, runId: "other-run" }), + ).toThrow(error); + + const suspend = core.queueCommand( + "runner.suspend", + {}, + "suspend-after-integrity-fault", + ); + const cleanup = (await authenticate(core, client.leaseToken!))!; + expect(cleanup.welcome.payload).toMatchObject({ + pendingCommands: [ + expect.objectContaining({ commandId: suspend.commandId }), + ], + }); + sendSecure(cleanup, { + protocol: "paperclip.runner", + version: 1, + kind: "command_result", + payload: { + commandId: suspend.commandId, + commandType: suspend.type, + controllerSeq: suspend.controllerSeq, + status: "completed", + result: { suspended: true }, + }, + }); + await expect(receiveSecure(cleanup)).resolves.toMatchObject({ + kind: "command_result_ack", + }); + expect(core.store.state.commands[0]?.status).toBe("completed"); + expect(core.store.state.ackedSourceSeq).toBe(0); + cleanup.socket.destroy(); + } finally { + await core.stop(); + rmSync(root, { recursive: true, force: true }); + } + }); + + it.each([ + "semantic_input_digest_mismatch", + "source_event_replay_conflict", + ] as const)( + "preserves a typed integrity fault from the trusted commit boundary (%s)", + async (reason) => { + const root = mkdtempSync( + resolve(tmpdir(), "paperclip-prp-typed-commit-integrity-"), + ); + const error = new NativeSessionProtocolIntegrityError(reason); + const onProtocolIntegrityError = vi.fn(); + const onCommittedEvent = vi.fn(async () => { + throw error; + }); + const onSemanticToolInput = vi.fn(async () => ({ result: { ok: true } })); + const core = new DurablePrpControlPlane({ + stateDirectory: root, + identity, + expectedRunnerVersion, + expectedRunnerDigest, + onProtocolIntegrityError, + onCommittedEvent, + onSemanticToolInput, + }); + try { + await core.start(); + const client = (await authenticate(core, core.issueBootstrapTicket()))!; + sendSecure(client, semanticInputEvent()); + await expect(receiveSecure(client)).resolves.toBeNull(); + expect(onProtocolIntegrityError).toHaveBeenCalledExactlyOnceWith(error); + expect(core.store.state.ackedSourceSeq).toBe(0); + expect(onSemanticToolInput).not.toHaveBeenCalled(); + const retry = (await authenticate(core, client.leaseToken!))!; + sendSecure(retry, semanticInputEvent()); + await expect(receiveSecure(retry)).resolves.toBeNull(); + expect(onProtocolIntegrityError).toHaveBeenCalledExactlyOnceWith(error); + expect(onCommittedEvent).toHaveBeenCalledTimes(1); + } finally { + await core.stop(); + rmSync(root, { recursive: true, force: true }); + } + }, + ); + + it("latches an authenticated replay conflict against exactly committed bytes", async () => { + const root = mkdtempSync( + resolve(tmpdir(), "paperclip-prp-replay-integrity-"), + ); + const onProtocolIntegrityError = vi.fn(); + const onCommittedEvent = vi.fn(async () => undefined); + const onSemanticToolInput = vi.fn( + async () => new Promise<{ result: unknown }>(() => undefined), + ); + const core = new DurablePrpControlPlane({ + stateDirectory: root, + identity, + expectedRunnerVersion, + expectedRunnerDigest, + onProtocolIntegrityError, + onCommittedEvent, + onSemanticToolInput, + }); + try { + await core.start(); + const client = (await authenticate(core, core.issueBootstrapTicket()))!; + sendSecure(client, semanticInputEvent()); + await expect(receiveSecure(client)).resolves.toMatchObject({ + kind: "ack", + payload: { ackedSourceSeq: 1 }, + }); + const changed = semanticInputEvent(); + const semantic = ( + (changed.payload as Record).payload as Record< + string, + unknown + > + ).semantic_tool as Record; + semantic.input = { changed: true }; + (semantic.content as Record).digest = + digestPaperclipSemanticContent(semantic.input); + sendSecure(client, changed); + await expect(receiveSecure(client)).resolves.toBeNull(); + expect(onProtocolIntegrityError).toHaveBeenCalledTimes(1); + expect(onProtocolIntegrityError.mock.calls[0]![0]).toBeInstanceOf( + NativeSessionProtocolIntegrityError, + ); + expect(onProtocolIntegrityError.mock.calls[0]![0]).toMatchObject({ + reason: "source_event_replay_conflict", + }); + expect(onCommittedEvent).toHaveBeenCalledTimes(1); + expect(onSemanticToolInput).toHaveBeenCalledTimes(1); + expect(core.store.state.ackedSourceSeq).toBe(1); + expect(core.store.state.committedEvents).toHaveLength(1); + expect(core.store.state.committedEvents[0]?.envelope).toEqual( + semanticInputEvent(), + ); + } finally { + await core.stop(); + rmSync(root, { recursive: true, force: true }); + } + }); + + it.each([ + "unauthenticated", + "envelope_runner", + "envelope_environment", + "envelope_run", + "envelope_session", + "envelope_turn", + "envelope_item", + "event_run", + "event_runner", + "event_session", + "event_turn", + "event_item", + "source_seq_gap", + "semantic_run", + "semantic_session", + "semantic_turn", + "semantic_item", + "malformed_semantic", + ])( + "rejects %s malformed input without poisoning the legitimate owner", + async (mismatch) => { + const root = mkdtempSync( + resolve(tmpdir(), "paperclip-prp-unbound-integrity-"), + ); + const onProtocolIntegrityError = vi.fn(); + const onCommittedEvent = vi.fn(async () => undefined); + const onSemanticToolInput = vi.fn(async () => ({ result: { ok: true } })); + const core = new DurablePrpControlPlane({ + stateDirectory: root, + identity, + expectedRunnerVersion, + expectedRunnerDigest, + onProtocolIntegrityError, + onCommittedEvent, + onSemanticToolInput, + }); + try { + await core.start(); + const event = corruptSemanticInputDigest(); + const canonical = event.payload as Record; + const semantic = (canonical.payload as Record) + .semantic_tool as Record; + if (mismatch.startsWith("envelope_")) { + const key = { + envelope_runner: "runnerInstanceId", + envelope_environment: "environmentLeaseId", + envelope_run: "runId", + envelope_session: "normalizedSessionId", + envelope_turn: "turnId", + envelope_item: "itemId", + }[mismatch]!; + event[key] = "another-owner"; + } else if (mismatch.startsWith("event_")) { + const key = { + event_runner: "sourceInstanceId", + event_run: "runId", + event_session: "normalizedSessionId", + event_turn: "turnId", + event_item: "itemId", + }[mismatch]!; + canonical[key] = "another-owner"; + } else if (mismatch === "source_seq_gap") canonical.sourceSeq = 2; + else if (mismatch === "malformed_semantic") delete semantic.callId; + else if (mismatch.startsWith("semantic_")) { + const key = { + semantic_run: "runId", + semantic_session: "normalizedSessionId", + semantic_turn: "turnId", + semantic_item: "itemId", + }[mismatch]!; + (semantic.correlation as Record)[key] = + "another-owner"; + } + if (mismatch === "unauthenticated") { + const client = await upgradeSocket(core.connectUrl); + sendMaskedJson(client.socket, event); + await expect(client.reader.next()).resolves.toBeNull(); + } else { + const client = (await authenticate( + core, + core.issueBootstrapTicket(), + ))!; + sendSecure(client, event); + await expect(receiveSecure(client)).resolves.toBeNull(); + } + expect(onProtocolIntegrityError).not.toHaveBeenCalled(); + expect(onCommittedEvent).not.toHaveBeenCalled(); + expect(onSemanticToolInput).not.toHaveBeenCalled(); + const legitimate = (await authenticate( + core, + core.issueBootstrapTicket(), + ))!; + sendSecure(legitimate, semanticInputEvent()); + await expect(receiveSecure(legitimate)).resolves.toMatchObject({ + kind: "ack", + payload: { ackedSourceSeq: 1 }, + }); + expect(onProtocolIntegrityError).not.toHaveBeenCalled(); + expect(onCommittedEvent).toHaveBeenCalledTimes(1); + expect(onSemanticToolInput).toHaveBeenCalledTimes(1); + legitimate.socket.destroy(); + } finally { + await core.stop(); + rmSync(root, { recursive: true, force: true }); + } + }, + ); + + it("does not dispatch an earlier in-flight commit after a replacement connection proves an integrity fault", async () => { + const root = mkdtempSync( + resolve(tmpdir(), "paperclip-prp-integrity-commit-race-"), + ); + let releaseCommit!: () => void; + const commitBarrier = new Promise((resolveCommit) => { + releaseCommit = resolveCommit; + }); + const onCommittedEvent = vi.fn(async () => commitBarrier); + const onProtocolIntegrityError = vi.fn(); + const onSemanticToolInput = vi.fn(async () => ({ result: { ok: true } })); + const core = new DurablePrpControlPlane({ + stateDirectory: root, + identity, + expectedRunnerVersion, + expectedRunnerDigest, + onCommittedEvent, + onProtocolIntegrityError, + onSemanticToolInput, + }); + try { + await core.start(); + const first = (await authenticate(core, core.issueBootstrapTicket()))!; + sendSecure(first, semanticInputEvent()); + await vi.waitFor(() => expect(onCommittedEvent).toHaveBeenCalledTimes(1)); + const replacement = (await authenticate(core, first.leaseToken!))!; + sendSecure(replacement, corruptSemanticInputDigest()); + await expect(receiveSecure(replacement)).resolves.toBeNull(); + expect(onProtocolIntegrityError).toHaveBeenCalledTimes(1); + releaseCommit(); + await new Promise((resolveTurn) => setImmediate(resolveTurn)); + expect(core.store.state.ackedSourceSeq).toBe(0); + expect(core.store.state.committedEvents).toEqual([]); + expect(onSemanticToolInput).not.toHaveBeenCalled(); + } finally { + releaseCommit(); + await core.stop(); + rmSync(root, { recursive: true, force: true }); + } + }); + + it("does not turn a missing semantic handler into an integrity fault", async () => { + const root = mkdtempSync( + resolve(tmpdir(), "paperclip-prp-integrity-no-handler-"), + ); + const onProtocolIntegrityError = vi.fn(); + const core = new DurablePrpControlPlane({ + stateDirectory: root, + identity, + expectedRunnerVersion, + expectedRunnerDigest, + onProtocolIntegrityError, + }); + try { + await core.start(); + const client = (await authenticate(core, core.issueBootstrapTicket()))!; + sendSecure(client, corruptSemanticInputDigest()); + await expect(receiveSecure(client)).resolves.toBeNull(); + expect(onProtocolIntegrityError).not.toHaveBeenCalled(); + expect(core.store.state.ackedSourceSeq).toBe(0); + } finally { + await core.stop(); + rmSync(root, { recursive: true, force: true }); + } + }); + + it.each([ + new Error("database temporarily unavailable"), + Object.assign(new Error("untrusted native_event_replay_conflict prose"), { + code: "native_event_replay_conflict", + }), + ])( + "keeps a non-typed commit failure retryable on the same authenticated authority (%s)", + async (commitFailure) => { + const root = mkdtempSync( + resolve(tmpdir(), "paperclip-prp-transient-commit-"), + ); + const onProtocolIntegrityError = vi.fn(); + const onCommittedEvent = vi + .fn() + .mockRejectedValueOnce(commitFailure) + .mockResolvedValue(undefined); + const onSemanticToolInput = vi.fn(async () => ({ result: { ok: true } })); + const core = new DurablePrpControlPlane({ + stateDirectory: root, + identity, + expectedRunnerVersion, + expectedRunnerDigest, + onProtocolIntegrityError, + onCommittedEvent, + onSemanticToolInput, + }); + try { + await core.start(); + const first = (await authenticate(core, core.issueBootstrapTicket()))!; + sendSecure(first, semanticInputEvent()); + await expect(receiveSecure(first)).resolves.toBeNull(); + expect(core.store.state.ackedSourceSeq).toBe(0); + expect(onSemanticToolInput).not.toHaveBeenCalled(); + const retry = (await authenticate(core, first.leaseToken!))!; + sendSecure(retry, semanticInputEvent()); + await expect(receiveSecure(retry)).resolves.toMatchObject({ + kind: "ack", + payload: { ackedSourceSeq: 1 }, + }); + expect(onProtocolIntegrityError).not.toHaveBeenCalled(); + expect(onCommittedEvent).toHaveBeenCalledTimes(2); + expect(onSemanticToolInput).toHaveBeenCalledTimes(1); + retry.socket.destroy(); + } finally { + await core.stop(); + rmSync(root, { recursive: true, force: true }); + } + }, + ); + it("exchanges a one-use bootstrap for a run-bound reconnect lease", async () => { const root = mkdtempSync(resolve(tmpdir(), "paperclip-prp-auth-")); const controlPlane = new DurablePrpControlPlane({ diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts index 839b6ce247..746e39e040 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts @@ -28,6 +28,7 @@ import { dirname, resolve } from "node:path"; import type { Duplex } from "node:stream"; import { fileURLToPath } from "node:url"; +import { NativeSessionProtocolIntegrityError } from "../contracts/native-session-backend.js"; import { githubCredentialEnvironment } from "../github-credential-environment.js"; import { validatePrpEvent, @@ -224,6 +225,10 @@ export interface DurablePrpControlPlaneOptions { }) => Promise<{ readonly result: unknown; readonly isError?: boolean }>; /** Persist the canonical event before the runner receives its cumulative ACK. */ onCommittedEvent?: (event: PrpEvent) => Promise; + /** Stop this exact owner after a proven, authenticated permanent integrity fault. */ + onProtocolIntegrityError?: ( + error: NativeSessionProtocolIntegrityError, + ) => void; connectionLeaseTtlMs?: number; } @@ -974,6 +979,9 @@ export class DurablePrpControlPlane { #port: number | null = null; #onSemanticToolInput?: DurablePrpControlPlaneOptions["onSemanticToolInput"]; #onCommittedEvent?: DurablePrpControlPlaneOptions["onCommittedEvent"]; + #onProtocolIntegrityError?: + DurablePrpControlPlaneOptions["onProtocolIntegrityError"]; + #protocolIntegrityError: NativeSessionProtocolIntegrityError | null = null; #connectionLeaseTtlMs: number; constructor(options: DurablePrpControlPlaneOptions) { @@ -999,6 +1007,7 @@ export class DurablePrpControlPlane { this.#expectedRunnerDigest = options.expectedRunnerDigest; this.#onSemanticToolInput = options.onSemanticToolInput; this.#onCommittedEvent = options.onCommittedEvent; + this.#onProtocolIntegrityError = options.onProtocolIntegrityError; this.#connectionLeaseTtlMs = options.connectionLeaseTtlMs ?? 60_000; } @@ -1075,6 +1084,8 @@ export class DurablePrpControlPlane { identity: DurableRecoveryIdentity, runAttachTemplate?: Record, ): void { + if (this.#protocolIntegrityError !== null) + throw this.#protocolIntegrityError; if ( !Object.values(identity).every( (value) => typeof value === "string" && stableIdPattern.test(value), @@ -1858,10 +1869,42 @@ export class DurablePrpControlPlane { ); } + #failProtocolIntegrity( + connection: AuthorityConnection, + error: NativeSessionProtocolIntegrityError, + ): void { + try { + if (this.#protocolIntegrityError === null) { + this.#protocolIntegrityError = error; + this.#onProtocolIntegrityError?.(error); + } + } finally { + connection.close(); + } + } + async #event( connection: AuthorityConnection, envelope: Record, ): Promise { + if (this.#protocolIntegrityError !== null) { + connection.close(); + return; + } + // Authentication binds the channel, but an authenticated sender can still + // submit an envelope for another run. Such frames must not poison this + // owner's session or turn an unrelated digest failure into its terminal fault. + if ( + envelope.runnerInstanceId !== this.#identity.runnerInstanceId || + envelope.environmentLeaseId !== this.#identity.environmentLeaseId || + envelope.runId !== this.#identity.runId || + envelope.normalizedSessionId !== this.#identity.normalizedSessionId || + envelope.turnId !== this.#identity.turnId || + envelope.itemId !== this.#identity.itemId + ) { + connection.close(); + return; + } const validated = validatePrpEvent(envelope.payload); if (!validated.ok) { connection.close(); @@ -1901,8 +1944,6 @@ export class DurablePrpControlPlane { !Object.prototype.hasOwnProperty.call(semantic, "input") || typeof semantic.content !== "object" || semantic.content === null || - (semantic.content as Record).digest !== - digestPaperclipSemanticContent(semantic.input) || semanticCorrelation?.runId !== this.#identity.runId || semanticCorrelation.normalizedSessionId !== this.#identity.normalizedSessionId || @@ -1915,15 +1956,43 @@ export class DurablePrpControlPlane { const existing = this.#store.state.committedEvents.find( (candidate) => candidate.sourceEventId === sourceEventId, ); - if (existing !== undefined) { - if (canonicalJson(existing.envelope) !== canonicalJson(envelope)) { - connection.close(); - return; - } - } else if (sourceSeq !== this.#store.state.ackedSourceSeq + 1) { + if ( + existing === undefined + ? sourceSeq !== this.#store.state.ackedSourceSeq + 1 + : sourceSeq !== existing.sourceSeq + ) { connection.close(); return; } + if ( + isSemanticInput && + semantic !== undefined && + (semantic.content as Record).digest !== + digestPaperclipSemanticContent(semantic.input) + ) { + // Only the authenticated, schema-valid, exactly correlated input may + // permanently fail its owner. Never commit, dispatch, or ACK these bytes. + // Keep the same error latched across reconnects; lifecycle command results + // remain available so the owner can still attempt a verified suspension. + this.#failProtocolIntegrity( + connection, + new NativeSessionProtocolIntegrityError( + "semantic_input_digest_mismatch", + ), + ); + return; + } + if (existing !== undefined) { + if (canonicalJson(existing.envelope) !== canonicalJson(envelope)) { + this.#failProtocolIntegrity( + connection, + new NativeSessionProtocolIntegrityError( + "source_event_replay_conflict", + ), + ); + return; + } + } // The caller's durable commit is the acknowledgement authority. A crash // after that idempotent commit but before the local cursor save is safe: @@ -1932,7 +2001,18 @@ export class DurablePrpControlPlane { // an uncommitted event disappear from the runner outbox permanently. try { await this.#onCommittedEvent?.(event); - } catch { + } catch (error) { + if (error instanceof NativeSessionProtocolIntegrityError) { + this.#failProtocolIntegrity(connection, error); + } else { + connection.close(); + } + return; + } + // Another authenticated connection can replace this one while its commit + // is in flight. Once that exact owner has faulted, even a prior successful + // commit cannot reopen delivery or invoke a new business operation. + if (this.#protocolIntegrityError !== null) { connection.close(); return; } diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts index cdb5332752..0db3d85d27 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts @@ -14,6 +14,7 @@ import type { PersistedHarnessSession, PersistedHarnessTurnTerminal, } from "../../contracts/harness-driver.js"; +import { NativeSessionProtocolIntegrityError } from "../../contracts/native-session-backend.js"; import { HarnessReconciliationError } from "../../contracts/harness-driver.js"; import { CODEX_CODEX_PROTOCOL_VERSION, @@ -307,6 +308,7 @@ export class CodexAppServerDriver implements HarnessDriver { // work during close; when no durable provider identity exists that // cleanup can fail independently. await cancellation.close().catch(() => {}); + if (error instanceof NativeSessionProtocolIntegrityError) throw error; if (input.signal?.aborted) input.signal.throwIfAborted(); throw error; } finally { @@ -596,6 +598,7 @@ export class CodexAppServerDriver implements HarnessDriver { }; } catch (error) { await cancellation.close().catch(() => {}); + if (error instanceof NativeSessionProtocolIntegrityError) throw error; if (options.signal.aborted) options.signal.throwIfAborted(); return { recovered: false, reason: redactCodexDiagnostic(String(error)) }; } finally { @@ -653,6 +656,7 @@ export class CodexAppServerDriver implements HarnessDriver { }, }; } catch (cause) { + if (cause instanceof NativeSessionProtocolIntegrityError) throw cause; const error = new Error( `planning_mode_unsupported: installed Codex app-server did not expose a usable native plan collaboration mode (${redactCodexDiagnostic(String(cause))})`, ); @@ -702,6 +706,7 @@ export class CodexAppServerDriver implements HarnessDriver { const response = await transport.request("thread/goal/get", { threadId }); return parseThreadGoal(response.goal); } catch (error) { + if (error instanceof NativeSessionProtocolIntegrityError) throw error; const policyDisabled = error instanceof CodexRpcError && (error.message.toLowerCase().includes("policy") diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.recovery.test.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.recovery.test.ts index 1f86ce2913..66c2ecb629 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.recovery.test.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.recovery.test.ts @@ -1,3 +1,4 @@ +import { NativeSessionProtocolIntegrityError } from "../../contracts/native-session-backend.js"; import { CODEX_BLOCK_RESULT_OUTPUT_SCHEMA, CODEX_INVALID_REQUEST, @@ -94,6 +95,59 @@ describe("Codex app-server Codex driver", () => { }); expect(second.calls.some((call) => call.method === "turn/start" || call.method === "thread/goal/set")).toBe(false); }); + it.each([ + "initial-read", + "goal-probe", + "plan-probe", + ] as const)( + "rethrows exact recovery integrity failure after cleanup at %s", + async (stage) => { + const originalTransport = new FakeCodexTransport(); + const recoveryTransport = new FakeCodexTransport(); + const fault = new NativeSessionProtocolIntegrityError( + "semantic_input_digest_mismatch", + ); + const request = recoveryTransport.request.bind(recoveryTransport); + let reads = 0; + vi.spyOn(recoveryTransport, "request").mockImplementation( + async (method, params) => { + if (method === "thread/read") reads += 1; + if ( + (stage === "initial-read" && method === "thread/read") || + (stage === "goal-probe" && method === "thread/goal/get") || + (stage === "plan-probe" && method === "collaborationMode/list") + ) + throw fault; + return request(method, params); + }, + ); + const close = vi.spyOn(recoveryTransport, "close"); + // The integrity error remains primary even when required cleanup rejects. + close.mockRejectedValue(new Error("secondary cleanup failure")); + const driver = makeDriver( + [originalTransport, recoveryTransport], + stage === "plan-probe" ? { requestedCollaborationMode: "plan" } : {}, + ); + const original = await driver.openSession({ + runId: "run-recovery-integrity", + normalizedSessionId: "session-recovery-integrity", + workingDirectory: WORKSPACE, + }); + await original.startTurn({ message: { role: "user", text: "Work." } }); + const checkpoint = await original.snapshot(); + await original.close({ reason: "fixture disconnect" }); + await expect( + driver.recoverSession!({ + ...checkpoint, + providerRecoveryPolicy: "allow_replacement_after_resume_failure", + }), + ).rejects.toBe(fault); + expect(close).toHaveBeenCalledTimes(1); + expect( + recoveryTransport.calls.some((call) => call.method === "thread/start"), + ).toBe(false); + }, + ); it("persists and verifies the tagged runnerd provider identity on recovery", async () => { const providerIdentity = { diff --git a/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts b/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts index d797aef3dd..03c8e04af3 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts @@ -1,4 +1,5 @@ import { randomUUID } from "node:crypto"; +import { NativeProviderTerminalFailure } from "../../contracts/native-session-backend.js"; import type { HarnessGoalOperation, @@ -78,6 +79,7 @@ export class CodexHarnessSession } async attachRun(input: { runId: string }): Promise { + this.assertProtocolIntegrity(); const transportOwnsQuiescence = this.transport.attachRun !== undefined; if ( this.turnStartPending || @@ -92,6 +94,7 @@ export class CodexHarnessSession turnId: `turn_attachment_${randomUUID().replaceAll("-", "")}`, itemId: `item_attachment_${randomUUID().replaceAll("-", "")}`, }); + this.assertProtocolIntegrity(); if (transportOwnsQuiescence) { // Runnerd's attachment contract performs two durable readiness probes, // drains the settled provider tail, and rotates authority atomically. @@ -133,6 +136,10 @@ export class CodexHarnessSession turnId: string; effectiveCollaborationMode: "default" | "plan"; }> { + this.assertProtocolIntegrity(); + if (this.protocolFailed && this.protocolFailureCode) { + throw new NativeProviderTerminalFailure(this.protocolFailureCode, false, this.protocolFailureMessage ?? undefined); + } if ( this.terminal || this.protocolFailed || @@ -237,6 +244,7 @@ export class CodexHarnessSession // never observes the terminal turn ahead of turn.accepted. releaseTurnStartSettled(); } + this.assertProtocolIntegrity(); const turn = record(response.turn); const turnId = text(turn.id); if (turnId.length === 0) @@ -268,6 +276,7 @@ export class CodexHarnessSession message: NativeUserMessage; correlationId?: string; }): Promise { + this.assertProtocolIntegrity(); this.requireCapability("steering"); this.requireActiveTurn(input.turnId, "steering"); if (input.correlationId) { @@ -312,6 +321,7 @@ export class CodexHarnessSession ); } catch (error) { if (error instanceof HarnessOperationAlreadyTerminalError) throw error; + this.rethrowProtocolIntegrity(error); const detail = redactCodexDiagnostic(String(error)); if (/unsupported|unavailable|capability|method not found/i.test(detail)) { throw this.unsupported("steering", detail); @@ -385,6 +395,7 @@ export class CodexHarnessSession turnId: string; resolution: HarnessRuntimeRequestResolution; }): Promise { + this.assertProtocolIntegrity(); this.requireCapability("runtimeRequestResolution"); const pending = this.pendingRuntimeRequestMap.get(input.requestId); if (pending === undefined) { @@ -590,6 +601,7 @@ export class CodexHarnessSession } async read(): Promise> { + this.assertProtocolIntegrity(); this.requireCapability("read"); try { return await this.transport.request("thread/read", { @@ -699,6 +711,7 @@ export class CodexHarnessSession } async snapshot(): Promise { + this.assertProtocolIntegrity(); return { driverKind: this.driverKind, driverSessionId: this.opened.threadId, diff --git a/packages/paperclip-runner/src/drivers/codex/codex-notification-identity.test.ts b/packages/paperclip-runner/src/drivers/codex/codex-notification-identity.test.ts new file mode 100644 index 0000000000..0c8be99f4d --- /dev/null +++ b/packages/paperclip-runner/src/drivers/codex/codex-notification-identity.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from "vitest"; +import { classifyCodexNotification } from "./codex-notification-identity.js"; +const classify = (method: string, params: Record) => + classifyCodexNotification({ + method, + params, + runId: "run", + rootThreadId: "root", + activeTurnId: "active", + knownThreads: new Set(["root", "child"]), + settledTurns: new Set(["old"]), + }).classification; +describe("provider notification authority", () => { + it.each([ + ["warning", { message: "buffered before replacement rejection" }, "root"], + ["paperclip/canonicalProviderEvent", { threadId: null, eventType: "provider.notice.recorded" }, "root"], + ["paperclip/canonicalProviderEvent", { threadId: null, eventType: "tool.execution.started" }, "invalid_authority"], + ["item/started", { threadId: "root", turnId: "active" }, "root"], + ["turn/completed", { threadId: "root", turnId: "old" }, "stale_turn"], + [ + "turn/completed", + { threadId: "child", turnId: "child-turn" }, + "descendant", + ], + [ + "thread/started", + { + thread: { + id: "new-child", + source: { subAgent: { thread_spawn: { parent_thread_id: "child" } } }, + }, + }, + "descendant", + ], + [ + "thread/started", + { thread: { id: "unrelated" } }, + "unrelated_information", + ], + [ + "thread/status/changed", + { threadId: "unrelated" }, + "unrelated_information", + ], + ["paperclip/runResult", { threadId: "child" }, "invalid_authority"], + ["turn/completed", { threadId: "unrelated" }, "invalid_authority"], + ["item/started", { threadId: 5 }, "invalid_authority"], + [ + "item/started", + { threadId: "root", thread: { id: "other" } }, + "invalid_authority", + ], + [ + "thread/status/changed", + { threadId: "root", runId: "other-task" }, + "invalid_authority", + ], + ] as const)("classifies %s %j as %s", (method, params, expected) => + expect(classify(method, params)).toBe(expected), + ); +}); diff --git a/packages/paperclip-runner/src/drivers/codex/codex-notification-identity.ts b/packages/paperclip-runner/src/drivers/codex/codex-notification-identity.ts new file mode 100644 index 0000000000..d5356e8ec8 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/codex/codex-notification-identity.ts @@ -0,0 +1,95 @@ +import { codexThreadLineage } from "./codex-thread-normalization.js"; + +export type CodexNotificationIdentity = + | "root" + | "descendant" + | "stale_turn" + | "unrelated_information" + | "invalid_authority"; +const informational = new Set([ + "thread/started", + "thread/status/changed", + "thread/closed", + "thread/tokenUsage/updated", + "warning", + "configWarning", + "guardianWarning", + "deprecationNotice", +]); +const record = (v: unknown): Record => + v !== null && typeof v === "object" && !Array.isArray(v) + ? (v as Record) + : {}; + +/** Only provider-originated lineage admits a descendant; tool requests never use this classifier. */ +export function classifyCodexNotification(input: { + method: string; + params: Record; + runId: string; + rootThreadId: string; + activeTurnId: string | null; + knownThreads: ReadonlySet; + settledTurns: ReadonlySet; +}): { + classification: CodexNotificationIdentity; + threadId: string | null; + turnId: string | null; +} { + const { params, method } = input; + const isInformation = informational.has(method) || + (method === "paperclip/canonicalProviderEvent" && + ["provider.notice.recorded", "harness.diagnostic"].includes(String(params.eventType))); + const threads = [ + params.threadId, + record(params.thread).id, + record(params.turn).threadId, + ].filter((v) => v !== undefined && v !== null); + const turns = [params.turnId, record(params.turn).id].filter( + (v) => v !== undefined && v !== null, + ); + const threadId = typeof threads[0] === "string" ? threads[0] : null; + const turnId = typeof turns[0] === "string" ? turns[0] : null; + const result = (classification: CodexNotificationIdentity) => ({ + classification, + threadId, + turnId, + }); + if ( + [...threads, ...turns].some( + (v) => typeof v !== "string" || v.length === 0, + ) || + new Set(threads).size > 1 || + new Set(turns).size > 1 || + [params.runId, params.paperclipRunId].some( + (v) => v !== undefined && v !== input.runId, + ) + ) + return result("invalid_authority"); + if (threadId === null && isInformation) return result("root"); + if (threadId === input.rootThreadId) { + if ( + input.activeTurnId !== null && + turnId && + turnId !== input.activeTurnId && + input.settledTurns.has(turnId) + ) + return result("stale_turn"); + return result("root"); + } + const lineage = codexThreadLineage(params.thread); + if ( + threadId && + (input.knownThreads.has(threadId) || + (method === "thread/started" && + lineage.parentThreadId !== null && + input.knownThreads.has(lineage.parentThreadId))) + ) { + // A descendant may report its own terminal, but never supply a root result or workspace authority. + return result( + method.startsWith("paperclip/") ? "invalid_authority" : "descendant", + ); + } + return result( + isInformation ? "unrelated_information" : "invalid_authority", + ); +} diff --git a/packages/paperclip-runner/src/drivers/codex/codex-protocol-integrity.test.ts b/packages/paperclip-runner/src/drivers/codex/codex-protocol-integrity.test.ts new file mode 100644 index 0000000000..7531d37fd7 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/codex/codex-protocol-integrity.test.ts @@ -0,0 +1,641 @@ +import { + createCipheriv, + createDecipheriv, + createHash, + createHmac, +} from "node:crypto"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { HarnessDriverBackend } from "../../backends/harness-driver-backend.js"; +import { createCodexTaskEnvelope } from "../../contracts/codex.js"; +import type { ControlPlanePort } from "../../contracts/control-plane-port.js"; +import type { NativeExecutionInputV1 } from "../../contracts/native-execution.js"; +import { NativeSessionProtocolIntegrityError } from "../../contracts/native-session-backend.js"; +import { + DurablePrpControlPlane, + durableRecoveryInternals, + type DurableRecoveryIdentity, +} from "../../control-plane/durable-prp-control-plane.js"; +import { createCapabilityRunnerdCodexTransport } from "../../live/runnerd-codex-transport.js"; +import { executeNativeSession } from "../../native-session-runtime.js"; +import { CodexAppServerDriver } from "./codex-app-server-driver.js"; +import { CodexSessionState } from "./codex-session-state.js"; +import { + FakeCodexTransport, + TestQueue, + WORKSPACE, + describe, + expect, + it, + makeDriver, + result, + vi, + type PrpEvent, +} from "./codex-app-server-driver.test-support.js"; + +// Synthetic runner process boundary; authentication and the encrypted wire are +// real. Keep this client local to this test rather than exporting test protocol +// machinery from the production controller. +async function authenticatedRunner( + core: DurablePrpControlPlane, + identity: DurableRecoveryIdentity, +) { + const framed = (domain: string, parts: Buffer[]) => { + const values = [Buffer.from(domain), Buffer.from([0])]; + for (const part of parts) { + const length = Buffer.alloc(8); + length.writeBigUInt64BE(BigInt(part.length)); + values.push(length, part); + } + return Buffer.concat(values); + }; + const digest = (domain: string, parts: Buffer[]) => + createHash("sha256").update(framed(domain, parts)).digest(); + const credential = Buffer.from(core.issueBootstrapTicket()); + const authKey = digest("paperclip-runner-auth-key-v1", [credential]); + const mac = (domain: string, parts: Buffer[]) => + createHmac("sha256", authKey).update(framed(domain, parts)).digest(); + const credentialId = `sha256:${digest("paperclip-runner-credential-id-v1", [credential]).toString("hex")}`; + const socket = new WebSocket(core.connectUrl); + const frames = new TestQueue>(); + const reader = frames[Symbol.asyncIterator](); + socket.addEventListener("message", (event) => + frames.push(JSON.parse(String(event.data))), + ); + socket.addEventListener("close", () => frames.close()); + socket.addEventListener("error", () => + frames.fail(new Error("Synthetic runner socket failed")), + ); + await new Promise((resolve, reject) => { + socket.addEventListener("open", () => resolve(), { once: true }); + socket.addEventListener("error", reject, { once: true }); + }); + socket.send( + JSON.stringify({ + protocol: "paperclip.runner", + version: 1, + kind: "auth_hello", + payload: { + credentialId, + clientNonce: "composed-integrity-client", + protocolMin: 1, + protocolMax: 1, + ...identity, + runnerVersion: "0.3.0", + runnerDigest: `sha256:${createHash("sha256").update(readFileSync(process.execPath)).digest("hex")}`, + }, + }), + ); + const challenge = (await reader.next()).value!.payload as Record< + string, + unknown + >; + const { serverProof, ...challengeFields } = challenge; + const canonical = Buffer.from( + durableRecoveryInternals.canonicalJson(challengeFields), + ); + expect(serverProof).toBe( + mac("paperclip-runner-server-proof-v1", [canonical]).toString("hex"), + ); + const clientProof = mac("paperclip-runner-client-proof-v1", [ + canonical, + Buffer.from(String(serverProof)), + ]).toString("hex"); + socket.send( + JSON.stringify({ + protocol: "paperclip.runner", + version: 1, + kind: "auth_response", + payload: { + credentialId, + clientNonce: challenge.clientNonce, + serverNonce: challenge.serverNonce, + clientProof, + }, + }), + ); + const binding = digest("paperclip-runner-session-binding-v1", [ + canonical, + Buffer.from(String(serverProof)), + Buffer.from(clientProof), + ]); + const sessionId = `sha256:${binding.toString("hex")}`; + const nonce = (prefix: string, counter: number) => { + const value = Buffer.alloc(12); + value.write(prefix, 0, "ascii"); + value.writeBigUInt64BE(BigInt(counter), 4); + return value; + }; + const aad = (direction: string, counter: number) => + Buffer.from( + `paperclip.runner.secure-frame.v1\0${sessionId}\0${direction}\0${counter}`, + ); + const welcome = (await reader.next()).value!; + expect(welcome.counter).toBe(0); + const sealed = Buffer.from(String(welcome.ciphertext), "hex"); + const decipher = createDecipheriv( + "aes-256-gcm", + mac("paperclip-runner-core-to-client-key-v1", [binding]), + nonce("P3S1", 0), + ); + decipher.setAAD(aad("core_to_client", 0)); + decipher.setAuthTag(sealed.subarray(-16)); + const opened = JSON.parse( + Buffer.concat([ + decipher.update(sealed.subarray(0, -16)), + decipher.final(), + ]).toString("utf8"), + ); + expect(opened.kind).toBe("welcome"); + let counter = 0; + return { + socket, + send(value: Record) { + const cipher = createCipheriv( + "aes-256-gcm", + mac("paperclip-runner-client-to-core-key-v1", [binding]), + nonce("P3C1", counter), + ); + cipher.setAAD(aad("client_to_core", counter)); + const ciphertext = Buffer.concat([ + cipher.update(JSON.stringify(value)), + cipher.final(), + cipher.getAuthTag(), + ]); + socket.send( + JSON.stringify({ + schema: "paperclip.runner.secure-frame.v1", + counter: counter++, + ciphertext: ciphertext.toString("hex"), + }), + ); + }, + }; +} + +describe("Codex protocol integrity propagation", () => { + it("rejects an authenticated controller fault through the real driver, backend, and admitted runtime without accepting a result", async () => { + const directory = mkdtempSync( + join(tmpdir(), "paperclip-composed-integrity-"), + ); + const identity: DurableRecoveryIdentity = { + runnerInstanceId: "composed-runner", + environmentLeaseId: "composed-lease", + runId: "composed-run", + normalizedSessionId: "composed-session", + turnId: "composed-turn", + itemId: "composed-item", + }; + const contract = { + revision: "1", + objective: "Validate the authenticated failure boundary", + criteria: [ + { + id: "objective", + requirement: "Do not accept corrupt provider input", + }, + ], + }; + const input: NativeExecutionInputV1 = { + schema: "paperclip.native-execution-input.v1", + binding: { + companyId: "composed-company", + issueId: "composed-issue", + agentId: "composed-agent", + runId: identity.runId, + executionWorkspaceId: "composed-workspace", + }, + task: { + identifier: "TEST-1", + title: contract.objective, + description: null, + prompt: contract.objective, + workMode: "standard", + }, + workspace: { + cwd: directory, + repoUrl: null, + repoRef: null, + branchName: null, + }, + session: { + normalizedSessionId: identity.normalizedSessionId, + driverKind: "codex_app_server", + protocolVersion: 1, + }, + provider: { kind: "codex", model: null }, + completionContract: { + id: "composed-contract", + sha256: "composed-contract-sha", + schemaVersion: "paperclip.completion-contract.v1", + contract, + }, + interactionResponses: [], + credentialBindings: [], + }; + const events: PrpEvent[] = []; + const controlPlane: ControlPlanePort = { + openRun: vi.fn(async () => undefined), + checkpointSession: vi.fn(async () => undefined), + appendEvent: vi.fn(async (event) => { + events.push(event as PrpEvent); + return { + cursor: events.length, + highestContiguousSourceSeq: events.length, + disposition: "committed" as const, + }; + }), + replayEvents: vi.fn(async () => ({ + events: [], + highestContiguousSourceSeq: 0, + })), + completeRun: vi.fn(async () => undefined), + }; + let authority: DurablePrpControlPlane | undefined; + let finishProcess!: (result: { + code: number; + signal: null; + stdout: string; + stderr: string; + }) => void; + const completion = new Promise<{ + code: number; + signal: null; + stdout: string; + stderr: string; + }>((resolve) => { + finishProcess = resolve; + }); + const kill = vi.fn(() => { + finishProcess({ code: 0, signal: null, stdout: "", stderr: "" }); + return true; + }); + const launch = vi.fn(() => ({ + child: { exitCode: null, kill }, + completion, + })); + const bundle = createCapabilityRunnerdCodexTransport({ + stateDirectory: directory, + prpIdentity: identity, + runnerBinary: process.execPath, + codexCommand: process.execPath, + codexArgs: [], + sourceCodexHome: null, + environment: {}, + runnerReconnectGraceMs: 900_000, + closeGraceMs: 50, + runnerProcessLauncher: launch, + controlPlaneRegistration: async (core) => { + authority = core; + await core.start(); + return { connectUrl: core.connectUrl, release: () => core.stop() }; + }, + }); + const driver = new CodexAppServerDriver({ + taskEnvelope: createCodexTaskEnvelope({ objective: contract.objective }), + environment: { PAPERCLIP_WORKSPACE_CWD: directory }, + approvalPolicy: "never", + transportFactory: () => bundle.transport, + }); + const backend = new HarnessDriverBackend(driver); + const admitted = vi.fn(); + const execution = executeNativeSession({ + input, + backend, + controlPlane, + runnerInstanceId: identity.runnerInstanceId, + controlPlaneInstanceId: "composed-core", + timeoutMs: 900_000, + requireSessionCloseBeforeReturn: true, + onSession: admitted, + }).catch((error: unknown) => error); + let client: Awaited> | undefined; + try { + await vi.waitFor(() => expect(launch).toHaveBeenCalledTimes(1)); + const core = authority!; + client = await authenticatedRunner(core, identity); + const commandResult = async ( + type: string, + result: Record = {}, + ) => { + await vi.waitFor(() => + expect( + core.store.state.commands.some((command) => command.type === type), + ).toBe(true), + ); + const command = core.store.state.commands.find( + (candidate) => candidate.type === type, + )!; + client!.send({ + protocol: "paperclip.runner", + version: 1, + kind: "command_result", + payload: { + commandId: command.commandId, + commandType: command.type, + controllerSeq: command.controllerSeq, + status: "completed", + result, + }, + }); + await vi.waitFor(() => expect(command.status).toBe("completed")); + }; + const event = ( + sourceSeq: number, + eventType: PrpEvent["eventType"], + payload: Record, + ) => ({ + protocol: "paperclip.runner", + version: 1, + kind: "event", + ...identity, + payload: { + schema: "paperclip.prp.event.v1", + schemaVersion: 1, + sourceEventId: `composed-event-${sourceSeq}`, + sourceSeq, + sourceInstanceId: identity.runnerInstanceId, + sourceKind: "runner", + runId: identity.runId, + normalizedSessionId: identity.normalizedSessionId, + turnId: identity.turnId, + itemId: identity.itemId, + eventType, + priority: 0, + emittedAt: "2026-09-08T00:00:00.000Z", + payload, + }, + }); + await commandResult("run.prepare"); + await commandResult("session.open"); + client.send( + event(1, "session.started", { + threadId: "composed-provider-thread", + sessionId: "composed-provider-session", + runtimeIdentity: { processId: process.pid }, + }), + ); + await commandResult("session.goal.get", { goal: null }); + await commandResult("turn.start", { + providerTurnId: "composed-provider-turn", + }); + client.send( + event(2, "turn.started", { + providerTurnId: "composed-provider-turn", + status: "inProgress", + }), + ); + await vi.waitFor(() => + expect(events.some((entry) => entry.eventType === "turn.started")).toBe( + true, + ), + ); + expect(controlPlane.openRun).toHaveBeenCalledTimes(1); + expect(admitted).toHaveBeenCalledWith(expect.anything()); + // Capture the actual transport fault, not a newly constructed lookalike. + // A pending read also proves that request and notification consumers see + // the very same object before the runtime closes its transport. + const transportFailure = bundle.transport + .request("thread/read", { threadId: "composed-provider-thread" }) + .catch((error: unknown) => error); + await vi.waitFor(() => + expect( + core.store.state.commands.some( + (command) => command.type === "session.snapshot", + ), + ).toBe(true), + ); + const faultAt = Date.now(); + client.send( + event(3, "semantic_tool.input", { + semantic_tool: { + schema: "paperclip.prp.semantic_tool.v1", + schemaVersion: 1, + phase: "input", + callId: "composed-call", + operationId: "get_task_context", + correlation: { + runId: identity.runId, + normalizedSessionId: identity.normalizedSessionId, + turnId: identity.turnId, + itemId: identity.itemId, + }, + idempotencyKey: null, + content: { + digest: `sha256:${"0".repeat(64)}`, + redactionDisposition: "digest_only", + references: [], + }, + input: { summary: "DO-NOT-LEAK-composed-test" }, + }, + }), + ); + const primary = await transportFailure; + expect(primary).toBeInstanceOf(NativeSessionProtocolIntegrityError); + expect(primary).toMatchObject({ + code: "native_event_replay_conflict", + reason: "semantic_input_digest_mismatch", + recovery: "operator_required", + }); + expect(await execution).toBe(primary); + expect(Date.now() - faultAt).toBeLessThan(5_000); + expect(core.store.state.ackedSourceSeq).toBe(2); + expect( + core.store.state.committedEvents.map((entry) => entry.eventType), + ).toEqual(["session.started", "turn.started"]); + expect(controlPlane.completeRun).not.toHaveBeenCalled(); + expect( + events.some((entry) => entry.eventType === "run.result.proposed"), + ).toBe(false); + expect(launch).toHaveBeenCalledTimes(1); + expect(kill).toHaveBeenCalled(); + expect(bundle.evidence().diagnostics.join("\n")).not.toContain( + "DO-NOT-LEAK", + ); + } finally { + client?.socket.close(); + kill(); + await bundle.detachControllerForRestart(); + await authority?.stop(); + await execution; + rmSync(directory, { recursive: true, force: true }); + } + }, 15_000); + + it.each(["pre-start", "pending-input", "buffered-terminal"] as const)( + "preserves the exact integrity fault through the composed backend at %s", + async (stage) => { + const transport = new FakeCodexTransport(); + const driver = makeDriver([transport]); + const harness = await driver.openSession({ + runId: "run-integrity", + normalizedSessionId: "session-integrity", + workingDirectory: WORKSPACE, + }); + if (!(harness instanceof CodexSessionState)) + throw new Error("Expected Codex state"); + const backend = new HarnessDriverBackend({ + descriptor: () => driver.descriptor(), + openSession: async () => harness, + }); + const session = await backend.openSession({ + identity: { + runId: "run-integrity", + sessionId: "session-integrity", + companyId: "company", + issueId: "issue", + agentId: "agent", + }, + workingDirectory: WORKSPACE, + }); + const fault = new NativeSessionProtocolIntegrityError( + "semantic_input_digest_mismatch", + ); + const events: PrpEvent[] = []; + let pending: Promise> | undefined; + let consumed: Promise | undefined; + const consume = async () => { + try { + for await (const event of session.events()) events.push(event); + return null; + } catch (error) { + return error; + } + }; + try { + if (stage !== "pre-start") { + const { turnId } = await session.startTurn({ + message: { role: "user", text: "Work safely." }, + }); + if (stage === "pending-input") { + consumed = consume(); + pending = transport.invoke({ + id: "input-integrity", + method: "item/tool/requestUserInput", + params: { + threadId: "thread-1", + turnId, + itemId: "input-integrity-item", + questions: [ + { + id: "color", + header: "Color", + question: "Which color?", + options: [{ label: "Amber" }, { label: "Cobalt" }], + }, + ], + }, + }); + await vi.waitFor(() => + expect( + events.some( + (event) => event.eventType === "runtime_request.created", + ), + ).toBe(true), + ); + } else { + transport.queue.push({ + method: "turn/completed", + params: { + threadId: "thread-1", + turn: { + id: turnId, + status: "completed", + items: [ + { + id: "final", + type: "agentMessage", + text: JSON.stringify(result), + }, + ], + }, + }, + }); + await vi.waitFor(() => expect(harness.terminal).toBe(true)); + expect(harness.result).not.toBeNull(); + } + } + transport.queue.fail(fault); + await vi.waitFor(() => expect(harness.protocolFailed).toBe(true)); + consumed ??= consume(); + expect(await consumed).toBe(fault); + expect( + events.filter((event) => + [ + "turn.completed", + "turn.failed", + "turn.interrupted", + "run.result.proposed", + "runtime_request.expired", + ].includes(event.eventType), + ), + ).toEqual([]); + if (pending) expect(await pending).toEqual({ answers: {} }); + await expect(session.result()).rejects.toBe(fault); + await expect(session.snapshot()).rejects.toBe(fault); + await expect( + session.startTurn({ + message: { role: "user", text: "Do not retry." }, + }), + ).rejects.toBe(fault); + await expect( + session.attachRun!({ + identity: { ...session.identity(), runId: "replacement-run" }, + }), + ).rejects.toBe(fault); + expect( + transport.calls.filter((call) => call.method === "turn/start"), + ).toHaveLength(stage === "pre-start" ? 0 : 1); + } finally { + await session.close({ reason: "test cleanup" }); + await consumed; + await pending; + } + }, + ); + + it("preserves a protocol failure received before turn start as a typed terminal", async () => { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ runId: "prestart", normalizedSessionId: "prestart-session", workingDirectory: WORKSPACE }); + transport.queue.push({ method: "turn/completed", params: { threadId: "unrelated", turn: { id: "wrong", status: "completed" } } }); + const events: PrpEvent[] = []; + for await (const event of session.events()) events.push(event); + await expect(session.startTurn({ message: { role: "user", text: "Work" } })).rejects.toMatchObject({ code: "native_provider_terminal_failed", providerCode: "thread_binding_mismatch", recoverable: false }); + await session.close({ reason: "test complete" }); + }); + + it("does not promote a message-and-field lookalike transport error", async () => { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ + runId: "run-generic", + normalizedSessionId: "session-generic", + workingDirectory: WORKSPACE, + }); + const fault = Object.assign(new Error("native_event_replay_conflict"), { + code: "native_event_replay_conflict", + reason: "semantic_input_digest_mismatch", + }); + const events: PrpEvent[] = []; + const consumed = (async () => { + for await (const event of session.events()) events.push(event); + })(); + try { + await session.startTurn({ message: { role: "user", text: "Work." } }); + transport.queue.fail(fault); + await consumed; + expect( + events.find((event) => event.eventType === "session.failed")?.payload + .code, + ).toBe("notification_transport_failed"); + expect(events.some((event) => event.eventType === "turn.failed")).toBe( + true, + ); + await expect(session.snapshot()).resolves.toBeDefined(); + } finally { + await session.close({ reason: "test cleanup" }); + await consumed; + } + }); +}); diff --git a/packages/paperclip-runner/src/drivers/codex/codex-session-notifications.ts b/packages/paperclip-runner/src/drivers/codex/codex-session-notifications.ts index 0470981119..cfccf479d8 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-session-notifications.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-session-notifications.ts @@ -1,13 +1,14 @@ +import { classifyCodexNotification } from "./codex-notification-identity.js"; import { paperclipWorkspaceFileReferencesFromText } from "../../live/workspace-file-reference.js"; -import { canonicalProviderEventsFromCodex } from "../../provider-events.js"; +import { canonicalProviderEventsFromCodex, isCanonicalProviderEventType } from "../../provider-events.js"; import { harnessRuntimeRequestOutcome } from "../../contracts/harness-driver.js"; +import { NativeSessionProtocolIntegrityError } from "../../contracts/native-session-backend.js"; import { validatePrpStructuredRunResult } from "../../protocol/replay-contract.js"; import type { CodexRpcNotification, CodexTraceInterpretation } from "./app-server-transport.js"; import { redactCodexDiagnostic } from "./app-server-transport.js"; import { boundedCodexPayload as boundedPayload, boundedCodexValue, isRetainableCodexPayload } from "./codex-boundaries.js"; import { runtimeRequestResponse } from "./codex-question-adapter.js"; import { - isBoundCodexNotification, isSupportedCodexNotificationMethod, codexThreadLineage as lineageFromThread, codexThreadStatus as threadStatus, @@ -40,6 +41,10 @@ export async function pumpNotifications(state: CodexSessionState): Promise await mapNotification(state, notification); } } catch (error) { + if (error instanceof NativeSessionProtocolIntegrityError) { + state.failProtocolIntegrity(error); + return; + } state.emit("harness.diagnostic", { code: "notification_transport_failed", message: redactCodexDiagnostic(String(error)), @@ -100,23 +105,29 @@ async function mapNotification(state: CodexSessionState, notification: CodexRpcN async function mapNotificationBody(state: CodexSessionState, notification: CodexRpcNotification): Promise { if (!isSupportedCodexNotificationMethod(notification.method)) return; - if (!isBoundCodexNotification(notification, { - runId: state.runId, - threadIds: [...state.lineageByThread.keys()], - })) { - const params = notification.params; - const claimedThreadId = text( - params.threadId, - text(record(params.thread).id, text(record(params.turn).threadId)), - ); - const claimedRunId = text(params.runId, text(params.paperclipRunId)); - if (claimedThreadId.length > 0 || claimedRunId.length > 0) { - state.failProtocol( - "thread_binding_mismatch", - `Provider ${notification.method} message did not name the active run or a known thread.`, - ); + if (notification.method === "item/completed" && notification.params.kind === "steering_acknowledgement" + && !notification.params.threadId && !notification.params.turnId && !notification.params.thread && !notification.params.turn) return; + const identity = classifyCodexNotification({ + method: notification.method, params: notification.params, runId: state.runId, + rootThreadId: state.opened.threadId, activeTurnId: state.activeTurnId, + knownThreads: new Set(state.lineageByThread.keys()), settledTurns: new Set(state.terminalTurns.keys()), + }); + if (identity.classification !== "root") { + if (state.notificationIdentityDiagnostics < 32) { + state.notificationIdentityDiagnostics += 1; + state.emit("harness.diagnostic", { + code: "provider_notification_identity", method: notification.method.slice(0, 128), + classification: identity.classification, + expectedThreadId: state.opened.threadId, receivedThreadId: identity.threadId?.slice(0, 256) ?? null, + expectedTurnId: state.activeTurnId, receivedTurnId: identity.turnId?.slice(0, 256) ?? null, + }); } - return; + if (identity.classification === "invalid_authority") { + state.failProtocol("thread_binding_mismatch", `Provider authoritative notification ${notification.method.slice(0, 128)} did not name its execution owner.`); + return; + } + if (identity.classification !== "descendant") return; + if (!["thread/started", "thread/status/changed", "thread/closed"].includes(notification.method)) return; } const params = notification.params; const turn = record(params.turn); @@ -124,6 +135,14 @@ async function mapNotificationBody(state: CodexSessionState, notification: Codex const threadId = text(params.threadId); const turnId = text(params.turnId, text(turn.id)); const itemId = text(item.id, text(params.itemId)); + if (notification.method === "paperclip/canonicalProviderEvent") { + if (!isCanonicalProviderEventType(params.eventType)) { + state.failProtocol("provider_event_type_invalid", "Unknown canonical provider event type."); + return; + } + state.emit(params.eventType, record(params.payload), { turnId: turnId || undefined, itemId: itemId || undefined }); + return; + } if (notification.method === "paperclip/workspaceChange/updated") { if (!state.notificationNamesActiveTurn(turnId, "workspace change")) return; if (threadId.length > 0 && threadId !== state.opened.threadId) return; diff --git a/packages/paperclip-runner/src/drivers/codex/codex-session-state.ts b/packages/paperclip-runner/src/drivers/codex/codex-session-state.ts index 2dbb35e7ae..4856acf46a 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-session-state.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-session-state.ts @@ -13,6 +13,7 @@ import { harnessRuntimeRequestOutcome, } from "../../contracts/harness-driver.js"; import type { CodexTaskEnvelope } from "../../contracts/codex.js"; +import { NativeSessionProtocolIntegrityError } from "../../contracts/native-session-backend.js"; import { validatePrpStructuredRunResult, type PrpEvent, @@ -32,21 +33,34 @@ import { canonicalJson, record } from "./codex-driver-values.js"; class AsyncQueue implements AsyncIterable { #values: T[] = []; - #waiters: Array<(value: IteratorResult) => void> = []; + #waiters: Array<{ + resolve: (value: IteratorResult) => void; + reject: (error: Error) => void; + }> = []; #closed = false; + #failure: Error | null = null; push(value: T): void { if (this.#closed) return; const waiter = this.#waiters.shift(); if (waiter === undefined) this.#values.push(value); - else waiter({ value, done: false }); + else waiter.resolve({ value, done: false }); } close(): void { if (this.#closed) return; this.#closed = true; for (const waiter of this.#waiters.splice(0)) - waiter({ value: undefined, done: true }); + waiter.resolve({ value: undefined, done: true }); + } + + fail(error: Error): void { + this.#failure ??= error; + this.#closed = true; + // Integrity failure takes precedence over a buffered semantic/terminal + // suffix, including one whose consumer has not started reading yet. + this.#values = []; + for (const waiter of this.#waiters.splice(0)) waiter.reject(this.#failure); } clear(): void { @@ -56,10 +70,13 @@ class AsyncQueue implements AsyncIterable { [Symbol.asyncIterator](): AsyncIterator { return { next: async () => { + if (this.#failure !== null) throw this.#failure; const value = this.#values.shift(); if (value !== undefined) return { value, done: false }; if (this.#closed) return { value: undefined, done: true }; - return new Promise((resolve) => this.#waiters.push(resolve)); + return new Promise((resolve, reject) => + this.#waiters.push({ resolve, reject }), + ); }, }; } @@ -104,6 +121,7 @@ export class CodexSessionState { protocolFailed = false; protocolFailureCode: string | null = null; protocolFailureMessage: string | null = null; + protocolIntegrityFailure: NativeSessionProtocolIntegrityError | null = null; terminal = false; dispositionOnlyRecoveryAvailable = false; dispositionOnlyRecoveryConsumed = false; @@ -117,6 +135,7 @@ export class CodexSessionState { "progress" | "final" | "summary" | "detail" | "unknown" >(); readonly pendingRuntimeRequestMap = new Map(); + notificationIdentityDiagnostics = 0; readonly lineageByThread = new Map(); currentGoal: HarnessThreadGoal | null = null; interruptQueued = false; @@ -317,6 +336,7 @@ export class CodexSessionState { operation: string, detail: unknown, ): HarnessCapabilityUnavailableError { + this.rethrowProtocolIntegrity(detail); const error = new HarnessCapabilityUnavailableError( operation, redactCodexDiagnostic(String(detail)), @@ -336,6 +356,34 @@ export class CodexSessionState { }); } + assertProtocolIntegrity(): void { + if (this.protocolIntegrityFailure !== null) + throw this.protocolIntegrityFailure; + } + + rethrowProtocolIntegrity(error: unknown): void { + if (!(error instanceof NativeSessionProtocolIntegrityError)) return; + this.failProtocolIntegrity(error); + this.assertProtocolIntegrity(); + } + + failProtocolIntegrity(error: NativeSessionProtocolIntegrityError): void { + this.protocolIntegrityFailure ??= error; + this.protocolFailed = true; + this.protocolFailureCode = this.protocolIntegrityFailure.code; + this.protocolFailureMessage = this.protocolIntegrityFailure.message; + this.terminal = true; + this.result = null; + this.resultFingerprint = null; + this.resultCallId = null; + this.resultTurnId = null; + // Fail the stream before settling pending local RPCs: a synthetic input + // expiration or terminal event must not turn corruption into a safe wait. + this.eventQueue.fail(this.protocolIntegrityFailure); + this.cancelPendingRequests("protocol_integrity_failed"); + // The owning runtime still performs and awaits exact transport cleanup. + } + failProtocol(code: string, message: string): void { if (this.protocolFailed) return; this.protocolFailed = true; @@ -351,7 +399,7 @@ export class CodexSessionState { const turnId = this.activeTurnId; this.emit( "turn.failed", - { status: "failed", error: { code } }, + { status: "failed", error: { code, message: this.protocolFailureMessage, recoverable: false } }, { turnId }, ); this.terminalTurns.set(turnId, canonicalJson({ protocolFailure: code })); diff --git a/packages/paperclip-runner/src/drivers/codex/codex-thread-normalization.ts b/packages/paperclip-runner/src/drivers/codex/codex-thread-normalization.ts index 00a8b5e1b8..355f9e2d04 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-thread-normalization.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-thread-normalization.ts @@ -140,6 +140,7 @@ export function isSupportedCodexNotificationMethod(method: string): boolean { method === "model/verification" || method === "model/safetyBuffering/updated" || method.startsWith("item/") || + method === "paperclip/canonicalProviderEvent" || method === "paperclip/workspaceChange/updated" || method === "paperclip/runResult" || method === "turn/diff/updated" || diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index 2b7d39aa33..61ed44ed75 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -15,6 +15,18 @@ import { join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { expect, it, vi } from "vitest"; +import type { ControlPlanePort } from "../contracts/control-plane-port.js"; +import type { NativeExecutionInputV1 } from "../contracts/native-execution.js"; +import type { + NativeSession, + NativeSessionBackend, +} from "../contracts/native-session-backend.js"; +import type { + PrpEvent, + PrpStructuredRunResult, + PrpTerminalState, +} from "../protocol/replay-contract.js"; +import { executeNativeSession } from "../native-session-runtime.js"; import type { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js"; import { @@ -46,6 +58,7 @@ import { latestRunnerdSessionReadiness, rehydrateRunnerdGoalNotification, rehydrateRunnerdItemNotification, + rehydrateRunnerdDeltaNotification, rehydrateRunnerdPlanNotification, rehydrateRunnerdResultNotification, rehydrateRunnerdThreadTokenUsage, @@ -1158,6 +1171,11 @@ it("binds a durable semantic result to the active provider turn", () => { }); }); +it("restores provider identity and streamed text from a canonical delta", () => { + expect(rehydrateRunnerdDeltaNotification({ text: "Reading Gmail", itemId: "message-1", turnId: "controller-turn" }, "root-thread", "provider-turn")) + .toMatchObject({ threadId: "root-thread", turnId: "provider-turn", delta: "Reading Gmail", itemId: "message-1" }); +}); + it("rehydrates a canonical agent item for the strict Codex facade", () => { expect( rehydrateRunnerdItemNotification( @@ -1937,6 +1955,247 @@ it("continues rehydrating events after the committed-event window slides", async } }, 30_000); +it("does not retry a real memoized transport close whose suspension proof is unavailable", async () => { + const identity = { + runId: "run-recovery", + sessionId: "session-recovery", + companyId: "company-recovery", + issueId: "issue-recovery", + agentId: "agent-recovery", + }; + + const result: PrpStructuredRunResult = { + schema: "paperclip.run_result.v1", + reportedWorkDisposition: "done", + summary: "Recovered native work completed.", + completionClaim: { + contractRevision: "1", + objectiveSatisfied: true, + criteria: [ + { criterionId: "objective", status: "satisfied", evidenceRefs: [] }, + ], + remainingWork: [], + }, + evidence: [], + verification: [{ commandOrCheck: "recovery", status: "passed" }], + attentionRequests: [], + artifacts: [], + }; + + const terminal: PrpTerminalState = { + schema: "paperclip.prp.terminal.v1", + turnTerminalState: "completed", + runTerminalState: "succeeded", + reportedWorkDisposition: "done", + }; + + const input: NativeExecutionInputV1 = { + schema: "paperclip.native-execution-input.v1", + binding: { + companyId: identity.companyId, + runId: identity.runId, + issueId: identity.issueId, + agentId: identity.agentId, + executionWorkspaceId: "workspace-recovery", + }, + task: { + identifier: "PAP-RECOVERY", + title: "Recover native work", + description: null, + prompt: "# PAP-RECOVERY: Recover native work", + workMode: "standard", + }, + workspace: { + cwd: "/workspace", + repoUrl: null, + repoRef: null, + branchName: null, + }, + session: { + normalizedSessionId: identity.sessionId, + driverKind: "codex_app_server", + protocolVersion: 1, + }, + provider: { kind: "codex", model: null }, + completionContract: { + id: "contract-recovery", + sha256: "contract-recovery-sha", + schemaVersion: "paperclip.completion-contract.v1", + contract: { + revision: "1", + objective: "Recover native work", + criteria: [{ id: "objective", requirement: "Complete after recovery" }], + }, + }, + interactionResponses: [], + credentialBindings: [], + }; + + function runnerEvent( + sourceSeq: number, + eventType: PrpEvent["eventType"], + payload: Record = {}, + ): PrpEvent { + return { + schema: "paperclip.prp.event.v1", + sourceEventId: `runner-recovery:${identity.runId}:${sourceSeq}`, + sourceSeq, + sourceInstanceId: "runner-recovery", + sourceKind: "runner", + runId: identity.runId, + normalizedSessionId: identity.sessionId, + turnId: "turn-recovery", + eventType, + schemaVersion: 1, + priority: 0, + emittedAt: "2026-08-09T00:00:00.000Z", + payload, + }; + } + const stateDirectory = await mkdtemp( + join(tmpdir(), "native-close-quarantine-"), + ); + const readRunnerState = vi.fn(async () => ({ + schema: "paperclip.runner.durable.state.v1", + runnerInstanceId: "runner-close-quarantine", + environmentLeaseId: "lease-close-quarantine", + runId: identity.runId, + normalizedSessionId: identity.sessionId, + turnId: "turn-close-quarantine", + itemId: "item-close-quarantine", + lifecycle: "ready", + })); + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: resolve( + import.meta.dirname, + "../../runner/target/debug/fake-codex-app-server", + ), + codexArgs: ["--state-file", join(stateDirectory, "fake-codex-state.json")], + stateDirectory, + closeGraceMs: 400, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, + prpIdentity: await readRunnerState(), + readRunnerState, + // A checkpoint owner requires durable suspension proof. A local transport + // without a checkpoint can simply terminate its process on close. + controlPlaneRegistration: async (authority) => { + await authority.start(); + return { + connectUrl: authority.connectUrl, + checkpoint: async () => {}, + release: async () => {}, + }; + }, + }); + try { + await bundle.transport.request("thread/start", { + cwd: stateDirectory, + dynamicTools: [], + }); + const failedClose = bundle.transport.close(); + const failure = await failedClose.catch((error: unknown) => error); + expect(failure).toMatchObject({ + code: "native_session_close_unrecoverable", + }); + expect(bundle.transport.close()).toBe(failedClose); + + vi.useFakeTimers(); + const close = vi.fn(({ reason }: { reason: string }) => + bundle.transport.close(reason), + ); + const capabilities = { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return capabilities; + }, + async *events() { + yield runnerEvent(1, "turn.completed"); + }, + async startTurn() { + return { turnId: "turn-recovery" }; + }, + async result() { + return { result, terminal, turnId: "turn-recovery" }; + }, + async snapshot() { + return { + backendKind: "mock", + sessionId: identity.sessionId, + identity, + providerSessionId: "provider-recovery", + cursor: null, + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const openSession = vi.fn(async () => session); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "real-memoized-close-quarantine", + version: "1", + capabilities, + }; + }, + openSession, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent() { + return { + cursor: 1, + highestContiguousSourceSeq: 1, + disposition: "committed", + }; + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, + async completeRun() {}, + }; + const execute = () => + executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + requireSessionCloseBeforeReturn: true, + }); + await expect(execute()).rejects.toBe(failure); + const readsAfterClose = readRunnerState.mock.calls.length; + await expect(execute()).rejects.toMatchObject({ + code: "native_session_cleanup_quarantined", + recovery: "operator_required", + }); + await vi.advanceTimersByTimeAsync(600_000); + await expect(execute()).rejects.toMatchObject({ + code: "native_session_cleanup_quarantined", + }); + expect(openSession).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalledOnce(); + expect(readRunnerState).toHaveBeenCalledTimes(readsAfterClose); + } finally { + vi.useRealTimers(); + await bundle.transport.close().catch(() => undefined); + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 10_000); + + it("binds an immediately failed durable turn before exposing its terminal", async () => { const stateDirectory = await mkdtemp( join(tmpdir(), "runnerd-fast-terminal-"), diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index 9f23830f55..a2cfcddaef 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -1,3 +1,4 @@ +import { isCanonicalProviderEventType } from "../provider-events.js"; import { execFileSync } from "node:child_process"; import { createHash, randomUUID } from "node:crypto"; import { @@ -34,6 +35,7 @@ import type { DurableRecoveryCommittedEvent, DurableRecoveryIdentity, } from "../contracts/durable-recovery.js"; +import { NativeSessionCloseUnrecoverableError } from "../contracts/native-session-backend.js"; import type { HarnessRuntimeRequestResolution, PersistedHarnessProviderIdentity, @@ -1565,6 +1567,22 @@ export function rehydrateRunnerdTurnNotification( }; } +export function rehydrateRunnerdDeltaNotification( + rawParams: Record, + openedThreadId: string, + activeTurnId: string, +): Record { + // Canonical PRP events already passed runner identity validation. Restore the + // provider binding just as for item starts/completions; the PRP controller + // turn is deliberately different from the provider's turn ID. + return { + ...rawParams, + threadId: openedThreadId, + turnId: activeTurnId, + delta: rawParams.delta ?? rawParams.text, + }; +} + export function rehydrateRunnerdItemNotification( rawParams: Record, openedThreadId: string, @@ -2152,6 +2170,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { #checkpointProviderIdentityConfirmed = false; #turnId = ""; #turnStartResponsePending = false; + #turnStartResponseSettled: Promise = Promise.resolve(); #turnStartResponseEpoch = 0; #observedTurnStartEpoch = 0; #expectedProviderTurnId: string | null = null; @@ -3009,9 +3028,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#controlPlaneRelease = null; } if (suspensionRequired && !runnerSuspended) { - throw new Error( - "provider_transport_failed: runner did not durably suspend before checkpoint", - ); + throw new NativeSessionCloseUnrecoverableError(); } if (this.#ownsRoot) rmSync(this.#root, { recursive: true, force: true }); this.#publish(); @@ -3043,8 +3060,13 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { identity, expectedRunnerVersion: runnerArtifact.version, expectedRunnerDigest: runnerArtifact.digest, - onSemanticToolInput: async (call) => - unwrapToolResponse( + onProtocolIntegrityError: (error) => this.#failTransport(error), + onSemanticToolInput: async (call) => { + // Semantic input may outrun the facade's turn/start response. Bind it + // only after the strict driver has accepted that same provider turn. + await this.#turnStartResponseSettled; + this.#throwIfFailed(); + return unwrapToolResponse( await this.#handler({ id: call.callId, method: "item/tool/call", @@ -3064,7 +3086,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } : {}), }), - ), + ); + }, connectionLeaseTtlMs: 60 * 60 * 1_000, }); this.#core = core; @@ -3694,8 +3717,13 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { identity, expectedRunnerVersion: runnerArtifact.version, expectedRunnerDigest: runnerArtifact.digest, - onSemanticToolInput: async (call) => - unwrapToolResponse( + onProtocolIntegrityError: (error) => this.#failTransport(error), + onSemanticToolInput: async (call) => { + // Semantic input may outrun the facade's turn/start response. Bind it + // only after the strict driver has accepted that same provider turn. + await this.#turnStartResponseSettled; + this.#throwIfFailed(); + return unwrapToolResponse( await this.#handler({ id: call.callId, method: "item/tool/call", @@ -3715,7 +3743,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } : {}), }), - ), + ); + }, connectionLeaseTtlMs: 60 * 60 * 1_000, }); this.#core = core; @@ -3933,6 +3962,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#turnId = pendingTurnId; const responseEpoch = ++this.#turnStartResponseEpoch; this.#turnStartResponsePending = true; + let releaseStartResponse!: () => void; + this.#turnStartResponseSettled = new Promise(resolve => { releaseStartResponse = resolve; }); this.#expectedProviderTurnId = null; let responseReady = false; try { @@ -4000,6 +4031,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { return { turn: { id: this.#turnId, status: "inProgress" } }; } finally { if (!responseReady) { + releaseStartResponse(); if (this.#turnStartResponseEpoch === responseEpoch) { this.#turnStartResponsePending = false; this.#expectedProviderTurnId = null; @@ -4010,6 +4042,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { // following task so the driver can bind and emit turn.accepted first. // The epoch prevents a late release from clearing a newer turn fence. const release = setTimeout(() => { + releaseStartResponse(); if (this.#turnStartResponseEpoch !== responseEpoch) return; this.#turnStartResponsePending = false; this.#expectedProviderTurnId = null; @@ -4294,7 +4327,14 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { sessionUpdatePayload, ); const notifications = - event.eventType === "provider.event" + isCanonicalProviderEventType(event.eventType) && !canonicalMethod + ? [{ method: "paperclip/canonicalProviderEvent", params: { + ...(this.#threadId ? { threadId: this.#threadId } : {}), + ...(this.#turnId ? { turnId: this.#turnId } : {}), + eventType: event.eventType, payload: eventPayload, + itemId: event.envelope.itemId, + } }] + : event.eventType === "provider.event" ? unwrapRunnerdProviderNotifications(eventPayload) : canonicalMethod ? expandRunnerdCanonicalNotifications(canonicalMethod, eventPayload) @@ -4355,6 +4395,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#threadId, method, ) + : event.eventType === "item.delta" + ? rehydrateRunnerdDeltaNotification(rawParams, this.#threadId, this.#turnId) : event.eventType !== "provider.event" && (method === "item/started" || method === "item/completed") ? rehydrateRunnerdItemNotification( @@ -4373,7 +4415,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { : rawParams; if ( params.turnId === undefined && - typeof event.envelope.turnId === "string" + typeof event.envelope.turnId === "string" && event.envelope.turnId.length > 0 ) { params.turnId = event.envelope.turnId; } @@ -4696,7 +4738,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { const detail = result.stderr.trim() || result.stdout.trim(); if (detail) this.#diagnostic(detail.slice(-4_096)); this.#publish(); - if (this.#closed || this.#handle !== handle) return; + if (this.#closed || this.#failure !== null || this.#handle !== handle) + return; // A per-turn runner exits after its terminal suffix is durably ACKed. // Drain that suffix into the provider-facing queue before classifying // process completion; clean terminal exit is the expected lifecycle. @@ -4735,7 +4778,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { ); }, (error) => { - if (this.#closed || this.#handle !== handle) return; + if (this.#closed || this.#failure !== null || this.#handle !== handle) + return; if ( this.#startupComplete && this.options.runnerReconnectGraceMs !== undefined && @@ -4763,6 +4807,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { if ( this.#runnerRecoveryInProgress || this.#closed || + this.#failure !== null || this.#handle !== failedHandle || this.#core === null || !failedHandle.restart @@ -4779,7 +4824,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { `runner process disconnected; recovery is allowed for ${graceMs}ms`, ); try { - while (!this.#closed && Date.now() < deadline) { + while (!this.#closed && this.#failure === null && Date.now() < deadline) { if (attempt > 0) { const base = delays[Math.min(attempt - 1, delays.length - 1)]!; const jittered = Math.max( @@ -4788,7 +4833,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { ); await new Promise((resolveWait) => setTimeout(resolveWait, jittered)); } - if (this.#closed) return; + if (this.#closed || this.#failure !== null) return; if (Date.now() >= deadline) break; const priorConnectionCount = this.#core.store.state.connectionCount; let recoveredHandle: RunnerProcessHandle; @@ -4825,7 +4870,12 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { }, ); const authenticated = (async () => { - while (!processSettled && !this.#closed && Date.now() < deadline) { + while ( + !processSettled && + !this.#closed && + this.#failure === null && + Date.now() < deadline + ) { if ( this.#core !== null && this.#core.store.state.connectionCount > priorConnectionCount && @@ -4861,8 +4911,13 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#rejectFailureSignal(error); if (this.#pump !== null) clearInterval(this.#pump); this.#pump = null; - this.#diagnostic(error.message); - this.#queue.close(error); + try { + this.#diagnostic(error.message); + } finally { + // An observer is not allowed to leave notification consumers waiting + // after the request path has already received this terminal failure. + this.#queue.close(error); + } } #throwIfFailed(): void { diff --git a/packages/paperclip-runner/src/native-session-runtime.test.ts b/packages/paperclip-runner/src/native-session-runtime.test.ts index 57046c6a4b..93aa995268 100644 --- a/packages/paperclip-runner/src/native-session-runtime.test.ts +++ b/packages/paperclip-runner/src/native-session-runtime.test.ts @@ -8,6 +8,11 @@ import type { NativeSessionBackend, PersistedNativeSession, } from "./contracts/native-session-backend.js"; +import { + NativeSessionCloseUnrecoverableError, + NativeSessionCleanupQuarantinedError, + NativeSessionProtocolIntegrityError, +} from "./contracts/native-session-backend.js"; import type { PrpEvent, PrpStructuredRunResult, @@ -195,6 +200,34 @@ function highestContiguous(events: PrpEvent[]): number { } describe("executeNativeSession recovery", () => { + it.each([false, true])("preserves a durable session failure when its stream closes (throws=%s)", async throws => { + const capabilities = { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + const session: NativeSession = { + identity: () => identity, + async capabilities() { return capabilities; }, + async *events() { + yield runnerEvent(1, "session.failed", { error: { code: "notification_transport_failed" }, recoverable: false }); + if (throws) throw new Error("provider stdout closed"); + }, + async startTurn() { return { turnId: "turn-recovery" }; }, + async result() { return null; }, + async snapshot() { return { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [] }; }, + async close() {}, + }; + const completeRun = vi.fn(); + const backend: NativeSessionBackend = { + async descriptor() { return { kind: "mock", name: "failure-fixture", version: "1", capabilities }; }, + async openSession() { return session; }, + }; + const port: ControlPlanePort = { + async openRun() {}, async checkpointSession() {}, + async appendEvent() { return { cursor: 1, highestContiguousSourceSeq: 1, disposition: "committed" }; }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, completeRun, + }; + await expect(executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery" })) + .rejects.toMatchObject({ code: "native_provider_terminal_failed", providerCode: "notification_transport_failed", recoverable: false }); + expect(completeRun).not.toHaveBeenCalled(); + }); it.each((["complete", "paused", "blocked", "limited", "usageLimited", "budgetLimited"] as const) .flatMap((status) => [false, true].map((snapshotBeforeUpdate) => ({ status, snapshotBeforeUpdate }))))( "handles a new chat turn instead of completing it from an existing $status goal (snapshot: $snapshotBeforeUpdate)", async ({ status, snapshotBeforeUpdate }) => { @@ -751,13 +784,13 @@ describe("executeNativeSession recovery", () => { }); }); - it("surfaces the provider's model rejection instead of missing semantic completion", async () => { + it.each([false, true])("preserves structured provider failure even when its message mentions a model (recoverable=%s)", async (recoverable) => { const capabilities = { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true }; const close = vi.fn(async () => {}); const session: NativeSession = { identity: () => identity, async capabilities() { return capabilities; }, - async *events() { yield runnerEvent(1, "turn.failed", { error: { code: "RUNTIME", message: "There's an issue with the selected model (custom-model). It may not exist or you may not have access to it." } }); }, + async *events() { yield runnerEvent(1, "turn.failed", { error: { code: "RUNTIME", recoverable, message: "There's an issue with the selected model (custom-model). It may not exist or you may not have access to it." } }); }, async startTurn() { return { turnId: "turn-recovery" }; }, async result() { return null; }, async snapshot() { return { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [] }; }, @@ -773,7 +806,11 @@ describe("executeNativeSession recovery", () => { async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; - await expect(executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery" })).rejects.toThrow("native_provider_model_rejected: There's an issue with the selected model (custom-model)"); + const result = executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery" }); + await expect(result).rejects.toThrow("There's an issue with the selected model (custom-model)"); + await expect(result).rejects.toMatchObject({ + code: "native_provider_terminal_failed", providerCode: "RUNTIME", recoverable, + }); expect(close).toHaveBeenCalled(); }); @@ -2500,6 +2537,247 @@ describe("executeNativeSession recovery", () => { }, ); + it.each([ + { boundary: "control-plane replay", fault: "typed", admitted: false }, + { boundary: "final event append", fault: "typed", admitted: false }, + { + boundary: "lost final event acknowledgement", + fault: "typed", + admitted: false, + }, + { boundary: "control-plane replay", fault: "lookalike", admitted: true }, + { boundary: "control-plane replay", fault: "generic", admitted: true }, + { + boundary: "completion invoked before commit", + fault: "typed", + admitted: true, + }, + { + boundary: "lost completion acknowledgement", + fault: "typed", + admitted: true, + }, + ] as const)( + "observes integrity before completion admission without revoking admitted completion (%j)", + async ({ boundary, fault, admitted }) => { + vi.useFakeTimers(); + let releaseBoundary = () => {}; + const boundaryReleased = new Promise((resolve) => { + releaseBoundary = resolve; + }); + let markBoundaryReached = () => {}; + const boundaryReached = new Promise((resolve) => { + markBoundaryReached = resolve; + }); + const failure = + fault === "typed" + ? new NativeSessionProtocolIntegrityError( + "semantic_input_digest_mismatch", + ) + : fault === "lookalike" + ? Object.assign(new Error("untrusted transport error"), { + code: "native_event_replay_conflict", + reason: "semantic_input_digest_mismatch", + }) + : new Error("snapshot temporarily unavailable"); + let latchedFailure: Error | null = null; + let boundaryBlocked = false; + const waitAtBoundary = async () => { + if (boundaryBlocked) return; + boundaryBlocked = true; + markBoundaryReached(); + await boundaryReleased; + }; + const events: PrpEvent[] = []; + let durableCompletion: unknown = null; + const close = vi.fn(async () => undefined); + const resolveResult = vi.fn(async () => ({ + result, + terminal, + turnId: "turn-recovery", + })); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; + }, + async *events() { + yield runnerEvent(1, "turn.completed"); + }, + async startTurn() { + return { turnId: "turn-recovery" }; + }, + result: resolveResult, + async snapshot() { + if (latchedFailure !== null) throw latchedFailure; + return { + backendKind: "mock", + sessionId: identity.sessionId, + identity, + providerSessionId: "provider-recovery", + cursor: "1", + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: `completion-integrity-${boundary}-${fault}`, + version: "1", + capabilities: await session.capabilities(), + }; + }, + async openSession() { + return session; + }, + }; + const completeRun = vi.fn( + async (completion) => { + if (boundary === "completion invoked before commit") + await waitAtBoundary(); + if (durableCompletion === null) + durableCompletion = structuredClone(completion); + else expect(completion).toEqual(durableCompletion); + if ( + boundary === "lost completion acknowledgement" && + !boundaryBlocked + ) { + await waitAtBoundary(); + await new Promise(() => undefined); + } + }, + ); + const checkpointSession = vi.fn( + async () => undefined, + ); + const port: ControlPlanePort = { + async openRun() {}, + checkpointSession, + async appendEvent(rawEvent) { + const event = structuredClone(rawEvent as PrpEvent); + const existing = events.some( + (candidate) => + candidate.sourceInstanceId === event.sourceInstanceId && + candidate.sourceSeq === event.sourceSeq, + ); + if (!existing) events.push(event); + if ( + boundary === "final event append" && + event.eventType === "run.terminal" + ) { + await waitAtBoundary(); + } + if ( + boundary === "lost final event acknowledgement" && + event.eventType === "run.terminal" && + !boundaryBlocked + ) { + await waitAtBoundary(); + await new Promise(() => undefined); + } + return { + cursor: events.length, + highestContiguousSourceSeq: highestContiguous( + events.filter( + (candidate) => + candidate.sourceInstanceId === event.sourceInstanceId, + ), + ), + disposition: existing ? "duplicate" : "committed", + }; + }, + async replayEvents(replay) { + if ( + boundary === "control-plane replay" && + replay.sourceInstanceId === "control-recovery" + ) { + await waitAtBoundary(); + } + const sourceEvents = events.filter( + (event) => event.sourceInstanceId === replay.sourceInstanceId, + ); + return { + events: structuredClone( + sourceEvents.filter( + (event) => event.sourceSeq > replay.afterSourceSeq, + ), + ), + highestContiguousSourceSeq: highestContiguous(sourceEvents), + }; + }, + completeRun, + }; + const outcome = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + timeoutMs: 10, + requireSessionCloseBeforeReturn: true, + }).then( + (value) => ({ value, error: null }), + (error: unknown) => ({ value: null, error }), + ); + try { + await boundaryReached; + const checkpointsBeforeFault = checkpointSession.mock.calls.length; + if (boundary === "completion invoked before commit") { + expect(completeRun).toHaveBeenCalledOnce(); + expect(durableCompletion).toBeNull(); + } + latchedFailure = failure; + releaseBoundary(); + if ( + boundary === "lost completion acknowledgement" || + boundary === "lost final event acknowledgement" + ) { + await vi.advanceTimersByTimeAsync(10); + } + const settled = await outcome; + if (!admitted) { + expect(settled.error).toBe(failure); + expect(settled.value).toBeNull(); + expect(completeRun).not.toHaveBeenCalled(); + expect(durableCompletion).toBeNull(); + } else { + expect(settled.error).toBeNull(); + expect(settled.value).toMatchObject({ + result, + terminal, + nativeEventCount: 3, + }); + expect(durableCompletion).toMatchObject({ result, terminal }); + expect(completeRun).toHaveBeenCalledTimes( + boundary === "lost completion acknowledgement" ? 2 : 1, + ); + } + expect( + events.filter((event) => event.sourceKind === "control_plane"), + ).toHaveLength(2); + expect(checkpointSession).toHaveBeenCalledTimes(checkpointsBeforeFault); + expect(resolveResult).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalledOnce(); + } finally { + releaseBoundary(); + await vi.advanceTimersByTimeAsync(30); + await outcome; + vi.useRealTimers(); + } + }, + ); + it.each([ "provider snapshot", "post-completion checkpoint", @@ -3383,6 +3661,122 @@ describe("executeNativeSession recovery", () => { }, ); + it.each([ + { typed: true, closeFails: false }, + { typed: true, closeFails: true }, + { typed: false, closeFails: true }, + { typed: true, closeFails: true, startupRace: true }, + ])( + "preserves a permanent integrity failure through required cleanup (%j)", + async ({ typed, closeFails, startupRace = false }) => { + const failure = typed + ? new NativeSessionProtocolIntegrityError( + "semantic_input_digest_mismatch", + ) + : Object.assign(new Error("ordinary provider connection failed"), { + code: "native_event_replay_conflict", + recovery: "operator_required", + }); + const closeFailure = new NativeSessionCloseUnrecoverableError(); + let observeClose = () => {}; + const closeStarted = new Promise((resolve) => { + observeClose = resolve; + }); + const close = vi.fn(async () => { + observeClose(); + if (closeFails) throw closeFailure; + }); + const capabilities = { + resume: true, + typedEvents: true, + steering: false, + interruption: false, + structuredResult: true, + }; + const session: NativeSession = { + identity: () => identity, + capabilities: async () => capabilities, + async *events() { + throw failure; + }, + startTurn: async () => { + if (startupRace) { + await closeStarted; + throw new Error( + "provider_transport_failed: startup raced with close", + ); + } + return { turnId: "turn-recovery" }; + }, + result: vi.fn(async () => null), + snapshot: async () => ({ + backendKind: "mock", + sessionId: "driver-integrity", + identity, + providerSessionId: "provider-integrity", + cursor: "0", + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }), + close, + }; + const backend: NativeSessionBackend = { + descriptor: async () => ({ + kind: "mock", + name: `integrity-${typed}-${closeFails}-${startupRace}`, + version: "1", + capabilities, + }), + openSession: vi.fn(async () => session), + }; + const events: PrpEvent[] = []; + const port: ControlPlanePort = { + openRun: async () => {}, + checkpointSession: async () => {}, + appendEvent: async (event) => { + events.push(structuredClone(event as PrpEvent)); + return { + cursor: events.length, + highestContiguousSourceSeq: highestContiguous(events), + disposition: "committed", + }; + }, + replayEvents: async () => ({ + events: [], + highestContiguousSourceSeq: 0, + }), + completeRun: vi.fn(async () => {}), + }; + const onSession = vi.fn(); + const options: ExecuteNativeSessionOptions = { + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + onSession, + requireSessionCloseBeforeReturn: true, + timeoutMs: 900_000, + }; + await expect(executeNativeSession(options)).rejects.toBe( + typed ? failure : closeFailure, + ); + expect(close).toHaveBeenCalledOnce(); + expect(onSession).toHaveBeenLastCalledWith(null); + expect(port.completeRun).not.toHaveBeenCalled(); + expect(session.result).not.toHaveBeenCalled(); + expect(events.some((event) => event.sourceKind === "runner")).toBe(false); + if (closeFails) { + await expect(executeNativeSession(options)).rejects.toBeInstanceOf( + NativeSessionCleanupQuarantinedError, + ); + expect(backend.openSession).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalledOnce(); + } + }, + ); + it("propagates an exhausted required backend checkpoint close", async () => { vi.useFakeTimers(); try { @@ -5428,9 +5822,11 @@ describe("executeNativeSession recovery", () => { runnerInstanceId: "runner-constrained-recovery", controlPlaneInstanceId: "control-constrained-recovery", }), - ).rejects.toThrow( - "native_session_recovery_failed: provider session ended with a failed terminal", - ); + ).rejects.toMatchObject({ + code: "native_provider_terminal_failed", + providerCode: "provider_checkpoint_failed_terminal", + recoverable: false, + }); expect(recoverSession).not.toHaveBeenCalled(); expect(openSession).not.toHaveBeenCalled(); diff --git a/packages/paperclip-runner/src/native-session-runtime.ts b/packages/paperclip-runner/src/native-session-runtime.ts index 47ad6289d9..e9210946b9 100644 --- a/packages/paperclip-runner/src/native-session-runtime.ts +++ b/packages/paperclip-runner/src/native-session-runtime.ts @@ -17,12 +17,18 @@ import type { NativeSessionBackend, } from "./contracts/native-session-backend.js"; import type { PersistedNativeSession } from "./contracts/native-session-backend.js"; -import type { HarnessThreadGoal } from "./contracts/harness-driver.js"; -import type { - PrpEvent, - PrpStructuredRunResult, - PrpTerminalState, +import { + NativeProviderTerminalFailure, + NativeSessionCloseUnrecoverableError, + NativeSessionCleanupQuarantinedError, + NativeSessionProtocolIntegrityError, +} from "./contracts/native-session-backend.js"; +import { + type PrpEvent, + type PrpStructuredRunResult, + type PrpTerminalState, } from "./protocol/replay-contract.js"; +import type { HarnessThreadGoal } from "./contracts/harness-driver.js"; import { validatePrpStructuredRunResult } from "./protocol/replay-contract.js"; import { parsePaperclipQuestionSet } from "./contracts/question-set.js"; @@ -70,6 +76,7 @@ interface QuarantinedSessionCleanup { recovery: Promise | null; recoveryMaxAttempts: number | null; timer: ReturnType | null; + operatorRecoveryRequired: boolean; } const quarantinedSessionCleanups = new Set(); @@ -349,6 +356,10 @@ function retainUnadmittedSessionCleanup( await attempt; return; } catch (error) { + if (error instanceof NativeSessionCloseUnrecoverableError) { + quarantineSessionCleanup(session, cleanupDomain, true); + throw error; + } if (retryCount >= MAX_FAILED_SESSION_CLOSE_RETRIES) { quarantineSessionCleanup(session, cleanupDomain); throw error; @@ -508,10 +519,17 @@ function retainFailedSessionCleanupOwner( function quarantineSessionCleanup( session: NativeSession, cleanupDomain: NativeSessionCleanupDomain, + operatorRecoveryRequired = false, ): void { - if ( - [...quarantinedSessionCleanups].some((entry) => entry.session === session) - ) { + const existing = [...quarantinedSessionCleanups].find( + (entry) => entry.session === session, + ); + if (existing) { + if (operatorRecoveryRequired) { + existing.operatorRecoveryRequired = true; + if (existing.timer) clearTimeout(existing.timer); + existing.timer = null; + } return; } const cleanup: QuarantinedSessionCleanup = { @@ -522,8 +540,10 @@ function quarantineSessionCleanup( recovery: null, recoveryMaxAttempts: null, timer: null, + operatorRecoveryRequired, }; quarantinedSessionCleanups.add(cleanup); + if (operatorRecoveryRequired) return; startQuarantinedSessionCleanupRecovery( cleanup, MAX_QUARANTINED_SESSION_CLOSE_RETRIES, @@ -537,6 +557,7 @@ function startQuarantinedSessionCleanupRecovery( reason: string, ): Promise { if (cleanup.recovery) return cleanup.recovery; + if (cleanup.operatorRecoveryRequired) return Promise.resolve(); const remainingAttempts = MAX_QUARANTINED_SESSION_AUTOMATIC_CLOSE_ATTEMPTS - cleanup.automaticAttempts; @@ -546,7 +567,8 @@ function startQuarantinedSessionCleanupRecovery( for ( let attemptCount = 0; attemptCount < boundedMaxAttempts && - quarantinedSessionCleanups.has(cleanup); + quarantinedSessionCleanups.has(cleanup) && + !cleanup.operatorRecoveryRequired; attemptCount += 1 ) { // The first retry starts immediately so an admission-triggered recovery @@ -563,7 +585,10 @@ function startQuarantinedSessionCleanupRecovery( try { await attempt; quarantinedSessionCleanups.delete(cleanup); - } catch { + } catch (error) { + if (error instanceof NativeSessionCloseUnrecoverableError) { + cleanup.operatorRecoveryRequired = true; + } // Retain the quarantine after this finite, sequential retry batch. } finally { if (cleanup.attempt === attempt) cleanup.attempt = null; @@ -591,6 +616,7 @@ function scheduleQuarantinedSessionCleanup( ): void { if ( !quarantinedSessionCleanups.has(cleanup) || + cleanup.operatorRecoveryRequired || cleanup.recovery || cleanup.attempt || cleanup.timer || @@ -628,6 +654,14 @@ async function retryQuarantinedSessionCleanups( let observedOwnerPhases = 0; while (true) { signal?.throwIfAborted(); + if ( + [...quarantinedSessionCleanups].some( + (cleanup) => + cleanup.domain === cleanupDomain && cleanup.operatorRecoveryRequired, + ) + ) { + throw new NativeSessionCleanupQuarantinedError(); + } const cleanupOwners = new Set>( [...failedSessionCleanupOwners] .filter(([, domain]) => domain === cleanupDomain) @@ -785,6 +819,7 @@ async function consumeTurn( typeof semanticResultGraceExpired > | null = null; let pendingNext: ReturnType | null = null; + let providerFailure: NativeProviderTerminalFailure | null = null; const settleDurableResult = ( event: PrpEvent, result: PrpStructuredRunResult, @@ -812,7 +847,7 @@ async function consumeTurn( }; }; while (true) { - pendingNext ??= eventIterator.next(); + pendingNext ??= eventIterator.next().catch(error => { throw providerFailure ?? error; }); const next = semanticResultDeadline === null ? await pendingNext @@ -843,6 +878,7 @@ async function consumeTurn( governedResult, }; } + if (providerFailure) throw providerFailure; throw new Error( "native event stream closed before a turn terminal fact", ); @@ -901,6 +937,15 @@ async function consumeTurn( const validation = validatePrpStructuredRunResult(event.payload); if (validation.ok) semanticResultProposal = validation.result; } + if (event.eventType === "session.failed") { + const failure = payload.error && typeof payload.error === "object" + ? payload.error as Record : payload; + providerFailure = new NativeProviderTerminalFailure( + typeof failure.code === "string" ? failure.code : "provider_session_failed", + failure.recoverable === true || payload.recoverable === true, + typeof failure.message === "string" ? failure.message : undefined, + ); + } const request = payload.request && typeof payload.request === "object" && @@ -1811,6 +1856,9 @@ export async function executeNativeSession( const failedProviderSession = providerRecoveryCheckpoint.terminal?.runTerminalState === "failed" && providerRecoveryCheckpoint.semanticResult === null; + if (failedProviderSession && !replacementAllowed) { + throw new NativeProviderTerminalFailure("provider_checkpoint_failed_terminal", false); + } const recovery = failedProviderSession ? { recovered: false as const, @@ -1969,6 +2017,10 @@ export async function executeNativeSession( await attempt; return; } catch (error) { + if (error instanceof NativeSessionCloseUnrecoverableError) { + quarantineSessionCleanup(session, cleanupDomain, true); + throw error; + } if (retryCount >= MAX_FAILED_SESSION_CLOSE_RETRIES) { quarantineSessionCleanup(session, cleanupDomain); throw error; @@ -2004,6 +2056,7 @@ export async function executeNativeSession( let goalCheckpointRequiresSuspension = Boolean( options.sessionGoalControl || options.resumeSessionGoalHeartbeat || persistedSession?.goal, ); + let protocolIntegrityFailure: NativeSessionProtocolIntegrityError | null = null; try { // Ownership publication is part of the execution-owned lifetime. If the // callback fails, the finally block below still quarantines and closes the @@ -2246,8 +2299,16 @@ export async function executeNativeSession( // before joining cleanup so the failed turn cannot commit late or // strand execution on a never-settling durability call. consumptionAbort.abort(error); - await consuming.catch(() => undefined); - throw error; + let startupFailure = error; + await consuming.catch((consumptionError) => { + // A failed iterator may have already initiated close while start or + // checkpoint was pending. Retain the authenticated integrity fault, + // not the resulting transport/cleanup error from that race. + if (consumptionError instanceof NativeSessionProtocolIntegrityError) { + startupFailure = consumptionError; + } + }); + throw startupFailure; } const terminalEvent = await consuming; consumed = terminalEvent; @@ -2279,6 +2340,17 @@ export async function executeNativeSession( turnId: terminalEvent.turnId ?? null, }; signal.throwIfAborted(); + if (settledCompletion === null && terminalEvent.eventType === "turn.failed") { + await checkpoint(signal); + const payload = terminalEvent.payload as Record; + const failure = payload.error && typeof payload.error === "object" ? payload.error as Record : payload; + const message = typeof failure.message === "string" ? failure.message.slice(0, 2_000) : "Provider turn failed"; + throw new NativeProviderTerminalFailure( + typeof failure.code === "string" ? failure.code : "provider_turn_failed", + failure.recoverable === true || payload.recoverable === true, + message, + ); + } if (settledCompletion === null && options.resolveMissingResult) { const recoveredResult = await options.resolveMissingResult({ turnId: terminalEvent.turnId ?? null, @@ -2304,13 +2376,6 @@ export async function executeNativeSession( completed = settledCompletion; } if (settledCompletion === null) { - if (consumed.event?.eventType === "turn.failed") { - const providerError = objectRecord(objectRecord(consumed.event.payload)?.error); - const message = typeof providerError?.message === "string" - ? providerError.message.slice(0, 2_000) : "Provider turn failed"; - const modelRejected = /issue with the selected model|model_not_found|invalid model|model[^\n]*(?:does not exist|not found|not supported)/i.test(message); - throw new Error(`${modelRejected ? "native_provider_model_rejected" : "native_provider_turn_failed"}: ${message}`); - } throw new Error( "native_finalization_missing: session returned no semantic result", ); @@ -2372,6 +2437,7 @@ export async function executeNativeSession( const baselineControlEventSequences = new Set(); const accountedControlEventSequences = new Set(); let baselineControlReplayCaptured = false; + let completionAdmissionStarted = false; const durableExecutionResult = await finalizeIdempotentControlPlaneWithin({ timeoutMs: finalizationTimeoutMs, operation: async (signal) => { @@ -2439,6 +2505,23 @@ export async function executeNativeSession( receipt.highestContiguousSourceSeq, ); } + if (!completionAdmissionStarted) { + // Replay/appends can yield after the prepared result's last snapshot. + // Observe the driver's latched integrity fault once more before + // admitting completion; Codex snapshots inspect local state only. + try { + await session.snapshot({ signal }); + } catch (error) { + if (error instanceof NativeSessionProtocolIntegrityError) throw error; + // Generic snapshot failures remain checkpoint enrichment failures, + // not evidence that the already validated result is invalid. + } + signal.throwIfAborted(); + // Invocation is the local admission boundary, not an atomic fence + // with the remote commit. A lost acknowledgement can mean committed + // success, so later faults must not veto its idempotent confirmation. + completionAdmissionStarted = true; + } await options.controlPlane.completeRun( { result: preparedFinalization.completed.result, @@ -2538,6 +2621,11 @@ export async function executeNativeSession( }; executionSucceeded = true; return { ...durableExecutionResult, ...enrichment }; + } catch (error) { + if (error instanceof NativeSessionProtocolIntegrityError) { + protocolIntegrityFailure = error; + } + throw error; } finally { const shouldClose = !options.keepSessionOpen || !executionSucceeded || sessionQuarantined || goalCheckpointRequiresSuspension; @@ -2555,7 +2643,14 @@ export async function executeNativeSession( // Unlike ordinary provider cleanup, this close owns required remote // checkpoint persistence. Exhausting its bounded recovery must fail // the execution instead of converting the rejection into success. - await requiredClose; + try { + await requiredClose; + } catch (closeError) { + // The exact cleanup owner remains retained/quarantined above. Its + // rejection must not turn permanent integrity failure into a + // generic retryable transport failure at the control-plane boundary. + throw protocolIntegrityFailure ?? closeError; + } } } else if (shouldClose && !failedCleanupDeferred) { // A provider that ignores close must not keep execution pending forever. diff --git a/packages/paperclip-runner/src/provider-events.ts b/packages/paperclip-runner/src/provider-events.ts index 94e384f7d1..05e6c3cbe0 100644 --- a/packages/paperclip-runner/src/provider-events.ts +++ b/packages/paperclip-runner/src/provider-events.ts @@ -76,32 +76,38 @@ export interface TypedEventFamilyCapability { detailLevel: "summary" | "structured"; } -export type CanonicalProviderEventType = - | "plan.updated" - | "tool.execution.started" - | "tool.execution.progressed" - | "tool.execution.completed" - | "research.started" - | "research.progressed" - | "research.completed" - | "delegation.started" - | "delegation.updated" - | "delegation.completed" - | "model.route.changed" - | "model.verification.updated" - | "context.compacted" - | "artifact.viewed" - | "artifact.generated" - | "review.mode.changed" - | "hook.started" - | "hook.completed" - | "memory.citation.referenced" - | "safety.review.started" - | "safety.review.completed" - | "terminal.input.sent" - | "wait.started" - | "wait.completed" - | "provider.notice.recorded"; +export const CANONICAL_PROVIDER_EVENT_TYPES = [ + "harness.diagnostic", + "plan.updated", + "tool.execution.started", + "tool.execution.progressed", + "tool.execution.completed", + "research.started", + "research.progressed", + "research.completed", + "delegation.started", + "delegation.updated", + "delegation.completed", + "model.route.changed", + "model.verification.updated", + "context.compacted", + "artifact.viewed", + "artifact.generated", + "review.mode.changed", + "hook.started", + "hook.completed", + "memory.citation.referenced", + "safety.review.started", + "safety.review.completed", + "terminal.input.sent", + "wait.started", + "wait.completed", + "provider.notice.recorded", +] as const; +export type CanonicalProviderEventType = (typeof CANONICAL_PROVIDER_EVENT_TYPES)[number]; +export function isCanonicalProviderEventType(value: unknown): value is CanonicalProviderEventType { + return CANONICAL_PROVIDER_EVENT_TYPES.some(type => value === type); +} export interface CanonicalProviderEvent { eventType: CanonicalProviderEventType; diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 116c497926..db3002bb8d 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -2750,3 +2750,7 @@ export { rewriteUrlHostToLoopback, } from "./runtime-exposure/loopback-bind.js"; export { ACCOUNT_HANDLE_MAX_LENGTH, toAccountHandle } from "./account-handle.js"; +export type { ExecutionContinuationEnvelope } from "./types/execution-continuation.js"; +export type { ExecutionProjection, ExecutionReconciliation } from "./types/execution-projection.js"; + +export { EXECUTION_RECONCILIATION_CAUSES, requiresExecutionReconciliation } from "./types/execution-projection.js"; diff --git a/packages/shared/src/types/execution-continuation.ts b/packages/shared/src/types/execution-continuation.ts new file mode 100644 index 0000000000..6505773947 --- /dev/null +++ b/packages/shared/src/types/execution-continuation.ts @@ -0,0 +1,52 @@ +/** Server-authored context. Each message retains its author and trust boundary. */ +export interface ExecutionContinuationEnvelope { + version: 1; + companyId: string; + issueId: string; + trigger: { + reason: string; + interactionId: string | null; + sourceRunId: string | null; + }; + originCommentIds: string[]; + objective: string; + messages: Array<{ + id: string; + authorType: string; + authorId: string | null; + /** Run-authored Local CLI comments retain user attribution but are not human direction. */ + createdByRunId?: string | null; + body: string; + createdAt: string; + updatedAt: string; + deleted: boolean; + sourceTrust: unknown; + }>; + interactionOutcomes: Array<{ + id: string; + kind: string; + status: string; + result: unknown; + }>; + /** Only valid when resuming the provider session associated with this run. */ + resumeDelta?: { + baseRunId: string; + messages: ExecutionContinuationEnvelope["messages"]; + }; + recoveryOutcomes?: Array<{ recoveryActionId: string; decision: unknown }>; + completedWork: string | null; + /** Completed mutations are context, never instructions to replay them. */ + completedActions?: Array<{ + runId: string; + receiptId: string; + operationId: string; + result: unknown; + }>; + unresolvedInteractionIds: string[]; + coverage: { + kind: "full_task_history" | "task_history_delta"; + baseRunId?: string; + throughCommentId: string | null; + summaryThroughCommentId: null; + }; +} diff --git a/packages/shared/src/types/execution-projection.ts b/packages/shared/src/types/execution-projection.ts new file mode 100644 index 0000000000..28db4cb066 --- /dev/null +++ b/packages/shared/src/types/execution-projection.ts @@ -0,0 +1,58 @@ +/** Presentation of existing execution records, not a second task status machine. */ +export interface ExecutionProjection { + phase: + | "working" + | "reconnecting" + | "retry_scheduled" + | "finishing" + | "recovery_needed" + | "waiting_for_access" + | "waiting_for_answer" + | "queued" + | "completed" + | "failed"; + label: string; + cause: string | null; + lastConfirmedActivityAt: string | null; + retryAt: string | null; + attempt: number; + maxAttempts: number; + recoveryOwner: "agent" | "board" | null; + nextAction: string | null; + permittedActions: Array<"inspect_run" | "inspect_recovery">; + predecessorRunId: string | null; + successorRunId: string | null; +} + +/** These incidents require an explicit reconciliation, never a generic Retry. */ +export const EXECUTION_RECONCILIATION_CAUSES = [ + "uncertain_provider_action", + "uncertain_external_action", + "uncertain_control_plane_action", + "completed_action_context_missing", + "continuation_evidence_incomplete", + "execution_finalization_deadline_exceeded", + "execution_recovery_budget_exhausted", + "provider_effect_inventory_unavailable", + "provider_failure_meaning_unverified", + "provider_ownership_unverified", + "native_provider_terminal_failed", + "native_event_replay_conflict", + "native_session_cleanup_quarantined", + "native_session_retry_exhausted", + "native_restart_recovery_blocked", + "native_continuation_requires_reconciliation", + "legacy_execution_requires_reconciliation", +] as const; +export function requiresExecutionReconciliation( + cause: string | null | undefined, +): boolean { + return EXECUTION_RECONCILIATION_CAUSES.some((value) => value === cause); +} + +export interface ExecutionReconciliation { + runId: string; + providerStopped: true; + actionOutcome: "completed" | "not_performed" | "mixed"; + outcomeEvidence: string; +} diff --git a/packages/shared/src/types/heartbeat.ts b/packages/shared/src/types/heartbeat.ts index ddaf2e9bd3..ce8affb0e4 100644 --- a/packages/shared/src/types/heartbeat.ts +++ b/packages/shared/src/types/heartbeat.ts @@ -159,6 +159,7 @@ export interface GitWorktreeBranchIncoherenceEvidence { } export interface HeartbeatRun { + execution?: import("./execution-projection.js").ExecutionProjection | null; id: string; companyId: string; agentId: string; diff --git a/packages/shared/src/types/issue.ts b/packages/shared/src/types/issue.ts index 15ad7c5e65..960e54280e 100644 --- a/packages/shared/src/types/issue.ts +++ b/packages/shared/src/types/issue.ts @@ -1,3 +1,4 @@ +import type { ExecutionProjection } from "./execution-projection.js"; import type { IssueCommentAuthorType, IssueCommentMetadataRowType, @@ -783,6 +784,9 @@ export interface IssueChangeReceiptEntry { export type IssueChanges = Record; export interface Issue { + activeRun?: { id: string; status: string; agentId: string; invocationSource: string; + triggerDetail: string | null; startedAt: Date | string | null; finishedAt: Date | string | null; + createdAt: Date | string; execution?: ExecutionProjection } | null; id: string; companyId: string; projectId: string | null; @@ -1474,6 +1478,7 @@ export interface IssueThreadInteractionBase extends IssueThreadInteractionActorF issueId: string; kind: IssueThreadInteractionKind; idempotencyKey?: string | null; + originCommentIds?: string[]; sourceCommentId?: string | null; sourceRunId?: string | null; sourceIdentityContextId?: string | null; diff --git a/packages/shared/src/validators/issue.ts b/packages/shared/src/validators/issue.ts index 54ddde4f4e..f0d135f838 100644 --- a/packages/shared/src/validators/issue.ts +++ b/packages/shared/src/validators/issue.ts @@ -375,6 +375,12 @@ const RESOLVE_ISSUE_RECOVERY_ACTION_OUTCOMES = [ ] as const; export const resolveIssueRecoveryActionSchema = z.object({ + executionReconciliation: z.object({ + runId: z.string().guid(), + providerStopped: z.literal(true), + actionOutcome: z.enum(["completed", "not_performed", "mixed"]), + outcomeEvidence: z.string().trim().min(20).max(12000), + }).strict().optional(), actionId: z.string().guid().optional(), outcome: z.enum(RESOLVE_ISSUE_RECOVERY_ACTION_OUTCOMES), sourceIssueStatus: z.enum(["todo", "done", "in_review", "blocked"]),