fix(runner): preserve provider identity and terminal failures (#13074)
## Thinking Path
> - Paperclip is the open source app people use to manage AI agents for
work.
> - The runner translates provider events into durable task execution.
> - Provider notifications can refer to another thread or a descendant.
> - Root validation treated these notifications as fatal, and later
layers could lose the original failure.
> - This pull request classifies event identity and preserves structured
terminal failures.
> - The server can then choose safe recovery without weakening tool
authority.
## Linked Issues or Issue Description
Refs #13038. This change incorporates the shared protocol-integrity and
bounded-cleanup prerequisites from that PR. It does not include the chat
feature. Related failure classification work: #13028.
**What happened?**
An informational provider notification for another thread could
terminate the root session. A failed stream could then become a
missing-result error and lose its cause.
**Expected behavior**
Ignore unrelated informational notices with bounded diagnostics. Reject
invalid authoritative events. Preserve the original failure code and
recovery meaning through cleanup.
**Steps to reproduce**
Run a native Codex task. Deliver a notification for an unrelated thread,
or close the stream after a structured failure. Inspect the root outcome
and recorded failure.
**Paperclip version or commit**
Reproduced before e20010472. This branch includes the current
session-goal contract from that commit.
**Deployment mode**
Built from source. Native runner with Codex.
## What Changed
- Classify root, provider-confirmed descendant, stale, unrelated, and
invalid provider events.
- Keep tool requests bound to their original execution authority.
- Preserve typed failures through transport, session, and durable
control-plane cleanup.
- Keep bounded cleanup failures separate from the primary execution
failure.
- Add compatible shared contracts for continuation context, execution
status, and explicit reconciliation. The dependent PR adds their server
and UI consumers.
## Verification
- Runner TypeScript and ACPX suites: 1,733 passed, 7 skipped. Node
contracts: 38 passed.
- Real provider-process fixtures cover 300 descendant identities across
restart, the 4,096-identity capacity boundary, and rejection of
continuation after terminal acknowledgement and restart.
- Repository build, typecheck, and full `pnpm test:run` passed on the
rebased stack (18,448 tests passed, 49 skipped). The full Rust workspace
passed with `--test-threads=1`; parallel execution exposed an existing
fixture port-reservation race. All latest-head CI checks passed. One
unchanged artifact-document concurrency test failed on the first CI run
and passed on its single rerun.
- Added notification, streaming failure, protocol integrity, cleanup
quarantine, and durable failure tests.
## Risks
Provider event classification must retain the new session-goal behavior
on master. Descendant notifications must never gain root tool authority.
Shared contract fields are additive. This PR does not migrate data or
start replacement provider work.
## Model Used
OpenAI GPT-6 through Codex. The exact deployment ID and context window
were not exposed. Used reasoning, tool use, code execution, and browser
automation.
## Checklist
- [x] I have included a thinking path that traces from project context
to this change
- [x] I have specified the model used (with version and capability
details)
- [x] I have checked ROADMAP.md and confirmed this PR does not duplicate
planned core work
- [x] I have searched GitHub for duplicate or related PRs and linked
them above
- [x] I have either (a) linked existing issues with `Fixes: #` / `Closes
#` / `Refs #` OR (b) described the issue in-PR following the relevant
issue template
- [x] I have not referenced internal/instance-local Paperclip issues or
links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip`
URLs)
- [x] My branch name describes the change (e.g. `docs/...`, `fix/...`)
and contains no internal Paperclip ticket id or instance-derived details
- [x] I have run tests locally and they pass
- [x] I have added or updated tests where applicable
- [x] I have updated relevant documentation to reflect my changes
- [x] I have considered and documented any risks above
- [x] All Paperclip CI gates are green
- [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups
- [x] I will address all Greptile and reviewer comments before
requesting merge
---------
Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
parent
5bb83e5b4b
commit
6681104692
|
|
@ -256,7 +256,7 @@ fn finish_turn(state_path: &Path, state: &mut FakeState, status: &str) -> io::Re
|
||||||
.unwrap_or_else(|| "provider-turn-1".to_owned());
|
.unwrap_or_else(|| "provider-turn-1".to_owned());
|
||||||
send(json!({
|
send(json!({
|
||||||
"method": "item/completed",
|
"method": "item/completed",
|
||||||
"params": {"item": {
|
"params": {"threadId": state.thread_id, "turnId": turn_id, "item": {
|
||||||
"id": "message-1",
|
"id": "message-1",
|
||||||
"type": "agentMessage",
|
"type": "agentMessage",
|
||||||
"status": "completed",
|
"status": "completed",
|
||||||
|
|
@ -307,10 +307,10 @@ fn emit_ambiguous_turn_evidence(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn emit_ambiguous_turn_item() -> io::Result<()> {
|
fn emit_ambiguous_turn_item(state: &FakeState) -> io::Result<()> {
|
||||||
send(json!({
|
send(json!({
|
||||||
"method": "item/completed",
|
"method": "item/completed",
|
||||||
"params": {"item": {
|
"params": {"threadId": state.thread_id, "item": {
|
||||||
"id": "replacement-message-before-terminal",
|
"id": "replacement-message-before-terminal",
|
||||||
"type": "agentMessage",
|
"type": "agentMessage",
|
||||||
"status": "completed",
|
"status": "completed",
|
||||||
|
|
@ -691,6 +691,9 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
.any(|value| value == "--question-before-failed-turn");
|
.any(|value| value == "--question-before-failed-turn");
|
||||||
let fail_turn_immediately = args.iter().any(|value| value == "--fail-turn-immediately");
|
let fail_turn_immediately = args.iter().any(|value| value == "--fail-turn-immediately");
|
||||||
let reuse_question_id = args.iter().any(|value| value == "--reuse-question-id");
|
let reuse_question_id = args.iter().any(|value| value == "--reuse-question-id");
|
||||||
|
let descendant_notifications = args
|
||||||
|
.iter()
|
||||||
|
.any(|value| value == "--descendant-notifications");
|
||||||
let pre_response_notification = args
|
let pre_response_notification = args
|
||||||
.iter()
|
.iter()
|
||||||
.any(|value| value == "--notification-before-response");
|
.any(|value| value == "--notification-before-response");
|
||||||
|
|
@ -942,6 +945,12 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
"id": id,
|
"id": id,
|
||||||
"result": {"thread": {"id": state.thread_id, "sessionId": "codex-account-session"}}
|
"result": {"thread": {"id": state.thread_id, "sessionId": "codex-account-session"}}
|
||||||
}))?;
|
}))?;
|
||||||
|
if descendant_notifications {
|
||||||
|
// Restoration must retain lineage without a replay of thread/started.
|
||||||
|
send(json!({"method": "turn/completed", "params": {
|
||||||
|
"threadId": "descendant-299", "turnId": "child-turn", "status": "completed"
|
||||||
|
}}))?;
|
||||||
|
}
|
||||||
if emit_tool_call_on_resume {
|
if emit_tool_call_on_resume {
|
||||||
if let Some(turn_id) = state.active_turn_id.as_deref() {
|
if let Some(turn_id) = state.active_turn_id.as_deref() {
|
||||||
send(json!({
|
send(json!({
|
||||||
|
|
@ -1175,7 +1184,7 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
}
|
}
|
||||||
if malformed_error_second_turn_start && turn_start_count == 2 {
|
if malformed_error_second_turn_start && turn_start_count == 2 {
|
||||||
if hold_ambiguous_second_turn_after_item {
|
if hold_ambiguous_second_turn_after_item {
|
||||||
emit_ambiguous_turn_item()?;
|
emit_ambiguous_turn_item(&state)?;
|
||||||
}
|
}
|
||||||
send(json!({"id": id, "error": {}}))?;
|
send(json!({"id": id, "error": {}}))?;
|
||||||
if emits_ambiguous_turn_evidence
|
if emits_ambiguous_turn_evidence
|
||||||
|
|
@ -1196,7 +1205,7 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
"result": {"turn": {"status": "inProgress"}}
|
"result": {"turn": {"status": "inProgress"}}
|
||||||
}))?;
|
}))?;
|
||||||
if hold_ambiguous_second_turn_after_item {
|
if hold_ambiguous_second_turn_after_item {
|
||||||
emit_ambiguous_turn_item()?;
|
emit_ambiguous_turn_item(&state)?;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if emits_ambiguous_turn_evidence
|
if emits_ambiguous_turn_evidence
|
||||||
|
|
@ -1226,6 +1235,23 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
"method": "turn/started",
|
"method": "turn/started",
|
||||||
"params": {"turn": {"id": provider_turn_id}}
|
"params": {"turn": {"id": provider_turn_id}}
|
||||||
}))?;
|
}))?;
|
||||||
|
if descendant_notifications {
|
||||||
|
for index in 0..300 {
|
||||||
|
send(json!({"method": "thread/started", "params": {"thread": {
|
||||||
|
"id": format!("descendant-{index}"),
|
||||||
|
"source": {"subAgent": {"thread_spawn": {"parent_thread_id": state.thread_id}}}
|
||||||
|
}}}))?;
|
||||||
|
}
|
||||||
|
send(json!({"method": "turn/completed", "params": {
|
||||||
|
"threadId": "descendant-299", "turnId": "child-turn", "status": "completed"
|
||||||
|
}}))?;
|
||||||
|
}
|
||||||
|
if args.iter().any(|value| value == "--descendant-overflow") {
|
||||||
|
send(json!({"method": "thread/started", "params": {"thread": {
|
||||||
|
"id": "descendant-overflow",
|
||||||
|
"source": {"subAgent": {"thread_spawn": {"parent_thread_id": state.thread_id}}}
|
||||||
|
}}}))?;
|
||||||
|
}
|
||||||
if fail_after_second_turn_start && turn_start_count == 2 {
|
if fail_after_second_turn_start && turn_start_count == 2 {
|
||||||
return Err("configured failure after second turn start".into());
|
return Err("configured failure after second turn start".into());
|
||||||
} else if fail_turn_immediately {
|
} else if fail_turn_immediately {
|
||||||
|
|
@ -1392,11 +1418,11 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
}),
|
}),
|
||||||
json!({
|
json!({
|
||||||
"method": "rawResponseItem/completed",
|
"method": "rawResponseItem/completed",
|
||||||
"params": {"item": {"id": "raw-tail", "type": "reasoning"}}
|
"params": {"threadId": state.thread_id, "turnId": provider_turn_id, "item": {"id": "raw-tail", "type": "reasoning"}}
|
||||||
}),
|
}),
|
||||||
json!({
|
json!({
|
||||||
"method": "rawResponse/completed",
|
"method": "rawResponse/completed",
|
||||||
"params": {"response": {"id": "response-tail"}}
|
"params": {"threadId": state.thread_id, "turnId": provider_turn_id, "response": {"id": "response-tail"}}
|
||||||
}),
|
}),
|
||||||
json!({
|
json!({
|
||||||
"method": "thread/goal/updated",
|
"method": "thread/goal/updated",
|
||||||
|
|
|
||||||
|
|
@ -54,6 +54,17 @@ const MAX_PENDING_RUNTIME_REQUESTS: usize = 128;
|
||||||
const MAX_PENDING_RUNTIME_REQUEST_BYTES: usize = 4 * 1024 * 1024;
|
const MAX_PENDING_RUNTIME_REQUEST_BYTES: usize = 4 * 1024 * 1024;
|
||||||
const OPENCODE_RUNTIME_REQUEST_METHOD: &str = "paperclip/runtimeRequest";
|
const OPENCODE_RUNTIME_REQUEST_METHOD: &str = "paperclip/runtimeRequest";
|
||||||
pub(crate) const MAX_SETTLED_PROVIDER_TURN_IDS: usize = 4_096;
|
pub(crate) const MAX_SETTLED_PROVIDER_TURN_IDS: usize = 4_096;
|
||||||
|
pub(crate) const MAX_DESCENDANT_THREAD_IDS: usize = 4_096;
|
||||||
|
|
||||||
|
fn remember_descendant_thread(ids: &mut BTreeSet<String>, id: &str) -> Result<bool, &'static str> {
|
||||||
|
if ids.contains(id) {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
if ids.len() >= MAX_DESCENDANT_THREAD_IDS {
|
||||||
|
return Err("provider_descendant_capacity_exhausted");
|
||||||
|
}
|
||||||
|
Ok(ids.insert(id.to_owned()))
|
||||||
|
}
|
||||||
type QuestionOptionLabels = BTreeMap<String, BTreeMap<String, String>>;
|
type QuestionOptionLabels = BTreeMap<String, BTreeMap<String, String>>;
|
||||||
type QuestionSetMapping = (String, Value, QuestionOptionLabels);
|
type QuestionSetMapping = (String, Value, QuestionOptionLabels);
|
||||||
|
|
||||||
|
|
@ -383,6 +394,19 @@ pub enum CodexProviderEvent {
|
||||||
method: String,
|
method: String,
|
||||||
params: Value,
|
params: Value,
|
||||||
},
|
},
|
||||||
|
/// Provider-confirmed child progress has no root terminal or tool authority.
|
||||||
|
DescendantNotification {
|
||||||
|
method: String,
|
||||||
|
params: Value,
|
||||||
|
},
|
||||||
|
/// An invalid authoritative event must retain its failure meaning across PRP.
|
||||||
|
ProtocolFailure {
|
||||||
|
diagnostic: Value,
|
||||||
|
},
|
||||||
|
/// A bounded provider resource was exhausted; this is not identity corruption.
|
||||||
|
ResourceLimit {
|
||||||
|
diagnostic: Value,
|
||||||
|
},
|
||||||
RuntimeRequest {
|
RuntimeRequest {
|
||||||
request_id: String,
|
request_id: String,
|
||||||
question_set: Value,
|
question_set: Value,
|
||||||
|
|
@ -537,6 +561,8 @@ pub struct CodexProvider {
|
||||||
goal_allows_autonomous_turns: bool,
|
goal_allows_autonomous_turns: bool,
|
||||||
ambiguous_turn_start_pending: bool,
|
ambiguous_turn_start_pending: bool,
|
||||||
settled_provider_turn_ids: SettledProviderTurnIds,
|
settled_provider_turn_ids: SettledProviderTurnIds,
|
||||||
|
descendant_thread_ids: BTreeSet<String>,
|
||||||
|
notification_identity_diagnostics: usize,
|
||||||
rejected_accepted_turn: Option<RejectedAcceptedTurn>,
|
rejected_accepted_turn: Option<RejectedAcceptedTurn>,
|
||||||
quarantined: bool,
|
quarantined: bool,
|
||||||
trace: Option<ProviderTraceSink>,
|
trace: Option<ProviderTraceSink>,
|
||||||
|
|
@ -798,6 +824,8 @@ impl CodexProvider {
|
||||||
goal_allows_autonomous_turns: false,
|
goal_allows_autonomous_turns: false,
|
||||||
ambiguous_turn_start_pending: false,
|
ambiguous_turn_start_pending: false,
|
||||||
settled_provider_turn_ids: SettledProviderTurnIds::default(),
|
settled_provider_turn_ids: SettledProviderTurnIds::default(),
|
||||||
|
descendant_thread_ids: BTreeSet::new(),
|
||||||
|
notification_identity_diagnostics: 0,
|
||||||
rejected_accepted_turn: None,
|
rejected_accepted_turn: None,
|
||||||
quarantined: false,
|
quarantined: false,
|
||||||
trace: ProviderTraceSink::from_environment(),
|
trace: ProviderTraceSink::from_environment(),
|
||||||
|
|
@ -1060,6 +1088,20 @@ impl CodexProvider {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Some(CodexProviderEvent::ResourceLimit { .. }) => {
|
||||||
|
return Err(LocalRunnerError::invalid("Codex descendant capacity requires reconciliation before fresh-session continuation"));
|
||||||
|
}
|
||||||
|
Some(CodexProviderEvent::ProtocolFailure { .. }) => {
|
||||||
|
return Err(LocalRunnerError::invalid(
|
||||||
|
"Codex protocol integrity failed during warm attachment",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Some(CodexProviderEvent::DescendantNotification { .. }) => {
|
||||||
|
// Child output cannot certify a quiescent root attachment.
|
||||||
|
return Err(LocalRunnerError::invalid(
|
||||||
|
"Codex descendant remains active during warm run attachment",
|
||||||
|
));
|
||||||
|
}
|
||||||
Some(CodexProviderEvent::ToolCall { .. })
|
Some(CodexProviderEvent::ToolCall { .. })
|
||||||
| Some(CodexProviderEvent::RuntimeRequest { .. }) => {
|
| Some(CodexProviderEvent::RuntimeRequest { .. }) => {
|
||||||
return Err(LocalRunnerError::invalid(
|
return Err(LocalRunnerError::invalid(
|
||||||
|
|
@ -1154,6 +1196,13 @@ impl CodexProvider {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn restore_descendant_thread_identities(&mut self, identities: &BTreeSet<String>) {
|
||||||
|
// Exact provider-confirmed lineage lives as long as the root session.
|
||||||
|
// Evicting it would turn later child progress into a root integrity fault.
|
||||||
|
self.descendant_thread_ids
|
||||||
|
.extend(identities.iter().cloned());
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn restore_settled_turn_identities(
|
pub(crate) fn restore_settled_turn_identities(
|
||||||
&mut self,
|
&mut self,
|
||||||
provider_turn_ids: impl IntoIterator<Item = String>,
|
provider_turn_ids: impl IntoIterator<Item = String>,
|
||||||
|
|
@ -1856,9 +1905,11 @@ impl CodexProvider {
|
||||||
if method == "item/tool/call" {
|
if method == "item/tool/call" {
|
||||||
let params = message.get("params").cloned().unwrap_or(Value::Null);
|
let params = message.get("params").cloned().unwrap_or(Value::Null);
|
||||||
if params.get("threadId").and_then(Value::as_str) != Some(self.thread_id.as_str()) {
|
if params.get("threadId").and_then(Value::as_str) != Some(self.thread_id.as_str()) {
|
||||||
return Err(LocalRunnerError::invalid(
|
return Ok(Some(self.identity_failure(
|
||||||
"Codex tool call named another thread",
|
method,
|
||||||
));
|
¶ms,
|
||||||
|
"thread_binding_mismatch",
|
||||||
|
)));
|
||||||
}
|
}
|
||||||
if request_targets_non_active_turn(
|
if request_targets_non_active_turn(
|
||||||
self.active_provider_turn_id.as_deref(),
|
self.active_provider_turn_id.as_deref(),
|
||||||
|
|
@ -1871,9 +1922,11 @@ impl CodexProvider {
|
||||||
LocalRunnerError::invalid("Codex tool call arrived outside an active turn")
|
LocalRunnerError::invalid("Codex tool call arrived outside an active turn")
|
||||||
})?;
|
})?;
|
||||||
if params.get("turnId").and_then(Value::as_str) != Some(active_turn_id) {
|
if params.get("turnId").and_then(Value::as_str) != Some(active_turn_id) {
|
||||||
return Err(LocalRunnerError::invalid(
|
return Ok(Some(self.identity_failure(
|
||||||
"Codex tool call named another turn",
|
method,
|
||||||
));
|
¶ms,
|
||||||
|
"turn_binding_mismatch",
|
||||||
|
)));
|
||||||
}
|
}
|
||||||
let call_id = bounded_identifier(
|
let call_id = bounded_identifier(
|
||||||
params.get("callId").and_then(Value::as_str),
|
params.get("callId").and_then(Value::as_str),
|
||||||
|
|
@ -1976,9 +2029,11 @@ impl CodexProvider {
|
||||||
&& params.get("threadId").and_then(Value::as_str)
|
&& params.get("threadId").and_then(Value::as_str)
|
||||||
!= Some(self.thread_id.as_str())
|
!= Some(self.thread_id.as_str())
|
||||||
{
|
{
|
||||||
return Err(LocalRunnerError::invalid(
|
return Ok(Some(self.identity_failure(
|
||||||
"Codex runtime request named another thread",
|
method,
|
||||||
));
|
¶ms,
|
||||||
|
"thread_binding_mismatch",
|
||||||
|
)));
|
||||||
}
|
}
|
||||||
if request_targets_non_active_turn(
|
if request_targets_non_active_turn(
|
||||||
self.active_provider_turn_id.as_deref(),
|
self.active_provider_turn_id.as_deref(),
|
||||||
|
|
@ -2088,30 +2143,110 @@ impl CodexProvider {
|
||||||
|
|
||||||
if let Some(method) = message.get("method").and_then(Value::as_str) {
|
if let Some(method) = message.get("method").and_then(Value::as_str) {
|
||||||
let params = message.get("params").cloned().unwrap_or(Value::Null);
|
let params = message.get("params").cloned().unwrap_or(Value::Null);
|
||||||
|
let identity = match classify_notification_thread(
|
||||||
|
method,
|
||||||
|
&self.thread_id,
|
||||||
|
&self.descendant_thread_ids,
|
||||||
|
¶ms,
|
||||||
|
) {
|
||||||
|
Ok(identity) => identity,
|
||||||
|
Err(_) => {
|
||||||
|
return Ok(Some(self.identity_failure(
|
||||||
|
method,
|
||||||
|
¶ms,
|
||||||
|
"thread_binding_mismatch",
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if identity == NotificationThread::Descendant {
|
||||||
|
let id =
|
||||||
|
notification_thread_id(¶ms).expect("classified descendant has an identity");
|
||||||
|
let newly_known = match remember_descendant_thread(
|
||||||
|
&mut self.descendant_thread_ids,
|
||||||
|
id,
|
||||||
|
) {
|
||||||
|
Ok(newly_known) => newly_known,
|
||||||
|
Err(code) => {
|
||||||
|
return Ok(Some(CodexProviderEvent::ResourceLimit {
|
||||||
|
diagnostic: json!({
|
||||||
|
"code": code, "recoverable": false, "classification": "resource_capacity",
|
||||||
|
"message": "Codex reached the child-thread inventory limit. Reconcile child work before continuing in a fresh provider session.",
|
||||||
|
"method": bounded_method(method), "limit": MAX_DESCENDANT_THREAD_IDS,
|
||||||
|
"expectedThreadId": self.thread_id, "receivedThreadId": id,
|
||||||
|
}),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
};
|
||||||
|
// Retain each discovered child's effect inventory independently of
|
||||||
|
// the informational diagnostic budget, then bound repeated progress.
|
||||||
|
if !newly_known && self.notification_identity_diagnostics >= 32 {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
self.notification_identity_diagnostics += 1;
|
||||||
|
// Descendant terminals are progress only. They never settle root authority.
|
||||||
|
return Ok(Some(CodexProviderEvent::DescendantNotification {
|
||||||
|
method: method.to_owned(),
|
||||||
|
params,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
if identity == NotificationThread::UnrelatedInformation {
|
||||||
|
self.notification_identity_diagnostics += 1;
|
||||||
|
if self.notification_identity_diagnostics > 32 {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
return Ok(Some(CodexProviderEvent::Notification {
|
||||||
|
method: "warning".to_owned(),
|
||||||
|
params: json!({
|
||||||
|
"threadId": self.thread_id,
|
||||||
|
"message": "ignored unrelated provider information",
|
||||||
|
"providerMethod": bounded_method(method),
|
||||||
|
"classification": "unrelated_information",
|
||||||
|
"expectedThreadId": self.thread_id,
|
||||||
|
"receivedThreadId": notification_thread_id(¶ms).map(|id| id.chars().take(256).collect::<String>()),
|
||||||
|
"expectedTurnId": self.active_provider_turn_id,
|
||||||
|
"receivedTurnId": notification_turn_id(¶ms).map(|id| id.chars().take(256).collect::<String>()),
|
||||||
|
}),
|
||||||
|
}));
|
||||||
|
}
|
||||||
let terminal_event_type = normalized_codex_terminal_event_type(method, ¶ms);
|
let terminal_event_type = normalized_codex_terminal_event_type(method, ¶ms);
|
||||||
let notification_turn_id = params
|
let notification_turn_id = params
|
||||||
.get("turnId")
|
.get("turnId")
|
||||||
.or_else(|| params.pointer("/turn/id"))
|
.or_else(|| params.pointer("/turn/id"))
|
||||||
.and_then(Value::as_str);
|
.and_then(Value::as_str);
|
||||||
if terminal_event_type.is_some()
|
if notification_turn_id.is_some()
|
||||||
&& notification_turn_id.is_some()
|
|
||||||
&& notification_turn_id != self.active_provider_turn_id.as_deref()
|
&& notification_turn_id != self.active_provider_turn_id.as_deref()
|
||||||
&& notification_turn_id
|
&& notification_turn_id
|
||||||
.is_some_and(|turn_id| self.settled_provider_turn_ids.contains(turn_id))
|
.is_some_and(|turn_id| self.settled_provider_turn_ids.contains(turn_id))
|
||||||
{
|
{
|
||||||
|
self.notification_identity_diagnostics += 1;
|
||||||
|
if self.notification_identity_diagnostics > 32 {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
return Ok(Some(CodexProviderEvent::Notification {
|
return Ok(Some(CodexProviderEvent::Notification {
|
||||||
method: "warning".to_owned(),
|
method: "warning".to_owned(),
|
||||||
params: json!({
|
params: json!({
|
||||||
"message": "ignored a terminal notification for a non-active Codex turn",
|
"message": "ignored a notification for a settled Codex turn",
|
||||||
"providerMethod": bounded_method(method),
|
"providerMethod": bounded_method(method), "classification": "stale_settled_turn",
|
||||||
|
"expectedThreadId": self.thread_id,
|
||||||
|
"receivedThreadId": self.thread_id,
|
||||||
|
"expectedTurnId": self.active_provider_turn_id,
|
||||||
|
"receivedTurnId": notification_turn_id.map(|id| id.chars().take(256).collect::<String>()),
|
||||||
}),
|
}),
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
validate_notification_binding(
|
if validate_notification_binding(
|
||||||
&self.thread_id,
|
&self.thread_id,
|
||||||
self.active_provider_turn_id.as_deref(),
|
self.active_provider_turn_id.as_deref(),
|
||||||
¶ms,
|
¶ms,
|
||||||
)?;
|
)
|
||||||
|
.is_err()
|
||||||
|
{
|
||||||
|
return Ok(Some(self.identity_failure(
|
||||||
|
method,
|
||||||
|
¶ms,
|
||||||
|
"turn_binding_mismatch",
|
||||||
|
)));
|
||||||
|
}
|
||||||
if let Some(terminal_event_type) = terminal_event_type {
|
if let Some(terminal_event_type) = terminal_event_type {
|
||||||
if self.active_provider_turn_id.is_none() {
|
if self.active_provider_turn_id.is_none() {
|
||||||
return Err(LocalRunnerError::invalid(
|
return Err(LocalRunnerError::invalid(
|
||||||
|
|
@ -2200,10 +2335,25 @@ impl CodexProvider {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn identity_failure(&self, method: &str, params: &Value, code: &str) -> CodexProviderEvent {
|
||||||
|
CodexProviderEvent::ProtocolFailure {
|
||||||
|
diagnostic: json!({
|
||||||
|
"code": code, "recoverable": false,
|
||||||
|
"message": "Codex rejected an event outside the active execution identity",
|
||||||
|
"classification": "invalid_authoritative", "method": bounded_method(method),
|
||||||
|
"expectedThreadId": self.thread_id.chars().take(256).collect::<String>(),
|
||||||
|
"receivedThreadId": notification_thread_id(params).map(|id| id.chars().take(256).collect::<String>()),
|
||||||
|
"expectedTurnId": self.active_provider_turn_id.as_ref().map(|id| id.chars().take(256).collect::<String>()),
|
||||||
|
"receivedTurnId": notification_turn_id(params).map(|id| id.chars().take(256).collect::<String>()),
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn shutdown(&mut self) -> Result<(), LocalRunnerError> {
|
pub fn shutdown(&mut self) -> Result<(), LocalRunnerError> {
|
||||||
self.expected_shutdown = true;
|
self.expected_shutdown = true;
|
||||||
self.cancel_pending_requests()?;
|
// A failed courtesy response must never prevent process fencing.
|
||||||
let result = self.process.terminate_group().map(|_| ());
|
let cancellation = self.cancel_pending_requests();
|
||||||
|
let result = self.process.terminate_group().map(|_| ()).and(cancellation);
|
||||||
if let Some(trace) = self.trace.as_mut() {
|
if let Some(trace) = self.trace.as_mut() {
|
||||||
trace.finish();
|
trace.finish();
|
||||||
}
|
}
|
||||||
|
|
@ -2695,6 +2845,119 @@ fn contains_provider_work_binding(value: &Value) -> bool {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, PartialEq)]
|
||||||
|
enum NotificationThread {
|
||||||
|
Root,
|
||||||
|
Descendant,
|
||||||
|
UnrelatedInformation,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn notification_thread_id(params: &Value) -> Option<&str> {
|
||||||
|
params
|
||||||
|
.get("threadId")
|
||||||
|
.or_else(|| params.pointer("/thread/id"))
|
||||||
|
.or_else(|| params.pointer("/turn/threadId"))
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn classify_notification_thread(
|
||||||
|
method: &str,
|
||||||
|
root: &str,
|
||||||
|
descendants: &BTreeSet<String>,
|
||||||
|
params: &Value,
|
||||||
|
) -> Result<NotificationThread, LocalRunnerError> {
|
||||||
|
let identities: Vec<&Value> = [
|
||||||
|
params.get("threadId"),
|
||||||
|
params.pointer("/thread/id"),
|
||||||
|
params.pointer("/turn/threadId"),
|
||||||
|
]
|
||||||
|
.into_iter()
|
||||||
|
.flatten()
|
||||||
|
.filter(|v| !v.is_null())
|
||||||
|
.collect();
|
||||||
|
if identities.iter().any(|id| {
|
||||||
|
id.as_str()
|
||||||
|
.is_none_or(|value| value.is_empty() || value.len() > 240)
|
||||||
|
}) || identities.windows(2).any(|ids| ids[0] != ids[1])
|
||||||
|
{
|
||||||
|
return Err(LocalRunnerError::invalid(
|
||||||
|
"Codex notification has malformed thread identity",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let turn_ids: Vec<&Value> = [params.get("turnId"), params.pointer("/turn/id")]
|
||||||
|
.into_iter()
|
||||||
|
.flatten()
|
||||||
|
.filter(|value| !value.is_null())
|
||||||
|
.collect();
|
||||||
|
if turn_ids.iter().any(|id| {
|
||||||
|
id.as_str()
|
||||||
|
.is_none_or(|value| value.is_empty() || value.len() > 240)
|
||||||
|
}) || turn_ids.windows(2).any(|ids| ids[0] != ids[1])
|
||||||
|
{
|
||||||
|
return Err(LocalRunnerError::invalid(
|
||||||
|
"Codex notification has malformed turn identity",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let thread = notification_thread_id(params);
|
||||||
|
if thread.is_none()
|
||||||
|
&& turn_ids.is_empty()
|
||||||
|
&& !matches!(
|
||||||
|
method,
|
||||||
|
"warning"
|
||||||
|
| "configWarning"
|
||||||
|
| "guardianWarning"
|
||||||
|
| "deprecationNotice"
|
||||||
|
| "remoteControl/status/changed"
|
||||||
|
| "mcpServer/startupStatus/updated"
|
||||||
|
| "account/rateLimits/updated"
|
||||||
|
)
|
||||||
|
{
|
||||||
|
return Err(LocalRunnerError::invalid(
|
||||||
|
"Codex authoritative notification omitted thread identity",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
// Unbound transport warnings belong to this provider connection. Preserve
|
||||||
|
// their diagnostic meaning; only a different named thread is unrelated.
|
||||||
|
if thread.is_none() || thread == Some(root) {
|
||||||
|
return Ok(NotificationThread::Root);
|
||||||
|
}
|
||||||
|
let parent = [
|
||||||
|
"/thread/source/subAgent/thread_spawn/parent_thread_id",
|
||||||
|
"/thread/source/subAgent/threadSpawn/parentThreadId",
|
||||||
|
"/thread/source/subagent/thread_spawn/parent_thread_id",
|
||||||
|
]
|
||||||
|
.iter()
|
||||||
|
.find_map(|path| params.pointer(path).and_then(Value::as_str));
|
||||||
|
if thread.is_some()
|
||||||
|
&& (thread.is_some_and(|id| descendants.contains(id))
|
||||||
|
|| (method == "thread/started"
|
||||||
|
&& parent.is_some_and(|id| id == root || descendants.contains(id))))
|
||||||
|
{
|
||||||
|
if method.starts_with("paperclip/") {
|
||||||
|
return Err(LocalRunnerError::invalid(
|
||||||
|
"Codex descendant cannot supply root execution authority",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
return Ok(NotificationThread::Descendant);
|
||||||
|
}
|
||||||
|
if matches!(
|
||||||
|
method,
|
||||||
|
"thread/started"
|
||||||
|
| "thread/status/changed"
|
||||||
|
| "thread/closed"
|
||||||
|
| "thread/tokenUsage/updated"
|
||||||
|
| "warning"
|
||||||
|
| "configWarning"
|
||||||
|
| "guardianWarning"
|
||||||
|
| "deprecationNotice"
|
||||||
|
) {
|
||||||
|
return Ok(NotificationThread::UnrelatedInformation);
|
||||||
|
}
|
||||||
|
Err(LocalRunnerError::invalid(
|
||||||
|
"Codex authoritative notification named another thread",
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
fn validate_notification_binding(
|
fn validate_notification_binding(
|
||||||
thread_id: &str,
|
thread_id: &str,
|
||||||
active_turn_id: Option<&str>,
|
active_turn_id: Option<&str>,
|
||||||
|
|
@ -3682,3 +3945,109 @@ mod tests {
|
||||||
assert_eq!(retain_buffered_message_bytes(usize::MAX, 1), None);
|
assert_eq!(retain_buffered_message_bytes(usize::MAX, 1), None);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod notification_identity_tests {
|
||||||
|
use super::*;
|
||||||
|
#[test]
|
||||||
|
fn bounds_lineage_without_eviction_or_misclassifying_capacity_as_integrity() {
|
||||||
|
let mut ids = BTreeSet::new();
|
||||||
|
for index in 0..MAX_DESCENDANT_THREAD_IDS {
|
||||||
|
assert_eq!(
|
||||||
|
remember_descendant_thread(&mut ids, &format!("child-{index}")),
|
||||||
|
Ok(true)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert_eq!(remember_descendant_thread(&mut ids, "child-0"), Ok(false));
|
||||||
|
assert_eq!(
|
||||||
|
remember_descendant_thread(&mut ids, "overflow"),
|
||||||
|
Err("provider_descendant_capacity_exhausted")
|
||||||
|
);
|
||||||
|
assert_eq!(ids.len(), MAX_DESCENDANT_THREAD_IDS);
|
||||||
|
assert!(ids.contains("child-0"));
|
||||||
|
assert!(!ids.contains("overflow"));
|
||||||
|
assert_eq!(
|
||||||
|
classify_notification_thread(
|
||||||
|
"turn/completed",
|
||||||
|
"root",
|
||||||
|
&ids,
|
||||||
|
&json!({"threadId":"child-0", "turnId":"child-turn"})
|
||||||
|
)
|
||||||
|
.unwrap(),
|
||||||
|
NotificationThread::Descendant
|
||||||
|
);
|
||||||
|
}
|
||||||
|
#[test]
|
||||||
|
fn rejects_missing_authority_and_malformed_turn_identities() {
|
||||||
|
for method in [
|
||||||
|
"item/started",
|
||||||
|
"item/completed",
|
||||||
|
"item/agentMessage/delta",
|
||||||
|
"thread/goal/updated",
|
||||||
|
"unknown/authority",
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
classify_notification_thread(method, "root", &BTreeSet::new(), &json!({})).is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for params in [
|
||||||
|
json!({"status":"completed"}),
|
||||||
|
json!({"threadId":"root","turnId":7}),
|
||||||
|
json!({"threadId":"root","turnId":"a","turn":{"id":"b"}}),
|
||||||
|
] {
|
||||||
|
assert!(classify_notification_thread(
|
||||||
|
"turn/completed",
|
||||||
|
"root",
|
||||||
|
&BTreeSet::new(),
|
||||||
|
¶ms
|
||||||
|
)
|
||||||
|
.is_err());
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
classify_notification_thread(
|
||||||
|
"configWarning",
|
||||||
|
"root",
|
||||||
|
&BTreeSet::new(),
|
||||||
|
&json!({"message":"warning"})
|
||||||
|
)
|
||||||
|
.unwrap(),
|
||||||
|
NotificationThread::Root
|
||||||
|
);
|
||||||
|
}
|
||||||
|
#[test]
|
||||||
|
fn classifies_provider_lineage_before_root_authority() {
|
||||||
|
let children = BTreeSet::from(["child".to_owned()]);
|
||||||
|
assert_eq!(
|
||||||
|
classify_notification_thread(
|
||||||
|
"thread/status/changed",
|
||||||
|
"root",
|
||||||
|
&children,
|
||||||
|
&json!({"threadId":"unrelated"})
|
||||||
|
)
|
||||||
|
.unwrap(),
|
||||||
|
NotificationThread::UnrelatedInformation
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
classify_notification_thread(
|
||||||
|
"turn/completed",
|
||||||
|
"root",
|
||||||
|
&children,
|
||||||
|
&json!({"threadId":"child", "turnId":"child-turn"})
|
||||||
|
)
|
||||||
|
.unwrap(),
|
||||||
|
NotificationThread::Descendant
|
||||||
|
);
|
||||||
|
assert_eq!(classify_notification_thread("thread/started", "root", &children, &json!({"thread":{"id":"grandchild","source":{"subAgent":{"thread_spawn":{"parent_thread_id":"child"}}}}})).unwrap(), NotificationThread::Descendant);
|
||||||
|
for (method, params) in [
|
||||||
|
("paperclip/runResult", json!({"threadId":"child"})),
|
||||||
|
("turn/completed", json!({"threadId":"unrelated"})),
|
||||||
|
("item/started", json!({"threadId":42})),
|
||||||
|
(
|
||||||
|
"item/started",
|
||||||
|
json!({"threadId":"root", "thread":{"id":"other"}}),
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
assert!(classify_notification_thread(method, "root", &children, ¶ms).is_err());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -797,6 +797,8 @@ struct CodexProviderState {
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
settled_provider_turn_ids: std::collections::BTreeSet<String>,
|
settled_provider_turn_ids: std::collections::BTreeSet<String>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
|
descendant_thread_ids: std::collections::BTreeSet<String>,
|
||||||
|
#[serde(default)]
|
||||||
settled_provider_turn_filter: DurableReplayFilter,
|
settled_provider_turn_filter: DurableReplayFilter,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
receipt_limit_diagnostic_emitted: bool,
|
receipt_limit_diagnostic_emitted: bool,
|
||||||
|
|
@ -882,6 +884,7 @@ impl CodexProviderState {
|
||||||
completed_turn_process_generation: None,
|
completed_turn_process_generation: None,
|
||||||
completed_provider_turn_id: None,
|
completed_provider_turn_id: None,
|
||||||
settled_provider_turn_ids: std::collections::BTreeSet::new(),
|
settled_provider_turn_ids: std::collections::BTreeSet::new(),
|
||||||
|
descendant_thread_ids: std::collections::BTreeSet::new(),
|
||||||
settled_provider_turn_filter: DurableReplayFilter::default(),
|
settled_provider_turn_filter: DurableReplayFilter::default(),
|
||||||
receipt_limit_diagnostic_emitted: false,
|
receipt_limit_diagnostic_emitted: false,
|
||||||
receipt_limit_interrupt_pending: false,
|
receipt_limit_interrupt_pending: false,
|
||||||
|
|
@ -908,10 +911,20 @@ impl CodexProviderState {
|
||||||
DurableRunnerError::invalid(format!("Codex semantic tool state is invalid: {error}"))
|
DurableRunnerError::invalid(format!("Codex semantic tool state is invalid: {error}"))
|
||||||
})?;
|
})?;
|
||||||
let mut pending_event_ids = HashSet::new();
|
let mut pending_event_ids = HashSet::new();
|
||||||
if self.schema != PROVIDER_STATE_SCHEMA
|
if self.descendant_thread_ids.len() > crate::codex_provider::MAX_DESCENDANT_THREAD_IDS
|
||||||
|
|| self
|
||||||
|
.descendant_thread_ids
|
||||||
|
.iter()
|
||||||
|
.any(|id| id.is_empty() || id.len() > 240)
|
||||||
|
|| self.schema != PROVIDER_STATE_SCHEMA
|
||||||
|| !matches!(
|
|| !matches!(
|
||||||
self.lifecycle.as_str(),
|
self.lifecycle.as_str(),
|
||||||
"prepared" | "session_open" | "turn_active" | "provider_exited" | "closed"
|
"prepared"
|
||||||
|
| "session_open"
|
||||||
|
| "turn_active"
|
||||||
|
| "provider_exited"
|
||||||
|
| "reconciliation_required"
|
||||||
|
| "closed"
|
||||||
)
|
)
|
||||||
|| self
|
|| self
|
||||||
.thread_id
|
.thread_id
|
||||||
|
|
@ -1468,6 +1481,7 @@ impl CodexCommandExecutor {
|
||||||
"failed to resume {provider_name} provider: {error}"
|
"failed to resume {provider_name} provider: {error}"
|
||||||
))
|
))
|
||||||
})?;
|
})?;
|
||||||
|
provider.restore_descendant_thread_identities(&state.descendant_thread_ids);
|
||||||
provider.enable_durable_tool_call_replays();
|
provider.enable_durable_tool_call_replays();
|
||||||
provider
|
provider
|
||||||
.restore_settled_turn_identities(
|
.restore_settled_turn_identities(
|
||||||
|
|
@ -1883,6 +1897,15 @@ impl CodexCommandExecutor {
|
||||||
|
|
||||||
fn ensure_provider(&mut self) -> Result<&mut CodexProvider, DurableRunnerError> {
|
fn ensure_provider(&mut self) -> Result<&mut CodexProvider, DurableRunnerError> {
|
||||||
self.restore_provider_if_needed()?;
|
self.restore_provider_if_needed()?;
|
||||||
|
if self
|
||||||
|
.state
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|state| state.lifecycle == "reconciliation_required")
|
||||||
|
{
|
||||||
|
return Err(DurableRunnerError::invalid(
|
||||||
|
"Codex provider session requires explicit reconciliation and a fresh session",
|
||||||
|
));
|
||||||
|
}
|
||||||
if self.provider.is_none() {
|
if self.provider.is_none() {
|
||||||
let state = self.state.as_ref().ok_or_else(|| {
|
let state = self.state.as_ref().ok_or_else(|| {
|
||||||
DurableRunnerError::invalid("Codex provider has not been prepared")
|
DurableRunnerError::invalid("Codex provider has not been prepared")
|
||||||
|
|
@ -1914,6 +1937,7 @@ impl CodexCommandExecutor {
|
||||||
.map_err(|error| {
|
.map_err(|error| {
|
||||||
DurableRunnerError::invalid(format!("failed to start Codex provider: {error}"))
|
DurableRunnerError::invalid(format!("failed to start Codex provider: {error}"))
|
||||||
})?;
|
})?;
|
||||||
|
provider.restore_descendant_thread_identities(&state.descendant_thread_ids);
|
||||||
provider.enable_durable_tool_call_replays();
|
provider.enable_durable_tool_call_replays();
|
||||||
provider
|
provider
|
||||||
.restore_settled_turn_identities(
|
.restore_settled_turn_identities(
|
||||||
|
|
@ -3447,6 +3471,95 @@ impl CodexCommandExecutor {
|
||||||
} => {
|
} => {
|
||||||
self.handle_tool_call(call_id, operation_id, input)?;
|
self.handle_tool_call(call_id, operation_id, input)?;
|
||||||
}
|
}
|
||||||
|
CodexProviderEvent::ProtocolFailure { diagnostic }
|
||||||
|
| CodexProviderEvent::ResourceLimit { diagnostic } => {
|
||||||
|
let resource_capacity = diagnostic["classification"] == "resource_capacity";
|
||||||
|
let state = self
|
||||||
|
.state
|
||||||
|
.as_mut()
|
||||||
|
.expect("Codex state available while polling");
|
||||||
|
// Neither an integrity failure nor a full lineage ledger can
|
||||||
|
// safely reopen this provider session, even after event ACK.
|
||||||
|
state.lifecycle = "reconciliation_required".to_owned();
|
||||||
|
state.completed_turn_authoritative = false;
|
||||||
|
state.completed_turn_process_generation = None;
|
||||||
|
state.completed_provider_turn_id = None;
|
||||||
|
state.settle_active_provider_turn_identity()?;
|
||||||
|
state.active_provider_turn_id = None;
|
||||||
|
state.push_terminal_event(NormalizedProviderEvent {
|
||||||
|
event_type: "harness.diagnostic".to_owned(),
|
||||||
|
priority: EventPriority::P0,
|
||||||
|
payload: diagnostic.clone(),
|
||||||
|
})?;
|
||||||
|
state.push_terminal_event(NormalizedProviderEvent {
|
||||||
|
event_type: "turn.failed".to_owned(),
|
||||||
|
priority: EventPriority::P0,
|
||||||
|
payload: json!({ "provider": state.config.provider, "status": "failed",
|
||||||
|
"code": diagnostic["code"], "recoverable": false,
|
||||||
|
"message": diagnostic["message"], "error": diagnostic }),
|
||||||
|
})?;
|
||||||
|
state.extend_terminal_events(terminal_events(state, "turn.failed", None))?;
|
||||||
|
// Commit the authoritative failure before best-effort provider cleanup.
|
||||||
|
self.save_state()?;
|
||||||
|
if let Some(mut provider) = self.provider.take() {
|
||||||
|
if let Some(frame_id) = trace_frame_id {
|
||||||
|
provider.record_provider_trace_interpretation(
|
||||||
|
frame_id,
|
||||||
|
if resource_capacity { "codex.resource_capacity" } else { "codex.identity.invalid_authoritative" },
|
||||||
|
"rejected",
|
||||||
|
Vec::new(),
|
||||||
|
if resource_capacity { "Provider resource capacity requires explicit reconciliation" } else { "Rejected provider authority outside the root execution identity" },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let _ = provider.shutdown();
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
CodexProviderEvent::DescendantNotification { method, params } => {
|
||||||
|
// Do not normalize a child's turn/completed as a root terminal.
|
||||||
|
// Retain bounded lineage evidence without credential-bearing payloads.
|
||||||
|
let child = params
|
||||||
|
.get("threadId")
|
||||||
|
.or_else(|| params.pointer("/thread/id"))
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.map(|id| id.chars().take(256).collect::<String>());
|
||||||
|
let root_thread = self
|
||||||
|
.provider
|
||||||
|
.as_ref()
|
||||||
|
.map(|provider| provider.thread_id().to_owned());
|
||||||
|
let root_turn = self
|
||||||
|
.provider
|
||||||
|
.as_ref()
|
||||||
|
.and_then(CodexProvider::active_provider_turn_id)
|
||||||
|
.map(str::to_owned);
|
||||||
|
let child_turn = params
|
||||||
|
.get("turnId")
|
||||||
|
.or_else(|| params.pointer("/turn/id"))
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.map(|id| id.chars().take(256).collect::<String>());
|
||||||
|
let state = self
|
||||||
|
.state
|
||||||
|
.as_mut()
|
||||||
|
.expect("Codex state remains available while polling");
|
||||||
|
if let Some(id) = child.as_ref() {
|
||||||
|
state.descendant_thread_ids.insert(id.clone());
|
||||||
|
}
|
||||||
|
state.extend_events(vec![NormalizedProviderEvent {
|
||||||
|
event_type: "harness.diagnostic".to_owned(),
|
||||||
|
priority: EventPriority::P1,
|
||||||
|
payload: json!({ "code": "provider_notification_identity", "classification": "descendant",
|
||||||
|
"method": method.chars().take(128).collect::<String>(), "receivedThreadId": child, "expectedThreadId": root_thread,
|
||||||
|
"receivedTurnId": child_turn, "expectedTurnId": root_turn }),
|
||||||
|
}])?;
|
||||||
|
self.save_state()?;
|
||||||
|
if let (Some(frame_id), Some(provider)) =
|
||||||
|
(trace_frame_id, self.provider.as_mut())
|
||||||
|
{
|
||||||
|
provider.record_provider_trace_interpretation(frame_id,
|
||||||
|
"codex.identity.descendant", "mapped", Vec::new(),
|
||||||
|
"Provider-confirmed descendant progress has no root terminal or tool authority");
|
||||||
|
}
|
||||||
|
}
|
||||||
CodexProviderEvent::Notification { method, params } => {
|
CodexProviderEvent::Notification { method, params } => {
|
||||||
let active_provider_turn_id = if method == "turn/started" {
|
let active_provider_turn_id = if method == "turn/started" {
|
||||||
self.provider
|
self.provider
|
||||||
|
|
@ -3806,6 +3919,27 @@ impl CodexCommandExecutor {
|
||||||
impl CommandExecutor for CodexCommandExecutor {
|
impl CommandExecutor for CodexCommandExecutor {
|
||||||
fn execute(&mut self, command: &Command) -> Result<CommandExecution, DurableRunnerError> {
|
fn execute(&mut self, command: &Command) -> Result<CommandExecution, DurableRunnerError> {
|
||||||
self.restore()?;
|
self.restore()?;
|
||||||
|
if self
|
||||||
|
.state
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|state| state.lifecycle == "reconciliation_required")
|
||||||
|
&& !matches!(
|
||||||
|
command.command_type.as_str(),
|
||||||
|
"session.snapshot"
|
||||||
|
| "session.close"
|
||||||
|
| "session.destroy"
|
||||||
|
| "runner.drain"
|
||||||
|
| "runner.suspend"
|
||||||
|
| "runner.shutdown"
|
||||||
|
| "turn.interrupt"
|
||||||
|
| "run.cancel"
|
||||||
|
| "turn.stop"
|
||||||
|
)
|
||||||
|
{
|
||||||
|
return Err(DurableRunnerError::invalid(
|
||||||
|
"Codex provider session requires explicit reconciliation and a fresh session",
|
||||||
|
));
|
||||||
|
}
|
||||||
match command.command_type.as_str() {
|
match command.command_type.as_str() {
|
||||||
"run.prepare" => self.prepare(&command.payload),
|
"run.prepare" => self.prepare(&command.payload),
|
||||||
"run.attach" => {
|
"run.attach" => {
|
||||||
|
|
@ -4291,6 +4425,7 @@ mod tests {
|
||||||
completed_turn_process_generation: None,
|
completed_turn_process_generation: None,
|
||||||
completed_provider_turn_id: None,
|
completed_provider_turn_id: None,
|
||||||
settled_provider_turn_ids: std::collections::BTreeSet::new(),
|
settled_provider_turn_ids: std::collections::BTreeSet::new(),
|
||||||
|
descendant_thread_ids: std::collections::BTreeSet::new(),
|
||||||
settled_provider_turn_filter: DurableReplayFilter::default(),
|
settled_provider_turn_filter: DurableReplayFilter::default(),
|
||||||
receipt_limit_diagnostic_emitted: false,
|
receipt_limit_diagnostic_emitted: false,
|
||||||
receipt_limit_interrupt_pending: false,
|
receipt_limit_interrupt_pending: false,
|
||||||
|
|
|
||||||
|
|
@ -1654,11 +1654,22 @@ fn ambiguous_replacement_turn_rejects_conflicting_later_identity() {
|
||||||
);
|
);
|
||||||
assert_eq!(provider.active_provider_turn_id(), Some("provider-turn-2"));
|
assert_eq!(provider.active_provider_turn_id(), Some("provider-turn-2"));
|
||||||
|
|
||||||
let conflicting_completion = wait_for_provider_error(&mut provider);
|
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
|
||||||
assert!(
|
let diagnostic = loop {
|
||||||
conflicting_completion.contains("another active turn"),
|
assert!(
|
||||||
"unexpected conflicting-identity error: {conflicting_completion}"
|
std::time::Instant::now() < deadline,
|
||||||
);
|
"expected structured integrity failure"
|
||||||
|
);
|
||||||
|
if let Some(CodexProviderEvent::ProtocolFailure { diagnostic }) =
|
||||||
|
provider.poll().expect("poll identity failure")
|
||||||
|
{
|
||||||
|
break diagnostic;
|
||||||
|
}
|
||||||
|
std::thread::sleep(std::time::Duration::from_millis(1));
|
||||||
|
};
|
||||||
|
assert_eq!(diagnostic["code"], "turn_binding_mismatch");
|
||||||
|
assert_eq!(diagnostic["recoverable"], false);
|
||||||
|
assert_eq!(diagnostic["expectedTurnId"], "provider-turn-2");
|
||||||
assert_eq!(provider.active_provider_turn_id(), Some("provider-turn-2"));
|
assert_eq!(provider.active_provider_turn_id(), Some("provider-turn-2"));
|
||||||
|
|
||||||
fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
|
fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
|
||||||
|
|
@ -4839,3 +4850,226 @@ fn codex_completion_emits_the_bound_result_before_the_terminal_event() {
|
||||||
executor.shutdown().expect("stop provider process");
|
executor.shutdown().expect("stop provider process");
|
||||||
fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
|
fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn durable_integrity_failure_preserves_code_and_stops_provider_authority() {
|
||||||
|
let directory = temporary_directory("durable-identity-failure");
|
||||||
|
let config = provider_config(
|
||||||
|
&directory,
|
||||||
|
&[
|
||||||
|
"--malformed-error-second-turn-start",
|
||||||
|
"--conflicting-ambiguous-second-turn",
|
||||||
|
],
|
||||||
|
);
|
||||||
|
let mut executor = CodexCommandExecutor::new(&directory);
|
||||||
|
executor
|
||||||
|
.execute(&command(
|
||||||
|
"prepare",
|
||||||
|
1,
|
||||||
|
"run.prepare",
|
||||||
|
json!({"provider": config}),
|
||||||
|
))
|
||||||
|
.expect("prepare");
|
||||||
|
executor
|
||||||
|
.execute(&command("open", 2, "session.open", json!({})))
|
||||||
|
.expect("open");
|
||||||
|
executor
|
||||||
|
.execute(&command(
|
||||||
|
"first",
|
||||||
|
3,
|
||||||
|
"turn.start",
|
||||||
|
json!({"text": "Complete the first turn."}),
|
||||||
|
))
|
||||||
|
.expect("first turn");
|
||||||
|
wait_for_executor_event(&mut executor, "turn.completed");
|
||||||
|
executor
|
||||||
|
.execute(&command(
|
||||||
|
"second",
|
||||||
|
4,
|
||||||
|
"turn.start",
|
||||||
|
json!({"text": "Start replacement work."}),
|
||||||
|
))
|
||||||
|
.expect_err("ambiguous response");
|
||||||
|
let failed = wait_for_executor_event(&mut executor, "turn.failed");
|
||||||
|
assert_eq!(failed.payload["code"], "turn_binding_mismatch");
|
||||||
|
assert_eq!(failed.payload["recoverable"], false);
|
||||||
|
let persisted: Value =
|
||||||
|
serde_json::from_slice(&fs::read(directory.join("codex-provider-state.json")).unwrap())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(persisted["lifecycle"], "reconciliation_required");
|
||||||
|
assert_eq!(persisted["completedTurnAuthoritative"], false);
|
||||||
|
executor.shutdown().expect("cleanup");
|
||||||
|
let mut restored = CodexCommandExecutor::new(&directory);
|
||||||
|
let error = restored
|
||||||
|
.execute(&command("retry", 5, "turn.start", json!({"text":"Retry"})))
|
||||||
|
.expect_err("an integrity failure must also remain fenced after restart");
|
||||||
|
assert!(error
|
||||||
|
.to_string()
|
||||||
|
.contains("requires explicit reconciliation"));
|
||||||
|
restored.shutdown().expect("restored cleanup");
|
||||||
|
fs::remove_dir_all(directory).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn durable_descendant_lineage_survives_capacity_and_provider_restoration() {
|
||||||
|
let directory = temporary_directory("descendant-restoration");
|
||||||
|
let config = provider_config(
|
||||||
|
&directory,
|
||||||
|
&["--descendant-notifications", "--durable-turn-ids"],
|
||||||
|
);
|
||||||
|
let runner_config = durable_config(&directory);
|
||||||
|
let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
|
||||||
|
first
|
||||||
|
.execute(&command(
|
||||||
|
"prepare",
|
||||||
|
1,
|
||||||
|
"run.prepare",
|
||||||
|
json!({
|
||||||
|
"provider": config, "authorizedTools": task_context_tool_set(),
|
||||||
|
"completionContract": {"revision": "lineage-contract", "criterionIds": ["lineage"]}
|
||||||
|
}),
|
||||||
|
))
|
||||||
|
.unwrap();
|
||||||
|
first
|
||||||
|
.execute(&command("open", 2, "session.open", json!({})))
|
||||||
|
.unwrap();
|
||||||
|
first
|
||||||
|
.execute(&command(
|
||||||
|
"turn",
|
||||||
|
3,
|
||||||
|
"turn.start",
|
||||||
|
json!({"text": "Read test context."}),
|
||||||
|
))
|
||||||
|
.unwrap();
|
||||||
|
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
|
||||||
|
let mut completed = false;
|
||||||
|
let mut children = std::collections::BTreeSet::new();
|
||||||
|
while std::time::Instant::now() < deadline && !completed {
|
||||||
|
for event in poll_and_ack(&mut first).unwrap() {
|
||||||
|
assert_ne!(event.event_type, "turn.failed");
|
||||||
|
if event.payload["classification"] == "descendant" {
|
||||||
|
children.insert(
|
||||||
|
event.payload["receivedThreadId"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap()
|
||||||
|
.to_owned(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
completed |= event.event_type == "run.terminal";
|
||||||
|
}
|
||||||
|
std::thread::sleep(std::time::Duration::from_millis(1));
|
||||||
|
}
|
||||||
|
assert!(completed);
|
||||||
|
assert_eq!(children.len(), 300);
|
||||||
|
first.shutdown().unwrap();
|
||||||
|
drop(first);
|
||||||
|
|
||||||
|
let mut restored = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
|
||||||
|
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
|
||||||
|
let mut child_seen = false;
|
||||||
|
while std::time::Instant::now() < deadline && !child_seen {
|
||||||
|
for event in poll_and_ack(&mut restored).unwrap() {
|
||||||
|
assert_ne!(event.event_type, "turn.failed");
|
||||||
|
assert_ne!(
|
||||||
|
event.event_type, "run.terminal",
|
||||||
|
"child completion cannot complete the root"
|
||||||
|
);
|
||||||
|
child_seen |= event.payload["classification"] == "descendant"
|
||||||
|
&& event.payload["receivedThreadId"] == "descendant-299";
|
||||||
|
}
|
||||||
|
std::thread::sleep(std::time::Duration::from_millis(1));
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
child_seen,
|
||||||
|
"restoration must recognize an existing child's terminal without rediscovery"
|
||||||
|
);
|
||||||
|
restored.shutdown().unwrap();
|
||||||
|
drop(restored);
|
||||||
|
|
||||||
|
// Seed the bounded persisted inventory instead of performing thousands of
|
||||||
|
// redundant disk writes, then exercise the real overflow event and fencing.
|
||||||
|
let state_path = directory.join("codex-provider-state.json");
|
||||||
|
let mut persisted: Value = serde_json::from_slice(&fs::read(&state_path).unwrap()).unwrap();
|
||||||
|
persisted["descendantThreadIds"] = json!((0..4096)
|
||||||
|
.map(|index| format!("descendant-{index}"))
|
||||||
|
.collect::<Vec<_>>());
|
||||||
|
persisted["config"]["args"]
|
||||||
|
.as_array_mut()
|
||||||
|
.unwrap()
|
||||||
|
.push(json!("--descendant-overflow"));
|
||||||
|
fs::write(&state_path, serde_json::to_vec(&persisted).unwrap()).unwrap();
|
||||||
|
let mut bounded = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
|
||||||
|
bounded
|
||||||
|
.execute(&command(
|
||||||
|
"bounded-turn",
|
||||||
|
4,
|
||||||
|
"turn.start",
|
||||||
|
json!({"text":"Continue bounded child work."}),
|
||||||
|
))
|
||||||
|
.unwrap();
|
||||||
|
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
|
||||||
|
let mut capacity_failed = false;
|
||||||
|
while std::time::Instant::now() < deadline && !capacity_failed {
|
||||||
|
for event in poll_and_ack(&mut bounded).unwrap() {
|
||||||
|
if event.event_type == "turn.failed" {
|
||||||
|
assert_eq!(
|
||||||
|
event.payload["code"],
|
||||||
|
"provider_descendant_capacity_exhausted"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
event.payload["error"]["classification"],
|
||||||
|
"resource_capacity"
|
||||||
|
);
|
||||||
|
capacity_failed = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
std::thread::sleep(std::time::Duration::from_millis(1));
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
capacity_failed,
|
||||||
|
"capacity exhaustion must become a specific durable recovery reason"
|
||||||
|
);
|
||||||
|
let persisted: Value = serde_json::from_slice(&fs::read(&state_path).unwrap()).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
persisted["descendantThreadIds"].as_array().unwrap().len(),
|
||||||
|
4096
|
||||||
|
);
|
||||||
|
assert_eq!(persisted["lifecycle"], "reconciliation_required");
|
||||||
|
// Acknowledging all terminal events must not authorize another provider turn.
|
||||||
|
while !poll_and_ack(&mut bounded).unwrap().is_empty() {}
|
||||||
|
for kind in ["turn.start", "session.open", "run.attach"] {
|
||||||
|
let error = bounded
|
||||||
|
.execute(&command("blocked", 5, kind, json!({"text":"Retry"})))
|
||||||
|
.expect_err("reconciliation cannot be bypassed in the current executor");
|
||||||
|
assert!(error
|
||||||
|
.to_string()
|
||||||
|
.contains("requires explicit reconciliation"));
|
||||||
|
}
|
||||||
|
bounded.shutdown().unwrap();
|
||||||
|
let mut restored = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
|
||||||
|
for kind in ["turn.start", "session.open", "run.attach"] {
|
||||||
|
let error = restored
|
||||||
|
.execute(&command(
|
||||||
|
"blocked-after-restart",
|
||||||
|
6,
|
||||||
|
kind,
|
||||||
|
json!({"text":"Retry"}),
|
||||||
|
))
|
||||||
|
.expect_err("restart must retain the reconciliation fence");
|
||||||
|
assert!(error
|
||||||
|
.to_string()
|
||||||
|
.contains("requires explicit reconciliation"));
|
||||||
|
}
|
||||||
|
let persisted_after_restart: Value =
|
||||||
|
serde_json::from_slice(&fs::read(&state_path).unwrap()).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
persisted_after_restart["providerProcessGeneration"],
|
||||||
|
persisted["providerProcessGeneration"]
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
persisted_after_restart["lifecycle"],
|
||||||
|
"reconciliation_required"
|
||||||
|
);
|
||||||
|
restored.shutdown().unwrap();
|
||||||
|
fs::remove_dir_all(directory).unwrap();
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@ import { describe, expect, it, vi } from "vitest";
|
||||||
|
|
||||||
import type { HarnessDriver, HarnessSession, PersistedHarnessSession } from "../contracts/harness-driver.js";
|
import type { HarnessDriver, HarnessSession, PersistedHarnessSession } from "../contracts/harness-driver.js";
|
||||||
import type { PrpEvent, PrpStructuredRunResult, PrpTerminalState } from "../protocol/replay-contract.js";
|
import type { PrpEvent, PrpStructuredRunResult, PrpTerminalState } from "../protocol/replay-contract.js";
|
||||||
|
import { NativeSessionProtocolIntegrityError } from "../contracts/native-session-backend.js";
|
||||||
import { HarnessDriverBackend } from "./harness-driver-backend.js";
|
import { HarnessDriverBackend } from "./harness-driver-backend.js";
|
||||||
|
|
||||||
const result: PrpStructuredRunResult = {
|
const result: PrpStructuredRunResult = {
|
||||||
|
|
@ -711,6 +712,121 @@ describe("HarnessDriverBackend", () => {
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it.each(["typed", "typed-snapshot", "lookalike", "generic"] as const)(
|
||||||
|
"only a typed integrity fault forbids pending-input fallback: %s",
|
||||||
|
async (kind) => {
|
||||||
|
const typed = kind === "typed" || kind === "typed-snapshot";
|
||||||
|
const fault = typed
|
||||||
|
? new NativeSessionProtocolIntegrityError(
|
||||||
|
"semantic_input_digest_mismatch",
|
||||||
|
)
|
||||||
|
: kind === "lookalike"
|
||||||
|
? Object.assign(new Error("native_event_replay_conflict"), {
|
||||||
|
code: "native_event_replay_conflict",
|
||||||
|
reason: "semantic_input_digest_mismatch",
|
||||||
|
})
|
||||||
|
: new Error("provider transport lost");
|
||||||
|
class PendingInputSession extends FakeHarnessSession {
|
||||||
|
override async *events() {
|
||||||
|
yield prpEvent(1, "runtime_request.created", {
|
||||||
|
request: {
|
||||||
|
schema: "paperclip.runtime_request.v2",
|
||||||
|
requestKind: "runtime",
|
||||||
|
requestId: "input-1",
|
||||||
|
type: "input",
|
||||||
|
status: "pending",
|
||||||
|
turnId: "turn-1",
|
||||||
|
itemId: "input-1",
|
||||||
|
input: {
|
||||||
|
schema: "paperclip.question_set.v1",
|
||||||
|
questions: [
|
||||||
|
{
|
||||||
|
id: "color",
|
||||||
|
prompt: "Which color?",
|
||||||
|
required: true,
|
||||||
|
answerMode: "text",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
throw kind === "typed-snapshot"
|
||||||
|
? new Error("ordinary stream failure")
|
||||||
|
: fault;
|
||||||
|
}
|
||||||
|
override async snapshot(): Promise<PersistedHarnessSession> {
|
||||||
|
if (kind === "typed-snapshot") throw fault;
|
||||||
|
return super.snapshot();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const session = await new HarnessDriverBackend({
|
||||||
|
...driver,
|
||||||
|
openSession: async () => new PendingInputSession(),
|
||||||
|
}).openSession({
|
||||||
|
identity: {
|
||||||
|
runId: "run-1",
|
||||||
|
sessionId: "session-1",
|
||||||
|
companyId: "company-1",
|
||||||
|
issueId: "issue-1",
|
||||||
|
agentId: "agent-1",
|
||||||
|
},
|
||||||
|
workingDirectory: "/workspace",
|
||||||
|
});
|
||||||
|
const events: PrpEvent[] = [];
|
||||||
|
const consumed = (async () => {
|
||||||
|
for await (const event of session.events()) events.push(event);
|
||||||
|
})();
|
||||||
|
if (typed) {
|
||||||
|
await expect(consumed).rejects.toBe(fault);
|
||||||
|
expect(events.map((event) => event.eventType)).toEqual([
|
||||||
|
"runtime_request.created",
|
||||||
|
]);
|
||||||
|
await expect(session.result()).rejects.toBe(fault);
|
||||||
|
await expect(session.snapshot()).rejects.toBe(fault);
|
||||||
|
} else {
|
||||||
|
await consumed;
|
||||||
|
expect(events.map((event) => event.eventType)).toEqual([
|
||||||
|
"runtime_request.created",
|
||||||
|
"runtime_request.expired",
|
||||||
|
"turn.interrupted",
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
await session.close({ reason: "fixture complete" });
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
it("never exposes an earlier terminal result after a later typed integrity fault", async () => {
|
||||||
|
const fault = new NativeSessionProtocolIntegrityError(
|
||||||
|
"semantic_input_digest_mismatch",
|
||||||
|
);
|
||||||
|
class TerminalThenIntegrityFailure extends FakeHarnessSession {
|
||||||
|
override async *events() {
|
||||||
|
yield* super.events();
|
||||||
|
throw fault;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const session = await new HarnessDriverBackend({
|
||||||
|
...driver,
|
||||||
|
openSession: async () => new TerminalThenIntegrityFailure(),
|
||||||
|
}).openSession({
|
||||||
|
identity: {
|
||||||
|
runId: "run-1",
|
||||||
|
sessionId: "session-1",
|
||||||
|
companyId: "company-1",
|
||||||
|
issueId: "issue-1",
|
||||||
|
agentId: "agent-1",
|
||||||
|
},
|
||||||
|
workingDirectory: "/workspace",
|
||||||
|
});
|
||||||
|
const iterator = session.events()[Symbol.asyncIterator]();
|
||||||
|
await iterator.next();
|
||||||
|
await iterator.next();
|
||||||
|
await expect(iterator.next()).rejects.toBe(fault);
|
||||||
|
await expect(session.result()).rejects.toBe(fault);
|
||||||
|
await expect(session.snapshot()).rejects.toBe(fault);
|
||||||
|
await session.close({ reason: "fixture complete" });
|
||||||
|
});
|
||||||
|
|
||||||
it("emits one non-replayable input expiration and terminal wait after provider loss", async () => {
|
it("emits one non-replayable input expiration and terminal wait after provider loss", async () => {
|
||||||
const questionSet = {
|
const questionSet = {
|
||||||
schema: "paperclip.question_set.v1" as const,
|
schema: "paperclip.question_set.v1" as const,
|
||||||
|
|
|
||||||
|
|
@ -12,6 +12,7 @@ import type {
|
||||||
OpenNativeSessionInput,
|
OpenNativeSessionInput,
|
||||||
PersistedNativeSession,
|
PersistedNativeSession,
|
||||||
} from "../contracts/native-session-backend.js";
|
} from "../contracts/native-session-backend.js";
|
||||||
|
import { NativeSessionProtocolIntegrityError } from "../contracts/native-session-backend.js";
|
||||||
import type {
|
import type {
|
||||||
PrpEvent,
|
PrpEvent,
|
||||||
PrpTerminalState,
|
PrpTerminalState,
|
||||||
|
|
@ -385,6 +386,31 @@ class HarnessNativeSession implements NativeSession {
|
||||||
readonly #session: HarnessSession;
|
readonly #session: HarnessSession;
|
||||||
#terminal: PrpTerminalState | null = null;
|
#terminal: PrpTerminalState | null = null;
|
||||||
#explicitlyCancelled = false;
|
#explicitlyCancelled = false;
|
||||||
|
#protocolIntegrityFailure: NativeSessionProtocolIntegrityError | null = null;
|
||||||
|
|
||||||
|
#assertProtocolIntegrity(): void {
|
||||||
|
if (this.#protocolIntegrityFailure !== null)
|
||||||
|
throw this.#protocolIntegrityFailure;
|
||||||
|
}
|
||||||
|
|
||||||
|
#rethrowProtocolIntegrity(error: unknown): void {
|
||||||
|
if (!(error instanceof NativeSessionProtocolIntegrityError)) return;
|
||||||
|
this.#protocolIntegrityFailure ??= error;
|
||||||
|
this.#terminal = null;
|
||||||
|
throw this.#protocolIntegrityFailure;
|
||||||
|
}
|
||||||
|
|
||||||
|
async #harnessSnapshot(): Promise<PersistedHarnessSession> {
|
||||||
|
this.#assertProtocolIntegrity();
|
||||||
|
try {
|
||||||
|
const snapshot = await this.#session.snapshot();
|
||||||
|
this.#assertProtocolIntegrity();
|
||||||
|
return snapshot;
|
||||||
|
} catch (error) {
|
||||||
|
this.#rethrowProtocolIntegrity(error);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
input: OpenNativeSessionInput,
|
input: OpenNativeSessionInput,
|
||||||
|
|
@ -421,6 +447,7 @@ class HarnessNativeSession implements NativeSession {
|
||||||
async attachRun(input: {
|
async attachRun(input: {
|
||||||
identity: OpenNativeSessionInput["identity"];
|
identity: OpenNativeSessionInput["identity"];
|
||||||
}): Promise<void> {
|
}): Promise<void> {
|
||||||
|
this.#assertProtocolIntegrity();
|
||||||
const currentIdentity = this.#input.identity;
|
const currentIdentity = this.#input.identity;
|
||||||
if (
|
if (
|
||||||
input.identity.sessionId !== currentIdentity.sessionId ||
|
input.identity.sessionId !== currentIdentity.sessionId ||
|
||||||
|
|
@ -433,7 +460,13 @@ class HarnessNativeSession implements NativeSession {
|
||||||
if (this.#session.attachRun === undefined) {
|
if (this.#session.attachRun === undefined) {
|
||||||
throw new Error("native_session_multi_run_unavailable");
|
throw new Error("native_session_multi_run_unavailable");
|
||||||
}
|
}
|
||||||
await this.#session.attachRun({ runId: input.identity.runId });
|
try {
|
||||||
|
await this.#session.attachRun({ runId: input.identity.runId });
|
||||||
|
this.#assertProtocolIntegrity();
|
||||||
|
} catch (error) {
|
||||||
|
this.#rethrowProtocolIntegrity(error);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
this.#input = { ...this.#input, identity: structuredClone(input.identity) };
|
this.#input = { ...this.#input, identity: structuredClone(input.identity) };
|
||||||
this.#terminal = null;
|
this.#terminal = null;
|
||||||
this.#explicitlyCancelled = false;
|
this.#explicitlyCancelled = false;
|
||||||
|
|
@ -445,6 +478,7 @@ class HarnessNativeSession implements NativeSession {
|
||||||
}
|
}
|
||||||
|
|
||||||
async *events(): AsyncIterable<PrpEvent> {
|
async *events(): AsyncIterable<PrpEvent> {
|
||||||
|
this.#assertProtocolIntegrity();
|
||||||
let sourceInstanceId: string | null = null;
|
let sourceInstanceId: string | null = null;
|
||||||
let lastSourceSequence = 0;
|
let lastSourceSequence = 0;
|
||||||
let sawTerminal = false;
|
let sawTerminal = false;
|
||||||
|
|
@ -497,7 +531,7 @@ class HarnessNativeSession implements NativeSession {
|
||||||
].includes(event.eventType)
|
].includes(event.eventType)
|
||||||
) {
|
) {
|
||||||
sawTerminal = true;
|
sawTerminal = true;
|
||||||
const snapshot = await this.#session.snapshot();
|
const snapshot = await this.#harnessSnapshot();
|
||||||
const disposition =
|
const disposition =
|
||||||
snapshot.semanticResult?.result.reportedWorkDisposition ??
|
snapshot.semanticResult?.result.reportedWorkDisposition ??
|
||||||
"yielded";
|
"yielded";
|
||||||
|
|
@ -523,6 +557,7 @@ class HarnessNativeSession implements NativeSession {
|
||||||
yield structuredClone(event);
|
yield structuredClone(event);
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
this.#rethrowProtocolIntegrity(error);
|
||||||
streamFailure = error;
|
streamFailure = error;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -531,7 +566,10 @@ class HarnessNativeSession implements NativeSession {
|
||||||
// governed wait; the control plane can materialize the continuation without
|
// governed wait; the control plane can materialize the continuation without
|
||||||
// ever trying to replay the dead provider request.
|
// ever trying to replay the dead provider request.
|
||||||
if (!sawTerminal && !this.#explicitlyCancelled && sourceInstanceId) {
|
if (!sawTerminal && !this.#explicitlyCancelled && sourceInstanceId) {
|
||||||
const snapshot = await this.#session.snapshot().catch(() => null);
|
const snapshot = await this.#harnessSnapshot().catch((error) => {
|
||||||
|
this.#rethrowProtocolIntegrity(error);
|
||||||
|
return null;
|
||||||
|
});
|
||||||
let governedWaitTurnId: string | undefined;
|
let governedWaitTurnId: string | undefined;
|
||||||
for (const request of observedPendingInputs.values()) {
|
for (const request of observedPendingInputs.values()) {
|
||||||
const sourceSeq =
|
const sourceSeq =
|
||||||
|
|
@ -604,8 +642,16 @@ class HarnessNativeSession implements NativeSession {
|
||||||
if (streamFailure && !synthesizedDurableWait) throw streamFailure;
|
if (streamFailure && !synthesizedDurableWait) throw streamFailure;
|
||||||
}
|
}
|
||||||
|
|
||||||
startTurn(input: Parameters<HarnessSession["startTurn"]>[0]) {
|
async startTurn(input: Parameters<HarnessSession["startTurn"]>[0]) {
|
||||||
return this.#session.startTurn(input);
|
this.#assertProtocolIntegrity();
|
||||||
|
try {
|
||||||
|
const started = await this.#session.startTurn(input);
|
||||||
|
this.#assertProtocolIntegrity();
|
||||||
|
return started;
|
||||||
|
} catch (error) {
|
||||||
|
this.#rethrowProtocolIntegrity(error);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
steer(input: {
|
steer(input: {
|
||||||
|
|
@ -681,8 +727,9 @@ class HarnessNativeSession implements NativeSession {
|
||||||
}
|
}
|
||||||
|
|
||||||
async result() {
|
async result() {
|
||||||
|
this.#assertProtocolIntegrity();
|
||||||
if (this.#explicitlyCancelled) return null;
|
if (this.#explicitlyCancelled) return null;
|
||||||
const snapshot = await this.#session.snapshot();
|
const snapshot = await this.#harnessSnapshot();
|
||||||
if (
|
if (
|
||||||
snapshot.semanticResult === undefined ||
|
snapshot.semanticResult === undefined ||
|
||||||
snapshot.semanticResult === null
|
snapshot.semanticResult === null
|
||||||
|
|
@ -700,7 +747,8 @@ class HarnessNativeSession implements NativeSession {
|
||||||
}
|
}
|
||||||
|
|
||||||
async snapshot(): Promise<PersistedNativeSession> {
|
async snapshot(): Promise<PersistedNativeSession> {
|
||||||
const snapshot = await this.#session.snapshot();
|
this.#assertProtocolIntegrity();
|
||||||
|
const snapshot = await this.#harnessSnapshot();
|
||||||
return {
|
return {
|
||||||
backendKind: "runner",
|
backendKind: "runner",
|
||||||
driverKind: snapshot.driverKind,
|
driverKind: snapshot.driverKind,
|
||||||
|
|
|
||||||
|
|
@ -90,6 +90,50 @@ export interface NativeSessionSnapshotOptions {
|
||||||
signal: AbortSignal;
|
signal: AbortSignal;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** The exact close owner has torn down its controller without proving suspension. */
|
||||||
|
export class NativeSessionCloseUnrecoverableError extends Error {
|
||||||
|
readonly code = "native_session_close_unrecoverable";
|
||||||
|
|
||||||
|
constructor() {
|
||||||
|
super(
|
||||||
|
"provider_transport_failed: runner did not durably suspend before checkpoint",
|
||||||
|
);
|
||||||
|
this.name = "NativeSessionCloseUnrecoverableError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** An authenticated, exactly bound runner event failed permanent integrity checks. */
|
||||||
|
export class NativeSessionProtocolIntegrityError extends Error {
|
||||||
|
readonly code = "native_event_replay_conflict";
|
||||||
|
readonly recovery = "operator_required";
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
readonly reason:
|
||||||
|
| "semantic_input_digest_mismatch"
|
||||||
|
| "source_event_replay_conflict",
|
||||||
|
) {
|
||||||
|
super(
|
||||||
|
reason === "semantic_input_digest_mismatch"
|
||||||
|
? "native_event_replay_conflict: authenticated runner semantic input failed integrity validation; automatic recovery is stopped."
|
||||||
|
: "native_event_replay_conflict: authenticated runner event conflicts with committed history; automatic recovery is stopped.",
|
||||||
|
);
|
||||||
|
this.name = "NativeSessionProtocolIntegrityError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Admission is blocked by a retained owner that has no safe automatic close retry. */
|
||||||
|
export class NativeSessionCleanupQuarantinedError extends Error {
|
||||||
|
readonly code = "native_session_cleanup_quarantined";
|
||||||
|
readonly recovery = "operator_required";
|
||||||
|
|
||||||
|
constructor() {
|
||||||
|
super(
|
||||||
|
"native_session_cleanup_quarantined: prior session cleanup requires operator recovery; verify its retained process ownership and checkpoint before a controlled restart. Clearing a task session does not resolve this quarantine.",
|
||||||
|
);
|
||||||
|
this.name = "NativeSessionCleanupQuarantinedError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export interface NativeSession {
|
export interface NativeSession {
|
||||||
identity(): NativeRunIdentity;
|
identity(): NativeRunIdentity;
|
||||||
capabilities(): Promise<NativeSessionCapabilities>;
|
capabilities(): Promise<NativeSessionCapabilities>;
|
||||||
|
|
@ -167,3 +211,12 @@ export interface NativeSessionBackend {
|
||||||
options: NativeSessionRecoveryOptions,
|
options: NativeSessionRecoveryOptions,
|
||||||
): Promise<NativeSessionRecoveryResult>;
|
): Promise<NativeSessionRecoveryResult>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** A provider failed terminal is not a missing completion proposal. */
|
||||||
|
export class NativeProviderTerminalFailure extends Error {
|
||||||
|
readonly code = "native_provider_terminal_failed";
|
||||||
|
constructor(readonly providerCode: string, readonly recoverable: boolean, message = "Provider session ended with a failed terminal") {
|
||||||
|
super(message);
|
||||||
|
this.name = "NativeProviderTerminalFailure";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -15,8 +15,10 @@ import { connect, type Socket } from "node:net";
|
||||||
import { tmpdir } from "node:os";
|
import { tmpdir } from "node:os";
|
||||||
import { resolve } from "node:path";
|
import { resolve } from "node:path";
|
||||||
|
|
||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
|
||||||
|
import { NativeSessionProtocolIntegrityError } from "../contracts/native-session-backend.js";
|
||||||
|
import { createCapabilityRunnerdCodexTransport } from "../live/runnerd-codex-transport.js";
|
||||||
import { validatePrpEvent } from "../protocol/replay-contract.js";
|
import { validatePrpEvent } from "../protocol/replay-contract.js";
|
||||||
import { digestPaperclipSemanticContent } from "../semantic-tools/receipts.js";
|
import { digestPaperclipSemanticContent } from "../semantic-tools/receipts.js";
|
||||||
import {
|
import {
|
||||||
|
|
@ -689,10 +691,13 @@ async function authenticate(
|
||||||
controlPlane: DurablePrpControlPlane,
|
controlPlane: DurablePrpControlPlane,
|
||||||
token: string,
|
token: string,
|
||||||
selectedIdentity: DurableRecoveryIdentity = identity,
|
selectedIdentity: DurableRecoveryIdentity = identity,
|
||||||
|
runnerDigest = expectedRunnerDigest,
|
||||||
): Promise<AuthenticatedClient | null> {
|
): Promise<AuthenticatedClient | null> {
|
||||||
const { socket, reader } = await upgradeSocket(controlPlane.connectUrl);
|
const { socket, reader } = await upgradeSocket(controlPlane.connectUrl);
|
||||||
const material = credentialMaterial(token);
|
const material = credentialMaterial(token);
|
||||||
sendMaskedJson(socket, authHello(material.credentialId, selectedIdentity));
|
const hello = authHello(material.credentialId, selectedIdentity);
|
||||||
|
(hello.payload as Record<string, unknown>).runnerDigest = runnerDigest;
|
||||||
|
sendMaskedJson(socket, hello);
|
||||||
const challenge = await reader.next();
|
const challenge = await reader.next();
|
||||||
if (challenge === null) return null;
|
if (challenge === null) return null;
|
||||||
const challengePayload = challenge.payload as Record<string, unknown>;
|
const challengePayload = challenge.payload as Record<string, unknown>;
|
||||||
|
|
@ -877,7 +882,529 @@ function semanticInputEvent(sourceSeq = 1): Record<string, unknown> {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function corruptSemanticInputDigest(
|
||||||
|
event = semanticInputEvent(),
|
||||||
|
): Record<string, unknown> {
|
||||||
|
const payload = (event.payload as Record<string, unknown>).payload as Record<
|
||||||
|
string,
|
||||||
|
unknown
|
||||||
|
>;
|
||||||
|
const semantic = payload.semantic_tool as Record<string, unknown>;
|
||||||
|
(semantic.content as Record<string, unknown>).digest =
|
||||||
|
`sha256:${"0".repeat(64)}`;
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
|
||||||
describe.sequential("DurablePrpControlPlane", () => {
|
describe.sequential("DurablePrpControlPlane", () => {
|
||||||
|
it.each([false, true])(
|
||||||
|
"promptly fails the real transport request and notification paths on authenticated bad semantic input (throwing observer: %s)",
|
||||||
|
async (throwingObserver) => {
|
||||||
|
const root = mkdtempSync(
|
||||||
|
resolve(tmpdir(), "paperclip-prp-transport-integrity-"),
|
||||||
|
);
|
||||||
|
let authority: DurablePrpControlPlane | undefined;
|
||||||
|
let launched = false;
|
||||||
|
const diagnostics: string[] = [];
|
||||||
|
// The launcher below is synthetic; use the current executable only as
|
||||||
|
// its artifact identity, without depending on a staged Rust build.
|
||||||
|
const runnerBinary = process.execPath;
|
||||||
|
const runnerDigest = `sha256:${createHash("sha256").update(readFileSync(runnerBinary)).digest("hex")}`;
|
||||||
|
const handler = vi.fn(async () => ({ success: true, contentItems: [] }));
|
||||||
|
const bundle = createCapabilityRunnerdCodexTransport({
|
||||||
|
stateDirectory: root,
|
||||||
|
prpIdentity: identity,
|
||||||
|
runnerBinary,
|
||||||
|
codexCommand: process.execPath,
|
||||||
|
codexArgs: [],
|
||||||
|
sourceCodexHome: null,
|
||||||
|
environment: {},
|
||||||
|
runnerReconnectGraceMs: 900_000,
|
||||||
|
onDiagnostic: (message) => {
|
||||||
|
diagnostics.push(message);
|
||||||
|
if (
|
||||||
|
throwingObserver &&
|
||||||
|
message.includes("native_event_replay_conflict")
|
||||||
|
) {
|
||||||
|
throw new Error("diagnostic observer failed");
|
||||||
|
}
|
||||||
|
},
|
||||||
|
controlPlaneRegistration: async (core) => {
|
||||||
|
authority = core;
|
||||||
|
await core.start();
|
||||||
|
return { connectUrl: core.connectUrl, release: () => core.stop() };
|
||||||
|
},
|
||||||
|
// Only the runner process is synthetic. Authentication, encrypted frames,
|
||||||
|
// authority validation, transport latching, and both consumers are real.
|
||||||
|
runnerProcessLauncher: () => {
|
||||||
|
launched = true;
|
||||||
|
return {
|
||||||
|
child: { exitCode: null, kill: () => true },
|
||||||
|
completion: new Promise(() => undefined),
|
||||||
|
};
|
||||||
|
},
|
||||||
|
});
|
||||||
|
bundle.transport.setServerRequestHandler(handler);
|
||||||
|
const requestFailure = bundle.transport
|
||||||
|
.request("thread/start", { cwd: tmpdir() })
|
||||||
|
.catch((error: unknown) => error);
|
||||||
|
const notificationFailure = (async () => {
|
||||||
|
for await (const _notification of bundle.transport.notifications()) {
|
||||||
|
// No provider content is expected before startup completes.
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
})().catch((error: unknown) => error);
|
||||||
|
try {
|
||||||
|
await vi.waitFor(() => expect(launched).toBe(true));
|
||||||
|
const core = authority!;
|
||||||
|
const client = (await authenticate(
|
||||||
|
core,
|
||||||
|
core.issueBootstrapTicket(),
|
||||||
|
identity,
|
||||||
|
runnerDigest,
|
||||||
|
))!;
|
||||||
|
const event = corruptSemanticInputDigest();
|
||||||
|
const semantic = (
|
||||||
|
(event.payload as Record<string, unknown>).payload as Record<
|
||||||
|
string,
|
||||||
|
unknown
|
||||||
|
>
|
||||||
|
).semantic_tool as Record<string, unknown>;
|
||||||
|
semantic.input = { summary: "DO-NOT-LEAK-integrity-test" };
|
||||||
|
sendSecure(client, event);
|
||||||
|
await expect(receiveSecure(client)).resolves.toBeNull();
|
||||||
|
const requestError = await requestFailure;
|
||||||
|
expect(requestError).toBeInstanceOf(
|
||||||
|
NativeSessionProtocolIntegrityError,
|
||||||
|
);
|
||||||
|
expect(await notificationFailure).toBe(requestError);
|
||||||
|
await expect(bundle.transport.request("initialize", {})).rejects.toBe(
|
||||||
|
requestError,
|
||||||
|
);
|
||||||
|
expect(handler).not.toHaveBeenCalled();
|
||||||
|
expect(core.store.state.ackedSourceSeq).toBe(0);
|
||||||
|
expect(core.store.state.committedEvents).toEqual([]);
|
||||||
|
expect(diagnostics.join("\n")).not.toContain("DO-NOT-LEAK");
|
||||||
|
expect(
|
||||||
|
diagnostics.filter((message) =>
|
||||||
|
message.includes("native_event_replay_conflict"),
|
||||||
|
),
|
||||||
|
).toHaveLength(1);
|
||||||
|
} finally {
|
||||||
|
await bundle.detachControllerForRestart();
|
||||||
|
await authority?.stop();
|
||||||
|
await Promise.allSettled([requestFailure, notificationFailure]);
|
||||||
|
rmSync(root, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
it("latches an authenticated semantic integrity fault without ACK or dispatch and still permits suspension", async () => {
|
||||||
|
const root = mkdtempSync(resolve(tmpdir(), "paperclip-prp-integrity-"));
|
||||||
|
const onProtocolIntegrityError = vi.fn();
|
||||||
|
const onCommittedEvent = vi.fn(async () => undefined);
|
||||||
|
const onSemanticToolInput = vi.fn(async () => ({ result: { ok: true } }));
|
||||||
|
const core = new DurablePrpControlPlane({
|
||||||
|
stateDirectory: root,
|
||||||
|
identity,
|
||||||
|
expectedRunnerVersion,
|
||||||
|
expectedRunnerDigest,
|
||||||
|
onProtocolIntegrityError,
|
||||||
|
onCommittedEvent,
|
||||||
|
onSemanticToolInput,
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
await core.start();
|
||||||
|
const client = (await authenticate(core, core.issueBootstrapTicket()))!;
|
||||||
|
sendSecure(client, corruptSemanticInputDigest());
|
||||||
|
await expect(receiveSecure(client)).resolves.toBeNull();
|
||||||
|
expect(onProtocolIntegrityError).toHaveBeenCalledTimes(1);
|
||||||
|
const error = onProtocolIntegrityError.mock.calls[0]![0];
|
||||||
|
expect(error).toBeInstanceOf(NativeSessionProtocolIntegrityError);
|
||||||
|
expect(error).toMatchObject({
|
||||||
|
code: "native_event_replay_conflict",
|
||||||
|
reason: "semantic_input_digest_mismatch",
|
||||||
|
recovery: "operator_required",
|
||||||
|
});
|
||||||
|
expect(core.store.state.ackedSourceSeq).toBe(0);
|
||||||
|
expect(core.store.state.committedEvents).toEqual([]);
|
||||||
|
expect(onCommittedEvent).not.toHaveBeenCalled();
|
||||||
|
expect(onSemanticToolInput).not.toHaveBeenCalled();
|
||||||
|
|
||||||
|
// A reconnect cannot reclassify this owner as healthy, even if it now
|
||||||
|
// supplies valid bytes for the failed sequence. Its callback is one-shot.
|
||||||
|
const replay = (await authenticate(core, client.leaseToken!))!;
|
||||||
|
sendSecure(replay, semanticInputEvent());
|
||||||
|
await expect(receiveSecure(replay)).resolves.toBeNull();
|
||||||
|
expect(onProtocolIntegrityError).toHaveBeenCalledTimes(1);
|
||||||
|
expect(core.store.state.ackedSourceSeq).toBe(0);
|
||||||
|
expect(onCommittedEvent).not.toHaveBeenCalled();
|
||||||
|
expect(onSemanticToolInput).not.toHaveBeenCalled();
|
||||||
|
expect(() =>
|
||||||
|
core.rotateRunIdentity({ ...identity, runId: "other-run" }),
|
||||||
|
).toThrow(error);
|
||||||
|
|
||||||
|
const suspend = core.queueCommand(
|
||||||
|
"runner.suspend",
|
||||||
|
{},
|
||||||
|
"suspend-after-integrity-fault",
|
||||||
|
);
|
||||||
|
const cleanup = (await authenticate(core, client.leaseToken!))!;
|
||||||
|
expect(cleanup.welcome.payload).toMatchObject({
|
||||||
|
pendingCommands: [
|
||||||
|
expect.objectContaining({ commandId: suspend.commandId }),
|
||||||
|
],
|
||||||
|
});
|
||||||
|
sendSecure(cleanup, {
|
||||||
|
protocol: "paperclip.runner",
|
||||||
|
version: 1,
|
||||||
|
kind: "command_result",
|
||||||
|
payload: {
|
||||||
|
commandId: suspend.commandId,
|
||||||
|
commandType: suspend.type,
|
||||||
|
controllerSeq: suspend.controllerSeq,
|
||||||
|
status: "completed",
|
||||||
|
result: { suspended: true },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await expect(receiveSecure(cleanup)).resolves.toMatchObject({
|
||||||
|
kind: "command_result_ack",
|
||||||
|
});
|
||||||
|
expect(core.store.state.commands[0]?.status).toBe("completed");
|
||||||
|
expect(core.store.state.ackedSourceSeq).toBe(0);
|
||||||
|
cleanup.socket.destroy();
|
||||||
|
} finally {
|
||||||
|
await core.stop();
|
||||||
|
rmSync(root, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
"semantic_input_digest_mismatch",
|
||||||
|
"source_event_replay_conflict",
|
||||||
|
] as const)(
|
||||||
|
"preserves a typed integrity fault from the trusted commit boundary (%s)",
|
||||||
|
async (reason) => {
|
||||||
|
const root = mkdtempSync(
|
||||||
|
resolve(tmpdir(), "paperclip-prp-typed-commit-integrity-"),
|
||||||
|
);
|
||||||
|
const error = new NativeSessionProtocolIntegrityError(reason);
|
||||||
|
const onProtocolIntegrityError = vi.fn();
|
||||||
|
const onCommittedEvent = vi.fn(async () => {
|
||||||
|
throw error;
|
||||||
|
});
|
||||||
|
const onSemanticToolInput = vi.fn(async () => ({ result: { ok: true } }));
|
||||||
|
const core = new DurablePrpControlPlane({
|
||||||
|
stateDirectory: root,
|
||||||
|
identity,
|
||||||
|
expectedRunnerVersion,
|
||||||
|
expectedRunnerDigest,
|
||||||
|
onProtocolIntegrityError,
|
||||||
|
onCommittedEvent,
|
||||||
|
onSemanticToolInput,
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
await core.start();
|
||||||
|
const client = (await authenticate(core, core.issueBootstrapTicket()))!;
|
||||||
|
sendSecure(client, semanticInputEvent());
|
||||||
|
await expect(receiveSecure(client)).resolves.toBeNull();
|
||||||
|
expect(onProtocolIntegrityError).toHaveBeenCalledExactlyOnceWith(error);
|
||||||
|
expect(core.store.state.ackedSourceSeq).toBe(0);
|
||||||
|
expect(onSemanticToolInput).not.toHaveBeenCalled();
|
||||||
|
const retry = (await authenticate(core, client.leaseToken!))!;
|
||||||
|
sendSecure(retry, semanticInputEvent());
|
||||||
|
await expect(receiveSecure(retry)).resolves.toBeNull();
|
||||||
|
expect(onProtocolIntegrityError).toHaveBeenCalledExactlyOnceWith(error);
|
||||||
|
expect(onCommittedEvent).toHaveBeenCalledTimes(1);
|
||||||
|
} finally {
|
||||||
|
await core.stop();
|
||||||
|
rmSync(root, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
it("latches an authenticated replay conflict against exactly committed bytes", async () => {
|
||||||
|
const root = mkdtempSync(
|
||||||
|
resolve(tmpdir(), "paperclip-prp-replay-integrity-"),
|
||||||
|
);
|
||||||
|
const onProtocolIntegrityError = vi.fn();
|
||||||
|
const onCommittedEvent = vi.fn(async () => undefined);
|
||||||
|
const onSemanticToolInput = vi.fn(
|
||||||
|
async () => new Promise<{ result: unknown }>(() => undefined),
|
||||||
|
);
|
||||||
|
const core = new DurablePrpControlPlane({
|
||||||
|
stateDirectory: root,
|
||||||
|
identity,
|
||||||
|
expectedRunnerVersion,
|
||||||
|
expectedRunnerDigest,
|
||||||
|
onProtocolIntegrityError,
|
||||||
|
onCommittedEvent,
|
||||||
|
onSemanticToolInput,
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
await core.start();
|
||||||
|
const client = (await authenticate(core, core.issueBootstrapTicket()))!;
|
||||||
|
sendSecure(client, semanticInputEvent());
|
||||||
|
await expect(receiveSecure(client)).resolves.toMatchObject({
|
||||||
|
kind: "ack",
|
||||||
|
payload: { ackedSourceSeq: 1 },
|
||||||
|
});
|
||||||
|
const changed = semanticInputEvent();
|
||||||
|
const semantic = (
|
||||||
|
(changed.payload as Record<string, unknown>).payload as Record<
|
||||||
|
string,
|
||||||
|
unknown
|
||||||
|
>
|
||||||
|
).semantic_tool as Record<string, unknown>;
|
||||||
|
semantic.input = { changed: true };
|
||||||
|
(semantic.content as Record<string, unknown>).digest =
|
||||||
|
digestPaperclipSemanticContent(semantic.input);
|
||||||
|
sendSecure(client, changed);
|
||||||
|
await expect(receiveSecure(client)).resolves.toBeNull();
|
||||||
|
expect(onProtocolIntegrityError).toHaveBeenCalledTimes(1);
|
||||||
|
expect(onProtocolIntegrityError.mock.calls[0]![0]).toBeInstanceOf(
|
||||||
|
NativeSessionProtocolIntegrityError,
|
||||||
|
);
|
||||||
|
expect(onProtocolIntegrityError.mock.calls[0]![0]).toMatchObject({
|
||||||
|
reason: "source_event_replay_conflict",
|
||||||
|
});
|
||||||
|
expect(onCommittedEvent).toHaveBeenCalledTimes(1);
|
||||||
|
expect(onSemanticToolInput).toHaveBeenCalledTimes(1);
|
||||||
|
expect(core.store.state.ackedSourceSeq).toBe(1);
|
||||||
|
expect(core.store.state.committedEvents).toHaveLength(1);
|
||||||
|
expect(core.store.state.committedEvents[0]?.envelope).toEqual(
|
||||||
|
semanticInputEvent(),
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
await core.stop();
|
||||||
|
rmSync(root, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
"unauthenticated",
|
||||||
|
"envelope_runner",
|
||||||
|
"envelope_environment",
|
||||||
|
"envelope_run",
|
||||||
|
"envelope_session",
|
||||||
|
"envelope_turn",
|
||||||
|
"envelope_item",
|
||||||
|
"event_run",
|
||||||
|
"event_runner",
|
||||||
|
"event_session",
|
||||||
|
"event_turn",
|
||||||
|
"event_item",
|
||||||
|
"source_seq_gap",
|
||||||
|
"semantic_run",
|
||||||
|
"semantic_session",
|
||||||
|
"semantic_turn",
|
||||||
|
"semantic_item",
|
||||||
|
"malformed_semantic",
|
||||||
|
])(
|
||||||
|
"rejects %s malformed input without poisoning the legitimate owner",
|
||||||
|
async (mismatch) => {
|
||||||
|
const root = mkdtempSync(
|
||||||
|
resolve(tmpdir(), "paperclip-prp-unbound-integrity-"),
|
||||||
|
);
|
||||||
|
const onProtocolIntegrityError = vi.fn();
|
||||||
|
const onCommittedEvent = vi.fn(async () => undefined);
|
||||||
|
const onSemanticToolInput = vi.fn(async () => ({ result: { ok: true } }));
|
||||||
|
const core = new DurablePrpControlPlane({
|
||||||
|
stateDirectory: root,
|
||||||
|
identity,
|
||||||
|
expectedRunnerVersion,
|
||||||
|
expectedRunnerDigest,
|
||||||
|
onProtocolIntegrityError,
|
||||||
|
onCommittedEvent,
|
||||||
|
onSemanticToolInput,
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
await core.start();
|
||||||
|
const event = corruptSemanticInputDigest();
|
||||||
|
const canonical = event.payload as Record<string, unknown>;
|
||||||
|
const semantic = (canonical.payload as Record<string, unknown>)
|
||||||
|
.semantic_tool as Record<string, unknown>;
|
||||||
|
if (mismatch.startsWith("envelope_")) {
|
||||||
|
const key = {
|
||||||
|
envelope_runner: "runnerInstanceId",
|
||||||
|
envelope_environment: "environmentLeaseId",
|
||||||
|
envelope_run: "runId",
|
||||||
|
envelope_session: "normalizedSessionId",
|
||||||
|
envelope_turn: "turnId",
|
||||||
|
envelope_item: "itemId",
|
||||||
|
}[mismatch]!;
|
||||||
|
event[key] = "another-owner";
|
||||||
|
} else if (mismatch.startsWith("event_")) {
|
||||||
|
const key = {
|
||||||
|
event_runner: "sourceInstanceId",
|
||||||
|
event_run: "runId",
|
||||||
|
event_session: "normalizedSessionId",
|
||||||
|
event_turn: "turnId",
|
||||||
|
event_item: "itemId",
|
||||||
|
}[mismatch]!;
|
||||||
|
canonical[key] = "another-owner";
|
||||||
|
} else if (mismatch === "source_seq_gap") canonical.sourceSeq = 2;
|
||||||
|
else if (mismatch === "malformed_semantic") delete semantic.callId;
|
||||||
|
else if (mismatch.startsWith("semantic_")) {
|
||||||
|
const key = {
|
||||||
|
semantic_run: "runId",
|
||||||
|
semantic_session: "normalizedSessionId",
|
||||||
|
semantic_turn: "turnId",
|
||||||
|
semantic_item: "itemId",
|
||||||
|
}[mismatch]!;
|
||||||
|
(semantic.correlation as Record<string, unknown>)[key] =
|
||||||
|
"another-owner";
|
||||||
|
}
|
||||||
|
if (mismatch === "unauthenticated") {
|
||||||
|
const client = await upgradeSocket(core.connectUrl);
|
||||||
|
sendMaskedJson(client.socket, event);
|
||||||
|
await expect(client.reader.next()).resolves.toBeNull();
|
||||||
|
} else {
|
||||||
|
const client = (await authenticate(
|
||||||
|
core,
|
||||||
|
core.issueBootstrapTicket(),
|
||||||
|
))!;
|
||||||
|
sendSecure(client, event);
|
||||||
|
await expect(receiveSecure(client)).resolves.toBeNull();
|
||||||
|
}
|
||||||
|
expect(onProtocolIntegrityError).not.toHaveBeenCalled();
|
||||||
|
expect(onCommittedEvent).not.toHaveBeenCalled();
|
||||||
|
expect(onSemanticToolInput).not.toHaveBeenCalled();
|
||||||
|
const legitimate = (await authenticate(
|
||||||
|
core,
|
||||||
|
core.issueBootstrapTicket(),
|
||||||
|
))!;
|
||||||
|
sendSecure(legitimate, semanticInputEvent());
|
||||||
|
await expect(receiveSecure(legitimate)).resolves.toMatchObject({
|
||||||
|
kind: "ack",
|
||||||
|
payload: { ackedSourceSeq: 1 },
|
||||||
|
});
|
||||||
|
expect(onProtocolIntegrityError).not.toHaveBeenCalled();
|
||||||
|
expect(onCommittedEvent).toHaveBeenCalledTimes(1);
|
||||||
|
expect(onSemanticToolInput).toHaveBeenCalledTimes(1);
|
||||||
|
legitimate.socket.destroy();
|
||||||
|
} finally {
|
||||||
|
await core.stop();
|
||||||
|
rmSync(root, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
it("does not dispatch an earlier in-flight commit after a replacement connection proves an integrity fault", async () => {
|
||||||
|
const root = mkdtempSync(
|
||||||
|
resolve(tmpdir(), "paperclip-prp-integrity-commit-race-"),
|
||||||
|
);
|
||||||
|
let releaseCommit!: () => void;
|
||||||
|
const commitBarrier = new Promise<void>((resolveCommit) => {
|
||||||
|
releaseCommit = resolveCommit;
|
||||||
|
});
|
||||||
|
const onCommittedEvent = vi.fn(async () => commitBarrier);
|
||||||
|
const onProtocolIntegrityError = vi.fn();
|
||||||
|
const onSemanticToolInput = vi.fn(async () => ({ result: { ok: true } }));
|
||||||
|
const core = new DurablePrpControlPlane({
|
||||||
|
stateDirectory: root,
|
||||||
|
identity,
|
||||||
|
expectedRunnerVersion,
|
||||||
|
expectedRunnerDigest,
|
||||||
|
onCommittedEvent,
|
||||||
|
onProtocolIntegrityError,
|
||||||
|
onSemanticToolInput,
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
await core.start();
|
||||||
|
const first = (await authenticate(core, core.issueBootstrapTicket()))!;
|
||||||
|
sendSecure(first, semanticInputEvent());
|
||||||
|
await vi.waitFor(() => expect(onCommittedEvent).toHaveBeenCalledTimes(1));
|
||||||
|
const replacement = (await authenticate(core, first.leaseToken!))!;
|
||||||
|
sendSecure(replacement, corruptSemanticInputDigest());
|
||||||
|
await expect(receiveSecure(replacement)).resolves.toBeNull();
|
||||||
|
expect(onProtocolIntegrityError).toHaveBeenCalledTimes(1);
|
||||||
|
releaseCommit();
|
||||||
|
await new Promise<void>((resolveTurn) => setImmediate(resolveTurn));
|
||||||
|
expect(core.store.state.ackedSourceSeq).toBe(0);
|
||||||
|
expect(core.store.state.committedEvents).toEqual([]);
|
||||||
|
expect(onSemanticToolInput).not.toHaveBeenCalled();
|
||||||
|
} finally {
|
||||||
|
releaseCommit();
|
||||||
|
await core.stop();
|
||||||
|
rmSync(root, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not turn a missing semantic handler into an integrity fault", async () => {
|
||||||
|
const root = mkdtempSync(
|
||||||
|
resolve(tmpdir(), "paperclip-prp-integrity-no-handler-"),
|
||||||
|
);
|
||||||
|
const onProtocolIntegrityError = vi.fn();
|
||||||
|
const core = new DurablePrpControlPlane({
|
||||||
|
stateDirectory: root,
|
||||||
|
identity,
|
||||||
|
expectedRunnerVersion,
|
||||||
|
expectedRunnerDigest,
|
||||||
|
onProtocolIntegrityError,
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
await core.start();
|
||||||
|
const client = (await authenticate(core, core.issueBootstrapTicket()))!;
|
||||||
|
sendSecure(client, corruptSemanticInputDigest());
|
||||||
|
await expect(receiveSecure(client)).resolves.toBeNull();
|
||||||
|
expect(onProtocolIntegrityError).not.toHaveBeenCalled();
|
||||||
|
expect(core.store.state.ackedSourceSeq).toBe(0);
|
||||||
|
} finally {
|
||||||
|
await core.stop();
|
||||||
|
rmSync(root, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
new Error("database temporarily unavailable"),
|
||||||
|
Object.assign(new Error("untrusted native_event_replay_conflict prose"), {
|
||||||
|
code: "native_event_replay_conflict",
|
||||||
|
}),
|
||||||
|
])(
|
||||||
|
"keeps a non-typed commit failure retryable on the same authenticated authority (%s)",
|
||||||
|
async (commitFailure) => {
|
||||||
|
const root = mkdtempSync(
|
||||||
|
resolve(tmpdir(), "paperclip-prp-transient-commit-"),
|
||||||
|
);
|
||||||
|
const onProtocolIntegrityError = vi.fn();
|
||||||
|
const onCommittedEvent = vi
|
||||||
|
.fn()
|
||||||
|
.mockRejectedValueOnce(commitFailure)
|
||||||
|
.mockResolvedValue(undefined);
|
||||||
|
const onSemanticToolInput = vi.fn(async () => ({ result: { ok: true } }));
|
||||||
|
const core = new DurablePrpControlPlane({
|
||||||
|
stateDirectory: root,
|
||||||
|
identity,
|
||||||
|
expectedRunnerVersion,
|
||||||
|
expectedRunnerDigest,
|
||||||
|
onProtocolIntegrityError,
|
||||||
|
onCommittedEvent,
|
||||||
|
onSemanticToolInput,
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
await core.start();
|
||||||
|
const first = (await authenticate(core, core.issueBootstrapTicket()))!;
|
||||||
|
sendSecure(first, semanticInputEvent());
|
||||||
|
await expect(receiveSecure(first)).resolves.toBeNull();
|
||||||
|
expect(core.store.state.ackedSourceSeq).toBe(0);
|
||||||
|
expect(onSemanticToolInput).not.toHaveBeenCalled();
|
||||||
|
const retry = (await authenticate(core, first.leaseToken!))!;
|
||||||
|
sendSecure(retry, semanticInputEvent());
|
||||||
|
await expect(receiveSecure(retry)).resolves.toMatchObject({
|
||||||
|
kind: "ack",
|
||||||
|
payload: { ackedSourceSeq: 1 },
|
||||||
|
});
|
||||||
|
expect(onProtocolIntegrityError).not.toHaveBeenCalled();
|
||||||
|
expect(onCommittedEvent).toHaveBeenCalledTimes(2);
|
||||||
|
expect(onSemanticToolInput).toHaveBeenCalledTimes(1);
|
||||||
|
retry.socket.destroy();
|
||||||
|
} finally {
|
||||||
|
await core.stop();
|
||||||
|
rmSync(root, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
it("exchanges a one-use bootstrap for a run-bound reconnect lease", async () => {
|
it("exchanges a one-use bootstrap for a run-bound reconnect lease", async () => {
|
||||||
const root = mkdtempSync(resolve(tmpdir(), "paperclip-prp-auth-"));
|
const root = mkdtempSync(resolve(tmpdir(), "paperclip-prp-auth-"));
|
||||||
const controlPlane = new DurablePrpControlPlane({
|
const controlPlane = new DurablePrpControlPlane({
|
||||||
|
|
|
||||||
|
|
@ -28,6 +28,7 @@ import { dirname, resolve } from "node:path";
|
||||||
import type { Duplex } from "node:stream";
|
import type { Duplex } from "node:stream";
|
||||||
import { fileURLToPath } from "node:url";
|
import { fileURLToPath } from "node:url";
|
||||||
|
|
||||||
|
import { NativeSessionProtocolIntegrityError } from "../contracts/native-session-backend.js";
|
||||||
import { githubCredentialEnvironment } from "../github-credential-environment.js";
|
import { githubCredentialEnvironment } from "../github-credential-environment.js";
|
||||||
import {
|
import {
|
||||||
validatePrpEvent,
|
validatePrpEvent,
|
||||||
|
|
@ -224,6 +225,10 @@ export interface DurablePrpControlPlaneOptions {
|
||||||
}) => Promise<{ readonly result: unknown; readonly isError?: boolean }>;
|
}) => Promise<{ readonly result: unknown; readonly isError?: boolean }>;
|
||||||
/** Persist the canonical event before the runner receives its cumulative ACK. */
|
/** Persist the canonical event before the runner receives its cumulative ACK. */
|
||||||
onCommittedEvent?: (event: PrpEvent) => Promise<void>;
|
onCommittedEvent?: (event: PrpEvent) => Promise<void>;
|
||||||
|
/** Stop this exact owner after a proven, authenticated permanent integrity fault. */
|
||||||
|
onProtocolIntegrityError?: (
|
||||||
|
error: NativeSessionProtocolIntegrityError,
|
||||||
|
) => void;
|
||||||
connectionLeaseTtlMs?: number;
|
connectionLeaseTtlMs?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -974,6 +979,9 @@ export class DurablePrpControlPlane {
|
||||||
#port: number | null = null;
|
#port: number | null = null;
|
||||||
#onSemanticToolInput?: DurablePrpControlPlaneOptions["onSemanticToolInput"];
|
#onSemanticToolInput?: DurablePrpControlPlaneOptions["onSemanticToolInput"];
|
||||||
#onCommittedEvent?: DurablePrpControlPlaneOptions["onCommittedEvent"];
|
#onCommittedEvent?: DurablePrpControlPlaneOptions["onCommittedEvent"];
|
||||||
|
#onProtocolIntegrityError?:
|
||||||
|
DurablePrpControlPlaneOptions["onProtocolIntegrityError"];
|
||||||
|
#protocolIntegrityError: NativeSessionProtocolIntegrityError | null = null;
|
||||||
#connectionLeaseTtlMs: number;
|
#connectionLeaseTtlMs: number;
|
||||||
|
|
||||||
constructor(options: DurablePrpControlPlaneOptions) {
|
constructor(options: DurablePrpControlPlaneOptions) {
|
||||||
|
|
@ -999,6 +1007,7 @@ export class DurablePrpControlPlane {
|
||||||
this.#expectedRunnerDigest = options.expectedRunnerDigest;
|
this.#expectedRunnerDigest = options.expectedRunnerDigest;
|
||||||
this.#onSemanticToolInput = options.onSemanticToolInput;
|
this.#onSemanticToolInput = options.onSemanticToolInput;
|
||||||
this.#onCommittedEvent = options.onCommittedEvent;
|
this.#onCommittedEvent = options.onCommittedEvent;
|
||||||
|
this.#onProtocolIntegrityError = options.onProtocolIntegrityError;
|
||||||
this.#connectionLeaseTtlMs = options.connectionLeaseTtlMs ?? 60_000;
|
this.#connectionLeaseTtlMs = options.connectionLeaseTtlMs ?? 60_000;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1075,6 +1084,8 @@ export class DurablePrpControlPlane {
|
||||||
identity: DurableRecoveryIdentity,
|
identity: DurableRecoveryIdentity,
|
||||||
runAttachTemplate?: Record<string, unknown>,
|
runAttachTemplate?: Record<string, unknown>,
|
||||||
): void {
|
): void {
|
||||||
|
if (this.#protocolIntegrityError !== null)
|
||||||
|
throw this.#protocolIntegrityError;
|
||||||
if (
|
if (
|
||||||
!Object.values(identity).every(
|
!Object.values(identity).every(
|
||||||
(value) => typeof value === "string" && stableIdPattern.test(value),
|
(value) => typeof value === "string" && stableIdPattern.test(value),
|
||||||
|
|
@ -1858,10 +1869,42 @@ export class DurablePrpControlPlane {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#failProtocolIntegrity(
|
||||||
|
connection: AuthorityConnection,
|
||||||
|
error: NativeSessionProtocolIntegrityError,
|
||||||
|
): void {
|
||||||
|
try {
|
||||||
|
if (this.#protocolIntegrityError === null) {
|
||||||
|
this.#protocolIntegrityError = error;
|
||||||
|
this.#onProtocolIntegrityError?.(error);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
connection.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async #event(
|
async #event(
|
||||||
connection: AuthorityConnection,
|
connection: AuthorityConnection,
|
||||||
envelope: Record<string, unknown>,
|
envelope: Record<string, unknown>,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
|
if (this.#protocolIntegrityError !== null) {
|
||||||
|
connection.close();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Authentication binds the channel, but an authenticated sender can still
|
||||||
|
// submit an envelope for another run. Such frames must not poison this
|
||||||
|
// owner's session or turn an unrelated digest failure into its terminal fault.
|
||||||
|
if (
|
||||||
|
envelope.runnerInstanceId !== this.#identity.runnerInstanceId ||
|
||||||
|
envelope.environmentLeaseId !== this.#identity.environmentLeaseId ||
|
||||||
|
envelope.runId !== this.#identity.runId ||
|
||||||
|
envelope.normalizedSessionId !== this.#identity.normalizedSessionId ||
|
||||||
|
envelope.turnId !== this.#identity.turnId ||
|
||||||
|
envelope.itemId !== this.#identity.itemId
|
||||||
|
) {
|
||||||
|
connection.close();
|
||||||
|
return;
|
||||||
|
}
|
||||||
const validated = validatePrpEvent(envelope.payload);
|
const validated = validatePrpEvent(envelope.payload);
|
||||||
if (!validated.ok) {
|
if (!validated.ok) {
|
||||||
connection.close();
|
connection.close();
|
||||||
|
|
@ -1901,8 +1944,6 @@ export class DurablePrpControlPlane {
|
||||||
!Object.prototype.hasOwnProperty.call(semantic, "input") ||
|
!Object.prototype.hasOwnProperty.call(semantic, "input") ||
|
||||||
typeof semantic.content !== "object" ||
|
typeof semantic.content !== "object" ||
|
||||||
semantic.content === null ||
|
semantic.content === null ||
|
||||||
(semantic.content as Record<string, unknown>).digest !==
|
|
||||||
digestPaperclipSemanticContent(semantic.input) ||
|
|
||||||
semanticCorrelation?.runId !== this.#identity.runId ||
|
semanticCorrelation?.runId !== this.#identity.runId ||
|
||||||
semanticCorrelation.normalizedSessionId !==
|
semanticCorrelation.normalizedSessionId !==
|
||||||
this.#identity.normalizedSessionId ||
|
this.#identity.normalizedSessionId ||
|
||||||
|
|
@ -1915,15 +1956,43 @@ export class DurablePrpControlPlane {
|
||||||
const existing = this.#store.state.committedEvents.find(
|
const existing = this.#store.state.committedEvents.find(
|
||||||
(candidate) => candidate.sourceEventId === sourceEventId,
|
(candidate) => candidate.sourceEventId === sourceEventId,
|
||||||
);
|
);
|
||||||
if (existing !== undefined) {
|
if (
|
||||||
if (canonicalJson(existing.envelope) !== canonicalJson(envelope)) {
|
existing === undefined
|
||||||
connection.close();
|
? sourceSeq !== this.#store.state.ackedSourceSeq + 1
|
||||||
return;
|
: sourceSeq !== existing.sourceSeq
|
||||||
}
|
) {
|
||||||
} else if (sourceSeq !== this.#store.state.ackedSourceSeq + 1) {
|
|
||||||
connection.close();
|
connection.close();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
if (
|
||||||
|
isSemanticInput &&
|
||||||
|
semantic !== undefined &&
|
||||||
|
(semantic.content as Record<string, unknown>).digest !==
|
||||||
|
digestPaperclipSemanticContent(semantic.input)
|
||||||
|
) {
|
||||||
|
// Only the authenticated, schema-valid, exactly correlated input may
|
||||||
|
// permanently fail its owner. Never commit, dispatch, or ACK these bytes.
|
||||||
|
// Keep the same error latched across reconnects; lifecycle command results
|
||||||
|
// remain available so the owner can still attempt a verified suspension.
|
||||||
|
this.#failProtocolIntegrity(
|
||||||
|
connection,
|
||||||
|
new NativeSessionProtocolIntegrityError(
|
||||||
|
"semantic_input_digest_mismatch",
|
||||||
|
),
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (existing !== undefined) {
|
||||||
|
if (canonicalJson(existing.envelope) !== canonicalJson(envelope)) {
|
||||||
|
this.#failProtocolIntegrity(
|
||||||
|
connection,
|
||||||
|
new NativeSessionProtocolIntegrityError(
|
||||||
|
"source_event_replay_conflict",
|
||||||
|
),
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// The caller's durable commit is the acknowledgement authority. A crash
|
// The caller's durable commit is the acknowledgement authority. A crash
|
||||||
// after that idempotent commit but before the local cursor save is safe:
|
// after that idempotent commit but before the local cursor save is safe:
|
||||||
|
|
@ -1932,7 +2001,18 @@ export class DurablePrpControlPlane {
|
||||||
// an uncommitted event disappear from the runner outbox permanently.
|
// an uncommitted event disappear from the runner outbox permanently.
|
||||||
try {
|
try {
|
||||||
await this.#onCommittedEvent?.(event);
|
await this.#onCommittedEvent?.(event);
|
||||||
} catch {
|
} catch (error) {
|
||||||
|
if (error instanceof NativeSessionProtocolIntegrityError) {
|
||||||
|
this.#failProtocolIntegrity(connection, error);
|
||||||
|
} else {
|
||||||
|
connection.close();
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Another authenticated connection can replace this one while its commit
|
||||||
|
// is in flight. Once that exact owner has faulted, even a prior successful
|
||||||
|
// commit cannot reopen delivery or invoke a new business operation.
|
||||||
|
if (this.#protocolIntegrityError !== null) {
|
||||||
connection.close();
|
connection.close();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -14,6 +14,7 @@ import type {
|
||||||
PersistedHarnessSession,
|
PersistedHarnessSession,
|
||||||
PersistedHarnessTurnTerminal,
|
PersistedHarnessTurnTerminal,
|
||||||
} from "../../contracts/harness-driver.js";
|
} from "../../contracts/harness-driver.js";
|
||||||
|
import { NativeSessionProtocolIntegrityError } from "../../contracts/native-session-backend.js";
|
||||||
import { HarnessReconciliationError } from "../../contracts/harness-driver.js";
|
import { HarnessReconciliationError } from "../../contracts/harness-driver.js";
|
||||||
import {
|
import {
|
||||||
CODEX_CODEX_PROTOCOL_VERSION,
|
CODEX_CODEX_PROTOCOL_VERSION,
|
||||||
|
|
@ -307,6 +308,7 @@ export class CodexAppServerDriver implements HarnessDriver {
|
||||||
// work during close; when no durable provider identity exists that
|
// work during close; when no durable provider identity exists that
|
||||||
// cleanup can fail independently.
|
// cleanup can fail independently.
|
||||||
await cancellation.close().catch(() => {});
|
await cancellation.close().catch(() => {});
|
||||||
|
if (error instanceof NativeSessionProtocolIntegrityError) throw error;
|
||||||
if (input.signal?.aborted) input.signal.throwIfAborted();
|
if (input.signal?.aborted) input.signal.throwIfAborted();
|
||||||
throw error;
|
throw error;
|
||||||
} finally {
|
} finally {
|
||||||
|
|
@ -596,6 +598,7 @@ export class CodexAppServerDriver implements HarnessDriver {
|
||||||
};
|
};
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
await cancellation.close().catch(() => {});
|
await cancellation.close().catch(() => {});
|
||||||
|
if (error instanceof NativeSessionProtocolIntegrityError) throw error;
|
||||||
if (options.signal.aborted) options.signal.throwIfAborted();
|
if (options.signal.aborted) options.signal.throwIfAborted();
|
||||||
return { recovered: false, reason: redactCodexDiagnostic(String(error)) };
|
return { recovered: false, reason: redactCodexDiagnostic(String(error)) };
|
||||||
} finally {
|
} finally {
|
||||||
|
|
@ -653,6 +656,7 @@ export class CodexAppServerDriver implements HarnessDriver {
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
} catch (cause) {
|
} catch (cause) {
|
||||||
|
if (cause instanceof NativeSessionProtocolIntegrityError) throw cause;
|
||||||
const error = new Error(
|
const error = new Error(
|
||||||
`planning_mode_unsupported: installed Codex app-server did not expose a usable native plan collaboration mode (${redactCodexDiagnostic(String(cause))})`,
|
`planning_mode_unsupported: installed Codex app-server did not expose a usable native plan collaboration mode (${redactCodexDiagnostic(String(cause))})`,
|
||||||
);
|
);
|
||||||
|
|
@ -702,6 +706,7 @@ export class CodexAppServerDriver implements HarnessDriver {
|
||||||
const response = await transport.request("thread/goal/get", { threadId });
|
const response = await transport.request("thread/goal/get", { threadId });
|
||||||
return parseThreadGoal(response.goal);
|
return parseThreadGoal(response.goal);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
if (error instanceof NativeSessionProtocolIntegrityError) throw error;
|
||||||
const policyDisabled =
|
const policyDisabled =
|
||||||
error instanceof CodexRpcError
|
error instanceof CodexRpcError
|
||||||
&& (error.message.toLowerCase().includes("policy")
|
&& (error.message.toLowerCase().includes("policy")
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
import { NativeSessionProtocolIntegrityError } from "../../contracts/native-session-backend.js";
|
||||||
import {
|
import {
|
||||||
CODEX_BLOCK_RESULT_OUTPUT_SCHEMA,
|
CODEX_BLOCK_RESULT_OUTPUT_SCHEMA,
|
||||||
CODEX_INVALID_REQUEST,
|
CODEX_INVALID_REQUEST,
|
||||||
|
|
@ -94,6 +95,59 @@ describe("Codex app-server Codex driver", () => {
|
||||||
});
|
});
|
||||||
expect(second.calls.some((call) => call.method === "turn/start" || call.method === "thread/goal/set")).toBe(false);
|
expect(second.calls.some((call) => call.method === "turn/start" || call.method === "thread/goal/set")).toBe(false);
|
||||||
});
|
});
|
||||||
|
it.each([
|
||||||
|
"initial-read",
|
||||||
|
"goal-probe",
|
||||||
|
"plan-probe",
|
||||||
|
] as const)(
|
||||||
|
"rethrows exact recovery integrity failure after cleanup at %s",
|
||||||
|
async (stage) => {
|
||||||
|
const originalTransport = new FakeCodexTransport();
|
||||||
|
const recoveryTransport = new FakeCodexTransport();
|
||||||
|
const fault = new NativeSessionProtocolIntegrityError(
|
||||||
|
"semantic_input_digest_mismatch",
|
||||||
|
);
|
||||||
|
const request = recoveryTransport.request.bind(recoveryTransport);
|
||||||
|
let reads = 0;
|
||||||
|
vi.spyOn(recoveryTransport, "request").mockImplementation(
|
||||||
|
async (method, params) => {
|
||||||
|
if (method === "thread/read") reads += 1;
|
||||||
|
if (
|
||||||
|
(stage === "initial-read" && method === "thread/read") ||
|
||||||
|
(stage === "goal-probe" && method === "thread/goal/get") ||
|
||||||
|
(stage === "plan-probe" && method === "collaborationMode/list")
|
||||||
|
)
|
||||||
|
throw fault;
|
||||||
|
return request(method, params);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const close = vi.spyOn(recoveryTransport, "close");
|
||||||
|
// The integrity error remains primary even when required cleanup rejects.
|
||||||
|
close.mockRejectedValue(new Error("secondary cleanup failure"));
|
||||||
|
const driver = makeDriver(
|
||||||
|
[originalTransport, recoveryTransport],
|
||||||
|
stage === "plan-probe" ? { requestedCollaborationMode: "plan" } : {},
|
||||||
|
);
|
||||||
|
const original = await driver.openSession({
|
||||||
|
runId: "run-recovery-integrity",
|
||||||
|
normalizedSessionId: "session-recovery-integrity",
|
||||||
|
workingDirectory: WORKSPACE,
|
||||||
|
});
|
||||||
|
await original.startTurn({ message: { role: "user", text: "Work." } });
|
||||||
|
const checkpoint = await original.snapshot();
|
||||||
|
await original.close({ reason: "fixture disconnect" });
|
||||||
|
await expect(
|
||||||
|
driver.recoverSession!({
|
||||||
|
...checkpoint,
|
||||||
|
providerRecoveryPolicy: "allow_replacement_after_resume_failure",
|
||||||
|
}),
|
||||||
|
).rejects.toBe(fault);
|
||||||
|
expect(close).toHaveBeenCalledTimes(1);
|
||||||
|
expect(
|
||||||
|
recoveryTransport.calls.some((call) => call.method === "thread/start"),
|
||||||
|
).toBe(false);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
it("persists and verifies the tagged runnerd provider identity on recovery", async () => {
|
it("persists and verifies the tagged runnerd provider identity on recovery", async () => {
|
||||||
const providerIdentity = {
|
const providerIdentity = {
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
import { randomUUID } from "node:crypto";
|
import { randomUUID } from "node:crypto";
|
||||||
|
import { NativeProviderTerminalFailure } from "../../contracts/native-session-backend.js";
|
||||||
|
|
||||||
import type {
|
import type {
|
||||||
HarnessGoalOperation,
|
HarnessGoalOperation,
|
||||||
|
|
@ -78,6 +79,7 @@ export class CodexHarnessSession
|
||||||
}
|
}
|
||||||
|
|
||||||
async attachRun(input: { runId: string }): Promise<void> {
|
async attachRun(input: { runId: string }): Promise<void> {
|
||||||
|
this.assertProtocolIntegrity();
|
||||||
const transportOwnsQuiescence = this.transport.attachRun !== undefined;
|
const transportOwnsQuiescence = this.transport.attachRun !== undefined;
|
||||||
if (
|
if (
|
||||||
this.turnStartPending ||
|
this.turnStartPending ||
|
||||||
|
|
@ -92,6 +94,7 @@ export class CodexHarnessSession
|
||||||
turnId: `turn_attachment_${randomUUID().replaceAll("-", "")}`,
|
turnId: `turn_attachment_${randomUUID().replaceAll("-", "")}`,
|
||||||
itemId: `item_attachment_${randomUUID().replaceAll("-", "")}`,
|
itemId: `item_attachment_${randomUUID().replaceAll("-", "")}`,
|
||||||
});
|
});
|
||||||
|
this.assertProtocolIntegrity();
|
||||||
if (transportOwnsQuiescence) {
|
if (transportOwnsQuiescence) {
|
||||||
// Runnerd's attachment contract performs two durable readiness probes,
|
// Runnerd's attachment contract performs two durable readiness probes,
|
||||||
// drains the settled provider tail, and rotates authority atomically.
|
// drains the settled provider tail, and rotates authority atomically.
|
||||||
|
|
@ -133,6 +136,10 @@ export class CodexHarnessSession
|
||||||
turnId: string;
|
turnId: string;
|
||||||
effectiveCollaborationMode: "default" | "plan";
|
effectiveCollaborationMode: "default" | "plan";
|
||||||
}> {
|
}> {
|
||||||
|
this.assertProtocolIntegrity();
|
||||||
|
if (this.protocolFailed && this.protocolFailureCode) {
|
||||||
|
throw new NativeProviderTerminalFailure(this.protocolFailureCode, false, this.protocolFailureMessage ?? undefined);
|
||||||
|
}
|
||||||
if (
|
if (
|
||||||
this.terminal ||
|
this.terminal ||
|
||||||
this.protocolFailed ||
|
this.protocolFailed ||
|
||||||
|
|
@ -237,6 +244,7 @@ export class CodexHarnessSession
|
||||||
// never observes the terminal turn ahead of turn.accepted.
|
// never observes the terminal turn ahead of turn.accepted.
|
||||||
releaseTurnStartSettled();
|
releaseTurnStartSettled();
|
||||||
}
|
}
|
||||||
|
this.assertProtocolIntegrity();
|
||||||
const turn = record(response.turn);
|
const turn = record(response.turn);
|
||||||
const turnId = text(turn.id);
|
const turnId = text(turn.id);
|
||||||
if (turnId.length === 0)
|
if (turnId.length === 0)
|
||||||
|
|
@ -268,6 +276,7 @@ export class CodexHarnessSession
|
||||||
message: NativeUserMessage;
|
message: NativeUserMessage;
|
||||||
correlationId?: string;
|
correlationId?: string;
|
||||||
}): Promise<void> {
|
}): Promise<void> {
|
||||||
|
this.assertProtocolIntegrity();
|
||||||
this.requireCapability("steering");
|
this.requireCapability("steering");
|
||||||
this.requireActiveTurn(input.turnId, "steering");
|
this.requireActiveTurn(input.turnId, "steering");
|
||||||
if (input.correlationId) {
|
if (input.correlationId) {
|
||||||
|
|
@ -312,6 +321,7 @@ export class CodexHarnessSession
|
||||||
);
|
);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (error instanceof HarnessOperationAlreadyTerminalError) throw error;
|
if (error instanceof HarnessOperationAlreadyTerminalError) throw error;
|
||||||
|
this.rethrowProtocolIntegrity(error);
|
||||||
const detail = redactCodexDiagnostic(String(error));
|
const detail = redactCodexDiagnostic(String(error));
|
||||||
if (/unsupported|unavailable|capability|method not found/i.test(detail)) {
|
if (/unsupported|unavailable|capability|method not found/i.test(detail)) {
|
||||||
throw this.unsupported("steering", detail);
|
throw this.unsupported("steering", detail);
|
||||||
|
|
@ -385,6 +395,7 @@ export class CodexHarnessSession
|
||||||
turnId: string;
|
turnId: string;
|
||||||
resolution: HarnessRuntimeRequestResolution;
|
resolution: HarnessRuntimeRequestResolution;
|
||||||
}): Promise<void> {
|
}): Promise<void> {
|
||||||
|
this.assertProtocolIntegrity();
|
||||||
this.requireCapability("runtimeRequestResolution");
|
this.requireCapability("runtimeRequestResolution");
|
||||||
const pending = this.pendingRuntimeRequestMap.get(input.requestId);
|
const pending = this.pendingRuntimeRequestMap.get(input.requestId);
|
||||||
if (pending === undefined) {
|
if (pending === undefined) {
|
||||||
|
|
@ -590,6 +601,7 @@ export class CodexHarnessSession
|
||||||
}
|
}
|
||||||
|
|
||||||
async read(): Promise<Record<string, unknown>> {
|
async read(): Promise<Record<string, unknown>> {
|
||||||
|
this.assertProtocolIntegrity();
|
||||||
this.requireCapability("read");
|
this.requireCapability("read");
|
||||||
try {
|
try {
|
||||||
return await this.transport.request("thread/read", {
|
return await this.transport.request("thread/read", {
|
||||||
|
|
@ -699,6 +711,7 @@ export class CodexHarnessSession
|
||||||
}
|
}
|
||||||
|
|
||||||
async snapshot(): Promise<PersistedHarnessSession> {
|
async snapshot(): Promise<PersistedHarnessSession> {
|
||||||
|
this.assertProtocolIntegrity();
|
||||||
return {
|
return {
|
||||||
driverKind: this.driverKind,
|
driverKind: this.driverKind,
|
||||||
driverSessionId: this.opened.threadId,
|
driverSessionId: this.opened.threadId,
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,61 @@
|
||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { classifyCodexNotification } from "./codex-notification-identity.js";
|
||||||
|
const classify = (method: string, params: Record<string, unknown>) =>
|
||||||
|
classifyCodexNotification({
|
||||||
|
method,
|
||||||
|
params,
|
||||||
|
runId: "run",
|
||||||
|
rootThreadId: "root",
|
||||||
|
activeTurnId: "active",
|
||||||
|
knownThreads: new Set(["root", "child"]),
|
||||||
|
settledTurns: new Set(["old"]),
|
||||||
|
}).classification;
|
||||||
|
describe("provider notification authority", () => {
|
||||||
|
it.each([
|
||||||
|
["warning", { message: "buffered before replacement rejection" }, "root"],
|
||||||
|
["paperclip/canonicalProviderEvent", { threadId: null, eventType: "provider.notice.recorded" }, "root"],
|
||||||
|
["paperclip/canonicalProviderEvent", { threadId: null, eventType: "tool.execution.started" }, "invalid_authority"],
|
||||||
|
["item/started", { threadId: "root", turnId: "active" }, "root"],
|
||||||
|
["turn/completed", { threadId: "root", turnId: "old" }, "stale_turn"],
|
||||||
|
[
|
||||||
|
"turn/completed",
|
||||||
|
{ threadId: "child", turnId: "child-turn" },
|
||||||
|
"descendant",
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"thread/started",
|
||||||
|
{
|
||||||
|
thread: {
|
||||||
|
id: "new-child",
|
||||||
|
source: { subAgent: { thread_spawn: { parent_thread_id: "child" } } },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"descendant",
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"thread/started",
|
||||||
|
{ thread: { id: "unrelated" } },
|
||||||
|
"unrelated_information",
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"thread/status/changed",
|
||||||
|
{ threadId: "unrelated" },
|
||||||
|
"unrelated_information",
|
||||||
|
],
|
||||||
|
["paperclip/runResult", { threadId: "child" }, "invalid_authority"],
|
||||||
|
["turn/completed", { threadId: "unrelated" }, "invalid_authority"],
|
||||||
|
["item/started", { threadId: 5 }, "invalid_authority"],
|
||||||
|
[
|
||||||
|
"item/started",
|
||||||
|
{ threadId: "root", thread: { id: "other" } },
|
||||||
|
"invalid_authority",
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"thread/status/changed",
|
||||||
|
{ threadId: "root", runId: "other-task" },
|
||||||
|
"invalid_authority",
|
||||||
|
],
|
||||||
|
] as const)("classifies %s %j as %s", (method, params, expected) =>
|
||||||
|
expect(classify(method, params)).toBe(expected),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
@ -0,0 +1,95 @@
|
||||||
|
import { codexThreadLineage } from "./codex-thread-normalization.js";
|
||||||
|
|
||||||
|
export type CodexNotificationIdentity =
|
||||||
|
| "root"
|
||||||
|
| "descendant"
|
||||||
|
| "stale_turn"
|
||||||
|
| "unrelated_information"
|
||||||
|
| "invalid_authority";
|
||||||
|
const informational = new Set([
|
||||||
|
"thread/started",
|
||||||
|
"thread/status/changed",
|
||||||
|
"thread/closed",
|
||||||
|
"thread/tokenUsage/updated",
|
||||||
|
"warning",
|
||||||
|
"configWarning",
|
||||||
|
"guardianWarning",
|
||||||
|
"deprecationNotice",
|
||||||
|
]);
|
||||||
|
const record = (v: unknown): Record<string, unknown> =>
|
||||||
|
v !== null && typeof v === "object" && !Array.isArray(v)
|
||||||
|
? (v as Record<string, unknown>)
|
||||||
|
: {};
|
||||||
|
|
||||||
|
/** Only provider-originated lineage admits a descendant; tool requests never use this classifier. */
|
||||||
|
export function classifyCodexNotification(input: {
|
||||||
|
method: string;
|
||||||
|
params: Record<string, unknown>;
|
||||||
|
runId: string;
|
||||||
|
rootThreadId: string;
|
||||||
|
activeTurnId: string | null;
|
||||||
|
knownThreads: ReadonlySet<string>;
|
||||||
|
settledTurns: ReadonlySet<string>;
|
||||||
|
}): {
|
||||||
|
classification: CodexNotificationIdentity;
|
||||||
|
threadId: string | null;
|
||||||
|
turnId: string | null;
|
||||||
|
} {
|
||||||
|
const { params, method } = input;
|
||||||
|
const isInformation = informational.has(method) ||
|
||||||
|
(method === "paperclip/canonicalProviderEvent" &&
|
||||||
|
["provider.notice.recorded", "harness.diagnostic"].includes(String(params.eventType)));
|
||||||
|
const threads = [
|
||||||
|
params.threadId,
|
||||||
|
record(params.thread).id,
|
||||||
|
record(params.turn).threadId,
|
||||||
|
].filter((v) => v !== undefined && v !== null);
|
||||||
|
const turns = [params.turnId, record(params.turn).id].filter(
|
||||||
|
(v) => v !== undefined && v !== null,
|
||||||
|
);
|
||||||
|
const threadId = typeof threads[0] === "string" ? threads[0] : null;
|
||||||
|
const turnId = typeof turns[0] === "string" ? turns[0] : null;
|
||||||
|
const result = (classification: CodexNotificationIdentity) => ({
|
||||||
|
classification,
|
||||||
|
threadId,
|
||||||
|
turnId,
|
||||||
|
});
|
||||||
|
if (
|
||||||
|
[...threads, ...turns].some(
|
||||||
|
(v) => typeof v !== "string" || v.length === 0,
|
||||||
|
) ||
|
||||||
|
new Set(threads).size > 1 ||
|
||||||
|
new Set(turns).size > 1 ||
|
||||||
|
[params.runId, params.paperclipRunId].some(
|
||||||
|
(v) => v !== undefined && v !== input.runId,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return result("invalid_authority");
|
||||||
|
if (threadId === null && isInformation) return result("root");
|
||||||
|
if (threadId === input.rootThreadId) {
|
||||||
|
if (
|
||||||
|
input.activeTurnId !== null &&
|
||||||
|
turnId &&
|
||||||
|
turnId !== input.activeTurnId &&
|
||||||
|
input.settledTurns.has(turnId)
|
||||||
|
)
|
||||||
|
return result("stale_turn");
|
||||||
|
return result("root");
|
||||||
|
}
|
||||||
|
const lineage = codexThreadLineage(params.thread);
|
||||||
|
if (
|
||||||
|
threadId &&
|
||||||
|
(input.knownThreads.has(threadId) ||
|
||||||
|
(method === "thread/started" &&
|
||||||
|
lineage.parentThreadId !== null &&
|
||||||
|
input.knownThreads.has(lineage.parentThreadId)))
|
||||||
|
) {
|
||||||
|
// A descendant may report its own terminal, but never supply a root result or workspace authority.
|
||||||
|
return result(
|
||||||
|
method.startsWith("paperclip/") ? "invalid_authority" : "descendant",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return result(
|
||||||
|
isInformation ? "unrelated_information" : "invalid_authority",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,641 @@
|
||||||
|
import {
|
||||||
|
createCipheriv,
|
||||||
|
createDecipheriv,
|
||||||
|
createHash,
|
||||||
|
createHmac,
|
||||||
|
} from "node:crypto";
|
||||||
|
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
|
|
||||||
|
import { HarnessDriverBackend } from "../../backends/harness-driver-backend.js";
|
||||||
|
import { createCodexTaskEnvelope } from "../../contracts/codex.js";
|
||||||
|
import type { ControlPlanePort } from "../../contracts/control-plane-port.js";
|
||||||
|
import type { NativeExecutionInputV1 } from "../../contracts/native-execution.js";
|
||||||
|
import { NativeSessionProtocolIntegrityError } from "../../contracts/native-session-backend.js";
|
||||||
|
import {
|
||||||
|
DurablePrpControlPlane,
|
||||||
|
durableRecoveryInternals,
|
||||||
|
type DurableRecoveryIdentity,
|
||||||
|
} from "../../control-plane/durable-prp-control-plane.js";
|
||||||
|
import { createCapabilityRunnerdCodexTransport } from "../../live/runnerd-codex-transport.js";
|
||||||
|
import { executeNativeSession } from "../../native-session-runtime.js";
|
||||||
|
import { CodexAppServerDriver } from "./codex-app-server-driver.js";
|
||||||
|
import { CodexSessionState } from "./codex-session-state.js";
|
||||||
|
import {
|
||||||
|
FakeCodexTransport,
|
||||||
|
TestQueue,
|
||||||
|
WORKSPACE,
|
||||||
|
describe,
|
||||||
|
expect,
|
||||||
|
it,
|
||||||
|
makeDriver,
|
||||||
|
result,
|
||||||
|
vi,
|
||||||
|
type PrpEvent,
|
||||||
|
} from "./codex-app-server-driver.test-support.js";
|
||||||
|
|
||||||
|
// Synthetic runner process boundary; authentication and the encrypted wire are
|
||||||
|
// real. Keep this client local to this test rather than exporting test protocol
|
||||||
|
// machinery from the production controller.
|
||||||
|
async function authenticatedRunner(
|
||||||
|
core: DurablePrpControlPlane,
|
||||||
|
identity: DurableRecoveryIdentity,
|
||||||
|
) {
|
||||||
|
const framed = (domain: string, parts: Buffer[]) => {
|
||||||
|
const values = [Buffer.from(domain), Buffer.from([0])];
|
||||||
|
for (const part of parts) {
|
||||||
|
const length = Buffer.alloc(8);
|
||||||
|
length.writeBigUInt64BE(BigInt(part.length));
|
||||||
|
values.push(length, part);
|
||||||
|
}
|
||||||
|
return Buffer.concat(values);
|
||||||
|
};
|
||||||
|
const digest = (domain: string, parts: Buffer[]) =>
|
||||||
|
createHash("sha256").update(framed(domain, parts)).digest();
|
||||||
|
const credential = Buffer.from(core.issueBootstrapTicket());
|
||||||
|
const authKey = digest("paperclip-runner-auth-key-v1", [credential]);
|
||||||
|
const mac = (domain: string, parts: Buffer[]) =>
|
||||||
|
createHmac("sha256", authKey).update(framed(domain, parts)).digest();
|
||||||
|
const credentialId = `sha256:${digest("paperclip-runner-credential-id-v1", [credential]).toString("hex")}`;
|
||||||
|
const socket = new WebSocket(core.connectUrl);
|
||||||
|
const frames = new TestQueue<Record<string, unknown>>();
|
||||||
|
const reader = frames[Symbol.asyncIterator]();
|
||||||
|
socket.addEventListener("message", (event) =>
|
||||||
|
frames.push(JSON.parse(String(event.data))),
|
||||||
|
);
|
||||||
|
socket.addEventListener("close", () => frames.close());
|
||||||
|
socket.addEventListener("error", () =>
|
||||||
|
frames.fail(new Error("Synthetic runner socket failed")),
|
||||||
|
);
|
||||||
|
await new Promise<void>((resolve, reject) => {
|
||||||
|
socket.addEventListener("open", () => resolve(), { once: true });
|
||||||
|
socket.addEventListener("error", reject, { once: true });
|
||||||
|
});
|
||||||
|
socket.send(
|
||||||
|
JSON.stringify({
|
||||||
|
protocol: "paperclip.runner",
|
||||||
|
version: 1,
|
||||||
|
kind: "auth_hello",
|
||||||
|
payload: {
|
||||||
|
credentialId,
|
||||||
|
clientNonce: "composed-integrity-client",
|
||||||
|
protocolMin: 1,
|
||||||
|
protocolMax: 1,
|
||||||
|
...identity,
|
||||||
|
runnerVersion: "0.3.0",
|
||||||
|
runnerDigest: `sha256:${createHash("sha256").update(readFileSync(process.execPath)).digest("hex")}`,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const challenge = (await reader.next()).value!.payload as Record<
|
||||||
|
string,
|
||||||
|
unknown
|
||||||
|
>;
|
||||||
|
const { serverProof, ...challengeFields } = challenge;
|
||||||
|
const canonical = Buffer.from(
|
||||||
|
durableRecoveryInternals.canonicalJson(challengeFields),
|
||||||
|
);
|
||||||
|
expect(serverProof).toBe(
|
||||||
|
mac("paperclip-runner-server-proof-v1", [canonical]).toString("hex"),
|
||||||
|
);
|
||||||
|
const clientProof = mac("paperclip-runner-client-proof-v1", [
|
||||||
|
canonical,
|
||||||
|
Buffer.from(String(serverProof)),
|
||||||
|
]).toString("hex");
|
||||||
|
socket.send(
|
||||||
|
JSON.stringify({
|
||||||
|
protocol: "paperclip.runner",
|
||||||
|
version: 1,
|
||||||
|
kind: "auth_response",
|
||||||
|
payload: {
|
||||||
|
credentialId,
|
||||||
|
clientNonce: challenge.clientNonce,
|
||||||
|
serverNonce: challenge.serverNonce,
|
||||||
|
clientProof,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const binding = digest("paperclip-runner-session-binding-v1", [
|
||||||
|
canonical,
|
||||||
|
Buffer.from(String(serverProof)),
|
||||||
|
Buffer.from(clientProof),
|
||||||
|
]);
|
||||||
|
const sessionId = `sha256:${binding.toString("hex")}`;
|
||||||
|
const nonce = (prefix: string, counter: number) => {
|
||||||
|
const value = Buffer.alloc(12);
|
||||||
|
value.write(prefix, 0, "ascii");
|
||||||
|
value.writeBigUInt64BE(BigInt(counter), 4);
|
||||||
|
return value;
|
||||||
|
};
|
||||||
|
const aad = (direction: string, counter: number) =>
|
||||||
|
Buffer.from(
|
||||||
|
`paperclip.runner.secure-frame.v1\0${sessionId}\0${direction}\0${counter}`,
|
||||||
|
);
|
||||||
|
const welcome = (await reader.next()).value!;
|
||||||
|
expect(welcome.counter).toBe(0);
|
||||||
|
const sealed = Buffer.from(String(welcome.ciphertext), "hex");
|
||||||
|
const decipher = createDecipheriv(
|
||||||
|
"aes-256-gcm",
|
||||||
|
mac("paperclip-runner-core-to-client-key-v1", [binding]),
|
||||||
|
nonce("P3S1", 0),
|
||||||
|
);
|
||||||
|
decipher.setAAD(aad("core_to_client", 0));
|
||||||
|
decipher.setAuthTag(sealed.subarray(-16));
|
||||||
|
const opened = JSON.parse(
|
||||||
|
Buffer.concat([
|
||||||
|
decipher.update(sealed.subarray(0, -16)),
|
||||||
|
decipher.final(),
|
||||||
|
]).toString("utf8"),
|
||||||
|
);
|
||||||
|
expect(opened.kind).toBe("welcome");
|
||||||
|
let counter = 0;
|
||||||
|
return {
|
||||||
|
socket,
|
||||||
|
send(value: Record<string, unknown>) {
|
||||||
|
const cipher = createCipheriv(
|
||||||
|
"aes-256-gcm",
|
||||||
|
mac("paperclip-runner-client-to-core-key-v1", [binding]),
|
||||||
|
nonce("P3C1", counter),
|
||||||
|
);
|
||||||
|
cipher.setAAD(aad("client_to_core", counter));
|
||||||
|
const ciphertext = Buffer.concat([
|
||||||
|
cipher.update(JSON.stringify(value)),
|
||||||
|
cipher.final(),
|
||||||
|
cipher.getAuthTag(),
|
||||||
|
]);
|
||||||
|
socket.send(
|
||||||
|
JSON.stringify({
|
||||||
|
schema: "paperclip.runner.secure-frame.v1",
|
||||||
|
counter: counter++,
|
||||||
|
ciphertext: ciphertext.toString("hex"),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("Codex protocol integrity propagation", () => {
|
||||||
|
it("rejects an authenticated controller fault through the real driver, backend, and admitted runtime without accepting a result", async () => {
|
||||||
|
const directory = mkdtempSync(
|
||||||
|
join(tmpdir(), "paperclip-composed-integrity-"),
|
||||||
|
);
|
||||||
|
const identity: DurableRecoveryIdentity = {
|
||||||
|
runnerInstanceId: "composed-runner",
|
||||||
|
environmentLeaseId: "composed-lease",
|
||||||
|
runId: "composed-run",
|
||||||
|
normalizedSessionId: "composed-session",
|
||||||
|
turnId: "composed-turn",
|
||||||
|
itemId: "composed-item",
|
||||||
|
};
|
||||||
|
const contract = {
|
||||||
|
revision: "1",
|
||||||
|
objective: "Validate the authenticated failure boundary",
|
||||||
|
criteria: [
|
||||||
|
{
|
||||||
|
id: "objective",
|
||||||
|
requirement: "Do not accept corrupt provider input",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
const input: NativeExecutionInputV1 = {
|
||||||
|
schema: "paperclip.native-execution-input.v1",
|
||||||
|
binding: {
|
||||||
|
companyId: "composed-company",
|
||||||
|
issueId: "composed-issue",
|
||||||
|
agentId: "composed-agent",
|
||||||
|
runId: identity.runId,
|
||||||
|
executionWorkspaceId: "composed-workspace",
|
||||||
|
},
|
||||||
|
task: {
|
||||||
|
identifier: "TEST-1",
|
||||||
|
title: contract.objective,
|
||||||
|
description: null,
|
||||||
|
prompt: contract.objective,
|
||||||
|
workMode: "standard",
|
||||||
|
},
|
||||||
|
workspace: {
|
||||||
|
cwd: directory,
|
||||||
|
repoUrl: null,
|
||||||
|
repoRef: null,
|
||||||
|
branchName: null,
|
||||||
|
},
|
||||||
|
session: {
|
||||||
|
normalizedSessionId: identity.normalizedSessionId,
|
||||||
|
driverKind: "codex_app_server",
|
||||||
|
protocolVersion: 1,
|
||||||
|
},
|
||||||
|
provider: { kind: "codex", model: null },
|
||||||
|
completionContract: {
|
||||||
|
id: "composed-contract",
|
||||||
|
sha256: "composed-contract-sha",
|
||||||
|
schemaVersion: "paperclip.completion-contract.v1",
|
||||||
|
contract,
|
||||||
|
},
|
||||||
|
interactionResponses: [],
|
||||||
|
credentialBindings: [],
|
||||||
|
};
|
||||||
|
const events: PrpEvent[] = [];
|
||||||
|
const controlPlane: ControlPlanePort = {
|
||||||
|
openRun: vi.fn(async () => undefined),
|
||||||
|
checkpointSession: vi.fn(async () => undefined),
|
||||||
|
appendEvent: vi.fn(async (event) => {
|
||||||
|
events.push(event as PrpEvent);
|
||||||
|
return {
|
||||||
|
cursor: events.length,
|
||||||
|
highestContiguousSourceSeq: events.length,
|
||||||
|
disposition: "committed" as const,
|
||||||
|
};
|
||||||
|
}),
|
||||||
|
replayEvents: vi.fn(async () => ({
|
||||||
|
events: [],
|
||||||
|
highestContiguousSourceSeq: 0,
|
||||||
|
})),
|
||||||
|
completeRun: vi.fn(async () => undefined),
|
||||||
|
};
|
||||||
|
let authority: DurablePrpControlPlane | undefined;
|
||||||
|
let finishProcess!: (result: {
|
||||||
|
code: number;
|
||||||
|
signal: null;
|
||||||
|
stdout: string;
|
||||||
|
stderr: string;
|
||||||
|
}) => void;
|
||||||
|
const completion = new Promise<{
|
||||||
|
code: number;
|
||||||
|
signal: null;
|
||||||
|
stdout: string;
|
||||||
|
stderr: string;
|
||||||
|
}>((resolve) => {
|
||||||
|
finishProcess = resolve;
|
||||||
|
});
|
||||||
|
const kill = vi.fn(() => {
|
||||||
|
finishProcess({ code: 0, signal: null, stdout: "", stderr: "" });
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
const launch = vi.fn(() => ({
|
||||||
|
child: { exitCode: null, kill },
|
||||||
|
completion,
|
||||||
|
}));
|
||||||
|
const bundle = createCapabilityRunnerdCodexTransport({
|
||||||
|
stateDirectory: directory,
|
||||||
|
prpIdentity: identity,
|
||||||
|
runnerBinary: process.execPath,
|
||||||
|
codexCommand: process.execPath,
|
||||||
|
codexArgs: [],
|
||||||
|
sourceCodexHome: null,
|
||||||
|
environment: {},
|
||||||
|
runnerReconnectGraceMs: 900_000,
|
||||||
|
closeGraceMs: 50,
|
||||||
|
runnerProcessLauncher: launch,
|
||||||
|
controlPlaneRegistration: async (core) => {
|
||||||
|
authority = core;
|
||||||
|
await core.start();
|
||||||
|
return { connectUrl: core.connectUrl, release: () => core.stop() };
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const driver = new CodexAppServerDriver({
|
||||||
|
taskEnvelope: createCodexTaskEnvelope({ objective: contract.objective }),
|
||||||
|
environment: { PAPERCLIP_WORKSPACE_CWD: directory },
|
||||||
|
approvalPolicy: "never",
|
||||||
|
transportFactory: () => bundle.transport,
|
||||||
|
});
|
||||||
|
const backend = new HarnessDriverBackend(driver);
|
||||||
|
const admitted = vi.fn();
|
||||||
|
const execution = executeNativeSession({
|
||||||
|
input,
|
||||||
|
backend,
|
||||||
|
controlPlane,
|
||||||
|
runnerInstanceId: identity.runnerInstanceId,
|
||||||
|
controlPlaneInstanceId: "composed-core",
|
||||||
|
timeoutMs: 900_000,
|
||||||
|
requireSessionCloseBeforeReturn: true,
|
||||||
|
onSession: admitted,
|
||||||
|
}).catch((error: unknown) => error);
|
||||||
|
let client: Awaited<ReturnType<typeof authenticatedRunner>> | undefined;
|
||||||
|
try {
|
||||||
|
await vi.waitFor(() => expect(launch).toHaveBeenCalledTimes(1));
|
||||||
|
const core = authority!;
|
||||||
|
client = await authenticatedRunner(core, identity);
|
||||||
|
const commandResult = async (
|
||||||
|
type: string,
|
||||||
|
result: Record<string, unknown> = {},
|
||||||
|
) => {
|
||||||
|
await vi.waitFor(() =>
|
||||||
|
expect(
|
||||||
|
core.store.state.commands.some((command) => command.type === type),
|
||||||
|
).toBe(true),
|
||||||
|
);
|
||||||
|
const command = core.store.state.commands.find(
|
||||||
|
(candidate) => candidate.type === type,
|
||||||
|
)!;
|
||||||
|
client!.send({
|
||||||
|
protocol: "paperclip.runner",
|
||||||
|
version: 1,
|
||||||
|
kind: "command_result",
|
||||||
|
payload: {
|
||||||
|
commandId: command.commandId,
|
||||||
|
commandType: command.type,
|
||||||
|
controllerSeq: command.controllerSeq,
|
||||||
|
status: "completed",
|
||||||
|
result,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await vi.waitFor(() => expect(command.status).toBe("completed"));
|
||||||
|
};
|
||||||
|
const event = (
|
||||||
|
sourceSeq: number,
|
||||||
|
eventType: PrpEvent["eventType"],
|
||||||
|
payload: Record<string, unknown>,
|
||||||
|
) => ({
|
||||||
|
protocol: "paperclip.runner",
|
||||||
|
version: 1,
|
||||||
|
kind: "event",
|
||||||
|
...identity,
|
||||||
|
payload: {
|
||||||
|
schema: "paperclip.prp.event.v1",
|
||||||
|
schemaVersion: 1,
|
||||||
|
sourceEventId: `composed-event-${sourceSeq}`,
|
||||||
|
sourceSeq,
|
||||||
|
sourceInstanceId: identity.runnerInstanceId,
|
||||||
|
sourceKind: "runner",
|
||||||
|
runId: identity.runId,
|
||||||
|
normalizedSessionId: identity.normalizedSessionId,
|
||||||
|
turnId: identity.turnId,
|
||||||
|
itemId: identity.itemId,
|
||||||
|
eventType,
|
||||||
|
priority: 0,
|
||||||
|
emittedAt: "2026-09-08T00:00:00.000Z",
|
||||||
|
payload,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await commandResult("run.prepare");
|
||||||
|
await commandResult("session.open");
|
||||||
|
client.send(
|
||||||
|
event(1, "session.started", {
|
||||||
|
threadId: "composed-provider-thread",
|
||||||
|
sessionId: "composed-provider-session",
|
||||||
|
runtimeIdentity: { processId: process.pid },
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
await commandResult("session.goal.get", { goal: null });
|
||||||
|
await commandResult("turn.start", {
|
||||||
|
providerTurnId: "composed-provider-turn",
|
||||||
|
});
|
||||||
|
client.send(
|
||||||
|
event(2, "turn.started", {
|
||||||
|
providerTurnId: "composed-provider-turn",
|
||||||
|
status: "inProgress",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
await vi.waitFor(() =>
|
||||||
|
expect(events.some((entry) => entry.eventType === "turn.started")).toBe(
|
||||||
|
true,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
expect(controlPlane.openRun).toHaveBeenCalledTimes(1);
|
||||||
|
expect(admitted).toHaveBeenCalledWith(expect.anything());
|
||||||
|
// Capture the actual transport fault, not a newly constructed lookalike.
|
||||||
|
// A pending read also proves that request and notification consumers see
|
||||||
|
// the very same object before the runtime closes its transport.
|
||||||
|
const transportFailure = bundle.transport
|
||||||
|
.request("thread/read", { threadId: "composed-provider-thread" })
|
||||||
|
.catch((error: unknown) => error);
|
||||||
|
await vi.waitFor(() =>
|
||||||
|
expect(
|
||||||
|
core.store.state.commands.some(
|
||||||
|
(command) => command.type === "session.snapshot",
|
||||||
|
),
|
||||||
|
).toBe(true),
|
||||||
|
);
|
||||||
|
const faultAt = Date.now();
|
||||||
|
client.send(
|
||||||
|
event(3, "semantic_tool.input", {
|
||||||
|
semantic_tool: {
|
||||||
|
schema: "paperclip.prp.semantic_tool.v1",
|
||||||
|
schemaVersion: 1,
|
||||||
|
phase: "input",
|
||||||
|
callId: "composed-call",
|
||||||
|
operationId: "get_task_context",
|
||||||
|
correlation: {
|
||||||
|
runId: identity.runId,
|
||||||
|
normalizedSessionId: identity.normalizedSessionId,
|
||||||
|
turnId: identity.turnId,
|
||||||
|
itemId: identity.itemId,
|
||||||
|
},
|
||||||
|
idempotencyKey: null,
|
||||||
|
content: {
|
||||||
|
digest: `sha256:${"0".repeat(64)}`,
|
||||||
|
redactionDisposition: "digest_only",
|
||||||
|
references: [],
|
||||||
|
},
|
||||||
|
input: { summary: "DO-NOT-LEAK-composed-test" },
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const primary = await transportFailure;
|
||||||
|
expect(primary).toBeInstanceOf(NativeSessionProtocolIntegrityError);
|
||||||
|
expect(primary).toMatchObject({
|
||||||
|
code: "native_event_replay_conflict",
|
||||||
|
reason: "semantic_input_digest_mismatch",
|
||||||
|
recovery: "operator_required",
|
||||||
|
});
|
||||||
|
expect(await execution).toBe(primary);
|
||||||
|
expect(Date.now() - faultAt).toBeLessThan(5_000);
|
||||||
|
expect(core.store.state.ackedSourceSeq).toBe(2);
|
||||||
|
expect(
|
||||||
|
core.store.state.committedEvents.map((entry) => entry.eventType),
|
||||||
|
).toEqual(["session.started", "turn.started"]);
|
||||||
|
expect(controlPlane.completeRun).not.toHaveBeenCalled();
|
||||||
|
expect(
|
||||||
|
events.some((entry) => entry.eventType === "run.result.proposed"),
|
||||||
|
).toBe(false);
|
||||||
|
expect(launch).toHaveBeenCalledTimes(1);
|
||||||
|
expect(kill).toHaveBeenCalled();
|
||||||
|
expect(bundle.evidence().diagnostics.join("\n")).not.toContain(
|
||||||
|
"DO-NOT-LEAK",
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
client?.socket.close();
|
||||||
|
kill();
|
||||||
|
await bundle.detachControllerForRestart();
|
||||||
|
await authority?.stop();
|
||||||
|
await execution;
|
||||||
|
rmSync(directory, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
}, 15_000);
|
||||||
|
|
||||||
|
it.each(["pre-start", "pending-input", "buffered-terminal"] as const)(
|
||||||
|
"preserves the exact integrity fault through the composed backend at %s",
|
||||||
|
async (stage) => {
|
||||||
|
const transport = new FakeCodexTransport();
|
||||||
|
const driver = makeDriver([transport]);
|
||||||
|
const harness = await driver.openSession({
|
||||||
|
runId: "run-integrity",
|
||||||
|
normalizedSessionId: "session-integrity",
|
||||||
|
workingDirectory: WORKSPACE,
|
||||||
|
});
|
||||||
|
if (!(harness instanceof CodexSessionState))
|
||||||
|
throw new Error("Expected Codex state");
|
||||||
|
const backend = new HarnessDriverBackend({
|
||||||
|
descriptor: () => driver.descriptor(),
|
||||||
|
openSession: async () => harness,
|
||||||
|
});
|
||||||
|
const session = await backend.openSession({
|
||||||
|
identity: {
|
||||||
|
runId: "run-integrity",
|
||||||
|
sessionId: "session-integrity",
|
||||||
|
companyId: "company",
|
||||||
|
issueId: "issue",
|
||||||
|
agentId: "agent",
|
||||||
|
},
|
||||||
|
workingDirectory: WORKSPACE,
|
||||||
|
});
|
||||||
|
const fault = new NativeSessionProtocolIntegrityError(
|
||||||
|
"semantic_input_digest_mismatch",
|
||||||
|
);
|
||||||
|
const events: PrpEvent[] = [];
|
||||||
|
let pending: Promise<Record<string, unknown>> | undefined;
|
||||||
|
let consumed: Promise<unknown> | undefined;
|
||||||
|
const consume = async () => {
|
||||||
|
try {
|
||||||
|
for await (const event of session.events()) events.push(event);
|
||||||
|
return null;
|
||||||
|
} catch (error) {
|
||||||
|
return error;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
try {
|
||||||
|
if (stage !== "pre-start") {
|
||||||
|
const { turnId } = await session.startTurn({
|
||||||
|
message: { role: "user", text: "Work safely." },
|
||||||
|
});
|
||||||
|
if (stage === "pending-input") {
|
||||||
|
consumed = consume();
|
||||||
|
pending = transport.invoke({
|
||||||
|
id: "input-integrity",
|
||||||
|
method: "item/tool/requestUserInput",
|
||||||
|
params: {
|
||||||
|
threadId: "thread-1",
|
||||||
|
turnId,
|
||||||
|
itemId: "input-integrity-item",
|
||||||
|
questions: [
|
||||||
|
{
|
||||||
|
id: "color",
|
||||||
|
header: "Color",
|
||||||
|
question: "Which color?",
|
||||||
|
options: [{ label: "Amber" }, { label: "Cobalt" }],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await vi.waitFor(() =>
|
||||||
|
expect(
|
||||||
|
events.some(
|
||||||
|
(event) => event.eventType === "runtime_request.created",
|
||||||
|
),
|
||||||
|
).toBe(true),
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
transport.queue.push({
|
||||||
|
method: "turn/completed",
|
||||||
|
params: {
|
||||||
|
threadId: "thread-1",
|
||||||
|
turn: {
|
||||||
|
id: turnId,
|
||||||
|
status: "completed",
|
||||||
|
items: [
|
||||||
|
{
|
||||||
|
id: "final",
|
||||||
|
type: "agentMessage",
|
||||||
|
text: JSON.stringify(result),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await vi.waitFor(() => expect(harness.terminal).toBe(true));
|
||||||
|
expect(harness.result).not.toBeNull();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
transport.queue.fail(fault);
|
||||||
|
await vi.waitFor(() => expect(harness.protocolFailed).toBe(true));
|
||||||
|
consumed ??= consume();
|
||||||
|
expect(await consumed).toBe(fault);
|
||||||
|
expect(
|
||||||
|
events.filter((event) =>
|
||||||
|
[
|
||||||
|
"turn.completed",
|
||||||
|
"turn.failed",
|
||||||
|
"turn.interrupted",
|
||||||
|
"run.result.proposed",
|
||||||
|
"runtime_request.expired",
|
||||||
|
].includes(event.eventType),
|
||||||
|
),
|
||||||
|
).toEqual([]);
|
||||||
|
if (pending) expect(await pending).toEqual({ answers: {} });
|
||||||
|
await expect(session.result()).rejects.toBe(fault);
|
||||||
|
await expect(session.snapshot()).rejects.toBe(fault);
|
||||||
|
await expect(
|
||||||
|
session.startTurn({
|
||||||
|
message: { role: "user", text: "Do not retry." },
|
||||||
|
}),
|
||||||
|
).rejects.toBe(fault);
|
||||||
|
await expect(
|
||||||
|
session.attachRun!({
|
||||||
|
identity: { ...session.identity(), runId: "replacement-run" },
|
||||||
|
}),
|
||||||
|
).rejects.toBe(fault);
|
||||||
|
expect(
|
||||||
|
transport.calls.filter((call) => call.method === "turn/start"),
|
||||||
|
).toHaveLength(stage === "pre-start" ? 0 : 1);
|
||||||
|
} finally {
|
||||||
|
await session.close({ reason: "test cleanup" });
|
||||||
|
await consumed;
|
||||||
|
await pending;
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
it("preserves a protocol failure received before turn start as a typed terminal", async () => {
|
||||||
|
const transport = new FakeCodexTransport();
|
||||||
|
const session = await makeDriver([transport]).openSession({ runId: "prestart", normalizedSessionId: "prestart-session", workingDirectory: WORKSPACE });
|
||||||
|
transport.queue.push({ method: "turn/completed", params: { threadId: "unrelated", turn: { id: "wrong", status: "completed" } } });
|
||||||
|
const events: PrpEvent[] = [];
|
||||||
|
for await (const event of session.events()) events.push(event);
|
||||||
|
await expect(session.startTurn({ message: { role: "user", text: "Work" } })).rejects.toMatchObject({ code: "native_provider_terminal_failed", providerCode: "thread_binding_mismatch", recoverable: false });
|
||||||
|
await session.close({ reason: "test complete" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not promote a message-and-field lookalike transport error", async () => {
|
||||||
|
const transport = new FakeCodexTransport();
|
||||||
|
const session = await makeDriver([transport]).openSession({
|
||||||
|
runId: "run-generic",
|
||||||
|
normalizedSessionId: "session-generic",
|
||||||
|
workingDirectory: WORKSPACE,
|
||||||
|
});
|
||||||
|
const fault = Object.assign(new Error("native_event_replay_conflict"), {
|
||||||
|
code: "native_event_replay_conflict",
|
||||||
|
reason: "semantic_input_digest_mismatch",
|
||||||
|
});
|
||||||
|
const events: PrpEvent[] = [];
|
||||||
|
const consumed = (async () => {
|
||||||
|
for await (const event of session.events()) events.push(event);
|
||||||
|
})();
|
||||||
|
try {
|
||||||
|
await session.startTurn({ message: { role: "user", text: "Work." } });
|
||||||
|
transport.queue.fail(fault);
|
||||||
|
await consumed;
|
||||||
|
expect(
|
||||||
|
events.find((event) => event.eventType === "session.failed")?.payload
|
||||||
|
.code,
|
||||||
|
).toBe("notification_transport_failed");
|
||||||
|
expect(events.some((event) => event.eventType === "turn.failed")).toBe(
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
await expect(session.snapshot()).resolves.toBeDefined();
|
||||||
|
} finally {
|
||||||
|
await session.close({ reason: "test cleanup" });
|
||||||
|
await consumed;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
@ -1,13 +1,14 @@
|
||||||
|
import { classifyCodexNotification } from "./codex-notification-identity.js";
|
||||||
import { paperclipWorkspaceFileReferencesFromText } from "../../live/workspace-file-reference.js";
|
import { paperclipWorkspaceFileReferencesFromText } from "../../live/workspace-file-reference.js";
|
||||||
import { canonicalProviderEventsFromCodex } from "../../provider-events.js";
|
import { canonicalProviderEventsFromCodex, isCanonicalProviderEventType } from "../../provider-events.js";
|
||||||
import { harnessRuntimeRequestOutcome } from "../../contracts/harness-driver.js";
|
import { harnessRuntimeRequestOutcome } from "../../contracts/harness-driver.js";
|
||||||
|
import { NativeSessionProtocolIntegrityError } from "../../contracts/native-session-backend.js";
|
||||||
import { validatePrpStructuredRunResult } from "../../protocol/replay-contract.js";
|
import { validatePrpStructuredRunResult } from "../../protocol/replay-contract.js";
|
||||||
import type { CodexRpcNotification, CodexTraceInterpretation } from "./app-server-transport.js";
|
import type { CodexRpcNotification, CodexTraceInterpretation } from "./app-server-transport.js";
|
||||||
import { redactCodexDiagnostic } from "./app-server-transport.js";
|
import { redactCodexDiagnostic } from "./app-server-transport.js";
|
||||||
import { boundedCodexPayload as boundedPayload, boundedCodexValue, isRetainableCodexPayload } from "./codex-boundaries.js";
|
import { boundedCodexPayload as boundedPayload, boundedCodexValue, isRetainableCodexPayload } from "./codex-boundaries.js";
|
||||||
import { runtimeRequestResponse } from "./codex-question-adapter.js";
|
import { runtimeRequestResponse } from "./codex-question-adapter.js";
|
||||||
import {
|
import {
|
||||||
isBoundCodexNotification,
|
|
||||||
isSupportedCodexNotificationMethod,
|
isSupportedCodexNotificationMethod,
|
||||||
codexThreadLineage as lineageFromThread,
|
codexThreadLineage as lineageFromThread,
|
||||||
codexThreadStatus as threadStatus,
|
codexThreadStatus as threadStatus,
|
||||||
|
|
@ -40,6 +41,10 @@ export async function pumpNotifications(state: CodexSessionState): Promise<void>
|
||||||
await mapNotification(state, notification);
|
await mapNotification(state, notification);
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
if (error instanceof NativeSessionProtocolIntegrityError) {
|
||||||
|
state.failProtocolIntegrity(error);
|
||||||
|
return;
|
||||||
|
}
|
||||||
state.emit("harness.diagnostic", {
|
state.emit("harness.diagnostic", {
|
||||||
code: "notification_transport_failed",
|
code: "notification_transport_failed",
|
||||||
message: redactCodexDiagnostic(String(error)),
|
message: redactCodexDiagnostic(String(error)),
|
||||||
|
|
@ -100,23 +105,29 @@ async function mapNotification(state: CodexSessionState, notification: CodexRpcN
|
||||||
|
|
||||||
async function mapNotificationBody(state: CodexSessionState, notification: CodexRpcNotification): Promise<void> {
|
async function mapNotificationBody(state: CodexSessionState, notification: CodexRpcNotification): Promise<void> {
|
||||||
if (!isSupportedCodexNotificationMethod(notification.method)) return;
|
if (!isSupportedCodexNotificationMethod(notification.method)) return;
|
||||||
if (!isBoundCodexNotification(notification, {
|
if (notification.method === "item/completed" && notification.params.kind === "steering_acknowledgement"
|
||||||
runId: state.runId,
|
&& !notification.params.threadId && !notification.params.turnId && !notification.params.thread && !notification.params.turn) return;
|
||||||
threadIds: [...state.lineageByThread.keys()],
|
const identity = classifyCodexNotification({
|
||||||
})) {
|
method: notification.method, params: notification.params, runId: state.runId,
|
||||||
const params = notification.params;
|
rootThreadId: state.opened.threadId, activeTurnId: state.activeTurnId,
|
||||||
const claimedThreadId = text(
|
knownThreads: new Set(state.lineageByThread.keys()), settledTurns: new Set(state.terminalTurns.keys()),
|
||||||
params.threadId,
|
});
|
||||||
text(record(params.thread).id, text(record(params.turn).threadId)),
|
if (identity.classification !== "root") {
|
||||||
);
|
if (state.notificationIdentityDiagnostics < 32) {
|
||||||
const claimedRunId = text(params.runId, text(params.paperclipRunId));
|
state.notificationIdentityDiagnostics += 1;
|
||||||
if (claimedThreadId.length > 0 || claimedRunId.length > 0) {
|
state.emit("harness.diagnostic", {
|
||||||
state.failProtocol(
|
code: "provider_notification_identity", method: notification.method.slice(0, 128),
|
||||||
"thread_binding_mismatch",
|
classification: identity.classification,
|
||||||
`Provider ${notification.method} message did not name the active run or a known thread.`,
|
expectedThreadId: state.opened.threadId, receivedThreadId: identity.threadId?.slice(0, 256) ?? null,
|
||||||
);
|
expectedTurnId: state.activeTurnId, receivedTurnId: identity.turnId?.slice(0, 256) ?? null,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
return;
|
if (identity.classification === "invalid_authority") {
|
||||||
|
state.failProtocol("thread_binding_mismatch", `Provider authoritative notification ${notification.method.slice(0, 128)} did not name its execution owner.`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (identity.classification !== "descendant") return;
|
||||||
|
if (!["thread/started", "thread/status/changed", "thread/closed"].includes(notification.method)) return;
|
||||||
}
|
}
|
||||||
const params = notification.params;
|
const params = notification.params;
|
||||||
const turn = record(params.turn);
|
const turn = record(params.turn);
|
||||||
|
|
@ -124,6 +135,14 @@ async function mapNotificationBody(state: CodexSessionState, notification: Codex
|
||||||
const threadId = text(params.threadId);
|
const threadId = text(params.threadId);
|
||||||
const turnId = text(params.turnId, text(turn.id));
|
const turnId = text(params.turnId, text(turn.id));
|
||||||
const itemId = text(item.id, text(params.itemId));
|
const itemId = text(item.id, text(params.itemId));
|
||||||
|
if (notification.method === "paperclip/canonicalProviderEvent") {
|
||||||
|
if (!isCanonicalProviderEventType(params.eventType)) {
|
||||||
|
state.failProtocol("provider_event_type_invalid", "Unknown canonical provider event type.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
state.emit(params.eventType, record(params.payload), { turnId: turnId || undefined, itemId: itemId || undefined });
|
||||||
|
return;
|
||||||
|
}
|
||||||
if (notification.method === "paperclip/workspaceChange/updated") {
|
if (notification.method === "paperclip/workspaceChange/updated") {
|
||||||
if (!state.notificationNamesActiveTurn(turnId, "workspace change")) return;
|
if (!state.notificationNamesActiveTurn(turnId, "workspace change")) return;
|
||||||
if (threadId.length > 0 && threadId !== state.opened.threadId) return;
|
if (threadId.length > 0 && threadId !== state.opened.threadId) return;
|
||||||
|
|
|
||||||
|
|
@ -13,6 +13,7 @@ import {
|
||||||
harnessRuntimeRequestOutcome,
|
harnessRuntimeRequestOutcome,
|
||||||
} from "../../contracts/harness-driver.js";
|
} from "../../contracts/harness-driver.js";
|
||||||
import type { CodexTaskEnvelope } from "../../contracts/codex.js";
|
import type { CodexTaskEnvelope } from "../../contracts/codex.js";
|
||||||
|
import { NativeSessionProtocolIntegrityError } from "../../contracts/native-session-backend.js";
|
||||||
import {
|
import {
|
||||||
validatePrpStructuredRunResult,
|
validatePrpStructuredRunResult,
|
||||||
type PrpEvent,
|
type PrpEvent,
|
||||||
|
|
@ -32,21 +33,34 @@ import { canonicalJson, record } from "./codex-driver-values.js";
|
||||||
|
|
||||||
class AsyncQueue<T> implements AsyncIterable<T> {
|
class AsyncQueue<T> implements AsyncIterable<T> {
|
||||||
#values: T[] = [];
|
#values: T[] = [];
|
||||||
#waiters: Array<(value: IteratorResult<T>) => void> = [];
|
#waiters: Array<{
|
||||||
|
resolve: (value: IteratorResult<T>) => void;
|
||||||
|
reject: (error: Error) => void;
|
||||||
|
}> = [];
|
||||||
#closed = false;
|
#closed = false;
|
||||||
|
#failure: Error | null = null;
|
||||||
|
|
||||||
push(value: T): void {
|
push(value: T): void {
|
||||||
if (this.#closed) return;
|
if (this.#closed) return;
|
||||||
const waiter = this.#waiters.shift();
|
const waiter = this.#waiters.shift();
|
||||||
if (waiter === undefined) this.#values.push(value);
|
if (waiter === undefined) this.#values.push(value);
|
||||||
else waiter({ value, done: false });
|
else waiter.resolve({ value, done: false });
|
||||||
}
|
}
|
||||||
|
|
||||||
close(): void {
|
close(): void {
|
||||||
if (this.#closed) return;
|
if (this.#closed) return;
|
||||||
this.#closed = true;
|
this.#closed = true;
|
||||||
for (const waiter of this.#waiters.splice(0))
|
for (const waiter of this.#waiters.splice(0))
|
||||||
waiter({ value: undefined, done: true });
|
waiter.resolve({ value: undefined, done: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
fail(error: Error): void {
|
||||||
|
this.#failure ??= error;
|
||||||
|
this.#closed = true;
|
||||||
|
// Integrity failure takes precedence over a buffered semantic/terminal
|
||||||
|
// suffix, including one whose consumer has not started reading yet.
|
||||||
|
this.#values = [];
|
||||||
|
for (const waiter of this.#waiters.splice(0)) waiter.reject(this.#failure);
|
||||||
}
|
}
|
||||||
|
|
||||||
clear(): void {
|
clear(): void {
|
||||||
|
|
@ -56,10 +70,13 @@ class AsyncQueue<T> implements AsyncIterable<T> {
|
||||||
[Symbol.asyncIterator](): AsyncIterator<T> {
|
[Symbol.asyncIterator](): AsyncIterator<T> {
|
||||||
return {
|
return {
|
||||||
next: async () => {
|
next: async () => {
|
||||||
|
if (this.#failure !== null) throw this.#failure;
|
||||||
const value = this.#values.shift();
|
const value = this.#values.shift();
|
||||||
if (value !== undefined) return { value, done: false };
|
if (value !== undefined) return { value, done: false };
|
||||||
if (this.#closed) return { value: undefined, done: true };
|
if (this.#closed) return { value: undefined, done: true };
|
||||||
return new Promise((resolve) => this.#waiters.push(resolve));
|
return new Promise((resolve, reject) =>
|
||||||
|
this.#waiters.push({ resolve, reject }),
|
||||||
|
);
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
@ -104,6 +121,7 @@ export class CodexSessionState {
|
||||||
protocolFailed = false;
|
protocolFailed = false;
|
||||||
protocolFailureCode: string | null = null;
|
protocolFailureCode: string | null = null;
|
||||||
protocolFailureMessage: string | null = null;
|
protocolFailureMessage: string | null = null;
|
||||||
|
protocolIntegrityFailure: NativeSessionProtocolIntegrityError | null = null;
|
||||||
terminal = false;
|
terminal = false;
|
||||||
dispositionOnlyRecoveryAvailable = false;
|
dispositionOnlyRecoveryAvailable = false;
|
||||||
dispositionOnlyRecoveryConsumed = false;
|
dispositionOnlyRecoveryConsumed = false;
|
||||||
|
|
@ -117,6 +135,7 @@ export class CodexSessionState {
|
||||||
"progress" | "final" | "summary" | "detail" | "unknown"
|
"progress" | "final" | "summary" | "detail" | "unknown"
|
||||||
>();
|
>();
|
||||||
readonly pendingRuntimeRequestMap = new Map<string, PendingRuntimeRequest>();
|
readonly pendingRuntimeRequestMap = new Map<string, PendingRuntimeRequest>();
|
||||||
|
notificationIdentityDiagnostics = 0;
|
||||||
readonly lineageByThread = new Map<string, HarnessThreadLineageEntry>();
|
readonly lineageByThread = new Map<string, HarnessThreadLineageEntry>();
|
||||||
currentGoal: HarnessThreadGoal | null = null;
|
currentGoal: HarnessThreadGoal | null = null;
|
||||||
interruptQueued = false;
|
interruptQueued = false;
|
||||||
|
|
@ -317,6 +336,7 @@ export class CodexSessionState {
|
||||||
operation: string,
|
operation: string,
|
||||||
detail: unknown,
|
detail: unknown,
|
||||||
): HarnessCapabilityUnavailableError {
|
): HarnessCapabilityUnavailableError {
|
||||||
|
this.rethrowProtocolIntegrity(detail);
|
||||||
const error = new HarnessCapabilityUnavailableError(
|
const error = new HarnessCapabilityUnavailableError(
|
||||||
operation,
|
operation,
|
||||||
redactCodexDiagnostic(String(detail)),
|
redactCodexDiagnostic(String(detail)),
|
||||||
|
|
@ -336,6 +356,34 @@ export class CodexSessionState {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
assertProtocolIntegrity(): void {
|
||||||
|
if (this.protocolIntegrityFailure !== null)
|
||||||
|
throw this.protocolIntegrityFailure;
|
||||||
|
}
|
||||||
|
|
||||||
|
rethrowProtocolIntegrity(error: unknown): void {
|
||||||
|
if (!(error instanceof NativeSessionProtocolIntegrityError)) return;
|
||||||
|
this.failProtocolIntegrity(error);
|
||||||
|
this.assertProtocolIntegrity();
|
||||||
|
}
|
||||||
|
|
||||||
|
failProtocolIntegrity(error: NativeSessionProtocolIntegrityError): void {
|
||||||
|
this.protocolIntegrityFailure ??= error;
|
||||||
|
this.protocolFailed = true;
|
||||||
|
this.protocolFailureCode = this.protocolIntegrityFailure.code;
|
||||||
|
this.protocolFailureMessage = this.protocolIntegrityFailure.message;
|
||||||
|
this.terminal = true;
|
||||||
|
this.result = null;
|
||||||
|
this.resultFingerprint = null;
|
||||||
|
this.resultCallId = null;
|
||||||
|
this.resultTurnId = null;
|
||||||
|
// Fail the stream before settling pending local RPCs: a synthetic input
|
||||||
|
// expiration or terminal event must not turn corruption into a safe wait.
|
||||||
|
this.eventQueue.fail(this.protocolIntegrityFailure);
|
||||||
|
this.cancelPendingRequests("protocol_integrity_failed");
|
||||||
|
// The owning runtime still performs and awaits exact transport cleanup.
|
||||||
|
}
|
||||||
|
|
||||||
failProtocol(code: string, message: string): void {
|
failProtocol(code: string, message: string): void {
|
||||||
if (this.protocolFailed) return;
|
if (this.protocolFailed) return;
|
||||||
this.protocolFailed = true;
|
this.protocolFailed = true;
|
||||||
|
|
@ -351,7 +399,7 @@ export class CodexSessionState {
|
||||||
const turnId = this.activeTurnId;
|
const turnId = this.activeTurnId;
|
||||||
this.emit(
|
this.emit(
|
||||||
"turn.failed",
|
"turn.failed",
|
||||||
{ status: "failed", error: { code } },
|
{ status: "failed", error: { code, message: this.protocolFailureMessage, recoverable: false } },
|
||||||
{ turnId },
|
{ turnId },
|
||||||
);
|
);
|
||||||
this.terminalTurns.set(turnId, canonicalJson({ protocolFailure: code }));
|
this.terminalTurns.set(turnId, canonicalJson({ protocolFailure: code }));
|
||||||
|
|
|
||||||
|
|
@ -140,6 +140,7 @@ export function isSupportedCodexNotificationMethod(method: string): boolean {
|
||||||
method === "model/verification" ||
|
method === "model/verification" ||
|
||||||
method === "model/safetyBuffering/updated" ||
|
method === "model/safetyBuffering/updated" ||
|
||||||
method.startsWith("item/") ||
|
method.startsWith("item/") ||
|
||||||
|
method === "paperclip/canonicalProviderEvent" ||
|
||||||
method === "paperclip/workspaceChange/updated" ||
|
method === "paperclip/workspaceChange/updated" ||
|
||||||
method === "paperclip/runResult" ||
|
method === "paperclip/runResult" ||
|
||||||
method === "turn/diff/updated" ||
|
method === "turn/diff/updated" ||
|
||||||
|
|
|
||||||
|
|
@ -15,6 +15,18 @@ import { join, resolve } from "node:path";
|
||||||
import { fileURLToPath } from "node:url";
|
import { fileURLToPath } from "node:url";
|
||||||
|
|
||||||
import { expect, it, vi } from "vitest";
|
import { expect, it, vi } from "vitest";
|
||||||
|
import type { ControlPlanePort } from "../contracts/control-plane-port.js";
|
||||||
|
import type { NativeExecutionInputV1 } from "../contracts/native-execution.js";
|
||||||
|
import type {
|
||||||
|
NativeSession,
|
||||||
|
NativeSessionBackend,
|
||||||
|
} from "../contracts/native-session-backend.js";
|
||||||
|
import type {
|
||||||
|
PrpEvent,
|
||||||
|
PrpStructuredRunResult,
|
||||||
|
PrpTerminalState,
|
||||||
|
} from "../protocol/replay-contract.js";
|
||||||
|
import { executeNativeSession } from "../native-session-runtime.js";
|
||||||
import type { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js";
|
import type { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js";
|
||||||
|
|
||||||
import {
|
import {
|
||||||
|
|
@ -46,6 +58,7 @@ import {
|
||||||
latestRunnerdSessionReadiness,
|
latestRunnerdSessionReadiness,
|
||||||
rehydrateRunnerdGoalNotification,
|
rehydrateRunnerdGoalNotification,
|
||||||
rehydrateRunnerdItemNotification,
|
rehydrateRunnerdItemNotification,
|
||||||
|
rehydrateRunnerdDeltaNotification,
|
||||||
rehydrateRunnerdPlanNotification,
|
rehydrateRunnerdPlanNotification,
|
||||||
rehydrateRunnerdResultNotification,
|
rehydrateRunnerdResultNotification,
|
||||||
rehydrateRunnerdThreadTokenUsage,
|
rehydrateRunnerdThreadTokenUsage,
|
||||||
|
|
@ -1158,6 +1171,11 @@ it("binds a durable semantic result to the active provider turn", () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("restores provider identity and streamed text from a canonical delta", () => {
|
||||||
|
expect(rehydrateRunnerdDeltaNotification({ text: "Reading Gmail", itemId: "message-1", turnId: "controller-turn" }, "root-thread", "provider-turn"))
|
||||||
|
.toMatchObject({ threadId: "root-thread", turnId: "provider-turn", delta: "Reading Gmail", itemId: "message-1" });
|
||||||
|
});
|
||||||
|
|
||||||
it("rehydrates a canonical agent item for the strict Codex facade", () => {
|
it("rehydrates a canonical agent item for the strict Codex facade", () => {
|
||||||
expect(
|
expect(
|
||||||
rehydrateRunnerdItemNotification(
|
rehydrateRunnerdItemNotification(
|
||||||
|
|
@ -1937,6 +1955,247 @@ it("continues rehydrating events after the committed-event window slides", async
|
||||||
}
|
}
|
||||||
}, 30_000);
|
}, 30_000);
|
||||||
|
|
||||||
|
it("does not retry a real memoized transport close whose suspension proof is unavailable", async () => {
|
||||||
|
const identity = {
|
||||||
|
runId: "run-recovery",
|
||||||
|
sessionId: "session-recovery",
|
||||||
|
companyId: "company-recovery",
|
||||||
|
issueId: "issue-recovery",
|
||||||
|
agentId: "agent-recovery",
|
||||||
|
};
|
||||||
|
|
||||||
|
const result: PrpStructuredRunResult = {
|
||||||
|
schema: "paperclip.run_result.v1",
|
||||||
|
reportedWorkDisposition: "done",
|
||||||
|
summary: "Recovered native work completed.",
|
||||||
|
completionClaim: {
|
||||||
|
contractRevision: "1",
|
||||||
|
objectiveSatisfied: true,
|
||||||
|
criteria: [
|
||||||
|
{ criterionId: "objective", status: "satisfied", evidenceRefs: [] },
|
||||||
|
],
|
||||||
|
remainingWork: [],
|
||||||
|
},
|
||||||
|
evidence: [],
|
||||||
|
verification: [{ commandOrCheck: "recovery", status: "passed" }],
|
||||||
|
attentionRequests: [],
|
||||||
|
artifacts: [],
|
||||||
|
};
|
||||||
|
|
||||||
|
const terminal: PrpTerminalState = {
|
||||||
|
schema: "paperclip.prp.terminal.v1",
|
||||||
|
turnTerminalState: "completed",
|
||||||
|
runTerminalState: "succeeded",
|
||||||
|
reportedWorkDisposition: "done",
|
||||||
|
};
|
||||||
|
|
||||||
|
const input: NativeExecutionInputV1 = {
|
||||||
|
schema: "paperclip.native-execution-input.v1",
|
||||||
|
binding: {
|
||||||
|
companyId: identity.companyId,
|
||||||
|
runId: identity.runId,
|
||||||
|
issueId: identity.issueId,
|
||||||
|
agentId: identity.agentId,
|
||||||
|
executionWorkspaceId: "workspace-recovery",
|
||||||
|
},
|
||||||
|
task: {
|
||||||
|
identifier: "PAP-RECOVERY",
|
||||||
|
title: "Recover native work",
|
||||||
|
description: null,
|
||||||
|
prompt: "# PAP-RECOVERY: Recover native work",
|
||||||
|
workMode: "standard",
|
||||||
|
},
|
||||||
|
workspace: {
|
||||||
|
cwd: "/workspace",
|
||||||
|
repoUrl: null,
|
||||||
|
repoRef: null,
|
||||||
|
branchName: null,
|
||||||
|
},
|
||||||
|
session: {
|
||||||
|
normalizedSessionId: identity.sessionId,
|
||||||
|
driverKind: "codex_app_server",
|
||||||
|
protocolVersion: 1,
|
||||||
|
},
|
||||||
|
provider: { kind: "codex", model: null },
|
||||||
|
completionContract: {
|
||||||
|
id: "contract-recovery",
|
||||||
|
sha256: "contract-recovery-sha",
|
||||||
|
schemaVersion: "paperclip.completion-contract.v1",
|
||||||
|
contract: {
|
||||||
|
revision: "1",
|
||||||
|
objective: "Recover native work",
|
||||||
|
criteria: [{ id: "objective", requirement: "Complete after recovery" }],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
interactionResponses: [],
|
||||||
|
credentialBindings: [],
|
||||||
|
};
|
||||||
|
|
||||||
|
function runnerEvent(
|
||||||
|
sourceSeq: number,
|
||||||
|
eventType: PrpEvent["eventType"],
|
||||||
|
payload: Record<string, unknown> = {},
|
||||||
|
): PrpEvent {
|
||||||
|
return {
|
||||||
|
schema: "paperclip.prp.event.v1",
|
||||||
|
sourceEventId: `runner-recovery:${identity.runId}:${sourceSeq}`,
|
||||||
|
sourceSeq,
|
||||||
|
sourceInstanceId: "runner-recovery",
|
||||||
|
sourceKind: "runner",
|
||||||
|
runId: identity.runId,
|
||||||
|
normalizedSessionId: identity.sessionId,
|
||||||
|
turnId: "turn-recovery",
|
||||||
|
eventType,
|
||||||
|
schemaVersion: 1,
|
||||||
|
priority: 0,
|
||||||
|
emittedAt: "2026-08-09T00:00:00.000Z",
|
||||||
|
payload,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const stateDirectory = await mkdtemp(
|
||||||
|
join(tmpdir(), "native-close-quarantine-"),
|
||||||
|
);
|
||||||
|
const readRunnerState = vi.fn(async () => ({
|
||||||
|
schema: "paperclip.runner.durable.state.v1",
|
||||||
|
runnerInstanceId: "runner-close-quarantine",
|
||||||
|
environmentLeaseId: "lease-close-quarantine",
|
||||||
|
runId: identity.runId,
|
||||||
|
normalizedSessionId: identity.sessionId,
|
||||||
|
turnId: "turn-close-quarantine",
|
||||||
|
itemId: "item-close-quarantine",
|
||||||
|
lifecycle: "ready",
|
||||||
|
}));
|
||||||
|
const bundle = createCapabilityRunnerdCodexTransport({
|
||||||
|
runnerBinary: defaultCapabilityRunnerdBinary(),
|
||||||
|
codexCommand: resolve(
|
||||||
|
import.meta.dirname,
|
||||||
|
"../../runner/target/debug/fake-codex-app-server",
|
||||||
|
),
|
||||||
|
codexArgs: ["--state-file", join(stateDirectory, "fake-codex-state.json")],
|
||||||
|
stateDirectory,
|
||||||
|
closeGraceMs: 400,
|
||||||
|
lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null },
|
||||||
|
prpIdentity: await readRunnerState(),
|
||||||
|
readRunnerState,
|
||||||
|
// A checkpoint owner requires durable suspension proof. A local transport
|
||||||
|
// without a checkpoint can simply terminate its process on close.
|
||||||
|
controlPlaneRegistration: async (authority) => {
|
||||||
|
await authority.start();
|
||||||
|
return {
|
||||||
|
connectUrl: authority.connectUrl,
|
||||||
|
checkpoint: async () => {},
|
||||||
|
release: async () => {},
|
||||||
|
};
|
||||||
|
},
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
await bundle.transport.request("thread/start", {
|
||||||
|
cwd: stateDirectory,
|
||||||
|
dynamicTools: [],
|
||||||
|
});
|
||||||
|
const failedClose = bundle.transport.close();
|
||||||
|
const failure = await failedClose.catch((error: unknown) => error);
|
||||||
|
expect(failure).toMatchObject({
|
||||||
|
code: "native_session_close_unrecoverable",
|
||||||
|
});
|
||||||
|
expect(bundle.transport.close()).toBe(failedClose);
|
||||||
|
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const close = vi.fn(({ reason }: { reason: string }) =>
|
||||||
|
bundle.transport.close(reason),
|
||||||
|
);
|
||||||
|
const capabilities = {
|
||||||
|
resume: true,
|
||||||
|
typedEvents: true,
|
||||||
|
steering: false,
|
||||||
|
interruption: true,
|
||||||
|
structuredResult: true,
|
||||||
|
};
|
||||||
|
const session: NativeSession = {
|
||||||
|
identity: () => identity,
|
||||||
|
async capabilities() {
|
||||||
|
return capabilities;
|
||||||
|
},
|
||||||
|
async *events() {
|
||||||
|
yield runnerEvent(1, "turn.completed");
|
||||||
|
},
|
||||||
|
async startTurn() {
|
||||||
|
return { turnId: "turn-recovery" };
|
||||||
|
},
|
||||||
|
async result() {
|
||||||
|
return { result, terminal, turnId: "turn-recovery" };
|
||||||
|
},
|
||||||
|
async snapshot() {
|
||||||
|
return {
|
||||||
|
backendKind: "mock",
|
||||||
|
sessionId: identity.sessionId,
|
||||||
|
identity,
|
||||||
|
providerSessionId: "provider-recovery",
|
||||||
|
cursor: null,
|
||||||
|
activeTurnId: null,
|
||||||
|
pendingRuntimeRequests: [],
|
||||||
|
lineage: [],
|
||||||
|
};
|
||||||
|
},
|
||||||
|
close,
|
||||||
|
};
|
||||||
|
const openSession = vi.fn(async () => session);
|
||||||
|
const backend: NativeSessionBackend = {
|
||||||
|
async descriptor() {
|
||||||
|
return {
|
||||||
|
kind: "mock",
|
||||||
|
name: "real-memoized-close-quarantine",
|
||||||
|
version: "1",
|
||||||
|
capabilities,
|
||||||
|
};
|
||||||
|
},
|
||||||
|
openSession,
|
||||||
|
};
|
||||||
|
const port: ControlPlanePort = {
|
||||||
|
async openRun() {},
|
||||||
|
async checkpointSession() {},
|
||||||
|
async appendEvent() {
|
||||||
|
return {
|
||||||
|
cursor: 1,
|
||||||
|
highestContiguousSourceSeq: 1,
|
||||||
|
disposition: "committed",
|
||||||
|
};
|
||||||
|
},
|
||||||
|
async replayEvents() {
|
||||||
|
return { events: [], highestContiguousSourceSeq: 0 };
|
||||||
|
},
|
||||||
|
async completeRun() {},
|
||||||
|
};
|
||||||
|
const execute = () =>
|
||||||
|
executeNativeSession({
|
||||||
|
input,
|
||||||
|
backend,
|
||||||
|
controlPlane: port,
|
||||||
|
runnerInstanceId: "runner-recovery",
|
||||||
|
controlPlaneInstanceId: "control-recovery",
|
||||||
|
requireSessionCloseBeforeReturn: true,
|
||||||
|
});
|
||||||
|
await expect(execute()).rejects.toBe(failure);
|
||||||
|
const readsAfterClose = readRunnerState.mock.calls.length;
|
||||||
|
await expect(execute()).rejects.toMatchObject({
|
||||||
|
code: "native_session_cleanup_quarantined",
|
||||||
|
recovery: "operator_required",
|
||||||
|
});
|
||||||
|
await vi.advanceTimersByTimeAsync(600_000);
|
||||||
|
await expect(execute()).rejects.toMatchObject({
|
||||||
|
code: "native_session_cleanup_quarantined",
|
||||||
|
});
|
||||||
|
expect(openSession).toHaveBeenCalledOnce();
|
||||||
|
expect(close).toHaveBeenCalledOnce();
|
||||||
|
expect(readRunnerState).toHaveBeenCalledTimes(readsAfterClose);
|
||||||
|
} finally {
|
||||||
|
vi.useRealTimers();
|
||||||
|
await bundle.transport.close().catch(() => undefined);
|
||||||
|
await rm(stateDirectory, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
}, 10_000);
|
||||||
|
|
||||||
|
|
||||||
it("binds an immediately failed durable turn before exposing its terminal", async () => {
|
it("binds an immediately failed durable turn before exposing its terminal", async () => {
|
||||||
const stateDirectory = await mkdtemp(
|
const stateDirectory = await mkdtemp(
|
||||||
join(tmpdir(), "runnerd-fast-terminal-"),
|
join(tmpdir(), "runnerd-fast-terminal-"),
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
import { isCanonicalProviderEventType } from "../provider-events.js";
|
||||||
import { execFileSync } from "node:child_process";
|
import { execFileSync } from "node:child_process";
|
||||||
import { createHash, randomUUID } from "node:crypto";
|
import { createHash, randomUUID } from "node:crypto";
|
||||||
import {
|
import {
|
||||||
|
|
@ -34,6 +35,7 @@ import type {
|
||||||
DurableRecoveryCommittedEvent,
|
DurableRecoveryCommittedEvent,
|
||||||
DurableRecoveryIdentity,
|
DurableRecoveryIdentity,
|
||||||
} from "../contracts/durable-recovery.js";
|
} from "../contracts/durable-recovery.js";
|
||||||
|
import { NativeSessionCloseUnrecoverableError } from "../contracts/native-session-backend.js";
|
||||||
import type {
|
import type {
|
||||||
HarnessRuntimeRequestResolution,
|
HarnessRuntimeRequestResolution,
|
||||||
PersistedHarnessProviderIdentity,
|
PersistedHarnessProviderIdentity,
|
||||||
|
|
@ -1565,6 +1567,22 @@ export function rehydrateRunnerdTurnNotification(
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function rehydrateRunnerdDeltaNotification(
|
||||||
|
rawParams: Record<string, unknown>,
|
||||||
|
openedThreadId: string,
|
||||||
|
activeTurnId: string,
|
||||||
|
): Record<string, unknown> {
|
||||||
|
// Canonical PRP events already passed runner identity validation. Restore the
|
||||||
|
// provider binding just as for item starts/completions; the PRP controller
|
||||||
|
// turn is deliberately different from the provider's turn ID.
|
||||||
|
return {
|
||||||
|
...rawParams,
|
||||||
|
threadId: openedThreadId,
|
||||||
|
turnId: activeTurnId,
|
||||||
|
delta: rawParams.delta ?? rawParams.text,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export function rehydrateRunnerdItemNotification(
|
export function rehydrateRunnerdItemNotification(
|
||||||
rawParams: Record<string, unknown>,
|
rawParams: Record<string, unknown>,
|
||||||
openedThreadId: string,
|
openedThreadId: string,
|
||||||
|
|
@ -2152,6 +2170,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
|
||||||
#checkpointProviderIdentityConfirmed = false;
|
#checkpointProviderIdentityConfirmed = false;
|
||||||
#turnId = "";
|
#turnId = "";
|
||||||
#turnStartResponsePending = false;
|
#turnStartResponsePending = false;
|
||||||
|
#turnStartResponseSettled: Promise<void> = Promise.resolve();
|
||||||
#turnStartResponseEpoch = 0;
|
#turnStartResponseEpoch = 0;
|
||||||
#observedTurnStartEpoch = 0;
|
#observedTurnStartEpoch = 0;
|
||||||
#expectedProviderTurnId: string | null = null;
|
#expectedProviderTurnId: string | null = null;
|
||||||
|
|
@ -3009,9 +3028,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
|
||||||
this.#controlPlaneRelease = null;
|
this.#controlPlaneRelease = null;
|
||||||
}
|
}
|
||||||
if (suspensionRequired && !runnerSuspended) {
|
if (suspensionRequired && !runnerSuspended) {
|
||||||
throw new Error(
|
throw new NativeSessionCloseUnrecoverableError();
|
||||||
"provider_transport_failed: runner did not durably suspend before checkpoint",
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
if (this.#ownsRoot) rmSync(this.#root, { recursive: true, force: true });
|
if (this.#ownsRoot) rmSync(this.#root, { recursive: true, force: true });
|
||||||
this.#publish();
|
this.#publish();
|
||||||
|
|
@ -3043,8 +3060,13 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
|
||||||
identity,
|
identity,
|
||||||
expectedRunnerVersion: runnerArtifact.version,
|
expectedRunnerVersion: runnerArtifact.version,
|
||||||
expectedRunnerDigest: runnerArtifact.digest,
|
expectedRunnerDigest: runnerArtifact.digest,
|
||||||
onSemanticToolInput: async (call) =>
|
onProtocolIntegrityError: (error) => this.#failTransport(error),
|
||||||
unwrapToolResponse(
|
onSemanticToolInput: async (call) => {
|
||||||
|
// Semantic input may outrun the facade's turn/start response. Bind it
|
||||||
|
// only after the strict driver has accepted that same provider turn.
|
||||||
|
await this.#turnStartResponseSettled;
|
||||||
|
this.#throwIfFailed();
|
||||||
|
return unwrapToolResponse(
|
||||||
await this.#handler({
|
await this.#handler({
|
||||||
id: call.callId,
|
id: call.callId,
|
||||||
method: "item/tool/call",
|
method: "item/tool/call",
|
||||||
|
|
@ -3064,7 +3086,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
|
||||||
}
|
}
|
||||||
: {}),
|
: {}),
|
||||||
}),
|
}),
|
||||||
),
|
);
|
||||||
|
},
|
||||||
connectionLeaseTtlMs: 60 * 60 * 1_000,
|
connectionLeaseTtlMs: 60 * 60 * 1_000,
|
||||||
});
|
});
|
||||||
this.#core = core;
|
this.#core = core;
|
||||||
|
|
@ -3694,8 +3717,13 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
|
||||||
identity,
|
identity,
|
||||||
expectedRunnerVersion: runnerArtifact.version,
|
expectedRunnerVersion: runnerArtifact.version,
|
||||||
expectedRunnerDigest: runnerArtifact.digest,
|
expectedRunnerDigest: runnerArtifact.digest,
|
||||||
onSemanticToolInput: async (call) =>
|
onProtocolIntegrityError: (error) => this.#failTransport(error),
|
||||||
unwrapToolResponse(
|
onSemanticToolInput: async (call) => {
|
||||||
|
// Semantic input may outrun the facade's turn/start response. Bind it
|
||||||
|
// only after the strict driver has accepted that same provider turn.
|
||||||
|
await this.#turnStartResponseSettled;
|
||||||
|
this.#throwIfFailed();
|
||||||
|
return unwrapToolResponse(
|
||||||
await this.#handler({
|
await this.#handler({
|
||||||
id: call.callId,
|
id: call.callId,
|
||||||
method: "item/tool/call",
|
method: "item/tool/call",
|
||||||
|
|
@ -3715,7 +3743,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
|
||||||
}
|
}
|
||||||
: {}),
|
: {}),
|
||||||
}),
|
}),
|
||||||
),
|
);
|
||||||
|
},
|
||||||
connectionLeaseTtlMs: 60 * 60 * 1_000,
|
connectionLeaseTtlMs: 60 * 60 * 1_000,
|
||||||
});
|
});
|
||||||
this.#core = core;
|
this.#core = core;
|
||||||
|
|
@ -3933,6 +3962,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
|
||||||
this.#turnId = pendingTurnId;
|
this.#turnId = pendingTurnId;
|
||||||
const responseEpoch = ++this.#turnStartResponseEpoch;
|
const responseEpoch = ++this.#turnStartResponseEpoch;
|
||||||
this.#turnStartResponsePending = true;
|
this.#turnStartResponsePending = true;
|
||||||
|
let releaseStartResponse!: () => void;
|
||||||
|
this.#turnStartResponseSettled = new Promise<void>(resolve => { releaseStartResponse = resolve; });
|
||||||
this.#expectedProviderTurnId = null;
|
this.#expectedProviderTurnId = null;
|
||||||
let responseReady = false;
|
let responseReady = false;
|
||||||
try {
|
try {
|
||||||
|
|
@ -4000,6 +4031,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
|
||||||
return { turn: { id: this.#turnId, status: "inProgress" } };
|
return { turn: { id: this.#turnId, status: "inProgress" } };
|
||||||
} finally {
|
} finally {
|
||||||
if (!responseReady) {
|
if (!responseReady) {
|
||||||
|
releaseStartResponse();
|
||||||
if (this.#turnStartResponseEpoch === responseEpoch) {
|
if (this.#turnStartResponseEpoch === responseEpoch) {
|
||||||
this.#turnStartResponsePending = false;
|
this.#turnStartResponsePending = false;
|
||||||
this.#expectedProviderTurnId = null;
|
this.#expectedProviderTurnId = null;
|
||||||
|
|
@ -4010,6 +4042,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
|
||||||
// following task so the driver can bind and emit turn.accepted first.
|
// following task so the driver can bind and emit turn.accepted first.
|
||||||
// The epoch prevents a late release from clearing a newer turn fence.
|
// The epoch prevents a late release from clearing a newer turn fence.
|
||||||
const release = setTimeout(() => {
|
const release = setTimeout(() => {
|
||||||
|
releaseStartResponse();
|
||||||
if (this.#turnStartResponseEpoch !== responseEpoch) return;
|
if (this.#turnStartResponseEpoch !== responseEpoch) return;
|
||||||
this.#turnStartResponsePending = false;
|
this.#turnStartResponsePending = false;
|
||||||
this.#expectedProviderTurnId = null;
|
this.#expectedProviderTurnId = null;
|
||||||
|
|
@ -4294,7 +4327,14 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
|
||||||
sessionUpdatePayload,
|
sessionUpdatePayload,
|
||||||
);
|
);
|
||||||
const notifications =
|
const notifications =
|
||||||
event.eventType === "provider.event"
|
isCanonicalProviderEventType(event.eventType) && !canonicalMethod
|
||||||
|
? [{ method: "paperclip/canonicalProviderEvent", params: {
|
||||||
|
...(this.#threadId ? { threadId: this.#threadId } : {}),
|
||||||
|
...(this.#turnId ? { turnId: this.#turnId } : {}),
|
||||||
|
eventType: event.eventType, payload: eventPayload,
|
||||||
|
itemId: event.envelope.itemId,
|
||||||
|
} }]
|
||||||
|
: event.eventType === "provider.event"
|
||||||
? unwrapRunnerdProviderNotifications(eventPayload)
|
? unwrapRunnerdProviderNotifications(eventPayload)
|
||||||
: canonicalMethod
|
: canonicalMethod
|
||||||
? expandRunnerdCanonicalNotifications(canonicalMethod, eventPayload)
|
? expandRunnerdCanonicalNotifications(canonicalMethod, eventPayload)
|
||||||
|
|
@ -4355,6 +4395,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
|
||||||
this.#threadId,
|
this.#threadId,
|
||||||
method,
|
method,
|
||||||
)
|
)
|
||||||
|
: event.eventType === "item.delta"
|
||||||
|
? rehydrateRunnerdDeltaNotification(rawParams, this.#threadId, this.#turnId)
|
||||||
: event.eventType !== "provider.event" &&
|
: event.eventType !== "provider.event" &&
|
||||||
(method === "item/started" || method === "item/completed")
|
(method === "item/started" || method === "item/completed")
|
||||||
? rehydrateRunnerdItemNotification(
|
? rehydrateRunnerdItemNotification(
|
||||||
|
|
@ -4373,7 +4415,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
|
||||||
: rawParams;
|
: rawParams;
|
||||||
if (
|
if (
|
||||||
params.turnId === undefined &&
|
params.turnId === undefined &&
|
||||||
typeof event.envelope.turnId === "string"
|
typeof event.envelope.turnId === "string" && event.envelope.turnId.length > 0
|
||||||
) {
|
) {
|
||||||
params.turnId = event.envelope.turnId;
|
params.turnId = event.envelope.turnId;
|
||||||
}
|
}
|
||||||
|
|
@ -4696,7 +4738,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
|
||||||
const detail = result.stderr.trim() || result.stdout.trim();
|
const detail = result.stderr.trim() || result.stdout.trim();
|
||||||
if (detail) this.#diagnostic(detail.slice(-4_096));
|
if (detail) this.#diagnostic(detail.slice(-4_096));
|
||||||
this.#publish();
|
this.#publish();
|
||||||
if (this.#closed || this.#handle !== handle) return;
|
if (this.#closed || this.#failure !== null || this.#handle !== handle)
|
||||||
|
return;
|
||||||
// A per-turn runner exits after its terminal suffix is durably ACKed.
|
// A per-turn runner exits after its terminal suffix is durably ACKed.
|
||||||
// Drain that suffix into the provider-facing queue before classifying
|
// Drain that suffix into the provider-facing queue before classifying
|
||||||
// process completion; clean terminal exit is the expected lifecycle.
|
// process completion; clean terminal exit is the expected lifecycle.
|
||||||
|
|
@ -4735,7 +4778,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
|
||||||
);
|
);
|
||||||
},
|
},
|
||||||
(error) => {
|
(error) => {
|
||||||
if (this.#closed || this.#handle !== handle) return;
|
if (this.#closed || this.#failure !== null || this.#handle !== handle)
|
||||||
|
return;
|
||||||
if (
|
if (
|
||||||
this.#startupComplete &&
|
this.#startupComplete &&
|
||||||
this.options.runnerReconnectGraceMs !== undefined &&
|
this.options.runnerReconnectGraceMs !== undefined &&
|
||||||
|
|
@ -4763,6 +4807,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
|
||||||
if (
|
if (
|
||||||
this.#runnerRecoveryInProgress ||
|
this.#runnerRecoveryInProgress ||
|
||||||
this.#closed ||
|
this.#closed ||
|
||||||
|
this.#failure !== null ||
|
||||||
this.#handle !== failedHandle ||
|
this.#handle !== failedHandle ||
|
||||||
this.#core === null ||
|
this.#core === null ||
|
||||||
!failedHandle.restart
|
!failedHandle.restart
|
||||||
|
|
@ -4779,7 +4824,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
|
||||||
`runner process disconnected; recovery is allowed for ${graceMs}ms`,
|
`runner process disconnected; recovery is allowed for ${graceMs}ms`,
|
||||||
);
|
);
|
||||||
try {
|
try {
|
||||||
while (!this.#closed && Date.now() < deadline) {
|
while (!this.#closed && this.#failure === null && Date.now() < deadline) {
|
||||||
if (attempt > 0) {
|
if (attempt > 0) {
|
||||||
const base = delays[Math.min(attempt - 1, delays.length - 1)]!;
|
const base = delays[Math.min(attempt - 1, delays.length - 1)]!;
|
||||||
const jittered = Math.max(
|
const jittered = Math.max(
|
||||||
|
|
@ -4788,7 +4833,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
|
||||||
);
|
);
|
||||||
await new Promise((resolveWait) => setTimeout(resolveWait, jittered));
|
await new Promise((resolveWait) => setTimeout(resolveWait, jittered));
|
||||||
}
|
}
|
||||||
if (this.#closed) return;
|
if (this.#closed || this.#failure !== null) return;
|
||||||
if (Date.now() >= deadline) break;
|
if (Date.now() >= deadline) break;
|
||||||
const priorConnectionCount = this.#core.store.state.connectionCount;
|
const priorConnectionCount = this.#core.store.state.connectionCount;
|
||||||
let recoveredHandle: RunnerProcessHandle;
|
let recoveredHandle: RunnerProcessHandle;
|
||||||
|
|
@ -4825,7 +4870,12 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
const authenticated = (async () => {
|
const authenticated = (async () => {
|
||||||
while (!processSettled && !this.#closed && Date.now() < deadline) {
|
while (
|
||||||
|
!processSettled &&
|
||||||
|
!this.#closed &&
|
||||||
|
this.#failure === null &&
|
||||||
|
Date.now() < deadline
|
||||||
|
) {
|
||||||
if (
|
if (
|
||||||
this.#core !== null &&
|
this.#core !== null &&
|
||||||
this.#core.store.state.connectionCount > priorConnectionCount &&
|
this.#core.store.state.connectionCount > priorConnectionCount &&
|
||||||
|
|
@ -4861,8 +4911,13 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
|
||||||
this.#rejectFailureSignal(error);
|
this.#rejectFailureSignal(error);
|
||||||
if (this.#pump !== null) clearInterval(this.#pump);
|
if (this.#pump !== null) clearInterval(this.#pump);
|
||||||
this.#pump = null;
|
this.#pump = null;
|
||||||
this.#diagnostic(error.message);
|
try {
|
||||||
this.#queue.close(error);
|
this.#diagnostic(error.message);
|
||||||
|
} finally {
|
||||||
|
// An observer is not allowed to leave notification consumers waiting
|
||||||
|
// after the request path has already received this terminal failure.
|
||||||
|
this.#queue.close(error);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#throwIfFailed(): void {
|
#throwIfFailed(): void {
|
||||||
|
|
|
||||||
|
|
@ -8,6 +8,11 @@ import type {
|
||||||
NativeSessionBackend,
|
NativeSessionBackend,
|
||||||
PersistedNativeSession,
|
PersistedNativeSession,
|
||||||
} from "./contracts/native-session-backend.js";
|
} from "./contracts/native-session-backend.js";
|
||||||
|
import {
|
||||||
|
NativeSessionCloseUnrecoverableError,
|
||||||
|
NativeSessionCleanupQuarantinedError,
|
||||||
|
NativeSessionProtocolIntegrityError,
|
||||||
|
} from "./contracts/native-session-backend.js";
|
||||||
import type {
|
import type {
|
||||||
PrpEvent,
|
PrpEvent,
|
||||||
PrpStructuredRunResult,
|
PrpStructuredRunResult,
|
||||||
|
|
@ -195,6 +200,34 @@ function highestContiguous(events: PrpEvent[]): number {
|
||||||
}
|
}
|
||||||
|
|
||||||
describe("executeNativeSession recovery", () => {
|
describe("executeNativeSession recovery", () => {
|
||||||
|
it.each([false, true])("preserves a durable session failure when its stream closes (throws=%s)", async throws => {
|
||||||
|
const capabilities = { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };
|
||||||
|
const session: NativeSession = {
|
||||||
|
identity: () => identity,
|
||||||
|
async capabilities() { return capabilities; },
|
||||||
|
async *events() {
|
||||||
|
yield runnerEvent(1, "session.failed", { error: { code: "notification_transport_failed" }, recoverable: false });
|
||||||
|
if (throws) throw new Error("provider stdout closed");
|
||||||
|
},
|
||||||
|
async startTurn() { return { turnId: "turn-recovery" }; },
|
||||||
|
async result() { return null; },
|
||||||
|
async snapshot() { return { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [] }; },
|
||||||
|
async close() {},
|
||||||
|
};
|
||||||
|
const completeRun = vi.fn();
|
||||||
|
const backend: NativeSessionBackend = {
|
||||||
|
async descriptor() { return { kind: "mock", name: "failure-fixture", version: "1", capabilities }; },
|
||||||
|
async openSession() { return session; },
|
||||||
|
};
|
||||||
|
const port: ControlPlanePort = {
|
||||||
|
async openRun() {}, async checkpointSession() {},
|
||||||
|
async appendEvent() { return { cursor: 1, highestContiguousSourceSeq: 1, disposition: "committed" }; },
|
||||||
|
async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, completeRun,
|
||||||
|
};
|
||||||
|
await expect(executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery" }))
|
||||||
|
.rejects.toMatchObject({ code: "native_provider_terminal_failed", providerCode: "notification_transport_failed", recoverable: false });
|
||||||
|
expect(completeRun).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
it.each((["complete", "paused", "blocked", "limited", "usageLimited", "budgetLimited"] as const)
|
it.each((["complete", "paused", "blocked", "limited", "usageLimited", "budgetLimited"] as const)
|
||||||
.flatMap((status) => [false, true].map((snapshotBeforeUpdate) => ({ status, snapshotBeforeUpdate }))))(
|
.flatMap((status) => [false, true].map((snapshotBeforeUpdate) => ({ status, snapshotBeforeUpdate }))))(
|
||||||
"handles a new chat turn instead of completing it from an existing $status goal (snapshot: $snapshotBeforeUpdate)", async ({ status, snapshotBeforeUpdate }) => {
|
"handles a new chat turn instead of completing it from an existing $status goal (snapshot: $snapshotBeforeUpdate)", async ({ status, snapshotBeforeUpdate }) => {
|
||||||
|
|
@ -751,13 +784,13 @@ describe("executeNativeSession recovery", () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("surfaces the provider's model rejection instead of missing semantic completion", async () => {
|
it.each([false, true])("preserves structured provider failure even when its message mentions a model (recoverable=%s)", async (recoverable) => {
|
||||||
const capabilities = { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true };
|
const capabilities = { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true };
|
||||||
const close = vi.fn(async () => {});
|
const close = vi.fn(async () => {});
|
||||||
const session: NativeSession = {
|
const session: NativeSession = {
|
||||||
identity: () => identity,
|
identity: () => identity,
|
||||||
async capabilities() { return capabilities; },
|
async capabilities() { return capabilities; },
|
||||||
async *events() { yield runnerEvent(1, "turn.failed", { error: { code: "RUNTIME", message: "There's an issue with the selected model (custom-model). It may not exist or you may not have access to it." } }); },
|
async *events() { yield runnerEvent(1, "turn.failed", { error: { code: "RUNTIME", recoverable, message: "There's an issue with the selected model (custom-model). It may not exist or you may not have access to it." } }); },
|
||||||
async startTurn() { return { turnId: "turn-recovery" }; },
|
async startTurn() { return { turnId: "turn-recovery" }; },
|
||||||
async result() { return null; },
|
async result() { return null; },
|
||||||
async snapshot() { return { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [] }; },
|
async snapshot() { return { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [] }; },
|
||||||
|
|
@ -773,7 +806,11 @@ describe("executeNativeSession recovery", () => {
|
||||||
async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },
|
async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },
|
||||||
async completeRun() {},
|
async completeRun() {},
|
||||||
};
|
};
|
||||||
await expect(executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery" })).rejects.toThrow("native_provider_model_rejected: There's an issue with the selected model (custom-model)");
|
const result = executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery" });
|
||||||
|
await expect(result).rejects.toThrow("There's an issue with the selected model (custom-model)");
|
||||||
|
await expect(result).rejects.toMatchObject({
|
||||||
|
code: "native_provider_terminal_failed", providerCode: "RUNTIME", recoverable,
|
||||||
|
});
|
||||||
expect(close).toHaveBeenCalled();
|
expect(close).toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -2500,6 +2537,247 @@ describe("executeNativeSession recovery", () => {
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
{ boundary: "control-plane replay", fault: "typed", admitted: false },
|
||||||
|
{ boundary: "final event append", fault: "typed", admitted: false },
|
||||||
|
{
|
||||||
|
boundary: "lost final event acknowledgement",
|
||||||
|
fault: "typed",
|
||||||
|
admitted: false,
|
||||||
|
},
|
||||||
|
{ boundary: "control-plane replay", fault: "lookalike", admitted: true },
|
||||||
|
{ boundary: "control-plane replay", fault: "generic", admitted: true },
|
||||||
|
{
|
||||||
|
boundary: "completion invoked before commit",
|
||||||
|
fault: "typed",
|
||||||
|
admitted: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
boundary: "lost completion acknowledgement",
|
||||||
|
fault: "typed",
|
||||||
|
admitted: true,
|
||||||
|
},
|
||||||
|
] as const)(
|
||||||
|
"observes integrity before completion admission without revoking admitted completion (%j)",
|
||||||
|
async ({ boundary, fault, admitted }) => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
let releaseBoundary = () => {};
|
||||||
|
const boundaryReleased = new Promise<void>((resolve) => {
|
||||||
|
releaseBoundary = resolve;
|
||||||
|
});
|
||||||
|
let markBoundaryReached = () => {};
|
||||||
|
const boundaryReached = new Promise<void>((resolve) => {
|
||||||
|
markBoundaryReached = resolve;
|
||||||
|
});
|
||||||
|
const failure =
|
||||||
|
fault === "typed"
|
||||||
|
? new NativeSessionProtocolIntegrityError(
|
||||||
|
"semantic_input_digest_mismatch",
|
||||||
|
)
|
||||||
|
: fault === "lookalike"
|
||||||
|
? Object.assign(new Error("untrusted transport error"), {
|
||||||
|
code: "native_event_replay_conflict",
|
||||||
|
reason: "semantic_input_digest_mismatch",
|
||||||
|
})
|
||||||
|
: new Error("snapshot temporarily unavailable");
|
||||||
|
let latchedFailure: Error | null = null;
|
||||||
|
let boundaryBlocked = false;
|
||||||
|
const waitAtBoundary = async () => {
|
||||||
|
if (boundaryBlocked) return;
|
||||||
|
boundaryBlocked = true;
|
||||||
|
markBoundaryReached();
|
||||||
|
await boundaryReleased;
|
||||||
|
};
|
||||||
|
const events: PrpEvent[] = [];
|
||||||
|
let durableCompletion: unknown = null;
|
||||||
|
const close = vi.fn(async () => undefined);
|
||||||
|
const resolveResult = vi.fn(async () => ({
|
||||||
|
result,
|
||||||
|
terminal,
|
||||||
|
turnId: "turn-recovery",
|
||||||
|
}));
|
||||||
|
const session: NativeSession = {
|
||||||
|
identity: () => identity,
|
||||||
|
async capabilities() {
|
||||||
|
return {
|
||||||
|
resume: true,
|
||||||
|
typedEvents: true,
|
||||||
|
steering: false,
|
||||||
|
interruption: true,
|
||||||
|
structuredResult: true,
|
||||||
|
};
|
||||||
|
},
|
||||||
|
async *events() {
|
||||||
|
yield runnerEvent(1, "turn.completed");
|
||||||
|
},
|
||||||
|
async startTurn() {
|
||||||
|
return { turnId: "turn-recovery" };
|
||||||
|
},
|
||||||
|
result: resolveResult,
|
||||||
|
async snapshot() {
|
||||||
|
if (latchedFailure !== null) throw latchedFailure;
|
||||||
|
return {
|
||||||
|
backendKind: "mock",
|
||||||
|
sessionId: identity.sessionId,
|
||||||
|
identity,
|
||||||
|
providerSessionId: "provider-recovery",
|
||||||
|
cursor: "1",
|
||||||
|
activeTurnId: null,
|
||||||
|
pendingRuntimeRequests: [],
|
||||||
|
lineage: [],
|
||||||
|
};
|
||||||
|
},
|
||||||
|
close,
|
||||||
|
};
|
||||||
|
const backend: NativeSessionBackend = {
|
||||||
|
async descriptor() {
|
||||||
|
return {
|
||||||
|
kind: "mock",
|
||||||
|
name: `completion-integrity-${boundary}-${fault}`,
|
||||||
|
version: "1",
|
||||||
|
capabilities: await session.capabilities(),
|
||||||
|
};
|
||||||
|
},
|
||||||
|
async openSession() {
|
||||||
|
return session;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const completeRun = vi.fn<ControlPlanePort["completeRun"]>(
|
||||||
|
async (completion) => {
|
||||||
|
if (boundary === "completion invoked before commit")
|
||||||
|
await waitAtBoundary();
|
||||||
|
if (durableCompletion === null)
|
||||||
|
durableCompletion = structuredClone(completion);
|
||||||
|
else expect(completion).toEqual(durableCompletion);
|
||||||
|
if (
|
||||||
|
boundary === "lost completion acknowledgement" &&
|
||||||
|
!boundaryBlocked
|
||||||
|
) {
|
||||||
|
await waitAtBoundary();
|
||||||
|
await new Promise<never>(() => undefined);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const checkpointSession = vi.fn<ControlPlanePort["checkpointSession"]>(
|
||||||
|
async () => undefined,
|
||||||
|
);
|
||||||
|
const port: ControlPlanePort = {
|
||||||
|
async openRun() {},
|
||||||
|
checkpointSession,
|
||||||
|
async appendEvent(rawEvent) {
|
||||||
|
const event = structuredClone(rawEvent as PrpEvent);
|
||||||
|
const existing = events.some(
|
||||||
|
(candidate) =>
|
||||||
|
candidate.sourceInstanceId === event.sourceInstanceId &&
|
||||||
|
candidate.sourceSeq === event.sourceSeq,
|
||||||
|
);
|
||||||
|
if (!existing) events.push(event);
|
||||||
|
if (
|
||||||
|
boundary === "final event append" &&
|
||||||
|
event.eventType === "run.terminal"
|
||||||
|
) {
|
||||||
|
await waitAtBoundary();
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
boundary === "lost final event acknowledgement" &&
|
||||||
|
event.eventType === "run.terminal" &&
|
||||||
|
!boundaryBlocked
|
||||||
|
) {
|
||||||
|
await waitAtBoundary();
|
||||||
|
await new Promise<never>(() => undefined);
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
cursor: events.length,
|
||||||
|
highestContiguousSourceSeq: highestContiguous(
|
||||||
|
events.filter(
|
||||||
|
(candidate) =>
|
||||||
|
candidate.sourceInstanceId === event.sourceInstanceId,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
disposition: existing ? "duplicate" : "committed",
|
||||||
|
};
|
||||||
|
},
|
||||||
|
async replayEvents(replay) {
|
||||||
|
if (
|
||||||
|
boundary === "control-plane replay" &&
|
||||||
|
replay.sourceInstanceId === "control-recovery"
|
||||||
|
) {
|
||||||
|
await waitAtBoundary();
|
||||||
|
}
|
||||||
|
const sourceEvents = events.filter(
|
||||||
|
(event) => event.sourceInstanceId === replay.sourceInstanceId,
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
events: structuredClone(
|
||||||
|
sourceEvents.filter(
|
||||||
|
(event) => event.sourceSeq > replay.afterSourceSeq,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
highestContiguousSourceSeq: highestContiguous(sourceEvents),
|
||||||
|
};
|
||||||
|
},
|
||||||
|
completeRun,
|
||||||
|
};
|
||||||
|
const outcome = executeNativeSession({
|
||||||
|
input,
|
||||||
|
backend,
|
||||||
|
controlPlane: port,
|
||||||
|
runnerInstanceId: "runner-recovery",
|
||||||
|
controlPlaneInstanceId: "control-recovery",
|
||||||
|
timeoutMs: 10,
|
||||||
|
requireSessionCloseBeforeReturn: true,
|
||||||
|
}).then(
|
||||||
|
(value) => ({ value, error: null }),
|
||||||
|
(error: unknown) => ({ value: null, error }),
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
await boundaryReached;
|
||||||
|
const checkpointsBeforeFault = checkpointSession.mock.calls.length;
|
||||||
|
if (boundary === "completion invoked before commit") {
|
||||||
|
expect(completeRun).toHaveBeenCalledOnce();
|
||||||
|
expect(durableCompletion).toBeNull();
|
||||||
|
}
|
||||||
|
latchedFailure = failure;
|
||||||
|
releaseBoundary();
|
||||||
|
if (
|
||||||
|
boundary === "lost completion acknowledgement" ||
|
||||||
|
boundary === "lost final event acknowledgement"
|
||||||
|
) {
|
||||||
|
await vi.advanceTimersByTimeAsync(10);
|
||||||
|
}
|
||||||
|
const settled = await outcome;
|
||||||
|
if (!admitted) {
|
||||||
|
expect(settled.error).toBe(failure);
|
||||||
|
expect(settled.value).toBeNull();
|
||||||
|
expect(completeRun).not.toHaveBeenCalled();
|
||||||
|
expect(durableCompletion).toBeNull();
|
||||||
|
} else {
|
||||||
|
expect(settled.error).toBeNull();
|
||||||
|
expect(settled.value).toMatchObject({
|
||||||
|
result,
|
||||||
|
terminal,
|
||||||
|
nativeEventCount: 3,
|
||||||
|
});
|
||||||
|
expect(durableCompletion).toMatchObject({ result, terminal });
|
||||||
|
expect(completeRun).toHaveBeenCalledTimes(
|
||||||
|
boundary === "lost completion acknowledgement" ? 2 : 1,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
expect(
|
||||||
|
events.filter((event) => event.sourceKind === "control_plane"),
|
||||||
|
).toHaveLength(2);
|
||||||
|
expect(checkpointSession).toHaveBeenCalledTimes(checkpointsBeforeFault);
|
||||||
|
expect(resolveResult).toHaveBeenCalledOnce();
|
||||||
|
expect(close).toHaveBeenCalledOnce();
|
||||||
|
} finally {
|
||||||
|
releaseBoundary();
|
||||||
|
await vi.advanceTimersByTimeAsync(30);
|
||||||
|
await outcome;
|
||||||
|
vi.useRealTimers();
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
it.each([
|
it.each([
|
||||||
"provider snapshot",
|
"provider snapshot",
|
||||||
"post-completion checkpoint",
|
"post-completion checkpoint",
|
||||||
|
|
@ -3383,6 +3661,122 @@ describe("executeNativeSession recovery", () => {
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
{ typed: true, closeFails: false },
|
||||||
|
{ typed: true, closeFails: true },
|
||||||
|
{ typed: false, closeFails: true },
|
||||||
|
{ typed: true, closeFails: true, startupRace: true },
|
||||||
|
])(
|
||||||
|
"preserves a permanent integrity failure through required cleanup (%j)",
|
||||||
|
async ({ typed, closeFails, startupRace = false }) => {
|
||||||
|
const failure = typed
|
||||||
|
? new NativeSessionProtocolIntegrityError(
|
||||||
|
"semantic_input_digest_mismatch",
|
||||||
|
)
|
||||||
|
: Object.assign(new Error("ordinary provider connection failed"), {
|
||||||
|
code: "native_event_replay_conflict",
|
||||||
|
recovery: "operator_required",
|
||||||
|
});
|
||||||
|
const closeFailure = new NativeSessionCloseUnrecoverableError();
|
||||||
|
let observeClose = () => {};
|
||||||
|
const closeStarted = new Promise<void>((resolve) => {
|
||||||
|
observeClose = resolve;
|
||||||
|
});
|
||||||
|
const close = vi.fn(async () => {
|
||||||
|
observeClose();
|
||||||
|
if (closeFails) throw closeFailure;
|
||||||
|
});
|
||||||
|
const capabilities = {
|
||||||
|
resume: true,
|
||||||
|
typedEvents: true,
|
||||||
|
steering: false,
|
||||||
|
interruption: false,
|
||||||
|
structuredResult: true,
|
||||||
|
};
|
||||||
|
const session: NativeSession = {
|
||||||
|
identity: () => identity,
|
||||||
|
capabilities: async () => capabilities,
|
||||||
|
async *events() {
|
||||||
|
throw failure;
|
||||||
|
},
|
||||||
|
startTurn: async () => {
|
||||||
|
if (startupRace) {
|
||||||
|
await closeStarted;
|
||||||
|
throw new Error(
|
||||||
|
"provider_transport_failed: startup raced with close",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return { turnId: "turn-recovery" };
|
||||||
|
},
|
||||||
|
result: vi.fn(async () => null),
|
||||||
|
snapshot: async () => ({
|
||||||
|
backendKind: "mock",
|
||||||
|
sessionId: "driver-integrity",
|
||||||
|
identity,
|
||||||
|
providerSessionId: "provider-integrity",
|
||||||
|
cursor: "0",
|
||||||
|
activeTurnId: null,
|
||||||
|
pendingRuntimeRequests: [],
|
||||||
|
lineage: [],
|
||||||
|
}),
|
||||||
|
close,
|
||||||
|
};
|
||||||
|
const backend: NativeSessionBackend = {
|
||||||
|
descriptor: async () => ({
|
||||||
|
kind: "mock",
|
||||||
|
name: `integrity-${typed}-${closeFails}-${startupRace}`,
|
||||||
|
version: "1",
|
||||||
|
capabilities,
|
||||||
|
}),
|
||||||
|
openSession: vi.fn(async () => session),
|
||||||
|
};
|
||||||
|
const events: PrpEvent[] = [];
|
||||||
|
const port: ControlPlanePort = {
|
||||||
|
openRun: async () => {},
|
||||||
|
checkpointSession: async () => {},
|
||||||
|
appendEvent: async (event) => {
|
||||||
|
events.push(structuredClone(event as PrpEvent));
|
||||||
|
return {
|
||||||
|
cursor: events.length,
|
||||||
|
highestContiguousSourceSeq: highestContiguous(events),
|
||||||
|
disposition: "committed",
|
||||||
|
};
|
||||||
|
},
|
||||||
|
replayEvents: async () => ({
|
||||||
|
events: [],
|
||||||
|
highestContiguousSourceSeq: 0,
|
||||||
|
}),
|
||||||
|
completeRun: vi.fn(async () => {}),
|
||||||
|
};
|
||||||
|
const onSession = vi.fn();
|
||||||
|
const options: ExecuteNativeSessionOptions = {
|
||||||
|
input,
|
||||||
|
backend,
|
||||||
|
controlPlane: port,
|
||||||
|
runnerInstanceId: "runner-recovery",
|
||||||
|
controlPlaneInstanceId: "control-recovery",
|
||||||
|
onSession,
|
||||||
|
requireSessionCloseBeforeReturn: true,
|
||||||
|
timeoutMs: 900_000,
|
||||||
|
};
|
||||||
|
await expect(executeNativeSession(options)).rejects.toBe(
|
||||||
|
typed ? failure : closeFailure,
|
||||||
|
);
|
||||||
|
expect(close).toHaveBeenCalledOnce();
|
||||||
|
expect(onSession).toHaveBeenLastCalledWith(null);
|
||||||
|
expect(port.completeRun).not.toHaveBeenCalled();
|
||||||
|
expect(session.result).not.toHaveBeenCalled();
|
||||||
|
expect(events.some((event) => event.sourceKind === "runner")).toBe(false);
|
||||||
|
if (closeFails) {
|
||||||
|
await expect(executeNativeSession(options)).rejects.toBeInstanceOf(
|
||||||
|
NativeSessionCleanupQuarantinedError,
|
||||||
|
);
|
||||||
|
expect(backend.openSession).toHaveBeenCalledOnce();
|
||||||
|
expect(close).toHaveBeenCalledOnce();
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
it("propagates an exhausted required backend checkpoint close", async () => {
|
it("propagates an exhausted required backend checkpoint close", async () => {
|
||||||
vi.useFakeTimers();
|
vi.useFakeTimers();
|
||||||
try {
|
try {
|
||||||
|
|
@ -5428,9 +5822,11 @@ describe("executeNativeSession recovery", () => {
|
||||||
runnerInstanceId: "runner-constrained-recovery",
|
runnerInstanceId: "runner-constrained-recovery",
|
||||||
controlPlaneInstanceId: "control-constrained-recovery",
|
controlPlaneInstanceId: "control-constrained-recovery",
|
||||||
}),
|
}),
|
||||||
).rejects.toThrow(
|
).rejects.toMatchObject({
|
||||||
"native_session_recovery_failed: provider session ended with a failed terminal",
|
code: "native_provider_terminal_failed",
|
||||||
);
|
providerCode: "provider_checkpoint_failed_terminal",
|
||||||
|
recoverable: false,
|
||||||
|
});
|
||||||
|
|
||||||
expect(recoverSession).not.toHaveBeenCalled();
|
expect(recoverSession).not.toHaveBeenCalled();
|
||||||
expect(openSession).not.toHaveBeenCalled();
|
expect(openSession).not.toHaveBeenCalled();
|
||||||
|
|
|
||||||
|
|
@ -17,12 +17,18 @@ import type {
|
||||||
NativeSessionBackend,
|
NativeSessionBackend,
|
||||||
} from "./contracts/native-session-backend.js";
|
} from "./contracts/native-session-backend.js";
|
||||||
import type { PersistedNativeSession } from "./contracts/native-session-backend.js";
|
import type { PersistedNativeSession } from "./contracts/native-session-backend.js";
|
||||||
import type { HarnessThreadGoal } from "./contracts/harness-driver.js";
|
import {
|
||||||
import type {
|
NativeProviderTerminalFailure,
|
||||||
PrpEvent,
|
NativeSessionCloseUnrecoverableError,
|
||||||
PrpStructuredRunResult,
|
NativeSessionCleanupQuarantinedError,
|
||||||
PrpTerminalState,
|
NativeSessionProtocolIntegrityError,
|
||||||
|
} from "./contracts/native-session-backend.js";
|
||||||
|
import {
|
||||||
|
type PrpEvent,
|
||||||
|
type PrpStructuredRunResult,
|
||||||
|
type PrpTerminalState,
|
||||||
} from "./protocol/replay-contract.js";
|
} from "./protocol/replay-contract.js";
|
||||||
|
import type { HarnessThreadGoal } from "./contracts/harness-driver.js";
|
||||||
import { validatePrpStructuredRunResult } from "./protocol/replay-contract.js";
|
import { validatePrpStructuredRunResult } from "./protocol/replay-contract.js";
|
||||||
import { parsePaperclipQuestionSet } from "./contracts/question-set.js";
|
import { parsePaperclipQuestionSet } from "./contracts/question-set.js";
|
||||||
|
|
||||||
|
|
@ -70,6 +76,7 @@ interface QuarantinedSessionCleanup {
|
||||||
recovery: Promise<void> | null;
|
recovery: Promise<void> | null;
|
||||||
recoveryMaxAttempts: number | null;
|
recoveryMaxAttempts: number | null;
|
||||||
timer: ReturnType<typeof setTimeout> | null;
|
timer: ReturnType<typeof setTimeout> | null;
|
||||||
|
operatorRecoveryRequired: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
const quarantinedSessionCleanups = new Set<QuarantinedSessionCleanup>();
|
const quarantinedSessionCleanups = new Set<QuarantinedSessionCleanup>();
|
||||||
|
|
@ -349,6 +356,10 @@ function retainUnadmittedSessionCleanup(
|
||||||
await attempt;
|
await attempt;
|
||||||
return;
|
return;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
if (error instanceof NativeSessionCloseUnrecoverableError) {
|
||||||
|
quarantineSessionCleanup(session, cleanupDomain, true);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
if (retryCount >= MAX_FAILED_SESSION_CLOSE_RETRIES) {
|
if (retryCount >= MAX_FAILED_SESSION_CLOSE_RETRIES) {
|
||||||
quarantineSessionCleanup(session, cleanupDomain);
|
quarantineSessionCleanup(session, cleanupDomain);
|
||||||
throw error;
|
throw error;
|
||||||
|
|
@ -508,10 +519,17 @@ function retainFailedSessionCleanupOwner(
|
||||||
function quarantineSessionCleanup(
|
function quarantineSessionCleanup(
|
||||||
session: NativeSession,
|
session: NativeSession,
|
||||||
cleanupDomain: NativeSessionCleanupDomain,
|
cleanupDomain: NativeSessionCleanupDomain,
|
||||||
|
operatorRecoveryRequired = false,
|
||||||
): void {
|
): void {
|
||||||
if (
|
const existing = [...quarantinedSessionCleanups].find(
|
||||||
[...quarantinedSessionCleanups].some((entry) => entry.session === session)
|
(entry) => entry.session === session,
|
||||||
) {
|
);
|
||||||
|
if (existing) {
|
||||||
|
if (operatorRecoveryRequired) {
|
||||||
|
existing.operatorRecoveryRequired = true;
|
||||||
|
if (existing.timer) clearTimeout(existing.timer);
|
||||||
|
existing.timer = null;
|
||||||
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const cleanup: QuarantinedSessionCleanup = {
|
const cleanup: QuarantinedSessionCleanup = {
|
||||||
|
|
@ -522,8 +540,10 @@ function quarantineSessionCleanup(
|
||||||
recovery: null,
|
recovery: null,
|
||||||
recoveryMaxAttempts: null,
|
recoveryMaxAttempts: null,
|
||||||
timer: null,
|
timer: null,
|
||||||
|
operatorRecoveryRequired,
|
||||||
};
|
};
|
||||||
quarantinedSessionCleanups.add(cleanup);
|
quarantinedSessionCleanups.add(cleanup);
|
||||||
|
if (operatorRecoveryRequired) return;
|
||||||
startQuarantinedSessionCleanupRecovery(
|
startQuarantinedSessionCleanupRecovery(
|
||||||
cleanup,
|
cleanup,
|
||||||
MAX_QUARANTINED_SESSION_CLOSE_RETRIES,
|
MAX_QUARANTINED_SESSION_CLOSE_RETRIES,
|
||||||
|
|
@ -537,6 +557,7 @@ function startQuarantinedSessionCleanupRecovery(
|
||||||
reason: string,
|
reason: string,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
if (cleanup.recovery) return cleanup.recovery;
|
if (cleanup.recovery) return cleanup.recovery;
|
||||||
|
if (cleanup.operatorRecoveryRequired) return Promise.resolve();
|
||||||
const remainingAttempts =
|
const remainingAttempts =
|
||||||
MAX_QUARANTINED_SESSION_AUTOMATIC_CLOSE_ATTEMPTS -
|
MAX_QUARANTINED_SESSION_AUTOMATIC_CLOSE_ATTEMPTS -
|
||||||
cleanup.automaticAttempts;
|
cleanup.automaticAttempts;
|
||||||
|
|
@ -546,7 +567,8 @@ function startQuarantinedSessionCleanupRecovery(
|
||||||
for (
|
for (
|
||||||
let attemptCount = 0;
|
let attemptCount = 0;
|
||||||
attemptCount < boundedMaxAttempts &&
|
attemptCount < boundedMaxAttempts &&
|
||||||
quarantinedSessionCleanups.has(cleanup);
|
quarantinedSessionCleanups.has(cleanup) &&
|
||||||
|
!cleanup.operatorRecoveryRequired;
|
||||||
attemptCount += 1
|
attemptCount += 1
|
||||||
) {
|
) {
|
||||||
// The first retry starts immediately so an admission-triggered recovery
|
// The first retry starts immediately so an admission-triggered recovery
|
||||||
|
|
@ -563,7 +585,10 @@ function startQuarantinedSessionCleanupRecovery(
|
||||||
try {
|
try {
|
||||||
await attempt;
|
await attempt;
|
||||||
quarantinedSessionCleanups.delete(cleanup);
|
quarantinedSessionCleanups.delete(cleanup);
|
||||||
} catch {
|
} catch (error) {
|
||||||
|
if (error instanceof NativeSessionCloseUnrecoverableError) {
|
||||||
|
cleanup.operatorRecoveryRequired = true;
|
||||||
|
}
|
||||||
// Retain the quarantine after this finite, sequential retry batch.
|
// Retain the quarantine after this finite, sequential retry batch.
|
||||||
} finally {
|
} finally {
|
||||||
if (cleanup.attempt === attempt) cleanup.attempt = null;
|
if (cleanup.attempt === attempt) cleanup.attempt = null;
|
||||||
|
|
@ -591,6 +616,7 @@ function scheduleQuarantinedSessionCleanup(
|
||||||
): void {
|
): void {
|
||||||
if (
|
if (
|
||||||
!quarantinedSessionCleanups.has(cleanup) ||
|
!quarantinedSessionCleanups.has(cleanup) ||
|
||||||
|
cleanup.operatorRecoveryRequired ||
|
||||||
cleanup.recovery ||
|
cleanup.recovery ||
|
||||||
cleanup.attempt ||
|
cleanup.attempt ||
|
||||||
cleanup.timer ||
|
cleanup.timer ||
|
||||||
|
|
@ -628,6 +654,14 @@ async function retryQuarantinedSessionCleanups(
|
||||||
let observedOwnerPhases = 0;
|
let observedOwnerPhases = 0;
|
||||||
while (true) {
|
while (true) {
|
||||||
signal?.throwIfAborted();
|
signal?.throwIfAborted();
|
||||||
|
if (
|
||||||
|
[...quarantinedSessionCleanups].some(
|
||||||
|
(cleanup) =>
|
||||||
|
cleanup.domain === cleanupDomain && cleanup.operatorRecoveryRequired,
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
throw new NativeSessionCleanupQuarantinedError();
|
||||||
|
}
|
||||||
const cleanupOwners = new Set<Promise<void>>(
|
const cleanupOwners = new Set<Promise<void>>(
|
||||||
[...failedSessionCleanupOwners]
|
[...failedSessionCleanupOwners]
|
||||||
.filter(([, domain]) => domain === cleanupDomain)
|
.filter(([, domain]) => domain === cleanupDomain)
|
||||||
|
|
@ -785,6 +819,7 @@ async function consumeTurn(
|
||||||
typeof semanticResultGraceExpired
|
typeof semanticResultGraceExpired
|
||||||
> | null = null;
|
> | null = null;
|
||||||
let pendingNext: ReturnType<typeof eventIterator.next> | null = null;
|
let pendingNext: ReturnType<typeof eventIterator.next> | null = null;
|
||||||
|
let providerFailure: NativeProviderTerminalFailure | null = null;
|
||||||
const settleDurableResult = (
|
const settleDurableResult = (
|
||||||
event: PrpEvent,
|
event: PrpEvent,
|
||||||
result: PrpStructuredRunResult,
|
result: PrpStructuredRunResult,
|
||||||
|
|
@ -812,7 +847,7 @@ async function consumeTurn(
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
while (true) {
|
while (true) {
|
||||||
pendingNext ??= eventIterator.next();
|
pendingNext ??= eventIterator.next().catch(error => { throw providerFailure ?? error; });
|
||||||
const next =
|
const next =
|
||||||
semanticResultDeadline === null
|
semanticResultDeadline === null
|
||||||
? await pendingNext
|
? await pendingNext
|
||||||
|
|
@ -843,6 +878,7 @@ async function consumeTurn(
|
||||||
governedResult,
|
governedResult,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
if (providerFailure) throw providerFailure;
|
||||||
throw new Error(
|
throw new Error(
|
||||||
"native event stream closed before a turn terminal fact",
|
"native event stream closed before a turn terminal fact",
|
||||||
);
|
);
|
||||||
|
|
@ -901,6 +937,15 @@ async function consumeTurn(
|
||||||
const validation = validatePrpStructuredRunResult(event.payload);
|
const validation = validatePrpStructuredRunResult(event.payload);
|
||||||
if (validation.ok) semanticResultProposal = validation.result;
|
if (validation.ok) semanticResultProposal = validation.result;
|
||||||
}
|
}
|
||||||
|
if (event.eventType === "session.failed") {
|
||||||
|
const failure = payload.error && typeof payload.error === "object"
|
||||||
|
? payload.error as Record<string, unknown> : payload;
|
||||||
|
providerFailure = new NativeProviderTerminalFailure(
|
||||||
|
typeof failure.code === "string" ? failure.code : "provider_session_failed",
|
||||||
|
failure.recoverable === true || payload.recoverable === true,
|
||||||
|
typeof failure.message === "string" ? failure.message : undefined,
|
||||||
|
);
|
||||||
|
}
|
||||||
const request =
|
const request =
|
||||||
payload.request &&
|
payload.request &&
|
||||||
typeof payload.request === "object" &&
|
typeof payload.request === "object" &&
|
||||||
|
|
@ -1811,6 +1856,9 @@ export async function executeNativeSession(
|
||||||
const failedProviderSession =
|
const failedProviderSession =
|
||||||
providerRecoveryCheckpoint.terminal?.runTerminalState === "failed" &&
|
providerRecoveryCheckpoint.terminal?.runTerminalState === "failed" &&
|
||||||
providerRecoveryCheckpoint.semanticResult === null;
|
providerRecoveryCheckpoint.semanticResult === null;
|
||||||
|
if (failedProviderSession && !replacementAllowed) {
|
||||||
|
throw new NativeProviderTerminalFailure("provider_checkpoint_failed_terminal", false);
|
||||||
|
}
|
||||||
const recovery = failedProviderSession
|
const recovery = failedProviderSession
|
||||||
? {
|
? {
|
||||||
recovered: false as const,
|
recovered: false as const,
|
||||||
|
|
@ -1969,6 +2017,10 @@ export async function executeNativeSession(
|
||||||
await attempt;
|
await attempt;
|
||||||
return;
|
return;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
if (error instanceof NativeSessionCloseUnrecoverableError) {
|
||||||
|
quarantineSessionCleanup(session, cleanupDomain, true);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
if (retryCount >= MAX_FAILED_SESSION_CLOSE_RETRIES) {
|
if (retryCount >= MAX_FAILED_SESSION_CLOSE_RETRIES) {
|
||||||
quarantineSessionCleanup(session, cleanupDomain);
|
quarantineSessionCleanup(session, cleanupDomain);
|
||||||
throw error;
|
throw error;
|
||||||
|
|
@ -2004,6 +2056,7 @@ export async function executeNativeSession(
|
||||||
let goalCheckpointRequiresSuspension = Boolean(
|
let goalCheckpointRequiresSuspension = Boolean(
|
||||||
options.sessionGoalControl || options.resumeSessionGoalHeartbeat || persistedSession?.goal,
|
options.sessionGoalControl || options.resumeSessionGoalHeartbeat || persistedSession?.goal,
|
||||||
);
|
);
|
||||||
|
let protocolIntegrityFailure: NativeSessionProtocolIntegrityError | null = null;
|
||||||
try {
|
try {
|
||||||
// Ownership publication is part of the execution-owned lifetime. If the
|
// Ownership publication is part of the execution-owned lifetime. If the
|
||||||
// callback fails, the finally block below still quarantines and closes the
|
// callback fails, the finally block below still quarantines and closes the
|
||||||
|
|
@ -2246,8 +2299,16 @@ export async function executeNativeSession(
|
||||||
// before joining cleanup so the failed turn cannot commit late or
|
// before joining cleanup so the failed turn cannot commit late or
|
||||||
// strand execution on a never-settling durability call.
|
// strand execution on a never-settling durability call.
|
||||||
consumptionAbort.abort(error);
|
consumptionAbort.abort(error);
|
||||||
await consuming.catch(() => undefined);
|
let startupFailure = error;
|
||||||
throw error;
|
await consuming.catch((consumptionError) => {
|
||||||
|
// A failed iterator may have already initiated close while start or
|
||||||
|
// checkpoint was pending. Retain the authenticated integrity fault,
|
||||||
|
// not the resulting transport/cleanup error from that race.
|
||||||
|
if (consumptionError instanceof NativeSessionProtocolIntegrityError) {
|
||||||
|
startupFailure = consumptionError;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
throw startupFailure;
|
||||||
}
|
}
|
||||||
const terminalEvent = await consuming;
|
const terminalEvent = await consuming;
|
||||||
consumed = terminalEvent;
|
consumed = terminalEvent;
|
||||||
|
|
@ -2279,6 +2340,17 @@ export async function executeNativeSession(
|
||||||
turnId: terminalEvent.turnId ?? null,
|
turnId: terminalEvent.turnId ?? null,
|
||||||
};
|
};
|
||||||
signal.throwIfAborted();
|
signal.throwIfAborted();
|
||||||
|
if (settledCompletion === null && terminalEvent.eventType === "turn.failed") {
|
||||||
|
await checkpoint(signal);
|
||||||
|
const payload = terminalEvent.payload as Record<string, unknown>;
|
||||||
|
const failure = payload.error && typeof payload.error === "object" ? payload.error as Record<string, unknown> : payload;
|
||||||
|
const message = typeof failure.message === "string" ? failure.message.slice(0, 2_000) : "Provider turn failed";
|
||||||
|
throw new NativeProviderTerminalFailure(
|
||||||
|
typeof failure.code === "string" ? failure.code : "provider_turn_failed",
|
||||||
|
failure.recoverable === true || payload.recoverable === true,
|
||||||
|
message,
|
||||||
|
);
|
||||||
|
}
|
||||||
if (settledCompletion === null && options.resolveMissingResult) {
|
if (settledCompletion === null && options.resolveMissingResult) {
|
||||||
const recoveredResult = await options.resolveMissingResult({
|
const recoveredResult = await options.resolveMissingResult({
|
||||||
turnId: terminalEvent.turnId ?? null,
|
turnId: terminalEvent.turnId ?? null,
|
||||||
|
|
@ -2304,13 +2376,6 @@ export async function executeNativeSession(
|
||||||
completed = settledCompletion;
|
completed = settledCompletion;
|
||||||
}
|
}
|
||||||
if (settledCompletion === null) {
|
if (settledCompletion === null) {
|
||||||
if (consumed.event?.eventType === "turn.failed") {
|
|
||||||
const providerError = objectRecord(objectRecord(consumed.event.payload)?.error);
|
|
||||||
const message = typeof providerError?.message === "string"
|
|
||||||
? providerError.message.slice(0, 2_000) : "Provider turn failed";
|
|
||||||
const modelRejected = /issue with the selected model|model_not_found|invalid model|model[^\n]*(?:does not exist|not found|not supported)/i.test(message);
|
|
||||||
throw new Error(`${modelRejected ? "native_provider_model_rejected" : "native_provider_turn_failed"}: ${message}`);
|
|
||||||
}
|
|
||||||
throw new Error(
|
throw new Error(
|
||||||
"native_finalization_missing: session returned no semantic result",
|
"native_finalization_missing: session returned no semantic result",
|
||||||
);
|
);
|
||||||
|
|
@ -2372,6 +2437,7 @@ export async function executeNativeSession(
|
||||||
const baselineControlEventSequences = new Set<number>();
|
const baselineControlEventSequences = new Set<number>();
|
||||||
const accountedControlEventSequences = new Set<number>();
|
const accountedControlEventSequences = new Set<number>();
|
||||||
let baselineControlReplayCaptured = false;
|
let baselineControlReplayCaptured = false;
|
||||||
|
let completionAdmissionStarted = false;
|
||||||
const durableExecutionResult = await finalizeIdempotentControlPlaneWithin({
|
const durableExecutionResult = await finalizeIdempotentControlPlaneWithin({
|
||||||
timeoutMs: finalizationTimeoutMs,
|
timeoutMs: finalizationTimeoutMs,
|
||||||
operation: async (signal) => {
|
operation: async (signal) => {
|
||||||
|
|
@ -2439,6 +2505,23 @@ export async function executeNativeSession(
|
||||||
receipt.highestContiguousSourceSeq,
|
receipt.highestContiguousSourceSeq,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
if (!completionAdmissionStarted) {
|
||||||
|
// Replay/appends can yield after the prepared result's last snapshot.
|
||||||
|
// Observe the driver's latched integrity fault once more before
|
||||||
|
// admitting completion; Codex snapshots inspect local state only.
|
||||||
|
try {
|
||||||
|
await session.snapshot({ signal });
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof NativeSessionProtocolIntegrityError) throw error;
|
||||||
|
// Generic snapshot failures remain checkpoint enrichment failures,
|
||||||
|
// not evidence that the already validated result is invalid.
|
||||||
|
}
|
||||||
|
signal.throwIfAborted();
|
||||||
|
// Invocation is the local admission boundary, not an atomic fence
|
||||||
|
// with the remote commit. A lost acknowledgement can mean committed
|
||||||
|
// success, so later faults must not veto its idempotent confirmation.
|
||||||
|
completionAdmissionStarted = true;
|
||||||
|
}
|
||||||
await options.controlPlane.completeRun(
|
await options.controlPlane.completeRun(
|
||||||
{
|
{
|
||||||
result: preparedFinalization.completed.result,
|
result: preparedFinalization.completed.result,
|
||||||
|
|
@ -2538,6 +2621,11 @@ export async function executeNativeSession(
|
||||||
};
|
};
|
||||||
executionSucceeded = true;
|
executionSucceeded = true;
|
||||||
return { ...durableExecutionResult, ...enrichment };
|
return { ...durableExecutionResult, ...enrichment };
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof NativeSessionProtocolIntegrityError) {
|
||||||
|
protocolIntegrityFailure = error;
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
} finally {
|
} finally {
|
||||||
const shouldClose =
|
const shouldClose =
|
||||||
!options.keepSessionOpen || !executionSucceeded || sessionQuarantined || goalCheckpointRequiresSuspension;
|
!options.keepSessionOpen || !executionSucceeded || sessionQuarantined || goalCheckpointRequiresSuspension;
|
||||||
|
|
@ -2555,7 +2643,14 @@ export async function executeNativeSession(
|
||||||
// Unlike ordinary provider cleanup, this close owns required remote
|
// Unlike ordinary provider cleanup, this close owns required remote
|
||||||
// checkpoint persistence. Exhausting its bounded recovery must fail
|
// checkpoint persistence. Exhausting its bounded recovery must fail
|
||||||
// the execution instead of converting the rejection into success.
|
// the execution instead of converting the rejection into success.
|
||||||
await requiredClose;
|
try {
|
||||||
|
await requiredClose;
|
||||||
|
} catch (closeError) {
|
||||||
|
// The exact cleanup owner remains retained/quarantined above. Its
|
||||||
|
// rejection must not turn permanent integrity failure into a
|
||||||
|
// generic retryable transport failure at the control-plane boundary.
|
||||||
|
throw protocolIntegrityFailure ?? closeError;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} else if (shouldClose && !failedCleanupDeferred) {
|
} else if (shouldClose && !failedCleanupDeferred) {
|
||||||
// A provider that ignores close must not keep execution pending forever.
|
// A provider that ignores close must not keep execution pending forever.
|
||||||
|
|
|
||||||
|
|
@ -76,32 +76,38 @@ export interface TypedEventFamilyCapability {
|
||||||
detailLevel: "summary" | "structured";
|
detailLevel: "summary" | "structured";
|
||||||
}
|
}
|
||||||
|
|
||||||
export type CanonicalProviderEventType =
|
export const CANONICAL_PROVIDER_EVENT_TYPES = [
|
||||||
| "plan.updated"
|
"harness.diagnostic",
|
||||||
| "tool.execution.started"
|
"plan.updated",
|
||||||
| "tool.execution.progressed"
|
"tool.execution.started",
|
||||||
| "tool.execution.completed"
|
"tool.execution.progressed",
|
||||||
| "research.started"
|
"tool.execution.completed",
|
||||||
| "research.progressed"
|
"research.started",
|
||||||
| "research.completed"
|
"research.progressed",
|
||||||
| "delegation.started"
|
"research.completed",
|
||||||
| "delegation.updated"
|
"delegation.started",
|
||||||
| "delegation.completed"
|
"delegation.updated",
|
||||||
| "model.route.changed"
|
"delegation.completed",
|
||||||
| "model.verification.updated"
|
"model.route.changed",
|
||||||
| "context.compacted"
|
"model.verification.updated",
|
||||||
| "artifact.viewed"
|
"context.compacted",
|
||||||
| "artifact.generated"
|
"artifact.viewed",
|
||||||
| "review.mode.changed"
|
"artifact.generated",
|
||||||
| "hook.started"
|
"review.mode.changed",
|
||||||
| "hook.completed"
|
"hook.started",
|
||||||
| "memory.citation.referenced"
|
"hook.completed",
|
||||||
| "safety.review.started"
|
"memory.citation.referenced",
|
||||||
| "safety.review.completed"
|
"safety.review.started",
|
||||||
| "terminal.input.sent"
|
"safety.review.completed",
|
||||||
| "wait.started"
|
"terminal.input.sent",
|
||||||
| "wait.completed"
|
"wait.started",
|
||||||
| "provider.notice.recorded";
|
"wait.completed",
|
||||||
|
"provider.notice.recorded",
|
||||||
|
] as const;
|
||||||
|
export type CanonicalProviderEventType = (typeof CANONICAL_PROVIDER_EVENT_TYPES)[number];
|
||||||
|
export function isCanonicalProviderEventType(value: unknown): value is CanonicalProviderEventType {
|
||||||
|
return CANONICAL_PROVIDER_EVENT_TYPES.some(type => value === type);
|
||||||
|
}
|
||||||
|
|
||||||
export interface CanonicalProviderEvent {
|
export interface CanonicalProviderEvent {
|
||||||
eventType: CanonicalProviderEventType;
|
eventType: CanonicalProviderEventType;
|
||||||
|
|
|
||||||
|
|
@ -2750,3 +2750,7 @@ export {
|
||||||
rewriteUrlHostToLoopback,
|
rewriteUrlHostToLoopback,
|
||||||
} from "./runtime-exposure/loopback-bind.js";
|
} from "./runtime-exposure/loopback-bind.js";
|
||||||
export { ACCOUNT_HANDLE_MAX_LENGTH, toAccountHandle } from "./account-handle.js";
|
export { ACCOUNT_HANDLE_MAX_LENGTH, toAccountHandle } from "./account-handle.js";
|
||||||
|
export type { ExecutionContinuationEnvelope } from "./types/execution-continuation.js";
|
||||||
|
export type { ExecutionProjection, ExecutionReconciliation } from "./types/execution-projection.js";
|
||||||
|
|
||||||
|
export { EXECUTION_RECONCILIATION_CAUSES, requiresExecutionReconciliation } from "./types/execution-projection.js";
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,52 @@
|
||||||
|
/** Server-authored context. Each message retains its author and trust boundary. */
|
||||||
|
export interface ExecutionContinuationEnvelope {
|
||||||
|
version: 1;
|
||||||
|
companyId: string;
|
||||||
|
issueId: string;
|
||||||
|
trigger: {
|
||||||
|
reason: string;
|
||||||
|
interactionId: string | null;
|
||||||
|
sourceRunId: string | null;
|
||||||
|
};
|
||||||
|
originCommentIds: string[];
|
||||||
|
objective: string;
|
||||||
|
messages: Array<{
|
||||||
|
id: string;
|
||||||
|
authorType: string;
|
||||||
|
authorId: string | null;
|
||||||
|
/** Run-authored Local CLI comments retain user attribution but are not human direction. */
|
||||||
|
createdByRunId?: string | null;
|
||||||
|
body: string;
|
||||||
|
createdAt: string;
|
||||||
|
updatedAt: string;
|
||||||
|
deleted: boolean;
|
||||||
|
sourceTrust: unknown;
|
||||||
|
}>;
|
||||||
|
interactionOutcomes: Array<{
|
||||||
|
id: string;
|
||||||
|
kind: string;
|
||||||
|
status: string;
|
||||||
|
result: unknown;
|
||||||
|
}>;
|
||||||
|
/** Only valid when resuming the provider session associated with this run. */
|
||||||
|
resumeDelta?: {
|
||||||
|
baseRunId: string;
|
||||||
|
messages: ExecutionContinuationEnvelope["messages"];
|
||||||
|
};
|
||||||
|
recoveryOutcomes?: Array<{ recoveryActionId: string; decision: unknown }>;
|
||||||
|
completedWork: string | null;
|
||||||
|
/** Completed mutations are context, never instructions to replay them. */
|
||||||
|
completedActions?: Array<{
|
||||||
|
runId: string;
|
||||||
|
receiptId: string;
|
||||||
|
operationId: string;
|
||||||
|
result: unknown;
|
||||||
|
}>;
|
||||||
|
unresolvedInteractionIds: string[];
|
||||||
|
coverage: {
|
||||||
|
kind: "full_task_history" | "task_history_delta";
|
||||||
|
baseRunId?: string;
|
||||||
|
throughCommentId: string | null;
|
||||||
|
summaryThroughCommentId: null;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,58 @@
|
||||||
|
/** Presentation of existing execution records, not a second task status machine. */
|
||||||
|
export interface ExecutionProjection {
|
||||||
|
phase:
|
||||||
|
| "working"
|
||||||
|
| "reconnecting"
|
||||||
|
| "retry_scheduled"
|
||||||
|
| "finishing"
|
||||||
|
| "recovery_needed"
|
||||||
|
| "waiting_for_access"
|
||||||
|
| "waiting_for_answer"
|
||||||
|
| "queued"
|
||||||
|
| "completed"
|
||||||
|
| "failed";
|
||||||
|
label: string;
|
||||||
|
cause: string | null;
|
||||||
|
lastConfirmedActivityAt: string | null;
|
||||||
|
retryAt: string | null;
|
||||||
|
attempt: number;
|
||||||
|
maxAttempts: number;
|
||||||
|
recoveryOwner: "agent" | "board" | null;
|
||||||
|
nextAction: string | null;
|
||||||
|
permittedActions: Array<"inspect_run" | "inspect_recovery">;
|
||||||
|
predecessorRunId: string | null;
|
||||||
|
successorRunId: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** These incidents require an explicit reconciliation, never a generic Retry. */
|
||||||
|
export const EXECUTION_RECONCILIATION_CAUSES = [
|
||||||
|
"uncertain_provider_action",
|
||||||
|
"uncertain_external_action",
|
||||||
|
"uncertain_control_plane_action",
|
||||||
|
"completed_action_context_missing",
|
||||||
|
"continuation_evidence_incomplete",
|
||||||
|
"execution_finalization_deadline_exceeded",
|
||||||
|
"execution_recovery_budget_exhausted",
|
||||||
|
"provider_effect_inventory_unavailable",
|
||||||
|
"provider_failure_meaning_unverified",
|
||||||
|
"provider_ownership_unverified",
|
||||||
|
"native_provider_terminal_failed",
|
||||||
|
"native_event_replay_conflict",
|
||||||
|
"native_session_cleanup_quarantined",
|
||||||
|
"native_session_retry_exhausted",
|
||||||
|
"native_restart_recovery_blocked",
|
||||||
|
"native_continuation_requires_reconciliation",
|
||||||
|
"legacy_execution_requires_reconciliation",
|
||||||
|
] as const;
|
||||||
|
export function requiresExecutionReconciliation(
|
||||||
|
cause: string | null | undefined,
|
||||||
|
): boolean {
|
||||||
|
return EXECUTION_RECONCILIATION_CAUSES.some((value) => value === cause);
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ExecutionReconciliation {
|
||||||
|
runId: string;
|
||||||
|
providerStopped: true;
|
||||||
|
actionOutcome: "completed" | "not_performed" | "mixed";
|
||||||
|
outcomeEvidence: string;
|
||||||
|
}
|
||||||
|
|
@ -159,6 +159,7 @@ export interface GitWorktreeBranchIncoherenceEvidence {
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface HeartbeatRun {
|
export interface HeartbeatRun {
|
||||||
|
execution?: import("./execution-projection.js").ExecutionProjection | null;
|
||||||
id: string;
|
id: string;
|
||||||
companyId: string;
|
companyId: string;
|
||||||
agentId: string;
|
agentId: string;
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
import type { ExecutionProjection } from "./execution-projection.js";
|
||||||
import type {
|
import type {
|
||||||
IssueCommentAuthorType,
|
IssueCommentAuthorType,
|
||||||
IssueCommentMetadataRowType,
|
IssueCommentMetadataRowType,
|
||||||
|
|
@ -783,6 +784,9 @@ export interface IssueChangeReceiptEntry {
|
||||||
export type IssueChanges = Record<string, IssueChangeReceiptEntry>;
|
export type IssueChanges = Record<string, IssueChangeReceiptEntry>;
|
||||||
|
|
||||||
export interface Issue {
|
export interface Issue {
|
||||||
|
activeRun?: { id: string; status: string; agentId: string; invocationSource: string;
|
||||||
|
triggerDetail: string | null; startedAt: Date | string | null; finishedAt: Date | string | null;
|
||||||
|
createdAt: Date | string; execution?: ExecutionProjection } | null;
|
||||||
id: string;
|
id: string;
|
||||||
companyId: string;
|
companyId: string;
|
||||||
projectId: string | null;
|
projectId: string | null;
|
||||||
|
|
@ -1474,6 +1478,7 @@ export interface IssueThreadInteractionBase extends IssueThreadInteractionActorF
|
||||||
issueId: string;
|
issueId: string;
|
||||||
kind: IssueThreadInteractionKind;
|
kind: IssueThreadInteractionKind;
|
||||||
idempotencyKey?: string | null;
|
idempotencyKey?: string | null;
|
||||||
|
originCommentIds?: string[];
|
||||||
sourceCommentId?: string | null;
|
sourceCommentId?: string | null;
|
||||||
sourceRunId?: string | null;
|
sourceRunId?: string | null;
|
||||||
sourceIdentityContextId?: string | null;
|
sourceIdentityContextId?: string | null;
|
||||||
|
|
|
||||||
|
|
@ -375,6 +375,12 @@ const RESOLVE_ISSUE_RECOVERY_ACTION_OUTCOMES = [
|
||||||
] as const;
|
] as const;
|
||||||
|
|
||||||
export const resolveIssueRecoveryActionSchema = z.object({
|
export const resolveIssueRecoveryActionSchema = z.object({
|
||||||
|
executionReconciliation: z.object({
|
||||||
|
runId: z.string().guid(),
|
||||||
|
providerStopped: z.literal(true),
|
||||||
|
actionOutcome: z.enum(["completed", "not_performed", "mixed"]),
|
||||||
|
outcomeEvidence: z.string().trim().min(20).max(12000),
|
||||||
|
}).strict().optional(),
|
||||||
actionId: z.string().guid().optional(),
|
actionId: z.string().guid().optional(),
|
||||||
outcome: z.enum(RESOLVE_ISSUE_RECOVERY_ACTION_OUTCOMES),
|
outcome: z.enum(RESOLVE_ISSUE_RECOVERY_ACTION_OUTCOMES),
|
||||||
sourceIssueStatus: z.enum(["todo", "done", "in_review", "blocked"]),
|
sourceIssueStatus: z.enum(["todo", "done", "in_review", "blocked"]),
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue