From bcc6fe7a442dae74ab0321ad472f7536ffa58f04 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sat, 5 Sep 2026 06:25:06 -0500 Subject: [PATCH] fix(runner): restore multi-turn remote sessions (#12840) ## Thinking Path > - Paperclip manages AI agents and their work. > - The runner executes agent turns on local and remote providers. > - A remote per-turn session must save its state before Paperclip releases its sandbox. > - The session runtime returned after 100 milliseconds while the remote checkpoint still ran. > - The next turn also checked the local state path instead of the verified remote backup. > - This pull request waits for the bounded remote close and accepts only a verified suspended backup. > - The benefit is reliable multi-turn execution without weaker identity checks. ## Linked Issues or Issue Description **What happened?** A successful remote agent turn released its sandbox before the runner saved the verified continuation backup. The next turn failed with `runner_state_identity_mismatch`. **Expected behavior** Paperclip must finish the bounded remote checkpoint before it releases the sandbox. A later turn must validate and restore the digest-matched suspended backup. **Steps to reproduce** 1. Run a native ACPX Claude Plan test in a non-reusable Daytona sandbox. 2. Reject the first plan to start a second turn. 3. Observe that the second turn fails before provider execution. **Paperclip version or commit** The failure reproduced at `13775a90b078ff64872f50961ea1b83d575e7bc6`. **Deployment mode** GitHub Actions with a Daytona sandbox. ## What Changed - Wait for the internally bounded remote runner close and checkpoint before the host returns. - Preserve the existing short cleanup bound for other providers. - Validate remote continuation lifecycle from a complete digest-verified backup when local runner state is absent. - Keep corrupt, non-suspended, mismatched, and unverified state fail-closed. - Make native Plan completion and accepted-Plan wake prompts deterministic. ## Verification - A prior 45-cell local campaign passed 44 cells. The only failure was the OpenCode Plan prompt variance fixed here. - A focused OpenCode local Plan rerun passed. - ACPX Claude Daytona message and question cells passed. - Focused regressions cover delayed checkpoint close and verified remote backup lifecycle. - GitHub Build and the focused ACPX Claude Daytona Plan cell will validate this exact head. ## Risks Remote runnerd sessions now wait for their internally bounded close/checkpoint path before returning; generic provider cleanup retains the existing 100 millisecond bound. Durable run success still cannot be reversed. The environment release guard still blocks sandbox destruction when no verified backup stamp exists. ## Model Used OpenAI Codex, GPT-5.6, extended reasoning, with code execution and GitHub Actions inspection. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change and contains no internal task id - [ ] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [ ] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open findings - [ ] I will address all Greptile and reviewer comments before requesting merge --- .github/workflows/runner-full-stack-e2e.yml | 16 +- .../runner-core/src/acpx_provider_backend.rs | 39 +- .../crates/runner-core/src/provider_events.rs | 20 +- .../tests/acpx_event_projection.rs | 70 +- .../tests/acpx_provider_resolutions.rs | 1 + .../runner-core/tests/acpx_provider_state.rs | 1 + .../tests/local_integrity_boundary_golden.rs | 1 + .../tests/native_provider_backend.rs | 115 +- .../durable-prp-control-plane.test.ts | 30 + .../durable-prp-control-plane.ts | 32 +- .../drivers/acpx/recovery-identity.test.ts | 16 + .../src/drivers/acpx/recovery-identity.ts | 14 +- packages/paperclip-runner/src/index.ts | 6 +- .../src/live/runnerd-codex-transport.test.ts | 559 ++++++++- .../src/live/runnerd-codex-transport.ts | 640 ++++++++-- .../src/native-session-runtime.test.ts | 326 +++++- .../src/native-session-runtime.ts | 213 +++- .../src/__tests__/environment-runtime.test.ts | 5 +- .../recovery-stale-issue-lock-sweep.test.ts | 44 + server/src/redaction.ts | 1 + server/src/services/heartbeat.ts | 2 +- .../native-harness-backup-stamp.ts | 153 ++- .../native-session-executor.test.ts | 730 +++++++++++- .../native-runtime/native-session-executor.ts | 1042 ++++++++++++----- server/src/services/recovery/service.ts | 21 +- server/src/vendor/paperclip-runner/index.ts | 7 +- tests/runner-e2e/README.md | 44 +- tests/runner-e2e/SECURITY.md | 47 +- tests/runner-e2e/catalog.test.ts | 52 +- tests/runner-e2e/catalog.ts | 6 +- tests/runner-e2e/dashboard-regenerate.ts | 2 + tests/runner-e2e/dashboard.ts | 24 +- tests/runner-e2e/evidence.ts | 6 +- tests/runner-e2e/history-index.ts | 28 +- tests/runner-e2e/history-publish.ts | 265 ++++- tests/runner-e2e/history.test.ts | 198 +++- tests/runner-e2e/public-summary-image.ts | 159 +++ tests/runner-e2e/report.test.ts | 8 +- tests/runner-e2e/report.ts | 4 +- tests/runner-e2e/run-observations.ts | 35 + tests/runner-e2e/runner.spec.ts | 42 +- tests/runner-e2e/selectors.ts | 11 +- tests/runner-e2e/support.test.ts | 40 + tests/runner-e2e/workflow-security.test.ts | 16 +- 44 files changed, 4494 insertions(+), 597 deletions(-) create mode 100644 tests/runner-e2e/public-summary-image.ts diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.yml index 1150d96d54..607d538c41 100644 --- a/.github/workflows/runner-full-stack-e2e.yml +++ b/.github/workflows/runner-full-stack-e2e.yml @@ -1104,21 +1104,20 @@ jobs: retention-days: 30 if-no-files-found: error - - name: Verify history source report and private screenshot evidence + - name: Verify normalized history source report id: history_source_ready if: always() run: | set -euo pipefail dashboard_root="runner-e2e-merged-report/normalized" - private_screenshot="$(find "$dashboard_root" -type f -name '*.png' -print -quit 2>/dev/null || true)" - if [ -f "$dashboard_root/index.html" ] && [ -n "$private_screenshot" ]; then + if [ -f "$dashboard_root/index.html" ] && [ -f "$dashboard_root/normalized-results.json" ]; then echo "ready=true" >> "$GITHUB_OUTPUT" else echo "ready=false" >> "$GITHUB_OUTPUT" fi publish_history: - name: Publish pruned immutable history and landing site + name: Publish trusted-summary S3 history and structured Pages bundle needs: [authorize, catalog, report] if: always() && needs.catalog.result == 'success' && needs.report.outputs.history_source_ready == 'true' runs-on: ubuntu-latest @@ -1154,6 +1153,9 @@ jobs: - run: pnpm install --frozen-lockfile + - name: Install publisher-only Chromium + run: pnpm exec playwright install --with-deps --only-shell chromium + - name: Download access-controlled normalized campaign uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: @@ -1166,7 +1168,7 @@ jobs: role-to-assume: ${{ vars.RUNNER_E2E_HISTORY_AWS_ROLE_ARN }} aws-region: ${{ vars.RUNNER_E2E_HISTORY_AWS_REGION }} - - name: Prune private evidence and publish immutable campaign history + - name: Publish trusted summary image to S3 and prune the Pages bundle env: PAPERCLIP_RUNNER_E2E_REPORT_DIR: ${{ github.workspace }}/runner-e2e-merged-report/normalized RUNNER_E2E_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_HISTORY_S3_BUCKET }} @@ -1179,12 +1181,12 @@ jobs: if: vars.RUNNER_FULL_STACK_E2E_PUBLISH_PAGES == 'true' run: echo "name=github-pages-${{ github.run_id }}-${{ github.run_attempt }}" >> "$GITHUB_OUTPUT" - - name: Package pruned structured dashboard for GitHub Pages + - name: Package structured-only dashboard for GitHub Pages if: vars.RUNNER_FULL_STACK_E2E_PUBLISH_PAGES == 'true' uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4 with: name: ${{ steps.pages_artifact_name.outputs.name }} - path: runner-e2e-merged-report/normalized + path: runner-e2e-merged-report/pages pages: name: Publish latest structured dashboard diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs index b14a2098c0..402b71a094 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs @@ -32,6 +32,7 @@ use crate::provider_events::{ project_acpx_state_event, AcpxEventProjectionContext, NormalizedProviderEvent, }; use crate::qualified_launch::verify_launch_artifact; +use crate::stable_identity::{is_stable_id, DURABLE_STABLE_ID_CHARS}; pub const ACPX_PROVIDER_STATE_FILE: &str = "acpx-provider-state.json"; const ACPX_PROVIDER_STATE_SCHEMA: &str = "paperclip.runner.acpx-provider-state.v3"; @@ -491,6 +492,7 @@ impl AcpxCommandExecutor { run_id: config.run_id.clone(), normalized_session_id: config.normalized_session_id.clone(), turn_id: config.turn_id.clone(), + provider_turn_id: None, item_id: config.item_id.clone(), }, state: None, @@ -594,6 +596,7 @@ impl AcpxCommandExecutor { let unsafe_active = matches!(state.lifecycle.as_str(), "turn_starting" | "turn_active"); let previous_turn = state.active_turn_id.clone(); if unsafe_active { + self.context.provider_turn_id = None; let state = self .state .as_mut() @@ -829,6 +832,7 @@ impl AcpxCommandExecutor { .as_ref() .and_then(|state| state.identity.as_ref()) .is_some(); + self.context.provider_turn_id = None; let state = self .state .as_mut() @@ -866,6 +870,16 @@ impl AcpxCommandExecutor { .get("text") .and_then(Value::as_str) .ok_or_else(|| DurableRunnerError::invalid("turn.start payload.text is required"))?; + let requested_provider_turn_id = payload + .get("turnId") + .and_then(Value::as_str) + .ok_or_else(|| DurableRunnerError::invalid("turn.start payload.turnId is required"))?; + let provider_turn_id = requested_provider_turn_id.to_owned(); + if !is_stable_id(&provider_turn_id, DURABLE_STABLE_ID_CHARS) { + return Err(DurableRunnerError::invalid( + "turn.start payload.turnId is invalid", + )); + } if self.session.is_none() { return Err(DurableRunnerError::invalid("ACPX session is not open")); } @@ -884,10 +898,13 @@ impl AcpxCommandExecutor { "ACPX provider cannot start a turn in its current lifecycle", )); } - state.active_turn_id = Some(self.context.turn_id.clone()); + state.active_turn_id = Some(provider_turn_id.clone()); state.semantic_result = None; state.lifecycle = "turn_starting".to_owned(); } + // ACPX provider events are scoped to the requested provider turn while + // semantic events remain correlated to the immutable durable PRP turn. + self.context.provider_turn_id = Some(provider_turn_id.clone()); self.save_state()?; let working_directory = self .state @@ -898,7 +915,7 @@ impl AcpxCommandExecutor { .session .as_mut() .expect("ACPX session exists before turn start") - .start_turn(&self.context.turn_id, text, &working_directory) + .start_turn(&provider_turn_id, text, &working_directory) { let state = self .state @@ -906,6 +923,7 @@ impl AcpxCommandExecutor { .expect("ACPX state remains available after failed turn start"); state.lifecycle = "closed".to_owned(); state.active_turn_id = None; + self.context.provider_turn_id = None; self.session = None; self.save_state()?; return Err(DurableRunnerError::invalid(format!( @@ -935,7 +953,7 @@ impl AcpxCommandExecutor { session.identity(), previous_process_id, session.process_id(), - &self.context.turn_id, + &provider_turn_id, ), )); } @@ -944,13 +962,13 @@ impl AcpxCommandExecutor { EventPriority::P0, json!({ "provider": "acpx", - "providerTurnId": self.context.turn_id, + "providerTurnId": provider_turn_id.clone(), "status": "inProgress", - "turn": {"id": self.context.turn_id, "status": "inProgress"}, + "turn": {"id": provider_turn_id.clone(), "status": "inProgress"}, }), )); Ok(CommandExecution { - result: json!({"status": "accepted", "providerTurnId": self.context.turn_id}), + result: json!({"status": "accepted", "providerTurnId": provider_turn_id}), events, }) } @@ -1019,6 +1037,7 @@ impl AcpxCommandExecutor { .as_mut() .expect("ACPX state remains available after provider termination"); state.active_turn_id = None; + self.context.provider_turn_id = None; // Persist a non-attachable, recoverable boundary before the fallible // lifetime proof. Terminal cleanup can then retry a timed-out fence // without reviving the stopped provider. @@ -1134,6 +1153,7 @@ impl AcpxCommandExecutor { .ok_or_else(|| DurableRunnerError::invalid("ACPX provider is not prepared"))?; state.lifecycle = "closed".to_owned(); state.active_turn_id = None; + self.context.provider_turn_id = None; let provider_session_id = state .identity .as_ref() @@ -1161,6 +1181,7 @@ impl AcpxCommandExecutor { state.identity = Some(identity); state.lifecycle = "suspended".to_owned(); state.active_turn_id = None; + self.context.provider_turn_id = None; self.session = None; self.save_state()?; } else if self.state.as_ref().is_some_and(|state| { @@ -1204,6 +1225,7 @@ impl AcpxCommandExecutor { DurableRunnerError::invalid(format!("ACPX provider failed: {error}")) })?; let Some(events) = events else { break }; + let mut provider_turn_settled = false; for event in events { let normalized = project_acpx_state_event(&self.context, &event) .map_err(|error| DurableRunnerError::invalid(error.to_string()))?; @@ -1227,6 +1249,7 @@ impl AcpxCommandExecutor { if let Some(event_type) = terminal { state.active_turn_id = None; state.lifecycle = "session_open".to_owned(); + provider_turn_settled = true; let status = match event_type.as_str() { "turn.completed" => "succeeded", "turn.cancelled" => "cancelled", @@ -1255,6 +1278,9 @@ impl AcpxCommandExecutor { })?; } } + if provider_turn_settled { + self.context.provider_turn_id = None; + } self.save_state()?; } Ok(()) @@ -1555,6 +1581,7 @@ mod tests { run_id: "run-1".to_owned(), normalized_session_id: "session-1".to_owned(), turn_id: "turn-1".to_owned(), + provider_turn_id: None, item_id: "item-1".to_owned(), } } diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs index 55918ed668..e8981c0733 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs @@ -46,7 +46,12 @@ pub(crate) fn normalized_codex_terminal_event_type( pub struct AcpxEventProjectionContext { pub run_id: String, pub normalized_session_id: String, + /// Immutable controller-owned turn identity used by durable PRP event + /// correlation. This must not be replaced by an ACP provider turn ID. pub turn_id: String, + /// Provider-owned turn identity used only to validate provider-originated + /// assistant and terminal events while an ACP turn is active. + pub provider_turn_id: Option, pub item_id: String, } @@ -67,9 +72,20 @@ impl AcpxEventProjectionContext { ] { validate_projection_identity(value, label, max_chars)?; } + if let Some(provider_turn_id) = self.provider_turn_id.as_deref() { + validate_projection_identity( + provider_turn_id, + "provider turn", + DURABLE_STABLE_ID_CHARS, + )?; + } Ok(()) } + fn active_provider_turn_id(&self) -> &str { + self.provider_turn_id.as_deref().unwrap_or(&self.turn_id) + } + fn correlation(&self) -> Value { json!({ "runId": self.run_id, @@ -383,9 +399,9 @@ fn require_projected_turn( context: &AcpxEventProjectionContext, turn_id: &str, ) -> Result<(), LocalRunnerError> { - if turn_id != context.turn_id { + if turn_id != context.active_provider_turn_id() { return Err(LocalRunnerError::invalid( - "ACPX state event does not match its durable turn projection", + "ACPX state event does not match its active provider turn projection", )); } Ok(()) diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_event_projection.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_event_projection.rs index 216d81b32c..c0f8092e9c 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_event_projection.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_event_projection.rs @@ -14,6 +14,7 @@ fn context() -> AcpxEventProjectionContext { run_id: "run-1".to_owned(), normalized_session_id: "session-1".to_owned(), turn_id: "turn-1".to_owned(), + provider_turn_id: None, item_id: "item-1".to_owned(), } } @@ -77,6 +78,68 @@ fn projects_authorized_tools_with_exact_durable_correlation() { .is_some_and(|value| value.starts_with("sha256:"))); } +#[test] +fn keeps_durable_correlation_separate_from_the_active_provider_turn() { + let mut context = context(); + context.provider_turn_id = Some("provider-turn-1".to_owned()); + + let semantic = project_acpx_state_event( + &context, + &AcpxProviderStateEvent::ToolCall { + call_id: "call-1".to_owned(), + operation_id: "issues.read".to_owned(), + input: json!({"taskId":"task-1"}), + }, + ) + .unwrap(); + assert_eq!( + semantic[0].payload["semantic_tool"]["correlation"]["turnId"], + "turn-1" + ); + + let request = project_acpx_state_event( + &context, + &AcpxProviderStateEvent::InputRequest { + request_id: "request-1".to_owned(), + question_set: json!({ + "schema":"paperclip.question_set.v1", + "questions":[], + }), + origin: None, + }, + ) + .unwrap(); + assert_eq!(request[0].payload["request"]["turnId"], "turn-1"); + + let assistant = project_acpx_state_event( + &context, + &AcpxProviderStateEvent::AssistantMessage { + turn_id: "provider-turn-1".to_owned(), + text: "Done".to_owned(), + }, + ) + .unwrap(); + assert_eq!(assistant[0].event_type, "item.completed"); + + let terminal = project_acpx_state_event( + &context, + &AcpxProviderStateEvent::TurnTerminal { + turn_id: "provider-turn-1".to_owned(), + status: AcpxTurnStatus::Completed, + error: None, + }, + ) + .unwrap(); + assert_eq!(terminal[0].event_type, "turn.completed"); + + let wrong_provider_turn = AcpxProviderStateEvent::TurnTerminal { + turn_id: "turn-1".to_owned(), + status: AcpxTurnStatus::Completed, + error: None, + }; + assert!(project_acpx_state_event(&context, &wrong_provider_turn).is_err()); +} + #[test] fn projects_terminal_tool_cancellations_as_correlated_results() { let events = project(AcpxProviderStateEvent::ToolResult(ToolResult { @@ -270,7 +333,7 @@ fn projects_assistant_terminal_and_diagnostic_events_fail_closed() { assert!(project_acpx_state_event(&context(), &wrong_turn) .unwrap_err() .to_string() - .contains("durable turn projection")); + .contains("active provider turn projection")); let permission = AcpxProviderStateEvent::PermissionRequest { request_id: "permission-1".to_owned(), kind: "write".to_owned(), @@ -300,6 +363,7 @@ fn rejects_invalid_durable_projection_identity() { ("run", 160), ("normalized session", 160), ("turn", 240), + ("provider turn", 240), ("item", 240), ] { let mut invalid = context(); @@ -308,6 +372,7 @@ fn rejects_invalid_durable_projection_identity() { "run" => invalid.run_id = oversized, "normalized session" => invalid.normalized_session_id = oversized, "turn" => invalid.turn_id = oversized, + "provider turn" => invalid.provider_turn_id = Some(oversized), "item" => invalid.item_id = oversized, _ => unreachable!(), } @@ -332,12 +397,13 @@ fn rejects_invalid_durable_projection_identity() { assert!(error.contains("request identity"), "{error}"); } - for field in ["run", "normalized session", "turn", "item"] { + for field in ["run", "normalized session", "turn", "provider turn", "item"] { let mut invalid = context(); match field { "run" => invalid.run_id = "run 1".to_owned(), "normalized session" => invalid.normalized_session_id = "session/1".to_owned(), "turn" => invalid.turn_id = "turn 1".to_owned(), + "provider turn" => invalid.provider_turn_id = Some("turn 1".to_owned()), "item" => invalid.item_id = "item/1".to_owned(), _ => unreachable!(), } diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_resolutions.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_resolutions.rs index 81d30d7376..7692f01060 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_resolutions.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_resolutions.rs @@ -123,6 +123,7 @@ fn projected_request_id_resolves_the_exact_upstream_sidecar_request() { run_id: "run-1".to_owned(), normalized_session_id: "session-1".to_owned(), turn_id: "turn-1".to_owned(), + provider_turn_id: None, item_id: "item-1".to_owned(), }, &input, diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_state.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_state.rs index 69fbb89c5f..9177443d93 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_state.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_state.rs @@ -325,6 +325,7 @@ fn correlates_projected_runtime_requests_to_the_upstream_input_id() { run_id: "run-1".to_owned(), normalized_session_id: "session-1".to_owned(), turn_id: "turn-1".to_owned(), + provider_turn_id: None, item_id: "item-1".to_owned(), }, &emitted[0], diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/local_integrity_boundary_golden.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/local_integrity_boundary_golden.rs index 553929a630..dda0278e83 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/local_integrity_boundary_golden.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/local_integrity_boundary_golden.rs @@ -189,6 +189,7 @@ fn acpx_projection_matches_shared_plan_question_final_and_terminal_identity() { run_id: "run-boundary".to_owned(), normalized_session_id: "session-boundary".to_owned(), turn_id: fixture["turnId"].as_str().expect("turn id").to_owned(), + provider_turn_id: None, item_id: "question-boundary-1".to_owned(), }; diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs index 2a2db7a20b..101e049f6d 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs @@ -10,7 +10,7 @@ use paperclip_runner_core::durable::{ QualifiedLaunchArtifact, }; use paperclip_runner_core::native_provider_backend::NativeProviderCommandExecutor; -use paperclip_runner_core::provider_bridge::authorized_tool_catalog_digest; +use paperclip_runner_core::provider_bridge::{authorized_tool_catalog_digest, AuthorizedTool}; use serde_json::{json, Value}; use sha2::{Digest, Sha256}; @@ -204,7 +204,11 @@ fn preserves_acpx_semantic_disposition_in_the_run_terminal() { .execute(&command(2, "session.open", json!({}))) .unwrap(); executor - .execute(&command(3, "turn.start", json!({"text": "Wait."}))) + .execute(&command( + 3, + "turn.start", + json!({"text": "Wait.", "turnId": "provider-turn-blocked"}), + )) .unwrap(); let events = executor.poll_events().unwrap(); @@ -341,7 +345,7 @@ fn executes_a_qualified_acpx_profile_through_the_native_selector() { .execute(&command( 3, "turn.start", - json!({"text": "Finish the task."}), + json!({"text": "Finish the task.", "turnId": "provider-turn-first"}), )) .unwrap(); assert_eq!(started.events[0].0, "turn.started"); @@ -364,6 +368,111 @@ fn executes_a_qualified_acpx_profile_through_the_native_selector() { fs::remove_dir_all(directory).unwrap(); } +#[test] +fn keeps_native_acpx_semantic_events_on_the_durable_controller_turn() { + let directory = temporary_directory("acpx-durable-turn-correlation"); + let config = acpx_config(&directory, "turns-tool"); + let mut executor = NativeProviderCommandExecutor::with_runner_config(&directory, &config); + let operations = vec![AuthorizedTool { + operation_id: "issues.read".to_owned(), + version: 1, + description: "Read an issue.".to_owned(), + input_schema: json!({"type":"object"}), + response_schema: json!({"type":"object"}), + }]; + let mut prepare = prepare_payload_with_mode(&directory, "codex", "turns-tool"); + prepare["authorizedTools"] = json!({ + "schema": "paperclip.runner.authorized-tools.v1", + "schemaVersion": 1, + "catalogDigest": authorized_tool_catalog_digest(&operations).unwrap(), + "operations": operations, + }); + + executor + .execute(&command(1, "run.prepare", prepare)) + .unwrap(); + executor + .execute(&command(2, "session.open", json!({}))) + .unwrap(); + let started = executor + .execute(&command( + 3, + "turn.start", + json!({"text": "Read the issue.", "turnId": "provider-turn-fresh"}), + )) + .unwrap(); + assert_eq!(started.result["providerTurnId"], "provider-turn-fresh"); + + let events = executor.poll_events().unwrap(); + let semantic = events + .iter() + .find(|event| event.event_type == "semantic_tool.input") + .expect("ACPX tool call must cross the native provider boundary"); + assert_eq!( + semantic.payload["semantic_tool"]["correlation"]["turnId"], + "turn-1" + ); + + executor.shutdown().unwrap(); + fs::remove_dir_all(directory).unwrap(); +} + +#[test] +fn starts_a_distinct_acpx_provider_turn_for_same_run_recovery() { + let directory = temporary_directory("acpx-same-run-recovery"); + let config = acpx_config(&directory, "turns-reserved-result-terminal"); + let mut executor = NativeProviderCommandExecutor::with_runner_config(&directory, &config); + + executor + .execute(&command( + 1, + "run.prepare", + prepare_payload(&directory, "codex"), + )) + .unwrap(); + executor + .execute(&command(2, "session.open", json!({}))) + .unwrap(); + let first = executor + .execute(&command( + 3, + "turn.start", + json!({"text": "First attempt.", "turnId": "provider-turn-first"}), + )) + .unwrap(); + assert_eq!(first.result["providerTurnId"], "provider-turn-first"); + + let first_events = executor.poll_events().unwrap(); + assert!(first_events + .iter() + .any(|event| event.event_type == "turn.completed")); + executor.acknowledge_events(first_events.len()).unwrap(); + + let recovered = executor + .execute(&command( + 4, + "turn.start", + json!({ + "text": "Recover the missing disposition.", + "turnId": "provider-turn-recovery", + }), + )) + .unwrap(); + assert_eq!(recovered.result["providerTurnId"], "provider-turn-recovery"); + assert!(recovered.events.iter().any(|(event_type, _, payload)| { + event_type == "turn.started" && payload["providerTurnId"] == "provider-turn-recovery" + })); + let recovered_events = executor.poll_events().unwrap(); + assert!(recovered_events.iter().any(|event| { + event.event_type == "turn.completed" + && event.payload["providerTurnId"] == "provider-turn-recovery" + })); + executor.acknowledge_events(recovered_events.len()).unwrap(); + + executor.shutdown().unwrap(); + fs::remove_dir_all(directory).unwrap(); +} + #[test] fn executes_opencode_through_the_local_facade_without_codex_event_labels() { let directory = temporary_directory("opencode"); diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts index e7d238c36d..47c655bbb8 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts @@ -1043,6 +1043,15 @@ describe.sequential("DurablePrpControlPlane", () => { controlPlane, controlPlane.issueBootstrapTicket(), ); + const nextAuthorityCommand = controlPlane.queueCommand( + "runner.drain", + {}, + "command-after-suspend-1", + true, + ); + expect( + controlPlane.store.state.commandDeliveryCounts[command.commandId], + ).toBe(1); const terminalResult = { protocol: "paperclip.runner", version: 1, @@ -1068,7 +1077,13 @@ describe.sequential("DurablePrpControlPlane", () => { }); expect(controlPlane.store.state.commands).toMatchObject([ { commandId: "command-suspend-1", status: "completed" }, + { commandId: "command-after-suspend-1", status: "pending" }, ]); + expect( + controlPlane.store.state.commandDeliveryCounts[ + nextAuthorityCommand.commandId + ], + ).toBeUndefined(); sendSecure(client!, terminalResult); await expect(receiveSecure(client!)).resolves.toMatchObject({ @@ -1076,7 +1091,22 @@ describe.sequential("DurablePrpControlPlane", () => { payload: { commandId: "command-suspend-1" }, }); expect(controlPlane.store.state.duplicateCommandResults).toBe(1); + expect( + controlPlane.store.state.commandDeliveryCounts[ + nextAuthorityCommand.commandId + ], + ).toBeUndefined(); + const leaseToken = client!.leaseToken!; client?.socket.destroy(); + const successor = await authenticate(controlPlane, leaseToken); + expect(successor?.welcome.payload).toMatchObject({ + pendingCommands: [ + expect.objectContaining({ + commandId: nextAuthorityCommand.commandId, + }), + ], + }); + successor?.socket.destroy(); } finally { await controlPlane.stop(); rmSync(root, { recursive: true, force: true }); diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts index 9978e474eb..fb76579f69 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts @@ -898,6 +898,7 @@ class AuthorityConnection { secureChannel: SecureChannel | null = null; lease: ConnectionLeaseRecord | null = null; connectionId: string | null = null; + terminalLifecycleCommandId: string | null = null; readonly wire: PrpWireConnection; #closed = false; #onClose: () => void; @@ -1537,6 +1538,11 @@ export class DurablePrpControlPlane { this.#store.state.lastLeaseExpiresAt = lease.expiresAt; const pending = this.#nextPendingCommand(); + const [pendingCommand] = pending; + connection.terminalLifecycleCommandId = + pendingCommand && this.#isTerminalLifecycleCommand(pendingCommand) + ? pendingCommand.commandId + : null; for (const command of pending) { this.#store.state.commandDeliveryCounts[command.commandId] = (this.#store.state.commandDeliveryCounts[command.commandId] ?? 0) + 1; @@ -1623,8 +1629,12 @@ export class DurablePrpControlPlane { } #sendNextCommand(connection: AuthorityConnection): void { + if (connection.terminalLifecycleCommandId !== null) return; const [command] = this.#nextPendingCommand(); if (command === undefined) return; + if (this.#isTerminalLifecycleCommand(command)) { + connection.terminalLifecycleCommandId = command.commandId; + } this.#store.state.commandDeliveryCounts[command.commandId] = (this.#store.state.commandDeliveryCounts[command.commandId] ?? 0) + 1; this.#store.save(); @@ -1655,6 +1665,9 @@ export class DurablePrpControlPlane { connection.close(); return; } + if (this.#isTerminalLifecycleCommand(command)) { + connection.terminalLifecycleCommandId = command.commandId; + } const status = result.status; // `indeterminate` is terminal too: a runner that crashed between journaling // a command and confirming its effect reports it on recovery and will not @@ -1678,24 +1691,31 @@ export class DurablePrpControlPlane { this.#store.state.duplicateCommandResults += 1; this.#store.save(); this.#ackTerminalCommandResult(connection, command); - this.#sendNextCommand(connection); + if (!this.#isTerminalLifecycleCommand(command)) { + this.#sendNextCommand(connection); + } return; } command.status = status; command.result = structuredClone(result); this.#store.save(); this.#ackTerminalCommandResult(connection, command); - this.#sendNextCommand(connection); + if (!this.#isTerminalLifecycleCommand(command)) { + this.#sendNextCommand(connection); + } + } + + #isTerminalLifecycleCommand(command: DurableRecoveryCoreCommand): boolean { + return ( + command.type === "runner.suspend" || command.type === "runner.shutdown" + ); } #ackTerminalCommandResult( connection: AuthorityConnection, command: DurableRecoveryCoreCommand, ): void { - if ( - command.type !== "runner.suspend" && - command.type !== "runner.shutdown" - ) { + if (!this.#isTerminalLifecycleCommand(command)) { return; } connection.sendJson( diff --git a/packages/paperclip-runner/src/drivers/acpx/recovery-identity.test.ts b/packages/paperclip-runner/src/drivers/acpx/recovery-identity.test.ts index 4e053e5f55..ca4b10e69a 100644 --- a/packages/paperclip-runner/src/drivers/acpx/recovery-identity.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/recovery-identity.test.ts @@ -8,6 +8,7 @@ import { afterEach, describe, expect, it } from "vitest"; import { resolveQualifiedAcpxProfile } from "./qualified-profiles.js"; import { ACPX_IDENTITY_RECORD_SCHEMA, + acpxRuntimeSessionDirectoryName, acpxProviderSessionIdentity, createAcpxIdentityRecord, createAcpxRecoveryBinding, @@ -25,6 +26,21 @@ afterEach(async () => { }); describe("ACPX recovery identity", () => { + it("derives one stable, filesystem-safe runtime directory name", () => { + expect(acpxRuntimeSessionDirectoryName("session/1")).toMatch( + /^session_1-[0-9a-f]{16}$/, + ); + expect(acpxRuntimeSessionDirectoryName("...")).toMatch( + /^session-[0-9a-f]{16}$/, + ); + expect(acpxRuntimeSessionDirectoryName("session/1")).toBe( + acpxRuntimeSessionDirectoryName("session/1"), + ); + expect(acpxRuntimeSessionDirectoryName("session/1")).not.toBe( + acpxRuntimeSessionDirectoryName("session_1"), + ); + }); + it("binds the canonical workspace, profile, model, policy, and session", async () => { const fixture = await recoveryFixture(); expect(fixture.binding.runtimeRoot).toContain("session-1-"); diff --git a/packages/paperclip-runner/src/drivers/acpx/recovery-identity.ts b/packages/paperclip-runner/src/drivers/acpx/recovery-identity.ts index e256632d95..0f0ae1bf8d 100644 --- a/packages/paperclip-runner/src/drivers/acpx/recovery-identity.ts +++ b/packages/paperclip-runner/src/drivers/acpx/recovery-identity.ts @@ -307,6 +307,18 @@ export async function resolveAcpxRuntimeRoot( throw new Error("ACPX runtime directory must be a directory"); if (root === dirname(root)) throw new Error("ACPX runtime directory must not be a filesystem root"); + return join( + resolve(root), + "acpx", + acpxRuntimeSessionDirectoryName(sessionId), + ); +} + +/** + * Return the stable, filesystem-safe directory name used for one normalized + * ACPX session below the runtime's `acpx` namespace. + */ +export function acpxRuntimeSessionDirectoryName(sessionId: string): string { const readable = sessionId .replace(/[^a-zA-Z0-9._-]/g, "_") .replace(/^\.+$/, "session") @@ -315,7 +327,7 @@ export async function resolveAcpxRuntimeRoot( .update(sessionId) .digest("hex") .slice(0, 16); - return join(resolve(root), "acpx", `${readable || "session"}-${suffix}`); + return `${readable || "session"}-${suffix}`; } function validateIdentity( diff --git a/packages/paperclip-runner/src/index.ts b/packages/paperclip-runner/src/index.ts index acc4c4f190..1b88311731 100644 --- a/packages/paperclip-runner/src/index.ts +++ b/packages/paperclip-runner/src/index.ts @@ -20,7 +20,10 @@ export { OpenCodeServerDriver, type OpenCodeServerDriverOptions, } from "./drivers/opencode/opencode-server-driver.js"; -export { parseCodexTurnDiff, summarizeCodexTurnDiff } from "./drivers/codex/codex-turn-diff.js"; +export { + parseCodexTurnDiff, + summarizeCodexTurnDiff, +} from "./drivers/codex/codex-turn-diff.js"; export * from "./native-session-runtime.js"; export { DurablePrpControlPlane, @@ -38,6 +41,7 @@ export * from "./drivers/codex/codex-app-server-driver.js"; export * from "./drivers/opencode/opencode-server-driver.js"; export * from "./drivers/opencode/mcp-bridge.js"; export * from "./drivers/acpx/qualified-profiles.js"; +export { acpxRuntimeSessionDirectoryName } from "./drivers/acpx/recovery-identity.js"; export { probeQualifiedAcpxEnvironment, type ProbeQualifiedAcpxEnvironmentOptions, diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index c8613723a4..dcbfc49159 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -131,6 +131,328 @@ it("identifies an active provider turn that must stop before suspension", () => }); }); +it("infers a remote provider turn until its own terminal event is durable", () => { + expect( + runnerdRecoveryInternals.providerTurnIsActiveFromCommittedEvents([ + { eventType: "turn.started" }, + { eventType: "run.result.proposed" }, + { eventType: "run.terminal" }, + ]), + ).toBe(true); + expect( + runnerdRecoveryInternals.providerTurnIsActiveFromCommittedEvents([ + { eventType: "turn.started" }, + { eventType: "run.terminal" }, + { eventType: "turn.interrupted" }, + ]), + ).toBe(false); +}); + +it("accepts only the observed provider start correlated by the command result", () => { + const requestedTurnId = "turn_lab_0123456789abcdef0123456789abcdef"; + expect( + runnerdRecoveryInternals.turnStartResponseReady({ + responseEpoch: 2, + observedEpoch: 2, + expectedProviderTurnId: requestedTurnId, + boundTurnId: requestedTurnId, + }), + ).toBe(true); + expect( + runnerdRecoveryInternals.turnStartResponseReady({ + responseEpoch: 2, + observedEpoch: 2, + expectedProviderTurnId: requestedTurnId, + boundTurnId: "provider-turn-different", + }), + ).toBe(false); + const providerAssignedTurnId = "provider-turn-assigned-for-this-command"; + expect( + runnerdRecoveryInternals.turnStartResponseReady({ + responseEpoch: 2, + observedEpoch: 2, + expectedProviderTurnId: providerAssignedTurnId, + boundTurnId: providerAssignedTurnId, + }), + ).toBe(true); + expect( + runnerdRecoveryInternals.turnStartResponseReady({ + responseEpoch: 2, + observedEpoch: 1, + expectedProviderTurnId: requestedTurnId, + boundTurnId: requestedTurnId, + }), + ).toBe(false); +}); + +it("defers turn starts until their command result and rejects stale identities", () => { + expect( + runnerdRecoveryInternals.turnStartNotificationDisposition({ + responsePending: true, + expectedProviderTurnId: null, + observedProviderTurnId: "provider-turn-early", + }), + ).toBe("defer"); + expect( + runnerdRecoveryInternals.turnStartNotificationDisposition({ + responsePending: true, + expectedProviderTurnId: "provider-turn-current", + observedProviderTurnId: "provider-turn-stale", + }), + ).toBe("reject"); + expect( + runnerdRecoveryInternals.turnStartNotificationDisposition({ + responsePending: true, + expectedProviderTurnId: "provider-turn-current", + observedProviderTurnId: "", + }), + ).toBe("reject"); + expect( + runnerdRecoveryInternals.turnStartNotificationDisposition({ + responsePending: true, + expectedProviderTurnId: "provider-turn-current", + observedProviderTurnId: "provider-turn-current", + }), + ).toBe("accept"); +}); + +it("requires ACPX command results to preserve the requested turn identity", () => { + expect( + runnerdRecoveryInternals.turnStartCommandResultValid({ + requestedTurnId: "turn-requested", + providerTurnId: "turn-requested", + requireRequestedIdentity: true, + }), + ).toBe(true); + expect( + runnerdRecoveryInternals.turnStartCommandResultValid({ + requestedTurnId: "turn-requested", + providerTurnId: "turn-different", + requireRequestedIdentity: true, + }), + ).toBe(false); + expect( + runnerdRecoveryInternals.turnStartCommandResultValid({ + requestedTurnId: "turn-requested", + providerTurnId: "provider-assigned-turn", + requireRequestedIdentity: false, + }), + ).toBe(true); +}); + +it.each(["before", "after"] as const)( + "retries external authority rotation after crashing %s the remote archive", + async (crashPoint) => { + const root = await mkdtemp(join(tmpdir(), "runner-external-rotation-")); + const priorIdentity = { + runnerInstanceId: "runner-external-rotation", + environmentLeaseId: "lease-external-rotation", + runId: "run-external-prior", + normalizedSessionId: "session-external-rotation", + turnId: "turn-external-prior", + itemId: "item-external-prior", + }; + const desiredIdentity = { + ...priorIdentity, + runId: "run-external-next", + turnId: "turn-external-next", + itemId: "item-external-next", + }; + const controlPlaneState = { + schema: "paperclip.runner.durable.control-plane-state.v1", + identity: priorIdentity, + }; + let activeRunnerState: Record | null = { + schema: "paperclip.runner.durable.state.v1", + ...priorIdentity, + lifecycle: "suspended", + }; + let archivedRunnerState: Record | null = null; + let readCount = 0; + let archiveCount = 0; + const readRunnerState = async () => { + readCount += 1; + if (activeRunnerState === null) throw new Error("runner state moved"); + return activeRunnerState; + }; + const archiveRunnerState = async () => { + archiveCount += 1; + if (archiveCount === 1) { + if (crashPoint === "after") { + archivedRunnerState = activeRunnerState; + activeRunnerState = null; + } + throw new Error(`crashed ${crashPoint} remote archive`); + } + if (activeRunnerState !== null) { + archivedRunnerState = activeRunnerState; + activeRunnerState = null; + } + if (archivedRunnerState === null) { + throw new Error("archived runner state unavailable"); + } + return archivedRunnerState; + }; + try { + await mkdir(join(root, "control-plane"), { recursive: true }); + await writeFile( + join(root, "control-plane", "control-plane-state.json"), + JSON.stringify(controlPlaneState), + ); + await expect( + runnerdRecoveryInternals.rotateExternalAuthorityEpoch( + root, + controlPlaneState, + desiredIdentity, + readRunnerState, + archiveRunnerState, + ), + ).rejects.toThrow(`crashed ${crashPoint} remote archive`); + await expect(stat(join(root, "control-plane"))).rejects.toMatchObject({ + code: "ENOENT", + }); + + await expect( + runnerdRecoveryInternals.rotateExternalAuthorityEpoch( + root, + controlPlaneState, + desiredIdentity, + readRunnerState, + archiveRunnerState, + ), + ).resolves.toEqual(controlPlaneState); + expect(readCount).toBe(1); + expect(archiveCount).toBe(2); + expect(activeRunnerState).toBeNull(); + expect(archivedRunnerState).toEqual( + expect.objectContaining(priorIdentity), + ); + } finally { + await rm(root, { recursive: true, force: true }); + } + }, +); + +it("quiesces the control route before checkpoint and containment regardless of process completion", async () => { + const settledSteps: string[] = []; + await runnerdRecoveryInternals.releaseRunnerProcessOwnership({ + runnerSettled: true, + checkpoint: async (settlement) => { + expect(settlement).toBe("settled"); + settledSteps.push("checkpoint"); + }, + forceKill: () => { + settledSteps.push("kill"); + }, + release: async () => { + settledSteps.push("release"); + }, + }); + expect(settledSteps).toEqual(["release", "checkpoint", "kill"]); + + const unsettledSteps: string[] = []; + await runnerdRecoveryInternals.releaseRunnerProcessOwnership({ + runnerSettled: false, + checkpoint: async (settlement) => { + expect(settlement).toBe("unsettled"); + unsettledSteps.push("checkpoint"); + }, + forceKill: () => { + unsettledSteps.push("kill"); + }, + release: async () => { + unsettledSteps.push("release"); + }, + }); + expect(unsettledSteps).toEqual(["release", "checkpoint", "kill"]); + + const failedCheckpointSteps: string[] = []; + await expect( + runnerdRecoveryInternals.releaseRunnerProcessOwnership({ + runnerSettled: true, + checkpoint: async () => { + failedCheckpointSteps.push("checkpoint"); + throw new Error("runner_remote_checkpoint_incomplete"); + }, + forceKill: () => { + failedCheckpointSteps.push("kill"); + }, + release: async () => { + failedCheckpointSteps.push("release"); + }, + }), + ).rejects.toThrow("runner_remote_checkpoint_incomplete"); + expect(failedCheckpointSteps).toEqual(["release", "checkpoint", "kill"]); +}); + +it("waits for the exact durable suspension command behind prior close work", async () => { + const commands = [ + { + commandId: "command_close_drain", + type: "runner.drain", + status: "pending", + }, + ]; + let lifecycle = "ready"; + let pumpCount = 0; + + await expect( + runnerdRecoveryInternals.awaitRunnerSuspensionBarrier({ + commands: () => commands, + queueSuspend: (commandId) => { + commands.push({ + commandId, + type: "runner.suspend", + status: "pending", + }); + }, + readRunnerState: async () => ({ lifecycle }), + runnerHasExited: async () => true, + pump: () => { + pumpCount += 1; + if (pumpCount === 1) commands[0]!.status = "completed"; + if (pumpCount === 2) { + commands[1]!.status = "completed"; + lifecycle = "suspended"; + } + }, + deadline: Date.now() + 1_000, + pollIntervalMs: 0, + }), + ).resolves.toBe(true); + expect(commands.map((command) => command.type)).toEqual([ + "runner.drain", + "runner.suspend", + ]); + expect(pumpCount).toBeGreaterThanOrEqual(2); +}); + +it("does not accept process exit without durable suspension", async () => { + const commands: Array<{ + commandId: string; + type: string; + status: string; + }> = []; + + await expect( + runnerdRecoveryInternals.awaitRunnerSuspensionBarrier({ + commands: () => commands, + queueSuspend: (commandId) => { + commands.push({ + commandId, + type: "runner.suspend", + status: "pending", + }); + }, + readRunnerState: async () => ({ lifecycle: "ready" }), + runnerHasExited: async () => true, + pump: () => undefined, + deadline: Date.now() + 5, + pollIntervalMs: 0, + }), + ).resolves.toBe(false); +}); + it("keeps ACPX terminal tools under the reserved runner-owned catalog", () => { const tools = [ { @@ -1131,6 +1453,29 @@ it("binds an immediately failed durable turn before exposing its terminal", asyn message: { role: "user", text: "Fail this test turn." }, }); expect(accepted.turnId).toBe("provider-turn-1"); + const durableState = JSON.parse( + await readFile( + join(stateDirectory, "control-plane", "control-plane-state.json"), + "utf8", + ), + ) as { + commands: Array<{ + type: string; + payload: Record; + }>; + }; + const durableTurnStart = durableState.commands.find( + (command) => command.type === "turn.start", + ); + expect(durableTurnStart).toMatchObject({ + payload: { + turnId: expect.stringMatching(/^turn_lab_[a-f0-9]{32}$/), + }, + }); + expect(JSON.parse(String(durableTurnStart?.payload.text))).toMatchObject({ + message: "Fail this test turn.", + task: { objective: "Exercise an immediate provider failure." }, + }); const events = []; for await (const event of session.events()) { events.push(event); @@ -1751,6 +2096,115 @@ it.each([ }, ); +it("probes an exact-authority resume and confirms its live provider identity", async () => { + const stateDirectory = await mkdtemp( + join(tmpdir(), "runnerd-exact-authority-resume-"), + ); + const identity = { + runnerInstanceId: "runner-exact-resume", + environmentLeaseId: "lease-exact-resume", + runId: "run-exact-resume", + normalizedSessionId: "session-exact-resume", + turnId: "turn-exact-resume", + itemId: "item-exact-resume", + }; + const options = { + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory, "--durable-turn-ids"), + stateDirectory, + lifecyclePolicy: { mode: "per_turn" as const, idleTimeoutMs: null }, + prpIdentity: identity, + }; + const first = createCapabilityRunnerdCodexTransport(options); + first.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + let providerThread: { id: string; sessionId: string } | null = null; + try { + const opened = await first.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [], + }); + const thread = opened.thread as Record; + providerThread = { + id: String(thread.id), + sessionId: String(thread.sessionId), + }; + } finally { + await first.transport.close(); + } + if (providerThread === null) { + throw new Error("exact-authority fixture did not return a provider thread"); + } + + const statePath = join( + stateDirectory, + "control-plane", + "control-plane-state.json", + ); + const beforeResume = JSON.parse(await readFile(statePath, "utf8")) as { + commands: Array<{ type: string }>; + committedEvents: Array<{ eventType: string }>; + }; + expect( + beforeResume.commands.some((command) => command.type === "run.attach"), + ).toBe(false); + const priorResumeEvents = beforeResume.committedEvents.filter( + (event) => event.eventType === "session.resumed", + ).length; + const priorSnapshots = beforeResume.commands.filter( + (command) => command.type === "session.snapshot", + ).length; + + const resumed = createCapabilityRunnerdCodexTransport({ + ...options, + resumeProviderSession: { + driverSessionId: providerThread.id, + providerSessionId: providerThread.sessionId, + }, + }); + resumed.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + try { + const read = await resumed.transport.request("thread/read", {}); + expect(read.thread).toMatchObject(providerThread); + const afterResume = JSON.parse(await readFile(statePath, "utf8")) as { + commands: Array<{ commandId: string; type: string; status: string }>; + committedEvents: Array<{ eventType: string }>; + }; + expect(afterResume.commands).toContainEqual( + expect.objectContaining({ + commandId: expect.stringMatching(/^command_resume_probe_/), + type: "runner.drain", + status: "completed", + }), + ); + expect(afterResume.commands).toContainEqual( + expect.objectContaining({ + type: "session.snapshot", + status: "completed", + }), + ); + expect( + afterResume.commands.filter( + (command) => command.type === "session.snapshot", + ), + ).toHaveLength(priorSnapshots + 2); + expect( + afterResume.committedEvents.filter( + (event) => event.eventType === "session.resumed", + ), + ).toHaveLength(priorResumeEvents + 1); + } finally { + await resumed.transport.close(); + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 30_000); + it("cold-restores a suspended provider session under its durable run binding", async () => { const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-cold-attach-")); const tracePath = join(stateDirectory, "provider-trace.ndjson"); @@ -1940,12 +2394,113 @@ it("cold-restores a suspended provider session under its durable run binding", a ); await mismatched.transport.close(); + const externalIdentity = { + ...secondIdentity, + runId: "run-cold-external", + turnId: "turn-cold-external", + itemId: "item-cold-external", + }; + const rejectedExternalRotationSteps: string[] = []; + let externalArchiveDirectory: string | null = null; + const rejectedExternalRotation = createCapabilityRunnerdCodexTransport({ + ...options, + runnerStateDirectory: externallyOwnedRunnerStateDirectory, + readRunnerState, + prepareExternalRunnerState: async () => { + rejectedExternalRotationSteps.push("prepared"); + }, + archiveExternalRunnerState: async ({ archiveKey }) => { + await expect( + stat(join(stateDirectory, "control-plane")), + ).rejects.toMatchObject({ code: "ENOENT" }); + expect( + await stat( + join( + stateDirectory, + "authority-epochs", + `epoch-${archiveKey}`, + "control-plane", + ), + ), + ).toBeDefined(); + externalArchiveDirectory = join( + stateDirectory, + "external-authority-epochs", + archiveKey, + ); + await mkdir(externalArchiveDirectory, { recursive: true }); + await rename( + join(externallyOwnedRunnerStateDirectory, "runner-state.json"), + join(externalArchiveDirectory, "runner-state.json"), + ); + rejectedExternalRotationSteps.push("remote-archived"); + throw new Error("controller crashed after external archive"); + }, + resumeDynamicTools: dynamicTools, + prpIdentity: externalIdentity, + }); + await expect( + rejectedExternalRotation.transport.request("thread/read", {}), + ).rejects.toThrow("controller crashed after external archive"); + await rejectedExternalRotation.transport.close(); + expect(rejectedExternalRotationSteps).toEqual([ + "prepared", + "remote-archived", + ]); + await expect(stat(join(stateDirectory, "control-plane"))).rejects.toThrow(); + await expect( + stat(join(externallyOwnedRunnerStateDirectory, "runner-state.json")), + ).rejects.toThrow(); + + const externalRotationSteps: string[] = []; + const externallyRotated = createCapabilityRunnerdCodexTransport({ + ...options, + runnerStateDirectory: externallyOwnedRunnerStateDirectory, + readRunnerState, + prepareExternalRunnerState: async () => { + throw new Error("retry must not prepare a new external runner"); + }, + archiveExternalRunnerState: async ({ archiveKey }) => { + expect(externalArchiveDirectory).toBe( + join(stateDirectory, "external-authority-epochs", archiveKey), + ); + externalRotationSteps.push("archived"); + return JSON.parse( + await readFile( + join(externalArchiveDirectory!, "runner-state.json"), + "utf8", + ), + ) as Record; + }, + resumeDynamicTools: dynamicTools, + resumeCompletionContract: { + revision: "contract-external", + criterionIds: ["criterion-external"], + }, + prpIdentity: externalIdentity, + }); + externallyRotated.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + try { + const read = await externallyRotated.transport.request("thread/read", {}); + expect(read.thread).toMatchObject({ + id: firstProviderThread.id, + sessionId: firstProviderThread.sessionId, + cwd: tmpdir(), + }); + expect(externalRotationSteps).toEqual(["archived"]); + } finally { + await externallyRotated.transport.close(); + } + const restored = createCapabilityRunnerdCodexTransport({ ...options, runnerStateDirectory: externallyOwnedRunnerStateDirectory, readRunnerState, resumeDynamicTools: dynamicTools, - prpIdentity: secondIdentity, + prpIdentity: externalIdentity, }); restored.transport.setServerRequestHandler(async () => ({ success: true, @@ -2141,7 +2696,7 @@ async function verifyLiveRunnerAdoption(mismatchedCheckpoint: boolean) { "restored adopted provider identity from the exact durable checkpoint after PRP event compaction; awaiting live confirmation", ); expect(adopted.evidence().diagnostics).toContain( - "confirmed adopted provider identity against authenticated session.snapshot", + "confirmed adopted provider identity against authenticated recovery session.snapshot", ); } finally { await adopted?.transport.close().catch(() => undefined); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index 4f893a89df..5ab150c78f 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -170,6 +170,33 @@ function controlPlaneIdentity( ); } +function recoveryIdentityMatches( + value: DurableRecoveryIdentity | Record, + expected: DurableRecoveryIdentity, +): boolean { + return ( + value.runnerInstanceId === expected.runnerInstanceId && + value.environmentLeaseId === expected.environmentLeaseId && + value.runId === expected.runId && + value.normalizedSessionId === expected.normalizedSessionId && + value.turnId === expected.turnId && + value.itemId === expected.itemId + ); +} + +function assertSuspendedRunnerState( + state: Record, + expected: DurableRecoveryIdentity, +): void { + if ( + state.schema !== "paperclip.runner.durable.state.v1" || + !recoveryIdentityMatches(state, expected) || + state.lifecycle !== "suspended" + ) { + throw new Error("native_runner_authority_rotation_requires_settled_state"); + } +} + function assertRealDirectory(path: string): void { const metadata = lstatSync(path); if (metadata.isSymbolicLink() || !metadata.isDirectory()) { @@ -270,6 +297,8 @@ function rotateLocalAuthorityEpoch( runnerState.environmentLeaseId !== priorIdentity.environmentLeaseId || runnerState.runId !== priorIdentity.runId || runnerState.normalizedSessionId !== priorIdentity.normalizedSessionId || + runnerState.turnId !== priorIdentity.turnId || + runnerState.itemId !== priorIdentity.itemId || runnerState.lifecycle !== "suspended" ) { throw new Error("native_runner_authority_rotation_requires_settled_state"); @@ -306,6 +335,70 @@ function rotateLocalAuthorityEpoch( return controlPlaneState; } +async function rotateExternalAuthorityEpoch( + root: string, + controlPlaneState: Record, + desired: DurableRecoveryIdentity, + readRunnerState: () => Promise>, + archiveRunnerState: (input: { + archiveKey: string; + priorIdentity: DurableRecoveryIdentity; + }) => Promise>, +): Promise> { + const priorIdentity = controlPlaneIdentity(controlPlaneState); + if ( + priorIdentity.runnerInstanceId !== desired.runnerInstanceId || + priorIdentity.environmentLeaseId !== desired.environmentLeaseId || + priorIdentity.normalizedSessionId !== desired.normalizedSessionId || + priorIdentity.runId === desired.runId + ) { + throw new Error( + "PRP recovery identity does not match the durable session binding", + ); + } + const archive = authorityArchiveDirectory(root, priorIdentity); + const archivedControlPlane = resolve(archive, "control-plane"); + const activeControlPlane = resolve(root, "control-plane"); + const archivesRoot = resolve(root, "authority-epochs"); + if (existsSync(archivesRoot)) { + assertRealDirectory(archivesRoot); + } else { + mkdirSync(archivesRoot, { mode: 0o700 }); + } + if (existsSync(archive)) { + assertRealDirectory(archive); + } else { + mkdirSync(archive, { mode: 0o700 }); + } + assertRealDirectory(archive); + if (existsSync(archivedControlPlane)) { + // This directory is the durable transaction marker. A prior controller + // may have stopped before or after the remote move, so resume the same + // idempotent archive instead of starting a new external runner. + assertRealDirectory(archivedControlPlane); + if (existsSync(activeControlPlane)) { + throw new Error("native_runner_authority_archive_conflict"); + } + const archivedIdentity = controlPlaneIdentity( + readControlPlaneState(archivedControlPlane), + ); + if (!recoveryIdentityMatches(archivedIdentity, priorIdentity)) { + throw new Error("native_runner_authority_archive_conflict"); + } + } else { + const runnerState = await readRunnerState(); + assertSuspendedRunnerState(runnerState, priorIdentity); + assertRealDirectory(activeControlPlane); + renameSync(activeControlPlane, archivedControlPlane); + } + const archivedRunnerState = await archiveRunnerState({ + archiveKey: basename(archive).replace(/^epoch-/, ""), + priorIdentity, + }); + assertSuspendedRunnerState(archivedRunnerState, priorIdentity); + return controlPlaneState; +} + function rotatedRunAttachPayload( state: Record, desired: DurableRecoveryIdentity, @@ -408,6 +501,152 @@ function providerDrainStateFromSnapshot(state: Record): { }; } +function providerTurnIsActiveFromCommittedEvents( + events: readonly { eventType: string }[], +): boolean { + let active = false; + for (const event of events) { + if (event.eventType === "turn.started") active = true; + else if ( + event.eventType === "turn.completed" || + event.eventType === "turn.failed" || + event.eventType === "turn.interrupted" || + event.eventType === "turn.cancelled" + ) { + active = false; + } + } + return active; +} + +function turnStartResponseReady(input: { + responseEpoch: number; + observedEpoch: number; + expectedProviderTurnId: string; + boundTurnId: string; +}): boolean { + return ( + input.responseEpoch === input.observedEpoch && + input.expectedProviderTurnId.length > 0 && + input.boundTurnId === input.expectedProviderTurnId + ); +} + +function turnStartNotificationDisposition(input: { + responsePending: boolean; + expectedProviderTurnId: string | null; + observedProviderTurnId: string; +}): "accept" | "defer" | "reject" { + if (!input.responsePending) return "accept"; + if (input.expectedProviderTurnId === null) return "defer"; + return input.observedProviderTurnId === input.expectedProviderTurnId + ? "accept" + : "reject"; +} + +function turnStartCommandResultValid(input: { + requestedTurnId: string; + providerTurnId: string; + requireRequestedIdentity: boolean; +}): boolean { + return ( + input.requestedTurnId.length > 0 && + input.providerTurnId.length > 0 && + (!input.requireRequestedIdentity || + input.providerTurnId === input.requestedTurnId) + ); +} + +async function releaseRunnerProcessOwnership(input: { + runnerSettled: boolean; + checkpoint: + ((settlement: "settled" | "unsettled") => Promise | void) | null; + forceKill: () => void; + release: (() => Promise | void) | null; +}): Promise { + let releaseFailure: unknown; + try { + if (input.release !== null) await input.release(); + } catch (error) { + releaseFailure = error; + } + let checkpointFailure: unknown; + try { + if (input.checkpoint !== null) { + // Release only the authenticated control route first. In provider-ingress + // mode this stops its reconnect loop; it does not release the sandbox + // lease or the runner process owner. The bounded process wait above may + // observe the remote exec before this route has fully quiesced, while the + // exact durable state becomes readable only after it has. Keep the + // independently verified checkpoint ahead of process containment. + await input.checkpoint(input.runnerSettled ? "settled" : "unsettled"); + } + } catch (error) { + checkpointFailure = error; + } finally { + input.forceKill(); + } + if (checkpointFailure !== undefined) throw checkpointFailure; + if (releaseFailure !== undefined) throw releaseFailure; +} + +async function awaitRunnerSuspensionBarrier(input: { + commands: () => readonly { + commandId: string; + type: string; + status: string; + }[]; + queueSuspend: (commandId: string) => void; + readRunnerState: () => Promise>; + runnerHasExited: () => Promise; + pump: () => void; + deadline: number; + pollIntervalMs?: number; +}): Promise { + const existing = [...input.commands()] + .reverse() + .find( + (command) => + command.type === "runner.suspend" && command.status === "pending", + ); + const commandId = + existing?.commandId ?? + `command_close_suspend_${randomUUID().replaceAll("-", "")}`; + if (!existing) input.queueSuspend(commandId); + + while (Date.now() < input.deadline) { + input.pump(); + const command = input + .commands() + .find((candidate) => candidate.commandId === commandId); + if ( + command !== undefined && + command.status !== "pending" && + command.status !== "completed" + ) { + return false; + } + let lifecycle: unknown; + try { + lifecycle = (await input.readRunnerState()).lifecycle; + } catch { + // A remote filesystem can lag the command-result delivery by a small + // amount. Keep the single close deadline as the fail-closed bound. + } + if (command?.status === "completed" && lifecycle === "suspended") { + return true; + } + // Process completion alone is not a suspension proof. The durable state + // write precedes the terminal command result and process exit, so allow + // either observation to arrive first while staying within the same bound. + await input.runnerHasExited(); + await new Promise((resolveWait) => + setTimeout(resolveWait, input.pollIntervalMs ?? 10), + ); + } + return false; +} + function bridgedCodexQuestionParams( request: Record, method: string, @@ -729,6 +968,8 @@ export interface CapabilityRunnerdCodexTransportOptions { | "runner_local_connect_failed" | "runner_direct_wss_failed" | "runner_ingress_unavailable"; + /** Persists only exact, independently verified suspended remote state. */ + checkpoint?: (settlement: "settled" | "unsettled") => Promise | void; release: () => Promise | void; }>; /** Optional remote process owner used only by the new runner coordinator. */ @@ -739,6 +980,13 @@ export interface CapabilityRunnerdCodexTransportOptions { runnerStateDirectory?: string; /** Read the live durable runner state when runnerd owns a remote filesystem. */ readRunnerState?: () => Promise>; + /** Materializes a verified external checkpoint before authority rotation. */ + prepareExternalRunnerState?: () => Promise; + /** Idempotently archives and returns the verified suspended runner binding. */ + archiveExternalRunnerState?: (input: { + archiveKey: string; + priorIdentity: DurableRecoveryIdentity; + }) => Promise>; /** Active-connection recovery budget. Omitted for the existing local mode. */ runnerReconnectGraceMs?: number; /** @@ -1679,6 +1927,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { #threadId = ""; #sessionId: string | null = null; #providerIdentity: Record | null = null; + #providerIdentityEventType: + "harness.ready" | "session.started" | "session.resumed" | null = null; #checkpointProviderIdentityExpectation: { driverSessionId: string; providerSessionId: string; @@ -1688,6 +1938,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { #turnId = ""; #turnStartResponsePending = false; #turnStartResponseEpoch = 0; + #observedTurnStartEpoch = 0; + #expectedProviderTurnId: string | null = null; #durableTurnId = ""; #authorizedTools: Record | null = null; #closed = false; @@ -1698,6 +1950,9 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { #runnerRecoveryInProgress = false; #startupComplete = false; #startupFailureCode = "native_runner_process_exited"; + #controlPlaneCheckpoint: + ((settlement: "settled" | "unsettled") => Promise | void) | null = + null; #controlPlaneRelease: (() => Promise | void) | null = null; #nextTraceDebugSequence = 1; #traceRehydrationSpoolOverflow = false; @@ -2068,13 +2323,24 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } } - async #stopActiveProviderTurnBeforeSuspend(): Promise { + async #stopActiveProviderTurnBeforeSuspend( + deadline: number, + ): Promise { const state = this.#providerDrainState(); const core = this.#core; + const inferredActiveProviderTurnId = + state === null && + core !== null && + providerTurnIsActiveFromCommittedEvents(core.store.state.committedEvents) + ? this.#turnId || this.#durableTurnId + : null; + const activeProviderTurnId = + state !== null && state !== "unreadable" + ? state.activeProviderTurnId + : inferredActiveProviderTurnId; if ( - state === null || state === "unreadable" || - state.activeProviderTurnId === null || + activeProviderTurnId === null || core === null ) { return false; @@ -2086,7 +2352,6 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { commandId, true, ); - const deadline = Date.now() + 1_000; while (Date.now() < deadline) { this.#pumpEventsSafely(); const command = core.store.state.commands.find( @@ -2094,7 +2359,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { ); if (command?.status === "completed") { this.#diagnostic( - `stopped active provider turn ${state.activeProviderTurnId} before runner suspension`, + `stopped active provider turn ${activeProviderTurnId} before runner suspension`, ); return true; } @@ -2200,6 +2465,12 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { async #closeOnce(): Promise { this.#closed = true; const adoptedRunner = this.options.adoptExistingRunner; + // `settled` is a durable-state assertion, not merely the absence of a + // process handle. Registration can install a remote checkpoint callback + // before process launch; a synchronous launch failure must therefore stay + // `unsettled` and preserve its original bootstrap diagnostic. + let runnerSuspended = false; + let suspensionRequired = false; if ( this.#core !== null && (this.#handle !== null || adoptedRunner !== undefined) && @@ -2208,33 +2479,52 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { // A terminal provider frame can become visible one control loop before // its durable provider suffix is ACKed. Drain it before suspension so a // fresh run authority never inherits the prior run's pending events. + const closeDeadline = Date.now() + (this.options.closeGraceMs ?? 10_000); if (!(await this.#runnerHasExited())) { const stoppedActiveTurn = - await this.#stopActiveProviderTurnBeforeSuspend(); + await this.#stopActiveProviderTurnBeforeSuspend(closeDeadline); await this.#drainSettledProviderEventsBeforeSuspend( - stoppedActiveTurn ? 5_000 : 1_000, + Math.min( + stoppedActiveTurn ? 5_000 : 1_000, + Math.max(0, closeDeadline - Date.now()), + ), ); } - const runnerAlreadyStopping = - (await this.#runnerHasExited()) || - this.#core.store.state.commands.some( - (command) => - (command.type === "runner.suspend" || - command.type === "runner.shutdown") && - command.status === "pending", - ); - // A terminal provider event settles the turn, but it does not stop the - // runner process. Close therefore needs an explicit lifecycle command - // unless one is already pending or the process has exited. Completed - // lifecycle commands can belong to an earlier restored runner process. - if (!runnerAlreadyStopping) { - this.#core.queueCommand("runner.suspend", {}, undefined, true); + suspensionRequired = this.#controlPlaneCheckpoint !== null; + if (suspensionRequired) { + runnerSuspended = await awaitRunnerSuspensionBarrier({ + commands: () => this.#core?.store.state.commands ?? [], + queueSuspend: (commandId) => { + this.#core?.queueCommand("runner.suspend", {}, commandId, true); + }, + readRunnerState: () => this.#readDurableRunnerState(), + runnerHasExited: () => this.#runnerHasExited(), + pump: () => this.#pumpEventsSafely(), + deadline: closeDeadline, + }); + if (!runnerSuspended) { + this.#diagnostic( + "runner did not prove durable suspension before checkpoint", + ); + } + } else { + const runnerAlreadyStopping = + (await this.#runnerHasExited()) || + this.#core.store.state.commands.some( + (command) => + (command.type === "runner.suspend" || + command.type === "runner.shutdown") && + command.status === "pending", + ); + if (!runnerAlreadyStopping) { + this.#core.queueCommand("runner.suspend", {}, undefined, true); + } } try { if (this.#handle) { const result = await waitForProcess( this.#handle, - this.options.closeGraceMs ?? 10_000, + Math.max(0, closeDeadline - Date.now()), ); this.#evidence.runnerExited = true; this.#evidence.runnerExitCode = result.code; @@ -2242,8 +2532,10 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { if (result.stderr.trim()) this.#diagnostic(result.stderr.trim().slice(-4_096)); } else if (adoptedRunner) { - const deadline = Date.now() + (this.options.closeGraceMs ?? 10_000); - while ((await adoptedRunner.isAlive()) && Date.now() < deadline) { + while ( + (await adoptedRunner.isAlive()) && + Date.now() < closeDeadline + ) { await new Promise((resolveWait) => setTimeout(resolveWait, 25)); } if (await adoptedRunner.isAlive()) { @@ -2286,12 +2578,29 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { clearInterval(this.#adoptedRunnerMonitor); this.#adoptedRunnerMonitor = null; this.#queue.close(); - // Ensure a runner that missed or could not finish the graceful lifecycle - // command cannot keep the control-plane server alive during teardown. - this.#handle?.child.kill("SIGKILL"); - if (this.#controlPlaneRelease !== null) await this.#controlPlaneRelease(); - await this.#core?.stop(); - this.#controlPlaneRelease = null; + // A suspended remote runner still owns the only readable copy of its + // provider state. Quiesce the authenticated route, then probe its + // independently verified durable state before releasing the process owner; + // an incomplete or identity-conflicting state remains fail-closed. + try { + await releaseRunnerProcessOwnership({ + runnerSettled: runnerSuspended, + checkpoint: this.#controlPlaneCheckpoint, + forceKill: () => { + this.#handle?.child.kill("SIGKILL"); + }, + release: this.#controlPlaneRelease, + }); + } finally { + await this.#core?.stop(); + this.#controlPlaneCheckpoint = null; + this.#controlPlaneRelease = null; + } + if (suspensionRequired && !runnerSuspended) { + throw new Error( + "provider_transport_failed: runner did not durably suspend before checkpoint", + ); + } if (this.#ownsRoot) rmSync(this.#root, { recursive: true, force: true }); this.#publish(); } @@ -2347,7 +2656,16 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { connectionLeaseTtlMs: 60 * 60 * 1_000, }); this.#core = core; - mkdirSync(resolve(this.#root, "runner"), { recursive: true, mode: 0o700 }); + // Externally launched runners own their state directory (for example in a + // Daytona sandbox). Do not create an empty controller-side placeholder: + // prior-run authority checks must be able to distinguish absent remote + // state from malformed direct state. + if (this.options.runnerStateDirectory === undefined) { + mkdirSync(resolve(this.#root, "runner"), { + recursive: true, + mode: 0o700, + }); + } const provider = this.options.provider ?? "codex"; const sourceRuntimeContext = this.options.runtimeContext ?? null; const runtimeContext = @@ -2654,7 +2972,10 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#startupFailureCode = registration?.startupFailureCode ?? "runner_local_connect_failed"; if (registration === null) await core.start(); - else this.#controlPlaneRelease = registration.release; + else { + this.#controlPlaneCheckpoint = registration.checkpoint ?? null; + this.#controlPlaneRelease = registration.release; + } const handle = spawnRunner({ connection: registration?.connection ?? { mode: "connect", @@ -2782,39 +3103,73 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { identity.itemId === desiredIdentity.itemId; let rotatedAuthority = false; if (controlPlaneState === null) { - if (!localProvider || !localStateOwner) { + if (!localProvider) { throw new Error("PRP provider resume state is unavailable"); } - try { - const archivedState = latestArchivedControlPlaneState( - this.#root, - desiredIdentity, - ); - if (!archivedState) { - throw new Error("PRP provider resume state is unavailable"); + const archivedState = latestArchivedControlPlaneState( + this.#root, + desiredIdentity, + ); + if (!archivedState) { + throw new Error("PRP provider resume state is unavailable"); + } + if (localStateOwner) { + try { + controlPlaneState = rotateLocalAuthorityEpoch( + this.#root, + archivedState, + desiredIdentity, + ); + } catch (error) { + quarantineLocalRuntimeState(this.#root, error); } - controlPlaneState = rotateLocalAuthorityEpoch( + } else { + if ( + this.options.readRunnerState === undefined || + this.options.archiveExternalRunnerState === undefined + ) { + throw new Error("native_runner_prp_run_rotation_unavailable"); + } + controlPlaneState = await rotateExternalAuthorityEpoch( this.#root, archivedState, desiredIdentity, + this.options.readRunnerState, + this.options.archiveExternalRunnerState, ); - } catch (error) { - quarantineLocalRuntimeState(this.#root, error); } identity = desiredIdentity; rotatedAuthority = true; } else if (!exactAuthority) { - if (!localProvider || !localStateOwner || controlPlaneState === null) { + if (!localProvider || controlPlaneState === null) { throw new Error("native_runner_prp_run_rotation_unavailable"); } - try { - controlPlaneState = rotateLocalAuthorityEpoch( + if (localStateOwner) { + try { + controlPlaneState = rotateLocalAuthorityEpoch( + this.#root, + controlPlaneState, + desiredIdentity, + ); + } catch (error) { + quarantineLocalRuntimeState(this.#root, error); + } + } else { + if ( + this.options.readRunnerState === undefined || + this.options.prepareExternalRunnerState === undefined || + this.options.archiveExternalRunnerState === undefined + ) { + throw new Error("native_runner_prp_run_rotation_unavailable"); + } + await this.options.prepareExternalRunnerState(); + controlPlaneState = await rotateExternalAuthorityEpoch( this.#root, controlPlaneState, desiredIdentity, + this.options.readRunnerState, + this.options.archiveExternalRunnerState, ); - } catch (error) { - quarantineLocalRuntimeState(this.#root, error); } identity = desiredIdentity; rotatedAuthority = true; @@ -2955,6 +3310,16 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } const committedEvents = core.store.state.committedEvents; const runAttachment = recoveredRunAttachment(core.store.state); + // Reconnecting the exact run authority has no run.attach command to wake + // provider restoration. Queue a unique, side-effect-free barrier before + // runnerd starts so every provider backend restores its durable session. + const recoveryProbeCommandId = + exactAuthority && runAttachment === null + ? `command_resume_probe_${randomUUID().replaceAll("-", "")}` + : null; + if (recoveryProbeCommandId !== null) { + core.queueCommand("runner.drain", {}, recoveryProbeCommandId); + } // A controller retry can open the exact authority after run.attach has // already reached a durable outcome. Re-observe that command instead of // silently waiting for an identity that a failed command can never emit. @@ -2976,9 +3341,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { committedEvents[adoptedProviderIdentityIndex]!, ); } else if ( - this.options.adoptExistingRunner && exactAuthority && - adoptedProviderIdentityIndex < 0 && this.options.resumeProviderSession?.driverSessionId.trim() && this.options.resumeProviderSession.providerSessionId?.trim() ) { @@ -2998,7 +3361,9 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { : structuredClone(this.#providerIdentity), }; this.#diagnostic( - "restored adopted provider identity from the exact durable checkpoint after PRP event compaction; awaiting live confirmation", + this.options.adoptExistingRunner && adoptedProviderIdentityIndex < 0 + ? "restored adopted provider identity from the exact durable checkpoint after PRP event compaction; awaiting live confirmation" + : "restored provider identity from the exact durable checkpoint; awaiting live confirmation", ); } const registration = this.options.controlPlaneRegistration @@ -3007,7 +3372,10 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#startupFailureCode = registration?.startupFailureCode ?? "runner_local_connect_failed"; if (registration === null) await core.start(); - else this.#controlPlaneRelease = registration.release; + else { + this.#controlPlaneCheckpoint = registration.checkpoint ?? null; + this.#controlPlaneRelease = registration.release; + } const adoptedRunner = this.options.adoptExistingRunner; const handle = adoptedRunner ? null @@ -3075,7 +3443,34 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { if (runAttachment) { await this.#waitCommand("run.attach", runAttachment.commandId); } - await this.#waitForProviderIdentity(); + if (recoveryProbeCommandId !== null) { + await this.#waitCommand("runner.drain", recoveryProbeCommandId); + if (this.#checkpointProviderIdentityExpectation !== null) { + // A replacement runner can restore the exact provider while its fresh + // session.resumed event is compacted or delayed behind the completed + // recovery barrier. Confirm the live session directly instead of + // waiting only on the bounded event replay. The authenticated command + // result is still checked against the exact database checkpoint, so a + // missing or changed provider identity continues to fail closed. + const snapshot = await this.#commandResult("session.snapshot", {}); + this.#confirmCheckpointProviderIdentity( + snapshot, + "authenticated recovery session.snapshot", + ); + } + } + // A relaunched executor proves its restored provider with either its fresh + // resume identity or the authenticated snapshot above. An adopted live + // executor keeps the already-verified provider session, so its + // authenticated drain plus the committed identity are the corresponding + // continuity proof. + await this.#waitForProviderIdentity( + recoveryProbeCommandId !== null && + adoptedRunner === undefined && + !this.#checkpointProviderIdentityConfirmed + ? "session.resumed" + : undefined, + ); this.#startupComplete = true; this.#diagnostic( rotatedAuthority @@ -3095,29 +3490,75 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#turnId = pendingTurnId; const responseEpoch = ++this.#turnStartResponseEpoch; this.#turnStartResponsePending = true; + this.#expectedProviderTurnId = null; let responseReady = false; try { - await this.#command("turn.start", { text: message }); - // Command completion only means runnerd accepted the command. Codex assigns - // the authoritative turn identity in the subsequent turn/started event, so - // do not expose the temporary transport identity to the strict driver. + // Persist a fresh requested identity with the durable command. ACPX uses + // it as its provider request identity, so a same-run recovery turn cannot + // alias an already-settled request from the retained provider session. + // Codex and OpenCode continue to return their provider-assigned identity. + const startResult = await this.#commandResult("turn.start", { + text: message, + turnId: pendingTurnId, + }); + const expectedProviderTurnId = + typeof startResult.providerTurnId === "string" && + startResult.providerTurnId.length > 0 + ? startResult.providerTurnId + : null; + if (expectedProviderTurnId === null) { + const error = new Error( + "runnerd turn.start omitted its provider turn identity", + ); + this.#failTransport(error); + throw error; + } + if ( + !turnStartCommandResultValid({ + requestedTurnId: pendingTurnId, + providerTurnId: expectedProviderTurnId, + requireRequestedIdentity: this.options.provider === "acpx", + }) + ) { + const error = new Error( + "runnerd ACPX turn.start changed its requested provider turn identity", + ); + this.#failTransport(error); + throw error; + } + this.#expectedProviderTurnId = expectedProviderTurnId; + // Command completion only means runnerd accepted the command. Bind the + // provider turn from the subsequent turn/started event before answering + // the strict driver. Correlate that event with the exact identity in this + // command's durable result so a delayed prior-turn event cannot satisfy + // the new response fence. ACPX echoes the requested identity, while + // Codex and OpenCode return their provider-assigned identity. const deadline = Date.now() + 30_000; - while (this.#turnId === pendingTurnId && Date.now() < deadline) { + const providerTurnStarted = () => + turnStartResponseReady({ + responseEpoch, + observedEpoch: this.#observedTurnStartEpoch, + expectedProviderTurnId, + boundTurnId: this.#turnId, + }); + while (!providerTurnStarted() && Date.now() < deadline) { this.#throwIfFailed(); this.#pumpEvents(); - if (this.#turnId !== pendingTurnId) break; + if (providerTurnStarted()) break; if (await this.#runnerHasExited()) throw new Error("runnerd exited before provider turn startup"); await new Promise((resolveWait) => setTimeout(resolveWait, 10)); } - if (this.#turnId === pendingTurnId) + if (!providerTurnStarted()) throw new Error("runnerd did not report the provider turn identity"); responseReady = true; return { turn: { id: this.#turnId, status: "inProgress" } }; } finally { if (!responseReady) { - if (this.#turnStartResponseEpoch === responseEpoch) + if (this.#turnStartResponseEpoch === responseEpoch) { this.#turnStartResponsePending = false; + this.#expectedProviderTurnId = null; + } } else { // Resolving this async method schedules the strict driver's response // continuation as a microtask. Keep terminal frames held until the @@ -3126,6 +3567,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { const release = setTimeout(() => { if (this.#turnStartResponseEpoch !== responseEpoch) return; this.#turnStartResponsePending = false; + this.#expectedProviderTurnId = null; if (!this.#closed) this.#pumpEventsSafely(); }, 0); release.unref(); @@ -3183,7 +3625,9 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { return record(record(command.result).result); } - async #waitForProviderIdentity(): Promise { + async #waitForProviderIdentity( + expectedEventType?: "harness.ready" | "session.started" | "session.resumed", + ): Promise { const deadline = Date.now() + 30_000; while (Date.now() < deadline) { this.#throwIfFailed(); @@ -3192,7 +3636,9 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#threadId.length > 0 && (this.#checkpointProviderIdentityExpectation !== null || this.#evidence.providerExecutionKind === "remote_service" || - this.#evidence.providerPid !== null) + this.#evidence.providerPid !== null) && + (expectedEventType === undefined || + this.#providerIdentityEventType === expectedEventType) ) return; if (await this.#runnerHasExited()) @@ -3232,6 +3678,18 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { while (this.#eventIndex < events.length) { const event = events[this.#eventIndex]!; const eventPayload = record(event.envelope.payload).payload; + const turnStartWhileCommandResultPending = + this.#turnStartResponsePending && + this.#expectedProviderTurnId === null && + (event.eventType === "turn.started" || + (event.eventType === "provider.event" && + unwrapRunnerdProviderNotifications(eventPayload).some( + (notification) => notification.method === "turn/started", + ))); + // The durable command result is the only correlation authority for a + // provider-assigned turn id. Leave an early start at the cursor until + // that exact expected identity is installed. + if (turnStartWhileCommandResultPending) return; const terminalWhileTurnStartPending = this.#turnStartResponsePending && ([ @@ -3356,6 +3814,23 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { const method = payload.method; if (typeof method !== "string") continue; const rawParams = record(payload.params); + const rawTurn = record(rawParams.turn); + // Correlate starts only with an explicit provider-native identity. + // The later rehydration fallback may use the durable controller turn, + // which is not evidence that the provider accepted this command. + const explicitProviderTurnId = + method === "turn/started" + ? typeof rawParams.providerTurnId === "string" && + rawParams.providerTurnId.length > 0 + ? rawParams.providerTurnId + : typeof rawTurn.id === "string" && rawTurn.id.length > 0 + ? rawTurn.id + : event.eventType === "provider.event" && + typeof rawParams.turnId === "string" && + rawParams.turnId.length > 0 + ? rawParams.turnId + : null + : null; const params = method === "thread/tokenUsage/updated" ? rehydrateRunnerdUsageNotification( @@ -3413,8 +3888,28 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { // the provider-native id before its terminal is rehydrated. if (method === "turn/started") { const providerTurnId = record(params.turn).id ?? params.turnId; + const disposition = turnStartNotificationDisposition({ + responsePending: this.#turnStartResponsePending, + expectedProviderTurnId: this.#expectedProviderTurnId, + observedProviderTurnId: explicitProviderTurnId ?? "", + }); + if (disposition === "defer") { + throw new Error( + "turn/started advanced before its durable command result", + ); + } + if (disposition === "reject") { + const error = new Error( + "turn/started identity disagreed with its durable command result", + ); + this.#failTransport(error); + throw error; + } if (typeof providerTurnId === "string" && providerTurnId.length > 0) { this.#turnId = providerTurnId; + if (this.#turnStartResponsePending) { + this.#observedTurnStartEpoch = this.#turnStartResponseEpoch; + } } } this.#queue.push({ @@ -3471,6 +3966,14 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } #applyProviderIdentityEvent(event: DurableRecoveryCommittedEvent): void { + if ( + event.eventType !== "harness.ready" && + event.eventType !== "session.started" && + event.eventType !== "session.resumed" + ) { + return; + } + this.#providerIdentityEventType = event.eventType; const started = record(record(event.envelope.payload).payload); const runtimeIdentity = record(started.runtimeIdentity); const descriptor = record(started.providerDescriptor); @@ -3707,9 +4210,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { result.code === 0 && (this.options.lifecyclePolicy?.mode ?? "per_turn") === "per_turn" && (this.#core?.store.state.committedEvents.some( - (event) => - event.eventType === "runner.suspending" || - event.eventType === "run.terminal", + (event) => event.eventType === "runner.suspending", ) ?? false); if (expectedPerTurnExit) return; @@ -3931,6 +4432,13 @@ export const runnerdLaunchProfileInternals = Object.freeze({ }); export const runnerdRecoveryInternals = Object.freeze({ + awaitRunnerSuspensionBarrier, providerDrainStateFromSnapshot, + providerTurnIsActiveFromCommittedEvents, recoveredRunAttachment, + releaseRunnerProcessOwnership, + rotateExternalAuthorityEpoch, + turnStartCommandResultValid, + turnStartNotificationDisposition, + turnStartResponseReady, }); diff --git a/packages/paperclip-runner/src/native-session-runtime.test.ts b/packages/paperclip-runner/src/native-session-runtime.test.ts index 3a6c5efe6b..54e34896cd 100644 --- a/packages/paperclip-runner/src/native-session-runtime.test.ts +++ b/packages/paperclip-runner/src/native-session-runtime.test.ts @@ -1388,9 +1388,9 @@ describe("executeNativeSession recovery", () => { // resources. A replacement bootstrap cannot start concurrently. await vi.advanceTimersByTimeAsync(101); const blockedAdmission = execute(); - const blockedAdmissionRejection = expect(blockedAdmission).rejects.toThrow( - "replacement bootstrap launched", - ); + const blockedAdmissionRejection = expect( + blockedAdmission, + ).rejects.toThrow("replacement bootstrap launched"); await vi.advanceTimersByTimeAsync(1_000); expect(openSession).toHaveBeenCalledOnce(); @@ -2679,6 +2679,215 @@ describe("executeNativeSession recovery", () => { await pendingClose; }); + it.each([false, true])( + "waits for a required backend checkpoint close when enrichment failure=%s", + async (enrichmentFails) => { + let releaseClose = () => {}; + const pendingClose = new Promise((resolve) => { + releaseClose = resolve; + }); + const close = vi.fn(() => pendingClose); + let runCompleted = false; + let enrichmentFailureObserved = false; + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { + resume: true, + typedEvents: true, + steering: false, + interruption: false, + structuredResult: true, + }; + }, + async *events() { + yield runnerEvent(1, "turn.completed"); + }, + async startTurn() { + return { turnId: "turn-recovery" }; + }, + async result() { + return { result, terminal, turnId: "turn-recovery" }; + }, + async snapshot() { + // Exercise only the best-effort enrichment snapshot after the + // control plane has durably committed the run result. + if (enrichmentFails && runCompleted) { + enrichmentFailureObserved = true; + throw new Error("checkpoint enrichment failed"); + } + return { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: "1", + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: false, + structuredResult: true, + }, + }; + }, + async openSession() { + return session; + }, + }; + const events: PrpEvent[] = []; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent(event) { + events.push(structuredClone(event as PrpEvent)); + return { + cursor: events.length, + highestContiguousSourceSeq: highestContiguous(events), + disposition: "committed", + }; + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, + async completeRun() { + runCompleted = true; + }, + }; + + let resolved = false; + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + requireSessionCloseBeforeReturn: true, + }).then((value) => { + resolved = true; + return value; + }); + await vi.waitFor(() => expect(close).toHaveBeenCalledOnce()); + await new Promise((resolve) => setTimeout(resolve, 150)); + expect(resolved).toBe(false); + releaseClose(); + await expect(execution).resolves.toMatchObject({ result, terminal }); + expect(resolved).toBe(true); + expect(enrichmentFailureObserved).toBe(enrichmentFails); + }, + ); + + it("propagates an exhausted required backend checkpoint close", async () => { + vi.useFakeTimers(); + try { + const closeFailure = new Error("required remote checkpoint close failed"); + const close = vi.fn(({ reason }: { reason: string }) => + reason === "native session quarantined cleanup recovery" + ? Promise.resolve() + : Promise.reject(closeFailure), + ); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { + resume: true, + typedEvents: true, + steering: false, + interruption: false, + structuredResult: true, + }; + }, + async *events() { + yield runnerEvent(1, "turn.completed"); + }, + async startTurn() { + return { turnId: "turn-recovery" }; + }, + async result() { + return { result, terminal, turnId: "turn-recovery" }; + }, + async snapshot() { + return { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: "1", + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: false, + structuredResult: true, + }, + }; + }, + async openSession() { + return session; + }, + }; + const events: PrpEvent[] = []; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent(event) { + events.push(structuredClone(event as PrpEvent)); + return { + cursor: events.length, + highestContiguousSourceSeq: highestContiguous(events), + disposition: "committed", + }; + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, + async completeRun() {}, + }; + + const execution = expect( + executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + requireSessionCloseBeforeReturn: true, + }), + ).rejects.toThrow(closeFailure); + await vi.advanceTimersByTimeAsync(3_000); + await execution; + expect(close).toHaveBeenCalledTimes(5); + } finally { + vi.useRealTimers(); + } + }); + it("closes after a synchronous governed-wait probe returns no result", async () => { const resolveGovernedWait = vi.fn(() => null); const lifecycle: string[] = []; @@ -4652,9 +4861,10 @@ describe("executeNativeSession recovery", () => { }); }); - it("retries disposition recovery when the driver releases an absent bound provider turn", async () => { + it("only replays the original ACPX envelope for a proven effect-free initial turn", async () => { const checkpoint: PersistedNativeSession = { backendKind: "mock", + driverKind: "acpx_runtime", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", @@ -4769,6 +4979,51 @@ describe("executeNativeSession recovery", () => { async completeRun() {}, }; + const submittedTurn = runnerEvent(1, "turn.submitted"); + delete submittedTurn.turnId; + const effectFreeTurn = [ + submittedTurn, + { + ...runnerEvent(2, "turn.started", { status: "inProgress" }), + turnId: "turn-work", + }, + { ...runnerEvent(3, "turn.accepted"), turnId: "turn-work" }, + { + ...runnerEvent(4, "item.completed", { + kind: "usage", + usage: { + total: { + requests: 1, + inputTokens: 0, + outputTokens: 0, + activeSeconds: 0, + providerCostUsd: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, + }, + runDelta: { + requests: 1, + inputTokens: 0, + outputTokens: 0, + activeSeconds: 0, + providerCostUsd: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, + }, + }, + }), + turnId: "turn-work", + }, + { + ...runnerEvent(5, "turn.completed", { + status: "completed", + error: null, + }), + turnId: "turn-work", + }, + ]; + bySource.set("runner-recovery", effectFreeTurn); + await expect( executeNativeSession({ input, @@ -4788,17 +5043,62 @@ describe("executeNativeSession recovery", () => { afterSourceSeq: 0, limit: 1_000, }); - expect(replayedPages.every((events) => events.length === 0)).toBe(true); + expect( + replayedPages.some( + (events) => + events.length === effectFreeTurn.length && + events.every( + (event, index) => + event.sourceSeq === effectFreeTurn[index]!.sourceSeq, + ), + ), + ).toBe(true); const recoveryEnvelope = JSON.parse( startTurn.mock.calls[0]![0].message.text, ) as { task: { prompt: string } }; - expect(recoveryEnvelope.task.prompt).toContain( + expect(recoveryEnvelope.task.prompt).toBe(input.task.prompt); + + startTurn.mockClear(); + bySource.set("runner-recovery", [ + ...effectFreeTurn.slice(0, 3), + { + ...runnerEvent(4, "item.completed", { + kind: "agentMessage", + text: "Work may already have been performed.", + }), + turnId: "turn-work", + }, + { + ...runnerEvent(5, "turn.completed", { + status: "completed", + error: null, + }), + turnId: "turn-work", + }, + ]); + + await expect( + executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + }), + ).resolves.toMatchObject({ + turnId: "turn-continuation", + providerSessionId: "provider-recovery", + }); + const dispositionEnvelope = JSON.parse( + startTurn.mock.calls[0]![0].message.text, + ) as { task: { prompt: string } }; + expect(dispositionEnvelope.task.prompt).toContain( "semantic-result recovery for a prior completed provider turn", ); - expect(recoveryEnvelope.task.prompt).toContain( + expect(dispositionEnvelope.task.prompt).toContain( "Do not repeat implementation, tests, research, or the final answer", ); - expect(recoveryEnvelope.task.prompt).not.toContain(input.task.prompt); + expect(dispositionEnvelope.task.prompt).not.toContain(input.task.prompt); checkpoint.dispositionOnlyRecoveryTurnId = undefined; recoveredSnapshot.dispositionOnlyRecoveryTurnId = undefined; @@ -5348,10 +5648,12 @@ describe("executeNativeSession recovery", () => { pendingRuntimeRequests: [], lineage: [], }; - const events = [{ - ...controlEvent(1, "run.result.accepted", { result }), - turnId: "turn-with-result", - }]; + const events = [ + { + ...controlEvent(1, "run.result.accepted", { result }), + turnId: "turn-with-result", + }, + ]; const checkpoints: PersistedNativeSession[] = []; const completeRun = vi.fn(async () => undefined); const startTurn = vi.fn(async () => ({ turnId: "unexpected-turn" })); diff --git a/packages/paperclip-runner/src/native-session-runtime.ts b/packages/paperclip-runner/src/native-session-runtime.ts index f59793d30c..5a1c72da05 100644 --- a/packages/paperclip-runner/src/native-session-runtime.ts +++ b/packages/paperclip-runner/src/native-session-runtime.ts @@ -86,6 +86,12 @@ export interface ExecuteNativeSessionOptions { existingSession?: NativeSession; persistedSession?: PersistedNativeSession | null; keepSessionOpen?: boolean; + /** + * Wait for the backend's close contract before returning a durable result. + * Use this only for backends whose close path is internally bounded and + * carries required persistence (for example, a remote runner checkpoint). + */ + requireSessionCloseBeforeReturn?: boolean; onCheckpoint?: ( snapshot: PersistedNativeSession, options?: CheckpointControlPlaneSessionOptions, @@ -763,9 +769,10 @@ async function consumeTurn( }; while (true) { pendingNext ??= eventIterator.next(); - const next = semanticResultDeadline === null - ? await pendingNext - : await Promise.race([pendingNext, semanticResultDeadline]); + const next = + semanticResultDeadline === null + ? await pendingNext + : await Promise.race([pendingNext, semanticResultDeadline]); if (next === semanticResultGraceExpired) { void pendingNext.catch(() => undefined); if (semanticResultEvent === null || governedResult === null) { @@ -886,7 +893,10 @@ async function consumeTurn( // Invalid structured inputs remain rejected by the driver and never become durable questions. } } - if (governedResult === null && event.eventType === "run.result.proposed") { + if ( + governedResult === null && + event.eventType === "run.result.proposed" + ) { const validation = validatePrpStructuredRunResult(event.payload); if (!validation.ok) { throw new Error("native_semantic_result_invalid"); @@ -1178,6 +1188,151 @@ async function replayCheckpointedTurnTerminal(input: { } } +const EFFECT_FREE_ACPX_USAGE_COUNTERS = [ + "inputTokens", + "outputTokens", + "activeSeconds", + "providerCostUsd", + "cacheReadTokens", + "cacheWriteTokens", +] as const; + +function objectRecord(value: unknown): Record | null { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Record) + : null; +} + +function isZeroWorkAcpxUsage(payload: Record): boolean { + if (payload.kind !== "usage") return false; + const usage = objectRecord(payload.usage); + if ( + usage === null || + Object.keys(usage).some((key) => key !== "total" && key !== "runDelta") + ) { + return false; + } + return ["total", "runDelta"].every((sectionName) => { + const section = objectRecord(usage[sectionName]); + if ( + section === null || + section.requests !== 1 || + Object.keys(section).some( + (key) => + key !== "requests" && + !EFFECT_FREE_ACPX_USAGE_COUNTERS.includes( + key as (typeof EFFECT_FREE_ACPX_USAGE_COUNTERS)[number], + ), + ) + ) { + return false; + } + return EFFECT_FREE_ACPX_USAGE_COUNTERS.every( + (counter) => section[counter] === 0, + ); + }); +} + +async function replayProvesEffectFreeInitialAcpxTurn(input: { + controlPlane: ControlPlanePort; + checkpoint: PersistedNativeSession; + runId: string; + sourceInstanceId: string; +}): Promise { + const terminalTurns = input.checkpoint.terminalTurns ?? []; + if ( + input.checkpoint.driverKind !== "acpx_runtime" || + input.checkpoint.semanticResult || + input.checkpoint.activeTurnId || + terminalTurns.length !== 1 || + terminalTurns[0]!.turnId.length === 0 + ) { + return false; + } + const targetTurnId = terminalTurns[0]!.turnId; + const events: PrpEvent[] = []; + let afterSourceSeq = 0; + try { + while (true) { + const replay = await input.controlPlane.replayEvents({ + runId: input.runId, + sourceInstanceId: input.sourceInstanceId, + afterSourceSeq, + limit: 1_000, + }); + if (replay.events.length === 0) break; + for (const event of replay.events) { + // An incomplete or reordered replay cannot prove absence of work. + if (event.sourceSeq !== afterSourceSeq + 1) return false; + events.push(structuredClone(event)); + afterSourceSeq = event.sourceSeq; + if (events.length > 10_000) return false; + } + } + } catch { + return false; + } + + const submittedIndexes = events.flatMap((event, index) => + event.eventType === "turn.submitted" ? [index] : [], + ); + const startedIndexes = events.flatMap((event, index) => + event.turnId === targetTurnId && event.eventType === "turn.started" + ? [index] + : [], + ); + const acceptedIndexes = events.flatMap((event, index) => + event.turnId === targetTurnId && event.eventType === "turn.accepted" + ? [index] + : [], + ); + const completedIndexes = events.flatMap((event, index) => + event.turnId === targetTurnId && event.eventType === "turn.completed" + ? [index] + : [], + ); + if ( + submittedIndexes.length !== 1 || + startedIndexes.length !== 1 || + acceptedIndexes.length !== 1 || + completedIndexes.length !== 1 + ) { + return false; + } + const submittedIndex = submittedIndexes[0]!; + const startedIndex = startedIndexes[0]!; + const acceptedIndex = acceptedIndexes[0]!; + const completedIndex = completedIndexes[0]!; + const completedPayload = objectRecord(events[completedIndex]!.payload); + if ( + startedIndex !== submittedIndex + 1 || + acceptedIndex !== startedIndex + 1 || + completedIndex <= acceptedIndex || + events[submittedIndex]!.turnId != null || + completedPayload?.status !== "completed" || + (completedPayload?.error !== undefined && completedPayload?.error !== null) + ) { + return false; + } + if ( + events.some( + (event, index) => + event.turnId === targetTurnId && + (index < startedIndex || index > completedIndex), + ) + ) { + return false; + } + return events + .slice(acceptedIndex + 1, completedIndex) + .every( + (event) => + event.turnId === targetTurnId && + event.eventType === "item.completed" && + isZeroWorkAcpxUsage(event.payload), + ); +} + function checkpointedResultlessDispositionFallback(input: { persisted: PersistedNativeSession; recovered: PersistedNativeSession; @@ -1729,7 +1884,16 @@ export async function executeNativeSession( (persistedSession?.terminalTurns?.length ?? 0) > 0 && !recoveredActiveTurnId, ); - if (dispositionOnlyRecovery) { + const effectFreeInitialAcpxTurn = + dispositionOnlyRecovery && persistedSession + ? await replayProvesEffectFreeInitialAcpxTurn({ + controlPlane: options.controlPlane, + checkpoint: persistedSession, + runId: input.binding.runId, + sourceInstanceId: options.runnerInstanceId, + }) + : false; + if (dispositionOnlyRecovery && !effectFreeInitialAcpxTurn) { modelEnvelope.task.prompt = [ "Paperclip semantic-result recovery for a prior completed provider turn.", "The prior turn already performed the work and its user-facing final answer is recorded.", @@ -2001,19 +2165,29 @@ export async function executeNativeSession( executionSucceeded = true; return { ...durableExecutionResult, ...enrichment }; } finally { - if ( - (!options.keepSessionOpen || !executionSucceeded || sessionQuarantined) && - !failedCleanupDeferred - ) { + const shouldClose = + !options.keepSessionOpen || !executionSucceeded || sessionQuarantined; + if (shouldClose && options.requireSessionCloseBeforeReturn) { + if (!failedCleanupDeferred) closeSession(); + // A remote runner close owns its suspension and verified checkpoint. + // Its implementation is finite, and the host must not release the + // environment until the complete close/retry owner has settled. + const requiredClose = sessionCloseRecoveryPromise ?? sessionClosePromise; + if (requiredClose !== null) { + // Unlike ordinary provider cleanup, this close owns required remote + // checkpoint persistence. Exhausting its bounded recovery must fail + // the execution instead of converting the rejection into success. + await requiredClose; + } + } else if (shouldClose && !failedCleanupDeferred) { // A provider that ignores close must not keep execution pending forever. // closeSession removes it from the caller before invoking the backend; // retain observation of the promise, but bound the final join. Provider // cleanup cannot reverse a result the control plane already committed; // after that durable boundary the session remains unavailable for reuse // and late close rejection stays observed without contradicting success. - const closeSettlement = Promise.allSettled([closeSession()]); await settlesWithin( - closeSettlement, + Promise.allSettled([closeSession()]), FAILED_OPERATION_SETTLEMENT_GRACE_MS, ); } @@ -2035,7 +2209,10 @@ function canonicalJson(value: unknown): string { function completedSemanticResultTurnId( snapshot: PersistedNativeSession, ): string | null { - if (snapshot.semanticResult === undefined || snapshot.semanticResult === null) { + if ( + snapshot.semanticResult === undefined || + snapshot.semanticResult === null + ) { return null; } const semanticFingerprint = canonicalJson(snapshot.semanticResult); @@ -2043,12 +2220,12 @@ function completedSemanticResultTurnId( try { const value: unknown = JSON.parse(terminal.fingerprint); if ( - typeof value === "object" - && value !== null - && !Array.isArray(value) - && (value as Record).status === "completed" - && (value as Record).semanticResult - === semanticFingerprint + typeof value === "object" && + value !== null && + !Array.isArray(value) && + (value as Record).status === "completed" && + (value as Record).semanticResult === + semanticFingerprint ) { return terminal.turnId; } diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index 087eea9577..e69e2fa9a9 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -4640,9 +4640,10 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { try { const normalizedSessionId = "native-replacement-session"; const runnerInstanceId = "native-replacement-runner"; + const sessionScopeId = "native-replacement-session-scope-v2"; const sessionRoot = path.join( backupBase, - createHash("sha256").update(normalizedSessionId).digest("hex"), + createHash("sha256").update(sessionScopeId).digest("hex"), ); const current = path.join(sessionRoot, "failover-backups", "current"); await mkdir(path.join(current, "runner"), { recursive: true }); @@ -4681,6 +4682,8 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { await writeFile(manifestPath, JSON.stringify(manifest)); const stamp = createNativeHarnessBackupStamp({ manifestPath, + sessionScopeId, + authorizedProviderLeaseId: seeded.reusableLease.providerLeaseId!, normalizedSessionId, runnerInstanceId, completedAt: manifest.completedAt, diff --git a/server/src/__tests__/recovery-stale-issue-lock-sweep.test.ts b/server/src/__tests__/recovery-stale-issue-lock-sweep.test.ts index 67fffd3815..7b23d65dec 100644 --- a/server/src/__tests__/recovery-stale-issue-lock-sweep.test.ts +++ b/server/src/__tests__/recovery-stale-issue-lock-sweep.test.ts @@ -22,6 +22,7 @@ const mockTelemetryClient = vi.hoisted(() => ({ track: vi.fn() })); vi.mock("../telemetry.ts", () => ({ getTelemetryClient: () => mockTelemetryClient })); import { heartbeatService } from "../services/heartbeat.ts"; +import { recoveryService } from "../services/recovery/service.ts"; const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; @@ -339,6 +340,49 @@ describeEmbeddedPostgres("recovery sweepStaleIssueLocks", () => { .resolves.toEqual([{ checkoutRunId: runningRunId, executionRunId: runningRunId }]); }); + it("preserves a process-less run while its in-process execution is still finalizing", async () => { + const { companyId, agentId, runningRunId } = await seed(); + const issueId = randomUUID(); + await db.insert(issues).values({ + id: issueId, + companyId, + title: "Native finalization remains live", + status: "in_progress", + priority: "high", + assigneeAgentId: agentId, + checkoutRunId: runningRunId, + executionRunId: runningRunId, + executionLockedAt: new Date(), + }); + await db + .update(heartbeatRuns) + .set({ + runtimeMode: "native", + processPid: 2_000_000_000, + }) + .where(eq(heartbeatRuns.id, runningRunId)); + + const result = await recoveryService(db, { + enqueueWakeup: vi.fn(), + liveRunExecutions: new Set([runningRunId]), + }).sweepStaleIssueLocks(); + + expect(result).toEqual({ + cleared: 0, + issueIds: [], + terminalizedRunIds: [], + }); + await expect(db.select({ status: heartbeatRuns.status }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, runningRunId))) + .resolves.toEqual([{ status: "running" }]); + await expect(db.select({ + checkoutRunId: issues.checkoutRunId, + executionRunId: issues.executionRunId, + }).from(issues).where(eq(issues.id, issueId))) + .resolves.toEqual([{ checkoutRunId: runningRunId, executionRunId: runningRunId }]); + }); + it("terminalizes a running run whose issue is terminal, even while the process stays alive (reuse-lease path)", async () => { // Reuse Lease ON stops the sandbox but keeps the server process alive, so // the in-memory handle and the recorded pid can both persist. The diff --git a/server/src/redaction.ts b/server/src/redaction.ts index 8b8871ac23..adba687753 100644 --- a/server/src/redaction.ts +++ b/server/src/redaction.ts @@ -94,6 +94,7 @@ export const PAPERCLIP_PUBLIC_SCHEMA_IDS = new Set([ "paperclip.native-finalization.v1", "paperclip.native-finalization.v2", "paperclip.native-harness-backup-stamp.v1", + "paperclip.native-harness-backup-stamp.v2", "paperclip.native-harness-backup.v1", "paperclip.native-model-envelope.v1", "paperclip.native-model-envelope.v2", diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 7c01771fef..195ad4810b 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -8330,7 +8330,7 @@ export function heartbeatService( cancelWorkForScope: cancelBudgetScopeWork, }; const budgets = budgetService(db, budgetHooks); - const recovery = recoveryService(db, { enqueueWakeup }); + const recovery = recoveryService(db, { enqueueWakeup, liveRunExecutions }); function isPlanApprovalConfirmationPayload(payload: unknown) { const target = parseObject(parseObject(payload).target); diff --git a/server/src/services/native-runtime/native-harness-backup-stamp.ts b/server/src/services/native-runtime/native-harness-backup-stamp.ts index e281bfce71..202bf8a9cf 100644 --- a/server/src/services/native-runtime/native-harness-backup-stamp.ts +++ b/server/src/services/native-runtime/native-harness-backup-stamp.ts @@ -1,36 +1,63 @@ import { createHash } from "node:crypto"; -import { - existsSync, - lstatSync, - readFileSync, - readdirSync, - readlinkSync, -} from "node:fs"; -import { resolve } from "node:path"; +import { lstatSync, readFileSync, readdirSync, readlinkSync } from "node:fs"; +import { dirname, resolve } from "node:path"; import { resolvePaperclipInstanceRoot } from "../../home-paths.js"; export interface NativeHarnessBackupStamp { - schema: "paperclip.native-harness-backup-stamp.v1"; + schema: "paperclip.native-harness-backup-stamp.v2"; normalizedSessionId: string; runnerInstanceId: string; + sessionScopeSha256: string; + sourceProviderLeaseId: string; + authorizedProviderLeaseId: string; manifestSha256: string; completedAt: string; } -function stateRoot(normalizedSessionId: string): string { +function stateBase(): string { return resolve( - process.env.PAPERCLIP_RUNNER_STATE_DIR - ?? resolve(resolvePaperclipInstanceRoot(), "runtime", "paperclip-runner", "durable-sessions"), - createHash("sha256").update(normalizedSessionId).digest("hex"), + process.env.PAPERCLIP_RUNNER_STATE_DIR ?? + resolve( + resolvePaperclipInstanceRoot(), + "runtime", + "paperclip-runner", + "durable-sessions", + ), ); } +function stateRootFromDigest(digest: string): string | null { + if (!/^[0-9a-f]{64}$/.test(digest)) return null; + const base = stateBase(); + const root = resolve(base, digest); + return dirname(root) === base ? root : null; +} + +function isRealDirectory(path: string): boolean { + try { + const metadata = lstatSync(path); + return metadata.isDirectory() && !metadata.isSymbolicLink(); + } catch { + return false; + } +} + +function isRealFile(path: string): boolean { + try { + const metadata = lstatSync(path); + return metadata.isFile() && !metadata.isSymbolicLink(); + } catch { + return false; + } +} + function digestDirectory(directory: string): { sha256: string; bytes: number } { const hash = createHash("sha256"); let bytes = 0; const visit = (current: string, relative: string) => { - const entries = readdirSync(current, { withFileTypes: true }) - .sort((left, right) => left.name.localeCompare(right.name)); + const entries = readdirSync(current, { withFileTypes: true }).sort( + (left, right) => left.name.localeCompare(right.name), + ); if (entries.length === 0) hash.update(`directory:${relative}\0`); for (const entry of entries) { const path = resolve(current, entry.name); @@ -44,10 +71,14 @@ function digestDirectory(directory: string): { sha256: string; bytes: number } { } else if (entry.isFile()) { const contents = readFileSync(path); bytes += contents.byteLength; - hash.update(`file:${relativePath}:${stats.mode & 0o777}:${contents.byteLength}\0`); + hash.update( + `file:${relativePath}:${stats.mode & 0o777}:${contents.byteLength}\0`, + ); hash.update(contents); } else { - throw new Error(`runner_harness_backup_unsupported_entry:${relativePath}`); + throw new Error( + `runner_harness_backup_unsupported_entry:${relativePath}`, + ); } } }; @@ -57,15 +88,55 @@ function digestDirectory(directory: string): { sha256: string; bytes: number } { export function createNativeHarnessBackupStamp(input: { manifestPath: string; + sessionScopeId: string; + authorizedProviderLeaseId: string; normalizedSessionId: string; runnerInstanceId: string; completedAt: string; }): NativeHarnessBackupStamp { + if (!input.authorizedProviderLeaseId) { + throw new Error("runner_harness_backup_lease_missing"); + } + const sessionScopeSha256 = createHash("sha256") + .update(input.sessionScopeId) + .digest("hex"); + const root = stateRootFromDigest(sessionScopeSha256); + if (!root || !isRealDirectory(root)) { + throw new Error("runner_harness_backup_scope_invalid"); + } + const resolvedManifestPath = resolve(input.manifestPath); + const allowedManifestPaths = ["current", "previous"].map((candidate) => + resolve(root, "failover-backups", candidate, "manifest.json"), + ); + if ( + !allowedManifestPaths.includes(resolvedManifestPath) || + !isRealDirectory(dirname(resolvedManifestPath)) || + !isRealFile(resolvedManifestPath) + ) { + throw new Error("runner_harness_backup_scope_invalid"); + } const manifestBytes = readFileSync(input.manifestPath); + const manifest = JSON.parse(manifestBytes.toString("utf8")) as Record< + string, + unknown + >; + if ( + manifest.schema !== "paperclip.native-harness-backup.v1" || + manifest.normalizedSessionId !== input.normalizedSessionId || + manifest.runnerInstanceId !== input.runnerInstanceId || + manifest.completedAt !== input.completedAt || + typeof manifest.sourceProviderLeaseId !== "string" || + !manifest.sourceProviderLeaseId + ) { + throw new Error("runner_harness_backup_scope_invalid"); + } return { - schema: "paperclip.native-harness-backup-stamp.v1", + schema: "paperclip.native-harness-backup-stamp.v2", normalizedSessionId: input.normalizedSessionId, runnerInstanceId: input.runnerInstanceId, + sessionScopeSha256, + sourceProviderLeaseId: manifest.sourceProviderLeaseId, + authorizedProviderLeaseId: input.authorizedProviderLeaseId, manifestSha256: `sha256:${createHash("sha256").update(manifestBytes).digest("hex")}`, completedAt: input.completedAt, }; @@ -79,28 +150,45 @@ export function verifyNativeHarnessBackupStamp( if (!value || typeof value !== "object" || Array.isArray(value)) return false; const stamp = value as Record; if ( - stamp.schema !== "paperclip.native-harness-backup-stamp.v1" || - typeof stamp.normalizedSessionId !== "string" || !stamp.normalizedSessionId || - typeof stamp.runnerInstanceId !== "string" || !stamp.runnerInstanceId || - typeof stamp.manifestSha256 !== "string" || !stamp.manifestSha256.startsWith("sha256:") - ) return false; - const backupRoot = resolve(stateRoot(stamp.normalizedSessionId), "failover-backups"); - for (const candidate of [resolve(backupRoot, "current"), resolve(backupRoot, "previous")]) { + stamp.schema !== "paperclip.native-harness-backup-stamp.v2" || + typeof stamp.normalizedSessionId !== "string" || + !stamp.normalizedSessionId || + typeof stamp.runnerInstanceId !== "string" || + !stamp.runnerInstanceId || + typeof stamp.sessionScopeSha256 !== "string" || + typeof stamp.sourceProviderLeaseId !== "string" || + !stamp.sourceProviderLeaseId || + stamp.authorizedProviderLeaseId !== expectedProviderLeaseId || + typeof stamp.manifestSha256 !== "string" || + !stamp.manifestSha256.startsWith("sha256:") + ) + return false; + const root = stateRootFromDigest(stamp.sessionScopeSha256); + if (!root || !isRealDirectory(root)) return false; + const backupRoot = resolve(root, "failover-backups"); + for (const candidate of [ + resolve(backupRoot, "current"), + resolve(backupRoot, "previous"), + ]) { const manifestPath = resolve(candidate, "manifest.json"); - if (!existsSync(manifestPath)) continue; + if (!isRealDirectory(candidate) || !isRealFile(manifestPath)) continue; try { const bytes = readFileSync(manifestPath); const manifestSha256 = `sha256:${createHash("sha256").update(bytes).digest("hex")}`; if (manifestSha256 !== stamp.manifestSha256) continue; - const manifest = JSON.parse(bytes.toString("utf8")) as Record; + const manifest = JSON.parse(bytes.toString("utf8")) as Record< + string, + unknown + >; if ( manifest.schema !== "paperclip.native-harness-backup.v1" || manifest.normalizedSessionId !== stamp.normalizedSessionId || manifest.runnerInstanceId !== stamp.runnerInstanceId || - manifest.sourceProviderLeaseId !== expectedProviderLeaseId || + manifest.sourceProviderLeaseId !== stamp.sourceProviderLeaseId || !Array.isArray(manifest.directories) || manifest.directories.length === 0 - ) continue; + ) + continue; let valid = true; for (const entry of manifest.directories) { if (!entry || typeof entry !== "object" || Array.isArray(entry)) { @@ -118,12 +206,15 @@ export function verifyNativeHarnessBackupStamp( break; } const directory = resolve(candidate, declared.name); - if (!existsSync(directory)) { + if (!isRealDirectory(directory)) { valid = false; break; } const actual = digestDirectory(directory); - if (actual.sha256 !== declared.sha256 || actual.bytes !== declared.bytes) { + if ( + actual.sha256 !== declared.sha256 || + actual.bytes !== declared.bytes + ) { valid = false; break; } diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index feb8f4cc4e..7a9c857026 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -17,9 +17,10 @@ import { nativeRunFinalizations, type Db, } from "@paperclipai/db"; -import type { - NativeExecutionInputV1, - PrpEvent, +import { + acpxRuntimeSessionDirectoryName, + type NativeExecutionInputV1, + type PrpEvent, } from "@paperclipai/paperclip-runner"; import { createHash } from "node:crypto"; import { @@ -65,6 +66,17 @@ type RunnerTransportOptions = { providerSessionId?: string | null; activeTurnId?: string | null; }; + archiveExternalRunnerState?: (input: { + archiveKey: string; + priorIdentity: { + runnerInstanceId: string; + environmentLeaseId: string; + runId: string; + normalizedSessionId: string; + turnId: string; + itemId: string; + }; + }) => Promise>; }; const durableControlPlaneState = (identity: Record) => ({ @@ -170,8 +182,10 @@ import { nativeUsageCostUsd, normalizeNativeUsage, readRemoteProviderPackManifest, + providerSessionIdentityFromDurableProviderState, providerSessionIdentityTransitionIsAllowed, providerPlanMarkdown, + remoteCheckpointIncompleteFailure, resolveRemoteRunnerTransportMode, renewNativeSessionExecutionLease, runtimeInputLifecycleMetric, @@ -482,9 +496,30 @@ describe("native harness persistence profiles", () => { expect( codex.directories.find((directory) => directory.name === "codex-home"), ).toMatchObject({ - excludeTopLevelEntries: ["tmp", ".tmp", "auth.json", "config.toml"], + excludeEntries: ["tmp", ".tmp", "auth.json", "config.toml"], }); }); + + it("excludes only nested Codex launch state from ACPX recovery", () => { + const acpx = profile({ kind: "acpx", agent: "codex" }, "acpx_runtime"); + const sessionDirectory = acpxRuntimeSessionDirectoryName("session"); + expect( + acpx.directories.find((directory) => directory.name === "acpx"), + ).toMatchObject({ + excludeEntries: [ + `acpx/${sessionDirectory}/codex-home/tmp`, + `acpx/${sessionDirectory}/codex-home/.tmp`, + `acpx/${sessionDirectory}/codex-home/auth.json`, + `acpx/${sessionDirectory}/codex-home/config.toml`, + ], + }); + expect( + profile( + { kind: "acpx", agent: "claude" }, + "acpx_runtime", + ).directories.find((directory) => directory.name === "acpx"), + ).toMatchObject({ excludeEntries: [] }); + }); }); describe("verified native harness backups", () => { @@ -723,9 +758,10 @@ describe("verified native harness backups", () => { const previousStateDirectory = process.env.PAPERCLIP_RUNNER_STATE_DIR; process.env.PAPERCLIP_RUNNER_STATE_DIR = stateBase; try { + const sessionScopeId = "native-session-scope-v2"; const sessionRoot = join( stateBase, - createHash("sha256").update("native-session").digest("hex"), + createHash("sha256").update(sessionScopeId).digest("hex"), ); const current = join(sessionRoot, "failover-backups", "current"); await mkdir(join(current, "runner"), { recursive: true }); @@ -753,6 +789,8 @@ describe("verified native harness backups", () => { await writeFile(manifestPath, JSON.stringify(manifest)); const stamp = createNativeHarnessBackupStamp({ manifestPath, + sessionScopeId, + authorizedProviderLeaseId: "sandbox-1", normalizedSessionId: "native-session", runnerInstanceId: "runner-1", completedAt: manifest.completedAt, @@ -760,6 +798,17 @@ describe("verified native harness backups", () => { expect(verifyNativeHarnessBackupStamp(stamp, "sandbox-1")).toBe(true); expect(verifyNativeHarnessBackupStamp(stamp, "sandbox-2")).toBe(false); + const reboundStamp = createNativeHarnessBackupStamp({ + manifestPath, + sessionScopeId, + authorizedProviderLeaseId: "sandbox-2", + normalizedSessionId: "native-session", + runnerInstanceId: "runner-1", + completedAt: manifest.completedAt, + }); + expect(verifyNativeHarnessBackupStamp(reboundStamp, "sandbox-2")).toBe( + true, + ); await writeFile(join(current, "runner", "runner-state.json"), "corrupt"); expect(verifyNativeHarnessBackupStamp(stamp, "sandbox-1")).toBe(false); } finally { @@ -771,6 +820,270 @@ describe("verified native harness backups", () => { await rm(stateBase, { recursive: true, force: true }); } }); + + it("rejects a digest-valid legacy stamp for remote lease authorization", async () => { + const stateBase = await mkdtemp( + join(tmpdir(), "paperclip-legacy-harness-stamp-"), + ); + const previousStateDirectory = process.env.PAPERCLIP_RUNNER_STATE_DIR; + process.env.PAPERCLIP_RUNNER_STATE_DIR = stateBase; + try { + const legacyRoot = join( + stateBase, + createHash("sha256").update("native-session").digest("hex"), + "failover-backups", + "current", + ); + await mkdir(join(legacyRoot, "runner"), { recursive: true }); + await mkdir(join(legacyRoot, "codex-home", "sessions"), { + recursive: true, + }); + await writeFile( + join(legacyRoot, "runner", "runner-state.json"), + "runner-state", + ); + await writeFile( + join(legacyRoot, "codex-home", "sessions", "thread.jsonl"), + "thread-state", + ); + const manifest = buildNativeHarnessBackupManifest({ + backupRoot: legacyRoot, + execution: backupExecution, + runnerInstanceId: "runner-1", + providerSessionIdentity: { + providerSessionId: "thread-1", + providerBackendSessionId: "session-1", + providerSessionIdentity: null, + }, + sourceProviderLeaseId: "sandbox-1", + }); + const manifestBytes = JSON.stringify(manifest); + await writeFile(join(legacyRoot, "manifest.json"), manifestBytes); + + expect( + verifyNativeHarnessBackupStamp( + { + schema: "paperclip.native-harness-backup-stamp.v1", + normalizedSessionId: "native-session", + runnerInstanceId: "runner-1", + manifestSha256: `sha256:${createHash("sha256").update(manifestBytes).digest("hex")}`, + completedAt: manifest.completedAt, + }, + "sandbox-1", + ), + ).toBe(false); + } finally { + if (previousStateDirectory === undefined) { + delete process.env.PAPERCLIP_RUNNER_STATE_DIR; + } else { + process.env.PAPERCLIP_RUNNER_STATE_DIR = previousStateDirectory; + } + await rm(stateBase, { recursive: true, force: true }); + } + }); +}); + +describe("split durable provider checkpoint identity", () => { + const execution = (provider: Record, driverKind: string) => + ({ + provider, + binding: { + companyId: "company", + runId: "run", + issueId: "issue", + agentId: "agent", + executionWorkspaceId: "workspace", + }, + workspace: { cwd: "/workspace" }, + session: { + normalizedSessionId: "native-session", + driverKind, + protocolVersion: 1, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, + }, + }) as unknown as NativeExecutionInputV1; + + it("reads ACPX identity from its provider-owned state after suspension", () => { + const profileDigest = `sha256:${"a".repeat(64)}`; + const identity = { + kind: "acpx", + normalizedSessionId: "native-session", + acpxRecordId: "record-1", + backendSessionId: "backend-1", + agentSessionId: "agent-session-1", + profileDigest, + workspaceDigest: `sha256:${"b".repeat(64)}`, + requestedModel: "claude-sonnet-5", + effectiveModel: "claude-sonnet-5", + permissionMode: "approve-all", + providerLifetimeFenceCandidates: [53001, 53002, 53003], + }; + expect( + providerSessionIdentityFromDurableProviderState({ + execution: execution( + { + kind: "acpx", + agent: "claude", + model: "claude-sonnet-5", + permissionMode: "approve-all", + }, + "acpx_runtime", + ), + providerState: { + schema: "paperclip.runner.acpx-provider-state.v3", + lifecycle: "suspended", + activeTurnId: null, + providerExitUnconfirmed: false, + descriptor: { + kind: "acpx", + provider: "acpx", + driver: "acpx_runtime", + agent: "claude", + model: "claude-sonnet-5", + commandDigest: profileDigest, + normalizedSessionId: "native-session", + }, + identity, + }, + }), + ).toEqual({ + providerSessionId: "record-1", + providerBackendSessionId: "backend-1", + providerSessionIdentity: identity, + }); + }); + + it.each([ + ["codex", "codex_app_server"], + ["opencode", "opencode_server"], + ] as const)( + "reads %s identity from the split Codex-provider state", + (provider, driverKind) => { + expect( + providerSessionIdentityFromDurableProviderState({ + execution: execution({ kind: provider }, driverKind), + providerState: { + schema: "paperclip.runner.codex-provider-state.v1", + lifecycle: "prepared", + config: { provider, driver: driverKind }, + threadId: "driver-session-1", + providerSessionId: "provider-session-1", + activeProviderTurnId: null, + ambiguousTurnStartPending: false, + }, + }), + ).toEqual({ + providerSessionId: "driver-session-1", + providerBackendSessionId: "provider-session-1", + providerSessionIdentity: null, + }); + }, + ); + + it("rejects active or scope-conflicting provider state", () => { + const profileDigest = `sha256:${"a".repeat(64)}`; + const acpxExecution = execution( + { + kind: "acpx", + agent: "claude", + model: "claude-sonnet-5", + permissionMode: "approve-all", + }, + "acpx_runtime", + ); + for (const providerState of [ + { + schema: "paperclip.runner.acpx-provider-state.v3", + lifecycle: "turn_active", + activeTurnId: "turn-1", + providerExitUnconfirmed: false, + descriptor: { + kind: "acpx", + provider: "acpx", + driver: "acpx_runtime", + agent: "claude", + model: "claude-sonnet-5", + commandDigest: profileDigest, + normalizedSessionId: "native-session", + }, + identity: { + kind: "acpx", + normalizedSessionId: "native-session", + acpxRecordId: "record-1", + backendSessionId: "backend-1", + agentSessionId: "agent-session-1", + profileDigest, + workspaceDigest: `sha256:${"b".repeat(64)}`, + requestedModel: "claude-sonnet-5", + effectiveModel: "claude-sonnet-5", + permissionMode: "approve-all", + providerLifetimeFenceCandidates: [53001, 53002, 53003], + }, + }, + { + schema: "paperclip.runner.acpx-provider-state.v3", + lifecycle: "suspended", + activeTurnId: null, + providerExitUnconfirmed: false, + descriptor: { + kind: "acpx", + provider: "acpx", + driver: "acpx_runtime", + agent: "claude", + model: "claude-sonnet-5", + commandDigest: profileDigest, + normalizedSessionId: "other-session", + }, + identity: { + kind: "acpx", + normalizedSessionId: "other-session", + acpxRecordId: "record-1", + backendSessionId: "backend-1", + agentSessionId: "agent-session-1", + profileDigest, + workspaceDigest: `sha256:${"b".repeat(64)}`, + requestedModel: "claude-sonnet-5", + effectiveModel: "claude-sonnet-5", + permissionMode: "approve-all", + providerLifetimeFenceCandidates: [53001, 53002, 53003], + }, + }, + ]) { + expect( + providerSessionIdentityFromDurableProviderState({ + execution: acpxExecution, + providerState, + }), + ).toEqual({ + providerSessionId: null, + providerBackendSessionId: null, + providerSessionIdentity: null, + }); + } + }); + + it.each(["claude_managed", "aws_agentcore"] as const)( + "reads %s identity from managed provider state", + (provider) => { + expect( + providerSessionIdentityFromDurableProviderState({ + execution: execution({ kind: provider }, `${provider}_driver`), + providerState: { + schema: "paperclip.runner.managed-provider-state.v1", + lifecycle: "suspended", + normalizedSessionId: "native-session", + descriptor: { kind: provider, config: {} }, + providerSessionId: "managed-session-1", + activeTurnId: null, + }, + }), + ).toEqual({ + providerSessionId: "managed-session-1", + providerBackendSessionId: "managed-session-1", + providerSessionIdentity: null, + }); + }, + ); }); describe("remote provider checkpoint snapshots", () => { @@ -813,7 +1126,7 @@ describe("remote provider checkpoint snapshots", () => { sourcePath: "/remote/session/filesystem/codex-home", targetPath: "/tmp/paperclip-checkpoint-test-codex-home", mode: 0o700, - excludeTopLevelEntries: ["tmp", ".tmp", "auth.json", "config.toml"], + excludeEntries: ["tmp", ".tmp", "auth.json", "config.toml"], }); expect(execute).toHaveBeenNthCalledWith( @@ -848,7 +1161,40 @@ describe("remote provider checkpoint snapshots", () => { ); }); - it("rejects non-top-level checkpoint exclusions", async () => { + it("omits nested ACPX-Codex scratch aliases without widening the exclusion", async () => { + const execute = vi.fn().mockResolvedValue({ + exitCode: 0, + timedOut: false, + stdout: "", + stderr: "", + }); + const syncOut = vi.fn(async () => undefined); + const sessionDirectory = acpxRuntimeSessionDirectoryName("session"); + const excluded = [ + `acpx/${sessionDirectory}/codex-home/tmp`, + `acpx/${sessionDirectory}/codex-home/.tmp`, + `acpx/${sessionDirectory}/codex-home/auth.json`, + `acpx/${sessionDirectory}/codex-home/config.toml`, + ]; + + await syncRemoteRunnerDirectoryOut({ + runner: { execute, syncOut } as never, + sourcePath: "/remote/session/filesystem/acpx", + targetPath: "/tmp/paperclip-checkpoint-test-acpx", + mode: 0o700, + excludeEntries: excluded, + }); + + const snapshotCommand = String(execute.mock.calls[1]?.[0]?.args?.[1]); + for (const entry of excluded) { + expect(snapshotCommand).toContain(`'--exclude=./${entry}'`); + } + expect(snapshotCommand).not.toContain("--exclude=./acpx-state"); + expect(snapshotCommand).not.toContain("--exclude=./codex-home"); + expect(syncOut).toHaveBeenCalledOnce(); + }); + + it("rejects unsafe relative checkpoint exclusions", async () => { const execute = vi.fn().mockResolvedValue({ exitCode: 0, timedOut: false, @@ -861,7 +1207,7 @@ describe("remote provider checkpoint snapshots", () => { sourcePath: "/remote/codex-home", targetPath: "/tmp/paperclip-checkpoint-invalid-codex-home", mode: 0o700, - excludeTopLevelEntries: ["../outside"], + excludeEntries: ["../outside"], }), ).rejects.toThrow("runner_remote_checkpoint_exclusion_invalid"); }); @@ -962,7 +1308,7 @@ describe("remote provider checkpoint restores", () => { sourcePath, targetPath: "/remote/codex-home", mode: 0o700, - excludeTopLevelEntries: ["tmp", ".tmp", "auth.json", "config.toml"], + excludeEntries: ["tmp", ".tmp", "auth.json", "config.toml"], }); expect(syncIn).toHaveBeenCalledOnce(); @@ -1068,6 +1414,25 @@ describe("remote runner transport authorization", () => { }); }); +describe("required remote checkpoint completion", () => { + it.each(["unavailable", "not_suspended"] as const)( + "fails a settled runner when its checkpoint is %s", + (incompleteReason) => { + expect( + remoteCheckpointIncompleteFailure("settled", incompleteReason), + ).toMatchObject({ + message: `runner_remote_checkpoint_incomplete: exact suspended harness state unavailable (${incompleteReason})`, + }); + }, + ); + + it("preserves the original startup error for a runner that never settled", () => { + expect( + remoteCheckpointIncompleteFailure("unsettled", "unavailable"), + ).toBeNull(); + }); +}); + describe("runtime question fallback", () => { const questionSet = { schema: "paperclip.question_set.v1" as const, @@ -3344,6 +3709,7 @@ describe("runnerd provider runtime wiring", () => { input: expect.objectContaining({ workspace: expect.objectContaining({ cwd: remoteCwd }), }), + requireSessionCloseBeforeReturn: true, }), ); const backendOptions = state.createBackend.mock.calls[0]![1]; @@ -3368,6 +3734,91 @@ describe("runnerd provider runtime wiring", () => { } }); + it("makes remote authority archival idempotent and returns the archived state", async () => { + const remoteExecute = vi.fn(); + const remoteTarget = { + kind: "remote" as const, + transport: "sandbox" as const, + providerKey: "daytona", + leaseId: "lease-authority-archive", + remoteCwd: "/home/daytona/paperclip-workspace", + runner: { execute: remoteExecute }, + } as never; + const normalizedSessionId = execution.session.normalizedSessionId; + if (!normalizedSessionId) { + throw new Error("fixture requires a normalized native session id"); + } + const archiveIdentity = { + runnerInstanceId: "runner-authority-archive", + environmentLeaseId: "lease-authority-archive", + runId: execution.binding.runId, + normalizedSessionId, + turnId: "turn-authority-archive", + itemId: "item-authority-archive", + }; + const archivedState = { + schema: "paperclip.runner.durable.state.v1", + ...archiveIdentity, + lifecycle: "suspended", + }; + state.createBackend.mockClear(); + state.createTransport.mockClear(); + await createRunnerdBackend({ + db: leaseDb(execution), + execution, + runnerInstanceId: archiveIdentity.runnerInstanceId, + runnerExecutionTarget: remoteTarget, + }); + state.createBackend.mock.calls[0]![1].codexTransportFactory!(); + const archiveExternalRunnerState = + state.createTransport.mock.calls[0]![0].archiveExternalRunnerState; + expect(archiveExternalRunnerState).toBeTypeOf("function"); + remoteExecute.mockClear(); + remoteExecute.mockResolvedValue({ + exitCode: 0, + timedOut: false, + stdout: Buffer.from(JSON.stringify(archivedState)).toString("base64"), + stderr: "", + }); + + await expect( + archiveExternalRunnerState!({ + archiveKey: "a".repeat(24), + priorIdentity: archiveIdentity, + }), + ).resolves.toEqual(archivedState); + await expect( + archiveExternalRunnerState!({ + archiveKey: "a".repeat(24), + priorIdentity: archiveIdentity, + }), + ).resolves.toEqual(archivedState); + expect(remoteExecute).toHaveBeenCalledTimes(2); + expect(remoteExecute.mock.calls[0]![0]).toEqual( + expect.objectContaining({ + command: "sh", + args: expect.arrayContaining([ + expect.stringContaining( + 'test ! -e "$1" && test ! -L "$1" && test -f "$3" && test ! -L "$3"', + ), + ]), + }), + ); + + remoteExecute.mockResolvedValueOnce({ + exitCode: 1, + timedOut: false, + stdout: "", + stderr: "source and archive both exist", + }); + await expect( + archiveExternalRunnerState!({ + archiveKey: "a".repeat(24), + priorIdentity: archiveIdentity, + }), + ).rejects.toThrow("runner_remote_authority_archive_failed"); + }); + it("uses the native execution workspace as the local provider containment root", async () => { state.createBackend.mockClear(); await createRunnerdBackend({ @@ -3672,6 +4123,220 @@ describe("runnerd provider runtime wiring", () => { } }); + it.each([ + { + directLifecycle: null, + backupLifecycle: "suspended", + corrupt: false, + accepted: true, + }, + { + directLifecycle: "empty", + backupLifecycle: "suspended", + corrupt: false, + accepted: true, + }, + { + directLifecycle: "ready", + backupLifecycle: "suspended", + corrupt: false, + accepted: false, + }, + { + directLifecycle: "malformed", + backupLifecycle: "suspended", + corrupt: false, + accepted: false, + }, + { + directLifecycle: "nonempty", + backupLifecycle: "suspended", + corrupt: false, + accepted: false, + }, + { + directLifecycle: null, + backupLifecycle: "ready", + corrupt: false, + accepted: false, + }, + { + directLifecycle: null, + backupLifecycle: "suspended", + corrupt: true, + accepted: false, + }, + ] as const)( + "uses remote prior-run backup when acceptance=$accepted direct=$directLifecycle backup=$backupLifecycle corrupt=$corrupt", + async ({ directLifecycle, backupLifecycle, corrupt, accepted }) => { + const stateBase = await mkdtemp( + join(tmpdir(), "paperclip-remote-prior-run-state-"), + ); + const previousStateDirectory = process.env.PAPERCLIP_RUNNER_STATE_DIR; + process.env.PAPERCLIP_RUNNER_STATE_DIR = stateBase; + const priorExecution = { + ...execution, + binding: { + ...execution.binding, + companyId: "company-remote-prior-scope", + runId: "run-remote-prior-scope", + agentId: "agent-remote-prior-scope", + executionWorkspaceId: "workspace-remote-prior-scope", + }, + session: { + ...execution.session, + normalizedSessionId: "session-remote-prior-scope", + }, + } as NativeExecutionInputV1; + const currentExecution = { + ...priorExecution, + binding: { + ...priorExecution.binding, + runId: "run-current-remote-scope", + }, + } as NativeExecutionInputV1; + const priorRunDb = { + select: () => ({ + from: () => ({ + where: () => ({ + limit: () => + Promise.resolve([ + { + status: "succeeded", + runnerProfileJson: { + nativeExecutionInput: priorExecution, + }, + }, + ]), + }), + }), + }), + } as unknown as Db; + const remoteTarget = { + kind: "remote" as const, + transport: "sandbox" as const, + providerKey: "daytona", + leaseId: "environment-lease-remote-prior-scope", + remoteCwd: "/home/daytona/paperclip-workspace", + runner: { + execute: vi.fn(), + }, + } as never; + const identity = { + runId: priorExecution.binding.runId, + normalizedSessionId: priorExecution.session.normalizedSessionId, + runnerInstanceId: "runner-remote-prior-scope", + environmentLeaseId: "lease-remote-prior-scope", + }; + try { + state.createBackend.mockClear(); + state.createTransport.mockClear(); + await createRunnerdBackend({ + db: leaseDb(priorExecution), + execution: priorExecution, + runnerInstanceId: identity.runnerInstanceId, + runnerExecutionTarget: remoteTarget, + }); + state.createBackend.mock.calls[0]![1].codexTransportFactory!(); + const scopedRoot = + state.createTransport.mock.calls[0]![0].stateDirectory!; + await mkdir(join(scopedRoot, "control-plane"), { recursive: true }); + await writeFile( + join(scopedRoot, "control-plane", "control-plane-state.json"), + JSON.stringify(durableControlPlaneState(identity)), + ); + if (directLifecycle === "empty") { + // Remote transports before the externally-owned-state fix left an + // empty local placeholder beside the controller state. It is not an + // authority record and must not mask a verified remote backup. + await mkdir(join(scopedRoot, "runner"), { recursive: true }); + } else if (directLifecycle === "nonempty") { + await mkdir(join(scopedRoot, "runner"), { recursive: true }); + await writeFile( + join(scopedRoot, "runner", "unexpected-state.json"), + "{}", + ); + } else if (directLifecycle !== null) { + await mkdir(join(scopedRoot, "runner"), { recursive: true }); + await writeFile( + join(scopedRoot, "runner", "runner-state.json"), + JSON.stringify( + directLifecycle === "malformed" + ? { + ...durableRunnerState(identity, "suspended"), + runId: "conflicting-direct-run", + } + : durableRunnerState(identity, directLifecycle), + ), + ); + } + const backupRoot = join(scopedRoot, "failover-backups", "current"); + await mkdir(join(backupRoot, "runner"), { recursive: true }); + await mkdir(join(backupRoot, "codex-home"), { recursive: true }); + await writeFile( + join(backupRoot, "runner", "runner-state.json"), + JSON.stringify(durableRunnerState(identity, backupLifecycle)), + ); + const manifest = buildNativeHarnessBackupManifest({ + backupRoot, + execution: priorExecution, + runnerInstanceId: identity.runnerInstanceId, + providerSessionIdentity: { + providerSessionId: "provider-remote-prior-scope", + providerBackendSessionId: null, + providerSessionIdentity: null, + }, + sourceProviderLeaseId: "sandbox-remote-prior-scope", + }); + await writeFile( + join(backupRoot, "manifest.json"), + JSON.stringify(manifest), + ); + if (corrupt) { + await writeFile( + join(backupRoot, "runner", "runner-state.json"), + JSON.stringify({ + ...durableRunnerState(identity, backupLifecycle), + x: 1, + }), + ); + } + state.createBackend.mockClear(); + state.createTransport.mockClear(); + + const continuation = createRunnerdBackend({ + db: priorRunDb, + execution: currentExecution, + runnerInstanceId: "runner-current-remote-scope", + runnerExecutionTarget: remoteTarget, + }); + if (!accepted) { + await expect(continuation).rejects.toThrow( + "runner_state_identity_mismatch", + ); + expect(state.createBackend).not.toHaveBeenCalled(); + return; + } + await expect(continuation).resolves.toBeDefined(); + state.createBackend.mock.calls[0]![1].codexTransportFactory!(); + expect(state.createTransport.mock.calls[0]![0].prpIdentity).toEqual( + expect.objectContaining({ + runId: currentExecution.binding.runId, + runnerInstanceId: identity.runnerInstanceId, + environmentLeaseId: identity.environmentLeaseId, + }), + ); + } finally { + if (previousStateDirectory === undefined) { + delete process.env.PAPERCLIP_RUNNER_STATE_DIR; + } else { + process.env.PAPERCLIP_RUNNER_STATE_DIR = previousStateDirectory; + } + await rm(stateBase, { recursive: true, force: true }); + } + }, + ); + it("quarantines legacy prior-run state only after the database proves a terminal owner in the same full scope", async () => { const stateBase = await mkdtemp( join(tmpdir(), "paperclip-legacy-terminal-unsuspended-state-"), @@ -3936,6 +4601,33 @@ describe("runnerd provider runtime wiring", () => { join(scopedRoot, "runner", "runner-state.json"), JSON.stringify(durableRunnerState(identity, "ready")), ); + await mkdir(join(scopedRoot, "codex-home", "sessions"), { + recursive: true, + }); + await mkdir(join(scopedRoot, "codex-home", "tmp"), { recursive: true }); + await mkdir(join(scopedRoot, "codex-home", ".tmp"), { + recursive: true, + }); + await writeFile( + join(scopedRoot, "codex-home", "auth.json"), + '{"OPENAI_API_KEY":"fixture-secret"}', + ); + await writeFile( + join(scopedRoot, "codex-home", "config.toml"), + 'bearer_token = "fixture-secret"', + ); + await writeFile( + join(scopedRoot, "codex-home", "tmp", "transient"), + "transient", + ); + await writeFile( + join(scopedRoot, "codex-home", ".tmp", "transient"), + "transient", + ); + await writeFile( + join(scopedRoot, "codex-home", "sessions", "rollout.jsonl"), + "durable session history", + ); state.createBackend.mockClear(); state.createTransport.mockClear(); @@ -3950,6 +4642,26 @@ describe("runnerd provider runtime wiring", () => { const quarantineEntries = await readdir(join(stateBase, "quarantine")); expect(quarantineEntries).toHaveLength(1); expect(quarantineEntries[0]).toContain(".identity_indeterminate."); + const quarantinedRoot = join( + stateBase, + "quarantine", + quarantineEntries[0]!, + ); + for (const entry of ["tmp", ".tmp", "auth.json", "config.toml"]) { + await expect( + access(join(quarantinedRoot, "codex-home", entry)), + ).rejects.toThrow(); + } + await expect( + access( + join(quarantinedRoot, "codex-home", "sessions", "rollout.jsonl"), + ), + ).resolves.toBeUndefined(); + await expect( + access( + join(quarantinedRoot, "control-plane", "control-plane-state.json"), + ), + ).resolves.toBeUndefined(); expect(state.createBackend).not.toHaveBeenCalled(); expect(state.createTransport).not.toHaveBeenCalled(); } finally { diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 7434f9ce55..f7fc0f2a3e 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -37,6 +37,7 @@ import type { PrpStructuredRunResult, } from "../../vendor/paperclip-runner/index.js"; import { + acpxRuntimeSessionDirectoryName, createNativeSessionBackend, createRunnerdCodexTransport, defaultCapabilityRunnerdBinary, @@ -152,9 +153,19 @@ const MAX_REMOTE_CHECKPOINT_ENTRIES = 20_000; const NATIVE_DURABLE_IDENTITY_MAX_BYTES = 2 * 1024 * 1024; const NATIVE_RUNNER_STATE_MAX_BYTES = 16 * 1024 * 1024; const NATIVE_WARM_CHECKPOINT_MAX_BYTES = 8 * 1024 * 1024; +const CODEX_HOME_NON_PERSISTENT_ENTRIES = [ + "tmp", + ".tmp", + "auth.json", + "config.toml", +] as const; const RUNNERD_CONTROL_PLANE_STATE_SCHEMA = "paperclip.runner.durable.control-plane-state.v1"; const RUNNERD_STATE_SCHEMA = "paperclip.runner.durable.state.v1"; +const CODEX_PROVIDER_STATE_SCHEMA = "paperclip.runner.codex-provider-state.v1"; +const ACPX_PROVIDER_STATE_SCHEMA = "paperclip.runner.acpx-provider-state.v3"; +const MANAGED_PROVIDER_STATE_SCHEMA = + "paperclip.runner.managed-provider-state.v1"; const RUNNERD_STATE_LIFECYCLES = new Set([ "connecting", "ready", @@ -1228,6 +1239,26 @@ function scopedRunnerdStateRoot(execution: NativeExecutionInput): string { ); } +function scrubRunnerdQuarantineLaunchState(root: string): void { + const codexHome = resolve(root, "codex-home"); + const codexHomeStats = lstatSync(codexHome, { throwIfNoEntry: false }); + if (!codexHomeStats) return; + if (!codexHomeStats.isDirectory() || codexHomeStats.isSymbolicLink()) { + throw new Error("runner_state_directory_unsafe"); + } + for (const name of CODEX_HOME_NON_PERSISTENT_ENTRIES) { + const entry = resolve(codexHome, name); + const stats = lstatSync(entry, { throwIfNoEntry: false }); + if (!stats) continue; + // Remove symlinks themselves, never their targets. Real temporary + // directories are safe to remove recursively inside the verified home. + rmSync(entry, { + recursive: stats.isDirectory() && !stats.isSymbolicLink(), + force: true, + }); + } +} + function quarantineRunnerdStateRoot( root: string, reason: "identity_indeterminate" | "identity_mismatch", @@ -1245,6 +1276,10 @@ function quarantineRunnerdStateRoot( ) { throw new Error("runner_state_directory_unsafe"); } + // Quarantine retains durable session history for diagnosis and recovery, but + // launch credentials and transient files are re-materializable and must not + // survive after this state loses authority. + scrubRunnerdQuarantineLaunchState(resolvedRoot); const quarantineRoot = resolve(stateBase, "quarantine"); mkdirSync(quarantineRoot, { recursive: true, mode: 0o700 }); if (!isSafeNativeStateDirectory(quarantineRoot)) { @@ -1305,10 +1340,12 @@ function migrateLegacyRunnerdStateRoot(input: { if ( exactRun && legacyIdentity && - runnerdAuthorityLifecycle( - input.legacy, - legacyIdentity as RunnerdDurableIdentity, - ) === "indeterminate" + ["absent", "indeterminate"].includes( + runnerdAuthorityLifecycle( + input.legacy, + legacyIdentity as RunnerdDurableIdentity, + ), + ) ) { quarantineRunnerdStateRoot(input.legacy, "identity_indeterminate"); throw new Error("runner_state_identity_mismatch"); @@ -1344,11 +1381,22 @@ function migrateLegacyRunnerdStateRoot(input: { function runnerdAuthorityLifecycle( root: string, identity: RunnerdDurableIdentity, -): "suspended" | "not_suspended" | "indeterminate" { +): "absent" | "suspended" | "not_suspended" | "indeterminate" { const runnerRoot = resolve(root, "runner"); + if (!existsSync(runnerRoot)) return "absent"; if (!isSafeNativeStateDirectory(runnerRoot)) return "indeterminate"; const statePath = resolve(runnerRoot, "runner-state.json"); - if (!existsSync(statePath)) return "indeterminate"; + if (!existsSync(statePath)) { + try { + // Older remote transports created this controller-side placeholder even + // though runner state was owned by the sandbox. It carries no authority, + // so a verified failover backup may be consulted. Any non-empty direct + // directory remains indeterminate and therefore blocks fallback. + return readdirSync(runnerRoot).length === 0 ? "absent" : "indeterminate"; + } catch { + return "indeterminate"; + } + } try { const state = record( JSON.parse( @@ -1381,6 +1429,28 @@ function runnerdAuthorityLifecycle( } } +function runnerdAuthorityLifecycleWithVerifiedBackup(input: { + root: string; + identity: RunnerdDurableIdentity; + execution: NativeExecutionInput; + allowVerifiedBackup: boolean; +}): "suspended" | "not_suspended" | "indeterminate" { + const direct = runnerdAuthorityLifecycle(input.root, input.identity); + if (direct !== "absent") return direct; + if (!input.allowVerifiedBackup) return "indeterminate"; + const backup = verifyNativeHarnessBackup({ + root: input.root, + execution: input.execution, + runnerInstanceId: input.identity.runnerInstanceId, + }); + if (!backup) return "indeterminate"; + const backupLifecycle = runnerdAuthorityLifecycle( + backup.root, + input.identity, + ); + return backupLifecycle === "absent" ? "indeterminate" : backupLifecycle; +} + type PriorRunnerdStateVerification = | "verified" | "active" @@ -1394,6 +1464,7 @@ async function verifyPriorRunnerdStateForSessionScope(input: { root: string; identity: RunnerdDurableIdentity; execution: NativeExecutionInput; + allowVerifiedBackup: boolean; }): Promise { let priorRun: { status: string; @@ -1433,7 +1504,7 @@ async function verifyPriorRunnerdStateForSessionScope(input: { nativeSessionScopeKey(priorExecution) === nativeSessionScopeKey(input.execution); if (!sameScope) return "scope_mismatch"; - const lifecycle = runnerdAuthorityLifecycle(input.root, input.identity); + const lifecycle = runnerdAuthorityLifecycleWithVerifiedBackup(input); return lifecycle === "suspended" ? "verified" : "terminal_state_indeterminate"; @@ -1445,6 +1516,7 @@ async function verifyPriorRunnerdStateForSessionScope(input: { async function migrateRunnerdStateRootForExecution(input: { db: Db; execution: NativeExecutionInput; + allowVerifiedBackup: boolean; restartRecovery?: NativeRestartRecoveryClaim; }): Promise { const scoped = scopedRunnerdStateRoot(input.execution); @@ -1476,7 +1548,14 @@ async function migrateRunnerdStateRootForExecution(input: { throw new Error("runner_state_identity_mismatch"); } if (durableIdentityMatchesExecution(identity, input.execution)) { - if (runnerdAuthorityLifecycle(scoped, identity) === "indeterminate") { + if ( + runnerdAuthorityLifecycleWithVerifiedBackup({ + root: scoped, + identity, + execution: input.execution, + allowVerifiedBackup: input.allowVerifiedBackup, + }) === "indeterminate" + ) { if (input.restartRecovery?.kind !== "reattach_existing_runner") { quarantineRunnerdStateRoot(scoped, "identity_indeterminate"); } @@ -1488,6 +1567,7 @@ async function migrateRunnerdStateRootForExecution(input: { root: scoped, identity, execution: input.execution, + allowVerifiedBackup: input.allowVerifiedBackup, }); if (verification !== "verified") { if (verification !== "active" && verification !== "unavailable") { @@ -1525,6 +1605,7 @@ async function migrateRunnerdStateRootForExecution(input: { root: legacy, identity, execution: input.execution, + allowVerifiedBackup: input.allowVerifiedBackup, }); if (verification !== "verified") { if (verification === "terminal_state_indeterminate") { @@ -1750,7 +1831,7 @@ type NativeDriverKind = NativeExecutionInput["session"]["driverKind"]; export interface NativeHarnessPersistenceDirectory { name: "runner" | "codex-home" | "opencode" | "acpx"; location: "runner" | "filesystem"; - excludeTopLevelEntries: readonly string[]; + excludeEntries: readonly string[]; } export interface NativeHarnessPersistenceProfile { @@ -1790,26 +1871,37 @@ export function resolveNativeHarnessPersistenceProfile( // credential source and config.toml can contain the native MCP // bearer token. Re-materialize both for a replacement sandbox // instead of putting credentials into the disaster-recovery copy. - excludeTopLevelEntries: ["tmp", ".tmp", "auth.json", "config.toml"], + excludeEntries: CODEX_HOME_NON_PERSISTENT_ENTRIES, } : execution.provider.kind === "opencode" ? { name: "opencode", location: "filesystem", - excludeTopLevelEntries: [], + excludeEntries: [], } : execution.provider.kind === "acpx" ? { name: "acpx", location: "filesystem", - excludeTopLevelEntries: [], + // ACPX stores each provider beneath a stable session directory. + // Codex creates process-local executable aliases in tmp/arg0; + // they may point outside the runtime tree and are neither safe + // nor necessary to restore. Credentials and launch-time config + // are also re-materialized in the replacement sandbox. + excludeEntries: + execution.provider.agent === "codex" + ? CODEX_HOME_NON_PERSISTENT_ENTRIES.map( + (entry) => + `acpx/${acpxRuntimeSessionDirectoryName(nativeSessionKey(execution))}/codex-home/${entry}`, + ) + : [], } : null; return { providerKind: execution.provider.kind, driverKind: execution.session.driverKind, directories: [ - { name: "runner", location: "runner", excludeTopLevelEntries: [] }, + { name: "runner", location: "runner", excludeEntries: [] }, ...(providerDirectory ? [providerDirectory] : []), ], }; @@ -1826,14 +1918,147 @@ function canonicalJson(value: unknown): string { return JSON.stringify(value) ?? "null"; } -function providerSessionIdentityFromRunnerState( - state: Record, -): Record { - return { - providerSessionId: state.providerSessionId ?? null, - providerBackendSessionId: state.providerBackendSessionId ?? null, - providerSessionIdentity: state.providerSessionIdentity ?? null, - }; +function runnerProviderStateFilename(execution: NativeExecutionInput): string { + switch (execution.provider.kind) { + case "codex": + case "opencode": + return "codex-provider-state.json"; + case "acpx": + return "acpx-provider-state.json"; + case "claude_managed": + case "aws_agentcore": + return "managed-provider-state.json"; + } +} + +/** + * The v2 runner owns PRP identity/lifecycle in runner-state.json and keeps + * provider recovery identity in a sibling provider state file. Never infer a + * resumable provider from the outer PRP journal alone. + */ +export function providerSessionIdentityFromDurableProviderState(input: { + execution: NativeExecutionInput; + providerState: unknown; +}): Record { + const state = record(input.providerState); + const expectedSessionId = nativeSessionKey(input.execution); + const nonEmptyString = (value: unknown): value is string => + typeof value === "string" && value.trim().length > 0; + const sha256 = (value: unknown): value is string => + typeof value === "string" && /^sha256:[0-9a-f]{64}$/.test(value); + const emptyIdentity = () => ({ + providerSessionId: null, + providerBackendSessionId: null, + providerSessionIdentity: null, + }); + switch (input.execution.provider.kind) { + case "acpx": { + const descriptor = record(state.descriptor); + const identity = record(state.identity); + const expectedModel = input.execution.provider.model; + const requiredIdentityFields = [ + "kind", + "normalizedSessionId", + "acpxRecordId", + "backendSessionId", + "agentSessionId", + "profileDigest", + "workspaceDigest", + "requestedModel", + "effectiveModel", + ] as const; + if ( + state.schema !== ACPX_PROVIDER_STATE_SCHEMA || + state.lifecycle !== "suspended" || + state.providerExitUnconfirmed !== false || + state.activeTurnId !== null || + descriptor.kind !== "acpx" || + descriptor.provider !== "acpx" || + descriptor.driver !== "acpx_runtime" || + descriptor.agent !== input.execution.provider.agent || + descriptor.model !== expectedModel || + descriptor.normalizedSessionId !== expectedSessionId || + identity.kind !== "acpx" || + identity.normalizedSessionId !== expectedSessionId || + requiredIdentityFields.some( + (field) => !nonEmptyString(identity[field]), + ) || + !sha256(identity.profileDigest) || + !sha256(identity.workspaceDigest) || + identity.profileDigest !== descriptor.commandDigest || + identity.requestedModel !== expectedModel || + identity.effectiveModel !== expectedModel || + identity.permissionMode !== input.execution.provider.permissionMode || + !["approve-all", "approve-reads", "deny-all"].includes( + String(identity.permissionMode), + ) || + !Array.isArray(identity.providerLifetimeFenceCandidates) || + identity.providerLifetimeFenceCandidates.length !== 3 || + new Set(identity.providerLifetimeFenceCandidates).size !== 3 || + identity.providerLifetimeFenceCandidates.some( + (port) => + !Number.isInteger(port) || + Number(port) < 49_152 || + Number(port) > 65_535, + ) + ) { + return emptyIdentity(); + } + return { + providerSessionId: identity.acpxRecordId ?? null, + providerBackendSessionId: identity.backendSessionId ?? null, + providerSessionIdentity: structuredClone(identity), + }; + } + case "codex": + case "opencode": { + const config = record(state.config); + const expectedDriver = + input.execution.provider.kind === "codex" + ? "codex_app_server" + : "opencode_server"; + if ( + state.schema !== CODEX_PROVIDER_STATE_SCHEMA || + !["prepared", "session_open", "provider_exited"].includes( + String(state.lifecycle), + ) || + !nonEmptyString(state.threadId) || + (state.providerSessionId !== null && + state.providerSessionId !== undefined && + !nonEmptyString(state.providerSessionId)) || + state.activeProviderTurnId !== null || + state.ambiguousTurnStartPending === true || + config.provider !== input.execution.provider.kind || + config.driver !== expectedDriver + ) { + return emptyIdentity(); + } + return { + providerSessionId: state.threadId ?? null, + providerBackendSessionId: state.providerSessionId ?? null, + providerSessionIdentity: null, + }; + } + case "claude_managed": + case "aws_agentcore": { + const descriptor = record(state.descriptor); + if ( + state.schema !== MANAGED_PROVIDER_STATE_SCHEMA || + state.lifecycle !== "suspended" || + state.normalizedSessionId !== expectedSessionId || + descriptor.kind !== input.execution.provider.kind || + !nonEmptyString(state.providerSessionId) || + state.activeTurnId !== null + ) { + return emptyIdentity(); + } + return { + providerSessionId: state.providerSessionId ?? null, + providerBackendSessionId: state.providerSessionId ?? null, + providerSessionIdentity: null, + }; + } + } } function providerSessionIdentityIsPresent(value: unknown): boolean { @@ -3427,6 +3652,9 @@ async function executePaperclipNativeSessionWithinScope( await migrateRunnerdStateRootForExecution({ db: input.db, execution: input.execution, + allowVerifiedBackup: + input.runnerExecutionTarget?.kind === "remote" && + input.runnerExecutionTarget.transport === "sandbox", restartRecovery: input.restartRecovery, }); } @@ -4093,6 +4321,9 @@ async function executePaperclipNativeSessionWithinScope( existingSession: existingWarmSession, persistedSession: persistedWarmSession, keepSessionOpen: warmSessionId !== null, + requireSessionCloseBeforeReturn: + runnerdBackend !== null && + input.runnerExecutionTarget?.kind === "remote", onCheckpoint: warmSessionId !== null && warmConfigDigest !== null ? async (snapshot) => @@ -4928,7 +5159,18 @@ async function stageRemoteRunnerFile(input: { function archiveExcludeArgs(entries: readonly string[]): string[] { for (const entry of entries) { - if (entry === "." || entry === ".." || !/^[A-Za-z0-9._-]+$/.test(entry)) { + const segments = entry.split("/"); + if ( + entry.length === 0 || + entry.startsWith("/") || + segments.some( + (segment) => + segment === "" || + segment === "." || + segment === ".." || + !/^[A-Za-z0-9._-]+$/.test(segment), + ) + ) { throw new Error("runner_remote_checkpoint_exclusion_invalid"); } } @@ -4941,9 +5183,9 @@ export async function stageRemoteRunnerDirectory(input: { sourcePath: string; targetPath: string; mode: number; - excludeTopLevelEntries?: readonly string[]; + excludeEntries?: readonly string[]; }): Promise { - const excludeArgs = archiveExcludeArgs(input.excludeTopLevelEntries ?? []); + const excludeArgs = archiveExcludeArgs(input.excludeEntries ?? []); if (input.runner.syncIn) { let stagingRoot: string | null = null; let sourcePath = input.sourcePath; @@ -5122,7 +5364,7 @@ export async function syncRemoteRunnerDirectoryOut(input: { sourcePath: string; targetPath: string; mode: number; - excludeTopLevelEntries?: readonly string[]; + excludeEntries?: readonly string[]; }): Promise { if ( !(await remoteRunnerPathExists({ @@ -5133,7 +5375,7 @@ export async function syncRemoteRunnerDirectoryOut(input: { ) return; mkdirSync(resolve(input.targetPath, ".."), { recursive: true, mode: 0o700 }); - const excluded = input.excludeTopLevelEntries ?? []; + const excluded = input.excludeEntries ?? []; const excludeArgs = archiveExcludeArgs(excluded) .map((argument) => `'${argument}'`) .join(" "); @@ -5288,6 +5530,32 @@ async function readRemoteRunnerState(input: { ); } +async function readRemoteRunnerProviderState(input: { + runner: CommandManagedRuntimeRunner; + stateDirectory: string; + execution: NativeExecutionInput; +}): Promise> { + const statePath = posix.join( + input.stateDirectory, + runnerProviderStateFilename(input.execution), + ); + const escapedPath = statePath.replaceAll("'", "'\\''"); + const result = await input.runner.execute({ + command: "sh", + args: ["-c", `test -f '${escapedPath}' && base64 < '${escapedPath}'`], + bypassSession: true, + timeoutMs: 10_000, + }); + if (result.exitCode !== 0 || result.timedOut) { + throw new Error("runner_remote_provider_state_unavailable"); + } + return record( + JSON.parse( + Buffer.from(result.stdout.replace(/\s+/g, ""), "base64").toString("utf8"), + ), + ); +} + function createRemoteRunnerProcessLauncher(input: { target: Extract; runner: CommandManagedRuntimeRunner; @@ -5404,6 +5672,21 @@ export function resolveRemoteRunnerTransportMode(input: { return requiredMode; } +export function remoteCheckpointIncompleteFailure( + settlement: "settled" | "unsettled", + incompleteReason: "unavailable" | "not_suspended" | null, +): Error | null { + // A transport that never completed provider bootstrap has no provider state + // to preserve; its original launch failure remains authoritative. Once + // runnerd has proved suspension, however, an unreadable or incomplete + // checkpoint must fail the required close so outer sandbox release is + // withheld. Process containment still happens in the transport finally. + if (settlement === "unsettled") return null; + return new Error( + `runner_remote_checkpoint_incomplete: exact suspended harness state unavailable (${incompleteReason ?? "unknown"})`, + ); +} + /** Production runnerd backend seam, exported so provider wiring can be regression tested. */ export async function createRunnerdBackend(input: { db: Db; @@ -5451,6 +5734,9 @@ export async function createRunnerdBackend(input: { await migrateRunnerdStateRootForExecution({ db: input.db, execution: input.execution, + allowVerifiedBackup: + input.runnerExecutionTarget?.kind === "remote" && + input.runnerExecutionTarget.transport === "sandbox", restartRecovery: input.restartRecovery, }); return await createRunnerdBackendWithinSessionClaim(input, sessionScopeId); @@ -5635,6 +5921,7 @@ async function createRunnerdBackendWithinSessionClaim( } : sourceRuntimeContext; let remotePrepared = false; + let remoteHarnessStatePrepared = false; let selectedRemoteMode: "dial_wss" | "listen_ws" | null = null; let remoteCaBundleMapping: { sourcePath: string; targetPath: string } | null = null; @@ -6104,9 +6391,16 @@ async function createRunnerdBackendWithinSessionClaim( const inspectRemoteHarnessState = async (): Promise<{ complete: boolean; runnerState: Record | null; + providerSessionIdentity: Record | null; + incompleteReason: "unavailable" | "not_suspended" | null; }> => { if (!remoteCommandRunner || !remoteStateDirectory) { - return { complete: false, runnerState: null }; + return { + complete: false, + runnerState: null, + providerSessionIdentity: null, + incompleteReason: "unavailable", + }; } const requirements = persistenceProfile.directories.flatMap((directory) => { const path = remotePersistencePath(directory); @@ -6119,17 +6413,25 @@ async function createRunnerdBackendWithinSessionClaim( const escapedRunnerState = posix .join(remoteStateDirectory, "runner-state.json") .replaceAll("'", "'\\''"); + const escapedProviderState = posix + .join(remoteStateDirectory, runnerProviderStateFilename(input.execution)) + .replaceAll("'", "'\\''"); const inspected = await remoteCommandRunner.execute({ command: "sh", args: [ "-c", - `${requirements.join(" && ")} && base64 < '${escapedRunnerState}'`, + `${requirements.join(" && ")} && test -f '${escapedProviderState}' && base64 < '${escapedRunnerState}'`, ], bypassSession: true, timeoutMs: 10_000, }); if (inspected.exitCode !== 0 || inspected.timedOut) { - return { complete: false, runnerState: null }; + return { + complete: false, + runnerState: null, + providerSessionIdentity: null, + incompleteReason: "unavailable", + }; } let runnerState: Record; try { @@ -6149,8 +6451,29 @@ async function createRunnerdBackendWithinSessionClaim( ) { throw new Error("runner_harness_state_mismatch"); } + if (runnerState.lifecycle !== "suspended") { + return { + complete: false, + runnerState: null, + providerSessionIdentity: null, + incompleteReason: "not_suspended", + }; + } + let providerState: Record; + try { + providerState = await readRemoteRunnerProviderState({ + runner: remoteCommandRunner, + stateDirectory: remoteStateDirectory, + execution: input.execution, + }); + } catch { + throw new Error("runner_harness_state_mismatch"); + } const providerSessionIdentity = - providerSessionIdentityFromRunnerState(runnerState); + providerSessionIdentityFromDurableProviderState({ + execution: input.execution, + providerState, + }); if (!providerSessionIdentityIsPresent(providerSessionIdentity)) { throw new Error("runner_harness_state_mismatch"); } @@ -6169,11 +6492,16 @@ async function createRunnerdBackendWithinSessionClaim( ) { throw new Error("runner_harness_state_mismatch"); } - return { complete: true, runnerState }; + return { + complete: true, + runnerState, + providerSessionIdentity, + incompleteReason: null, + }; }; const recordInPlaceHarnessReuse = async ( - runnerState: Record, + providerSessionIdentity: Record, startedAtMs = Date.now(), ) => { const now = Date.now(); @@ -6198,7 +6526,7 @@ async function createRunnerdBackendWithinSessionClaim( provider: input.execution.provider.kind, harness: input.execution.session.driverKind, identityPresent: providerSessionIdentityIsPresent( - providerSessionIdentityFromRunnerState(runnerState), + providerSessionIdentity, ), }, }); @@ -6211,14 +6539,27 @@ async function createRunnerdBackendWithinSessionClaim( return; } const leaseRow = await input.db - .select({ metadata: environmentLeases.metadata }) + .select({ + metadata: environmentLeases.metadata, + providerLeaseId: environmentLeases.providerLeaseId, + }) .from(environmentLeases) .where(eq(environmentLeases.id, remoteTarget.leaseId)) .limit(1) .then((rows) => rows[0] ?? null); - if (!leaseRow) throw new Error("runner_harness_backup_lease_missing"); + if (!leaseRow?.providerLeaseId) { + throw new Error("runner_harness_backup_lease_missing"); + } + if ( + sandboxLeaseAcquisition?.providerLeaseId && + sandboxLeaseAcquisition.providerLeaseId !== leaseRow.providerLeaseId + ) { + throw new Error("runner_harness_backup_lease_mismatch"); + } const stamp = createNativeHarnessBackupStamp({ manifestPath: resolve(backup.root, "manifest.json"), + sessionScopeId, + authorizedProviderLeaseId: leaseRow.providerLeaseId, normalizedSessionId: backup.manifest.normalizedSessionId, runnerInstanceId: backup.manifest.runnerInstanceId, completedAt: backup.manifest.completedAt, @@ -6278,9 +6619,9 @@ async function createRunnerdBackendWithinSessionClaim( if ( !restored.complete || !restored.runnerState || - canonicalJson( - providerSessionIdentityFromRunnerState(restored.runnerState), - ) !== canonicalJson(backup.manifest.providerSessionIdentity) + !restored.providerSessionIdentity || + canonicalJson(restored.providerSessionIdentity) !== + canonicalJson(backup.manifest.providerSessionIdentity) ) { throw new Error("runner_harness_state_mismatch"); } @@ -6351,159 +6692,174 @@ async function createRunnerdBackendWithinSessionClaim( remotePrepared = false; await prepareRemoteRunner(selectedRemoteMode); } - await measureNativeRunnerSpan(input.trace, "stage.asset.home", () => - measureNativeRunnerSpan( - input.trace, - "session.checkpoint.restore", - async () => { - if ( - remoteTarget?.transport === "sandbox" && - remoteCommandRunner - ) { - const acquisitionRecordedAtMs = Date.now(); - const backupAvailable = harnessBackupCandidates(root).some( - (candidate) => - existsSync(resolve(candidate, "manifest.json")), - ); - const restoreIntoCreatedSandbox = - shouldRestoreNativeHarnessBackupIntoSandbox({ - acquisitionOutcome: - sandboxLeaseAcquisition?.outcome ?? null, - reusableLeaseConfigured: - remoteTarget.reusableLeaseConfigured, - backupAvailable, + if (!remoteHarnessStatePrepared) { + await measureNativeRunnerSpan(input.trace, "stage.asset.home", () => + measureNativeRunnerSpan( + input.trace, + "session.checkpoint.restore", + async () => { + if ( + remoteTarget?.transport === "sandbox" && + remoteCommandRunner + ) { + const acquisitionRecordedAtMs = Date.now(); + const backupAvailable = harnessBackupCandidates(root).some( + (candidate) => + existsSync(resolve(candidate, "manifest.json")), + ); + const restoreIntoCreatedSandbox = + shouldRestoreNativeHarnessBackupIntoSandbox({ + acquisitionOutcome: + sandboxLeaseAcquisition?.outcome ?? null, + reusableLeaseConfigured: + remoteTarget.reusableLeaseConfigured, + backupAvailable, + }); + await input.trace?.record({ + name: "sandbox.lease.acquisition", + startedAtMs: acquisitionRecordedAtMs, + endedAtMs: acquisitionRecordedAtMs, + attributes: { + provider: remoteTarget.providerKey ?? "sandbox", + harness: input.execution.session.driverKind, + lifecycleMode: + input.execution.session.lifecyclePolicy.mode, + outcome: sandboxLeaseAcquisition?.outcome ?? "unknown", + stateSource: + sandboxLeaseAcquisition?.outcome === "replacement" || + restoreIntoCreatedSandbox + ? "verified_failover_backup" + : sandboxLeaseAcquisition?.outcome === "resumed" + ? "sandbox_filesystem" + : "new_sandbox", + bytesTransferred: 0, + }, }); - await input.trace?.record({ - name: "sandbox.lease.acquisition", - startedAtMs: acquisitionRecordedAtMs, - endedAtMs: acquisitionRecordedAtMs, - attributes: { - provider: remoteTarget.providerKey ?? "sandbox", - harness: input.execution.session.driverKind, - lifecycleMode: input.execution.session.lifecyclePolicy.mode, - outcome: sandboxLeaseAcquisition?.outcome ?? "unknown", - stateSource: - sandboxLeaseAcquisition?.outcome === "replacement" || - restoreIntoCreatedSandbox - ? "verified_failover_backup" - : sandboxLeaseAcquisition?.outcome === "resumed" - ? "sandbox_filesystem" - : "new_sandbox", - bytesTransferred: 0, - }, - }); - if (sandboxLeaseAcquisition?.outcome === "resumed") { - const reuseStartedAtMs = Date.now(); - const state = await measureNativeRunnerSpan( - input.trace, - "sandbox.lease.resume", - inspectRemoteHarnessState, - { - attributes: { - provider: remoteTarget.providerKey ?? "sandbox", - harness: input.execution.session.driverKind, - lifecycleMode: - input.execution.session.lifecyclePolicy.mode, - outcome: "resumed", + if (sandboxLeaseAcquisition?.outcome === "resumed") { + const reuseStartedAtMs = Date.now(); + const state = await measureNativeRunnerSpan( + input.trace, + "sandbox.lease.resume", + inspectRemoteHarnessState, + { + attributes: { + provider: remoteTarget.providerKey ?? "sandbox", + harness: input.execution.session.driverKind, + lifecycleMode: + input.execution.session.lifecyclePolicy.mode, + outcome: "resumed", + }, }, - }, - ); - if (!state.complete || !state.runnerState) { - throw new Error("runner_harness_state_mismatch"); - } - await recordInPlaceHarnessReuse( - state.runnerState, - reuseStartedAtMs, - ); - } else if (sandboxLeaseAcquisition?.outcome === "replacement") { - await measureNativeRunnerSpan( - input.trace, - "sandbox.lease.replacement", - restoreVerifiedHarnessBackup, - { - attributes: { - provider: remoteTarget.providerKey ?? "sandbox", - harness: input.execution.session.driverKind, - lifecycleMode: - input.execution.session.lifecyclePolicy.mode, - outcome: "replacement", - reason: sandboxLeaseAcquisition.reason ?? "unknown", - }, - }, - ); - } else if (restoreIntoCreatedSandbox) { - await measureNativeRunnerSpan( - input.trace, - "sandbox.lease.replacement", - restoreVerifiedHarnessBackup, - { - attributes: { - provider: remoteTarget.providerKey ?? "sandbox", - harness: input.execution.session.driverKind, - lifecycleMode: - input.execution.session.lifecyclePolicy.mode, - outcome: "created", - reason: "reuse_disabled", - }, - }, - ); - } else { - const reuseStartedAtMs = Date.now(); - const state = await inspectRemoteHarnessState(); - if (state.complete && state.runnerState) { - // Re-entry while this newly-created lease is already running (for - // example a transport reconnect) still uses the in-place state. + ); + if ( + !state.complete || + !state.runnerState || + !state.providerSessionIdentity + ) { + throw new Error("runner_harness_state_mismatch"); + } await recordInPlaceHarnessReuse( - state.runnerState, + state.providerSessionIdentity, reuseStartedAtMs, ); - } else if (backupAvailable) { - // A continuation that has a durable backup but no recorded reusable - // lease was not provider-confirmed lost. Never silently create a new - // provider session from that ambiguous state. - throw new Error("runner_harness_state_mismatch"); - } - } - await materializeRemoteHarnessLaunchState(); - } else if (remoteTarget && remoteCommandRunner) { - // Local and generic SSH execution retain their existing checkpoint - // behavior. The manifest-only failover gate applies to managed sandbox - // replacement, where provider lease provenance is available. - for (const directory of persistenceProfile.directories) { - const localDirectory = resolve(root, directory.name); - const remoteDirectory = remotePersistencePath(directory); - if ( - remoteDirectory && - existsSync(localDirectory) && - !(await remoteRunnerPathExists({ - runner: remoteCommandRunner, - path: - directory.name === "runner" - ? posix.join(remoteDirectory, "runner-state.json") - : remoteDirectory, - kind: directory.name === "runner" ? "file" : "directory", - })) + } else if ( + sandboxLeaseAcquisition?.outcome === "replacement" ) { - await stageRemoteRunnerDirectory({ - target: remoteTarget, - runner: remoteCommandRunner, - sourcePath: localDirectory, - targetPath: remoteDirectory, - mode: 0o700, - excludeTopLevelEntries: directory.excludeTopLevelEntries, - }); + await measureNativeRunnerSpan( + input.trace, + "sandbox.lease.replacement", + restoreVerifiedHarnessBackup, + { + attributes: { + provider: remoteTarget.providerKey ?? "sandbox", + harness: input.execution.session.driverKind, + lifecycleMode: + input.execution.session.lifecyclePolicy.mode, + outcome: "replacement", + reason: sandboxLeaseAcquisition.reason ?? "unknown", + }, + }, + ); + } else if (restoreIntoCreatedSandbox) { + await measureNativeRunnerSpan( + input.trace, + "sandbox.lease.replacement", + restoreVerifiedHarnessBackup, + { + attributes: { + provider: remoteTarget.providerKey ?? "sandbox", + harness: input.execution.session.driverKind, + lifecycleMode: + input.execution.session.lifecyclePolicy.mode, + outcome: "created", + reason: "reuse_disabled", + }, + }, + ); + } else { + const reuseStartedAtMs = Date.now(); + const state = await inspectRemoteHarnessState(); + if ( + state.complete && + state.runnerState && + state.providerSessionIdentity + ) { + // Re-entry while this newly-created lease is already running (for + // example a transport reconnect) still uses the in-place state. + await recordInPlaceHarnessReuse( + state.providerSessionIdentity, + reuseStartedAtMs, + ); + } else if (backupAvailable) { + // A continuation that has a durable backup but no recorded reusable + // lease was not provider-confirmed lost. Never silently create a new + // provider session from that ambiguous state. + throw new Error("runner_harness_state_mismatch"); + } + } + await materializeRemoteHarnessLaunchState(); + } else if (remoteTarget && remoteCommandRunner) { + // Local and generic SSH execution retain their existing checkpoint + // behavior. The manifest-only failover gate applies to managed sandbox + // replacement, where provider lease provenance is available. + for (const directory of persistenceProfile.directories) { + const localDirectory = resolve(root, directory.name); + const remoteDirectory = remotePersistencePath(directory); + if ( + remoteDirectory && + existsSync(localDirectory) && + !(await remoteRunnerPathExists({ + runner: remoteCommandRunner, + path: + directory.name === "runner" + ? posix.join(remoteDirectory, "runner-state.json") + : remoteDirectory, + kind: + directory.name === "runner" ? "file" : "directory", + })) + ) { + await stageRemoteRunnerDirectory({ + target: remoteTarget, + runner: remoteCommandRunner, + sourcePath: localDirectory, + targetPath: remoteDirectory, + mode: 0o700, + excludeEntries: directory.excludeEntries, + }); + } } } - } - }, - { - attributes: { - mode: remoteTarget?.transport ?? "local", - lifecycleMode: input.execution.session.lifecyclePolicy.mode, }, - }, - ), - ); + { + attributes: { + mode: remoteTarget?.transport ?? "local", + lifecycleMode: input.execution.session.lifecyclePolicy.mode, + }, + }, + ), + ); + remoteHarnessStatePrepared = true; + } if ( remoteTarget && remoteCommandRunner && @@ -6575,7 +6931,9 @@ async function createRunnerdBackendWithinSessionClaim( ); }; - const checkpointRemoteRunner = async () => { + const checkpointRemoteRunner = async ( + settlement: "settled" | "unsettled", + ) => { if ( !remoteCommandRunner || !remoteStateDirectory || @@ -6589,8 +6947,27 @@ async function createRunnerdBackendWithinSessionClaim( // `runner_harness_state_mismatch`. Only checkpoint a harness that runnerd // has proved complete. A malformed or identity-conflicting state still // throws from inspectRemoteHarnessState and therefore fails closed. - const checkpointable = await inspectRemoteHarnessState(); - if (!checkpointable.complete) return; + let checkpointable = await inspectRemoteHarnessState(); + // The remote runner writes its suspended lifecycle and provider state + // before the outer process-owner RPC necessarily observes completion. + // Allow a very small bounded visibility window without ever accepting an + // active, incomplete, malformed, or identity-conflicting checkpoint. + for (let attempt = 1; !checkpointable.complete && attempt < 3; attempt++) { + await new Promise((resolve) => setTimeout(resolve, 100)); + checkpointable = await inspectRemoteHarnessState(); + } + if (!checkpointable.complete) { + const incompleteFailure = remoteCheckpointIncompleteFailure( + settlement, + checkpointable.incompleteReason, + ); + await input.onLog?.( + "stderr", + `[paperclip-runner] remote checkpoint ${incompleteFailure ? "failed" : "skipped"}: exact suspended harness state unavailable (process=${settlement} reason=${checkpointable.incompleteReason})\n`, + ); + if (incompleteFailure) throw incompleteFailure; + return; + } const backupSpanAttributes = { provider: input.execution.provider.kind, harness: input.execution.session.driverKind, @@ -6598,125 +6975,194 @@ async function createRunnerdBackendWithinSessionClaim( stateSource: "sandbox_filesystem", bytesTransferred: 0, }; - await measureNativeRunnerSpan( - input.trace, - "session.checkpoint.persist", - () => - measureNativeRunnerSpan( - input.trace, - "harness_state.backup.persist", - async () => { - const runnerState = await readRemoteRunnerState({ - runner: remoteCommandRunner, - stateDirectory: remoteStateDirectory, - }); - if ( - runnerState.runnerInstanceId !== input.runnerInstanceId || - runnerState.normalizedSessionId !== - nativeSessionKey(input.execution) - ) { - throw new Error("runner_harness_state_mismatch"); - } - const providerSessionIdentity = - providerSessionIdentityFromRunnerState(runnerState); - if (!providerSessionIdentityIsPresent(providerSessionIdentity)) { - throw new Error("runner_harness_state_mismatch"); - } - - const backupRoot = harnessBackupRoot(root); - mkdirSync(backupRoot, { recursive: true, mode: 0o700 }); - const pendingRoot = resolve(backupRoot, `.pending-${randomUUID()}`); - mkdirSync(pendingRoot, { recursive: true, mode: 0o700 }); - try { - for (const directory of persistenceProfile.directories) { - const sourcePath = remotePersistencePath(directory); - if (!sourcePath) - throw new Error("runner_harness_state_mismatch"); - const targetPath = resolve(pendingRoot, directory.name); - await syncRemoteRunnerDirectoryOut({ - runner: remoteCommandRunner, - sourcePath, - targetPath, - mode: 0o700, - excludeTopLevelEntries: directory.excludeTopLevelEntries, - }); - if (!existsSync(targetPath)) { - throw new Error("runner_harness_state_mismatch"); - } + try { + await measureNativeRunnerSpan( + input.trace, + "session.checkpoint.persist", + () => + measureNativeRunnerSpan( + input.trace, + "harness_state.backup.persist", + async () => { + const verified = await inspectRemoteHarnessState(); + if ( + !verified.complete || + !verified.runnerState || + !verified.providerSessionIdentity + ) { + throw new Error("runner_harness_state_mismatch"); } - const manifest = buildNativeHarnessBackupManifest({ - backupRoot: pendingRoot, - execution: input.execution, - runnerInstanceId: input.runnerInstanceId, - providerSessionIdentity, - sourceProviderLeaseId: - sandboxLeaseAcquisition?.providerLeaseId ?? - remoteTarget?.leaseId ?? - input.durableEnvironmentLeaseId ?? - "unknown", - }); - backupSpanAttributes.bytesTransferred = - manifest.directories.reduce( - (total, directory) => total + directory.bytes, - 0, - ); - const temporaryManifest = resolve( - pendingRoot, - "manifest.json.tmp", + const providerSessionIdentity = verified.providerSessionIdentity; + + const backupRoot = harnessBackupRoot(root); + mkdirSync(backupRoot, { recursive: true, mode: 0o700 }); + const pendingRoot = resolve( + backupRoot, + `.pending-${randomUUID()}`, ); - const manifestPath = resolve(pendingRoot, "manifest.json"); - writeFileSync(temporaryManifest, JSON.stringify(manifest), { - encoding: "utf8", - mode: 0o600, - }); - renameSync(temporaryManifest, manifestPath); - - const currentRoot = resolve(backupRoot, "current"); - const previousRoot = resolve(backupRoot, "previous"); - rmSync(previousRoot, { recursive: true, force: true }); - let movedCurrent = false; - if (existsSync(currentRoot)) { - renameSync(currentRoot, previousRoot); - movedCurrent = true; - } + mkdirSync(pendingRoot, { recursive: true, mode: 0o700 }); try { - renameSync(pendingRoot, currentRoot); - if ( - remoteTarget?.transport === "sandbox" && - remoteTarget.leaseId - ) { - await recordHarnessBackupStampForCurrentLease({ - root: currentRoot, - manifest, - bytes: backupSpanAttributes.bytesTransferred, + for (const directory of persistenceProfile.directories) { + const sourcePath = remotePersistencePath(directory); + if (!sourcePath) + throw new Error("runner_harness_state_mismatch"); + const targetPath = resolve(pendingRoot, directory.name); + await syncRemoteRunnerDirectoryOut({ + runner: remoteCommandRunner, + sourcePath, + targetPath, + mode: 0o700, + excludeEntries: directory.excludeEntries, }); + if (!existsSync(targetPath)) { + throw new Error("runner_harness_state_mismatch"); + } } - } catch (error) { + const manifest = buildNativeHarnessBackupManifest({ + backupRoot: pendingRoot, + execution: input.execution, + runnerInstanceId: input.runnerInstanceId, + providerSessionIdentity, + sourceProviderLeaseId: + sandboxLeaseAcquisition?.providerLeaseId ?? + remoteTarget?.leaseId ?? + input.durableEnvironmentLeaseId ?? + "unknown", + }); + backupSpanAttributes.bytesTransferred = + manifest.directories.reduce( + (total, directory) => total + directory.bytes, + 0, + ); + const temporaryManifest = resolve( + pendingRoot, + "manifest.json.tmp", + ); + const manifestPath = resolve(pendingRoot, "manifest.json"); + writeFileSync(temporaryManifest, JSON.stringify(manifest), { + encoding: "utf8", + mode: 0o600, + }); + renameSync(temporaryManifest, manifestPath); + + const currentRoot = resolve(backupRoot, "current"); + const previousRoot = resolve(backupRoot, "previous"); + rmSync(previousRoot, { recursive: true, force: true }); + let movedCurrent = false; if (existsSync(currentRoot)) { - rmSync(currentRoot, { recursive: true, force: true }); + renameSync(currentRoot, previousRoot); + movedCurrent = true; } - if ( - movedCurrent && - existsSync(previousRoot) && - !existsSync(currentRoot) - ) { - renameSync(previousRoot, currentRoot); + try { + renameSync(pendingRoot, currentRoot); + if ( + remoteTarget?.transport === "sandbox" && + remoteTarget.leaseId + ) { + await recordHarnessBackupStampForCurrentLease({ + root: currentRoot, + manifest, + bytes: backupSpanAttributes.bytesTransferred, + }); + } + } catch (error) { + if (existsSync(currentRoot)) { + rmSync(currentRoot, { recursive: true, force: true }); + } + if ( + movedCurrent && + existsSync(previousRoot) && + !existsSync(currentRoot) + ) { + renameSync(previousRoot, currentRoot); + } + throw error; } - throw error; + rmSync(previousRoot, { recursive: true, force: true }); + } finally { + rmSync(pendingRoot, { recursive: true, force: true }); } - rmSync(previousRoot, { recursive: true, force: true }); - } finally { - rmSync(pendingRoot, { recursive: true, force: true }); - } - }, - { - attributes: backupSpanAttributes, - }, - ), - { parentName: "task.settle" }, - ); + }, + { + attributes: backupSpanAttributes, + }, + ), + { parentName: "task.settle" }, + ); + } catch (error) { + const detail = redactSensitiveText( + error instanceof Error ? error.message : String(error), + ) + .replace(/[\r\n]+/g, " ") + .slice(0, 512); + await input.onLog?.( + "stderr", + `[paperclip-runner] remote checkpoint failed: ${detail || "unknown failure"}\n`, + ); + throw error; + } }; + const prepareExternalRunnerState = + remoteTarget && remoteCommandRunner + ? async () => { + selectedRemoteMode ??= resolveRemoteRunnerTransportMode({ + target: remoteTarget, + runnerIngressAuthorized: input.runnerIngressAuthorized === true, + }); + await ensureRemoteRunner(); + } + : undefined; + const archiveExternalRunnerState = + remoteCommandRunner && remoteStateDirectory && remoteSessionRoot + ? async (archive: { archiveKey: string }) => { + if (!/^[0-9a-f]{24}$/.test(archive.archiveKey)) { + throw new Error("runner_remote_authority_archive_invalid"); + } + const sourcePath = posix.join( + remoteStateDirectory, + "runner-state.json", + ); + const archiveDirectory = posix.join( + remoteSessionRoot, + "authority-epochs", + `epoch-${archive.archiveKey}`, + ); + const archivedStatePath = posix.join( + archiveDirectory, + "runner-state.json", + ); + const result = await remoteCommandRunner.execute({ + command: "sh", + args: [ + "-c", + 'set -eu; if test -L "$2" || { test -e "$2" && test ! -d "$2"; }; then exit 1; fi; if test -f "$1" && test ! -L "$1" && test ! -e "$3" && test ! -L "$3"; then umask 077; install -d -m 0700 "$2"; mv -- "$1" "$3"; elif test ! -e "$1" && test ! -L "$1" && test -f "$3" && test ! -L "$3"; then :; else exit 1; fi; base64 < "$3"', + "paperclip-runner-authority-archive", + sourcePath, + archiveDirectory, + archivedStatePath, + ], + bypassSession: true, + timeoutMs: 10_000, + }); + if (result.exitCode !== 0 || result.timedOut) { + throw new Error("runner_remote_authority_archive_failed"); + } + try { + return record( + JSON.parse( + Buffer.from( + result.stdout.replace(/\s+/g, ""), + "base64", + ).toString("utf8"), + ), + ); + } catch { + throw new Error("runner_remote_authority_archive_failed"); + } + } + : undefined; + const remoteProcessLauncher = remoteTarget && remoteCommandRunner && remoteBinary ? createRemoteRunnerProcessLauncher({ @@ -6918,6 +7364,8 @@ async function createRunnerdBackendWithinSessionClaim( stateDirectory: remoteStateDirectory, }) : undefined, + prepareExternalRunnerState, + archiveExternalRunnerState, runnerBinary: controllerRunnerBinary, codexCommand: remoteCodexBinary ?? undefined, sourceCodexHome: remoteTarget @@ -7117,10 +7565,8 @@ async function createRunnerdBackendWithinSessionClaim( ...(caBundlePath ? { caBundlePath } : {}), }, startupFailureCode: "runner_direct_wss_failed" as const, - release: async () => { - await inbound.release(); - await checkpointRemoteRunner(); - }, + checkpoint: checkpointRemoteRunner, + release: () => inbound.release(), }; } @@ -7182,10 +7628,10 @@ async function createRunnerdBackendWithinSessionClaim( return outbound?.failure; }, startupFailureCode: "runner_ingress_unavailable" as const, + checkpoint: checkpointRemoteRunner, release: async () => { if (outbound) await outbound.close(); else await transport.ingress.close(); - await checkpointRemoteRunner(); }, }; }, diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 371daee650..b3b1a92fd2 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -666,7 +666,13 @@ function isRepeatedProductiveContinuationRecovery(latestRun: SuccessfulLatestIss isProductiveContinuationRun(latestRun); } -export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) { +export function recoveryService( + db: Db, + deps: { + enqueueWakeup: RecoveryWakeup; + liveRunExecutions?: Readonly<{ has(id: string): boolean }>; + }, +) { const issuesSvc = issueService(db); const recoveryActionsSvc = issueRecoveryActionService(db); const treeControlSvc = issueTreeControlService(db); @@ -4581,12 +4587,15 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) else if (issueStatus === "cancelled") issueTerminalStatus = "cancelled"; } - // Process-death authority. The run is live only when a process still backs - // it. Check the in-memory handle first, then the recorded pid and process - // group. Require recorded process metadata, so this authority never fires on - // a run that has not yet stored its pid. + // Process-death authority. The run is live while either its adapter process + // or the enclosing heartbeat execution/finalization still owns it. Check + // that full in-process lifecycle first, then the recorded pid and process + // group. Require recorded process metadata, so this authority never fires + // on a run that has not yet stored its pid. let processGone = false; - if (!runningProcesses.get(run.id)) { + const hasLiveExecution = + deps.liveRunExecutions?.has(run.id) ?? runningProcesses.has(run.id); + if (!hasLiveExecution) { if (typeof pid === "number" || typeof processGroupId === "number") { const processAlive = (typeof pid === "number" && isPidAlive(pid)) || diff --git a/server/src/vendor/paperclip-runner/index.ts b/server/src/vendor/paperclip-runner/index.ts index b3ac878531..690f220309 100644 --- a/server/src/vendor/paperclip-runner/index.ts +++ b/server/src/vendor/paperclip-runner/index.ts @@ -64,7 +64,7 @@ const sourceUrl = new URL( "../../../../packages/paperclip-runner/src/index.ts", import.meta.url, ); -const runner = await import(sourceUrl.href) as RunnerModule; +const runner = (await import(sourceUrl.href)) as RunnerModule; export const DurablePrpControlPlane = runner.DurablePrpControlPlane; export const PaperclipSemanticDispatcher = runner.PaperclipSemanticDispatcher; @@ -76,6 +76,8 @@ export const NATIVE_RUNTIME_ASSET_SCHEMA = runner.NATIVE_RUNTIME_ASSET_SCHEMA; export const PAPERCLIP_EXECUTION_PROMPT = runner.PAPERCLIP_EXECUTION_PROMPT; export const PAPERCLIP_EXECUTION_PROMPT_REVISION = runner.PAPERCLIP_EXECUTION_PROMPT_REVISION; +export const acpxRuntimeSessionDirectoryName = + runner.acpxRuntimeSessionDirectoryName; export const canonicalNativeRuntimeContextDigest = runner.canonicalNativeRuntimeContextDigest; export const createNativeSessionBackend = runner.createNativeSessionBackend; @@ -96,7 +98,8 @@ export const parseHarnessRuntimeRequestResolution = export const parseNativeExecutionInput = runner.parseNativeExecutionInput; export const parseNativeRuntimeContext = runner.parseNativeRuntimeContext; export const parsePaperclipQuestionSet = runner.parsePaperclipQuestionSet; -export const parsePaperclipQuestionResponse = runner.parsePaperclipQuestionResponse; +export const parsePaperclipQuestionResponse = + runner.parsePaperclipQuestionResponse; export const resolveQualifiedAcpxProfile = runner.resolveQualifiedAcpxProfile; export const resolveSourceCodexHome = runner.resolveSourceCodexHome; export const validatePrpEvent = runner.validatePrpEvent; diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index ad62738f60..cbf336f459 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -189,9 +189,10 @@ Packaged, access-controlled evidence is written beneath outcomes, sanitized fixture/API metadata, a result record, JUnit, HTML, and a blob report. Failures additionally retain the Playwright trace/video, browser diagnostics, failure screenshot, and sanitized Paperclip/run logs when -produced. PNG and WebM files are not pixel-inspected, so they are suitable only -for the local results directory and access-controlled GitHub Actions artifacts. -SVG is active content and is rejected from the packaged evidence entirely. +produced. Provider/UI PNG and WebM files remain limited to the local results +directory and access-controlled GitHub Actions artifact because secrets can be +rendered into pixels. SVG is active content and is rejected from the packaged +evidence entirely. Every completed local campaign also writes `tests/runner-e2e/results//dashboard.html`. The self-contained page @@ -204,16 +205,22 @@ usage is labeled `unavailable` or `unpriced`; it is never presented as zero cost. The CI report job stages the same portable site at `normalized/index.html` inside the access-controlled merged report artifact. -Permanent public history has a narrower boundary. Before uploading to S3 or -packaging the optional GitHub Pages artifact, the publisher removes raster and -video evidence, archives, and the generated Playwright/blob/HTML report trees. -It then regenerates the dashboard against only the remaining allowlisted, -inert structured per-attempt evidence (`.json`, `.log`, `.md`, and `.txt`). -Per-attempt XML is excluded because browsers can process XML/XSLT. The root -`junit.xml` remains public because the report aggregator builds it from fixed -markup and XML-escaped fields. Public dashboards therefore contain results and -accounting but no attempt screenshots, videos, traces, or generated Playwright -reports. +Permanent publication uses two explicit bundles. The CloudFront-backed S3 +history contains one publisher-generated `public-images/campaign-summary.png`. +Trusted publisher code renders it offline from fixed catalog labels and +sanitized status/count/duration fields; provider output, error text, comments, +and target-produced pixels are never inputs. The PNG must pass a 12 MiB bound +and signature validation before entering the immutable manifest. S3 also +retains allowlisted inert per-attempt evidence (`.json`, `.log`, `.md`, and +`.txt`); `.log` copies have already passed exact-value/key-shape scanning and +redaction. The GitHub Pages bundle is regenerated separately and remains +structured-only. + +Both public bundles exclude video, archives, raw/unallowlisted logs, SVG or +other active content, generated Playwright/blob/HTML report trees, and +per-attempt XML. The root `junit.xml` remains public because the report +aggregator builds it from fixed markup and XML-escaped fields. Full evidence +remains available only in the access-controlled workflow artifact. ### Billing interpretation @@ -380,13 +387,10 @@ bundle digest fails closed. GitHub Pages remains the stable latest dashboard. Enable Pages with GitHub Actions as its source and set `RUNNER_FULL_STACK_E2E_PUBLISH_PAGES=true`. -The publisher prunes screenshots, video, archives, and generated report trees, -then regenerates the public dashboard before either the CloudFront-backed S3 -history or optional Pages artifact is created. Public per-attempt evidence is -limited to allowlisted inert structured text. Databases, Paperclip homes, -workspaces, raw/unredacted logs, credentials, and visual evidence are never -published. Sanitized allowlisted `.log` copies may be public only after -exact-value/key-shape scanning and redaction. +The publisher creates an S3 stage with the trusted synthetic summary PNG and a +separate structured-only Pages stage. Neither surface publishes provider/UI +screenshots, video, archives, generated reports, SVG/active content, databases, +Paperclip homes, workspaces, raw/unallowlisted logs, or credentials. See [FIXTURES.md](./FIXTURES.md) before adding or changing a profile, environment, task, matcher, or future Paperclip object fixture. diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md index 518e9f1a47..06f01adfd4 100644 --- a/tests/runner-e2e/SECURITY.md +++ b/tests/runner-e2e/SECURITY.md @@ -180,29 +180,38 @@ latest pointers are mutable, and S3 versioning makes those updates recoverable. CloudFront and GitHub Pages are public. Fixture identifiers, timing, token usage, costs, normalized results, and allowlisted inert structured per-attempt -evidence are expected public data. Screenshots, video, archives, generated -Playwright/blob/HTML report trees, credentials, Paperclip homes, databases, -workspaces, master keys, raw/unredacted logs, and unallowlisted files are not. -Only allowlisted `.log` copies that passed exact-value/key-shape scanning and -redaction may cross the public boundary. +evidence are expected public data. The CloudFront-backed S3 history also +publishes one synthetic campaign-summary PNG generated by trusted publisher +code solely from fixed catalog labels and sanitized numeric/status fields. +Provider output, comments, error text, and provider/UI screenshots are never +rendered into it. Video, archives, generated +Playwright/blob/HTML report trees, SVG or other active content, credentials, +Paperclip homes, databases, workspaces, master keys, raw/unredacted logs, and +unallowlisted files are not public. Allowlisted `.log` copies must pass the +existing exact-value/key-shape scan and redaction boundary. The packaged evidence uploaded as a 30-day GitHub Actions artifact has a different, access-controlled boundary. Text is exact-value and key-shape -scanned and redacted. PNG and WebM are raw-byte scanned but cannot be inspected -for credentials rendered as pixels, so they remain only in local evidence and -the access-controlled artifact. SVG is rejected during packaging because it is -active content. +scanned and redacted. PNG and WebM are raw-byte scanned; SVG is rejected during +packaging because it is active content. A raster image cannot be exhaustively +secret-scanned by bytes, so every target/provider-produced image remains +access-controlled and is removed at the public boundary. -Before permanent publication, the campaign publisher prunes raster/video -files, archives, and generated report trees. It then regenerates the dashboard -from the remaining allowlisted `.json`, `.log`, `.md`, and `.txt` evidence and -accepts only that dashboard, normalized JSON/JUnit/summary, fixed -branding assets, and the inert structured evidence paths. Per-attempt XML is -excluded because browsers can process XML/XSLT; the only public XML is the -root `junit.xml`, which the report aggregator constructs from fixed markup and -XML-escaped fields. The same pruned tree feeds both S3/CloudFront history and -the optional GitHub Pages artifact. A leak fails the cell and withholds the -unsafe file. +Before permanent publication, the campaign publisher creates a separate S3 +stage and retains only allowlisted `.json`, `.log`, `.md`, and `.txt` evidence. +It then launches publisher-only Chromium with networking blocked to render one +`public-images/campaign-summary.png`. That fixed-path PNG is capped at 12 MiB +and its signature is validated. Per-attempt XML is excluded because browsers +can process XML/XSLT; +the only public XML is the root `junit.xml`, which the report aggregator +constructs from fixed markup and XML-escaped fields. Videos, archives, +raw/unallowlisted logs, SVG, undeclared images, generated reports, and symlinks +fail closed or are removed before the immutable manifest is calculated. + +GitHub Pages is built from a second structured-only stage without the summary +PNG. This keeps Pages small while the CloudFront-backed S3 dashboard can show a +useful visual generated without target content. A leak fails the cell and +withholds the unsafe file. Rotate the affected credential immediately if a secret-scanning failure or unexpected public object is observed. Preserve the access-controlled Actions diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index dd87dd3494..fe758e7e8b 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -318,10 +318,46 @@ describe("runner E2E catalog", () => { expect(task!.buildPrompt("nonce")).toContain( 'summary:"PAPERCLIP_E2E_PLAN_DONE_nonce"', ); + expect(task!.buildPrompt("nonce")).toContain("first call get_task_context"); + expect(task!.buildPrompt("nonce")).toContain( + "identifies the exact revised Plan revision used as the confirmation target as accepted", + ); + expect(task!.buildPrompt("nonce")).toContain( + "After that verification succeeds, your immediate next action must be the paperclip_finish tool call", + ); + expect(task!.buildPrompt("nonce")).not.toContain( + "trust that inline acceptance", + ); + expect(task!.buildPrompt("nonce")).toContain( + "those two tool calls form one indivisible response sequence", + ); + expect(task!.buildPrompt("nonce")).toContain( + "Do not emit assistant text, end the heartbeat, or stop after write_document alone", + ); expect(task!.buildPrompt("nonce")).toContain( "one atomic issue PATCH with status `done` and that exact comment", ); - expect(task!.buildRevisionRequest?.("nonce")).toContain("baseRevisionId"); + const revisionRequest = task!.buildRevisionRequest?.("nonce"); + expect(revisionRequest).toContain("baseRevisionId"); + expect(revisionRequest).toContain( + "request_human_input must be your immediate next action", + ); + }); + + it("requires one atomic legacy Ask completion write", () => { + const task = runnerTasks.find( + (candidate) => candidate.id === "ask-question", + ); + expect(task).toBeDefined(); + const prompt = task!.buildPrompt("nonce"); + expect(prompt).toContain( + "make exactly one public-API write containing the marker", + ); + expect(prompt).toContain( + 'PATCH /api/issues/$PAPERCLIP_TASK_ID with {"status":"done","comment":"E2E_ASK_12_nonce"}', + ); + expect(prompt).toContain("Do not POST to /comments"); + expect(prompt).toContain("do not PATCH the status separately"); }); it("accepts only complete immutable Daytona digests", () => { @@ -416,6 +452,20 @@ describe("runner E2E selectors", () => { expect(jobs.filter((job) => job.needsDaytona)).toHaveLength(21); expect(jobs.filter((job) => !job.needsDaytona)).toHaveLength(45); expect(new Set(jobs.map((job) => job.executionId)).size).toBe(66); + expect( + jobs.find( + (job) => + job.executionId === + "core-compatibility.runner-acpx-claude.local.plan-revise-accept", + )?.timeoutMinutes, + ).toBe(48); + expect( + jobs.find( + (job) => + job.executionId === + "local-session-integrity.runner-acpx-codex.local.structured-question-restart-resume", + )?.timeoutMinutes, + ).toBe(32); expect( jobs.every((job) => runnerMatrix.some( diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 4752beab55..a606e4db91 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -448,7 +448,7 @@ export const runnerTasks: readonly RunnerTaskFixture[] = [ `Remove PAPERCLIP_E2E_PLAN_DRAFT_${nonce} and include PAPERCLIP_E2E_PLAN_REVISED_${nonce}.`, "Change the plan from two steps to exactly three numbered steps, with verification as step 3.", "Publish the revised canonical Plan revision and request confirmation for that new revision.", - "In a native runner, call write_document for key `plan`, then call request_human_input exactly once with interactionKind `confirmation`, targetRevisionId set to the returned latest Plan revision, and continuationPolicy `wake_assignee`; do not call paperclip_finish while waiting.", + "In a native runner, call write_document for key `plan`, then call request_human_input exactly once with interactionKind `confirmation`, targetRevisionId set to the returned latest Plan revision, and continuationPolicy `wake_assignee`; do not call paperclip_finish while waiting. After write_document succeeds, request_human_input must be your immediate next action using that call's returned latestRevisionId; do not emit assistant text, end the response or heartbeat, or stop after write_document alone.", "In a legacy runner, first GET the current `plan` issue document, then PUT the revised Plan with `baseRevisionId` set to that response's `latestRevisionId`; after the update succeeds, create the equivalent request_confirmation targeting the newly returned `latestRevisionId` through the public Paperclip API.", ].join(" "), buildPrompt: (nonce) => @@ -461,7 +461,7 @@ export const runnerTasks: readonly RunnerTaskFixture[] = [ "Only after the revised plan is accepted, implement it by posting one final visible task-thread response containing exactly " + `PAPERCLIP_E2E_PLAN_DONE_${nonce}` + " and mark the task Done.", - `For a native runner, remain in the requested planning collaboration mode. Call write_document for key \`plan\`, then call request_human_input exactly once with interactionKind \`confirmation\`, targetRevisionId set to the returned latest Plan revision, and continuationPolicy \`wake_assignee\`. Do not call paperclip_finish while waiting for either Plan confirmation. After the revised Plan is accepted, call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"PAPERCLIP_E2E_PLAN_DONE_${nonce}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Wait for that tool call to succeed, then emit exactly PAPERCLIP_E2E_PLAN_DONE_${nonce} as the complete final response. Do not write a user-facing final response before paperclip_finish succeeds, and do not call another tool.`, + `For a native runner, remain in the requested planning collaboration mode. Call write_document for key \`plan\`, then call request_human_input exactly once with interactionKind \`confirmation\`, targetRevisionId set to the returned latest Plan revision, and continuationPolicy \`wake_assignee\`. For both the initial Plan and the revised Plan, those two tool calls form one indivisible response sequence: immediately after write_document succeeds, request_human_input must be your next action using that call's returned latestRevisionId. Do not emit assistant text, end the response or heartbeat, or stop after write_document alone before the matching confirmation request succeeds. Do not call paperclip_finish while waiting for either Plan confirmation. When an acceptance wake arrives, first call get_task_context. Treat the wake as valid only when that control-plane result is for the current task and identifies the exact revised Plan revision used as the confirmation target as accepted; otherwise do not finish and continue waiting for the matching revision-bound confirmation. After that verification succeeds, your immediate next action must be the paperclip_finish tool call. Do not call list_documents or any other tool, and do not emit any assistant text, acknowledgement, progress note, or preamble between verification and paperclip_finish. Call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"PAPERCLIP_E2E_PLAN_DONE_${nonce}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Wait for that tool call to succeed, then emit only PAPERCLIP_E2E_PLAN_DONE_${nonce} as the complete final response. Do not write a user-facing final response before paperclip_finish succeeds, and do not call another tool.`, `For a legacy runner, use the public Paperclip API. The first PUT of the \`plan\` issue document creates it. For every later PUT, first GET the current document and set \`baseRevisionId\` to its \`latestRevisionId\`; a 409 means you must GET again and retry with the new latest revision. Create a \`request_confirmation\` targeting the successful PUT response's \`latestRevisionId\` with \`continuationPolicy: wake_assignee\`, and move the issue to \`in_review\` while waiting. After the revised Plan is accepted, write PAPERCLIP_E2E_PLAN_DONE_${nonce} exactly once through one atomic issue PATCH with status \`done\` and that exact comment; do not POST a separate comment or perform a second write.`, "Do not create files, child tasks, or unrelated work, and do not expose credentials.", ].join("\n"), @@ -513,7 +513,7 @@ export const runnerTasks: readonly RunnerTaskFixture[] = [ "Do not create or modify files, do not create a plan or additional work, and do not expose credentials.", "After posting the direct answer, mark this task Done.", `In a native runner, call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"E2E_ASK_12_${nonce}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. For the entire run, paperclip_finish must be your only tool call: never call report_progress or any other tool before or after it. Wait for that tool call to succeed, then emit exactly E2E_ASK_12_${nonce} as the complete final response. Do not write a user-facing final response before paperclip_finish succeeds.`, - `In a legacy runner, post exactly E2E_ASK_12_${nonce} as the task comment body and mark the task Done through the public API.`, + `In a legacy runner, make exactly one public-API write containing the marker: PATCH /api/issues/$PAPERCLIP_TASK_ID with {"status":"done","comment":"E2E_ASK_12_${nonce}"}. Do not POST to /comments, do not PATCH the status separately, and do not include the marker in any other API write.`, ].join("\n"), buildMatchers(nonce, execution) { return [ diff --git a/tests/runner-e2e/dashboard-regenerate.ts b/tests/runner-e2e/dashboard-regenerate.ts index 2a66ed3a39..0643656a9a 100644 --- a/tests/runner-e2e/dashboard-regenerate.ts +++ b/tests/runner-e2e/dashboard-regenerate.ts @@ -56,6 +56,7 @@ export async function regenerateRunnerDashboard(input: { historyFile?: string | null; outputDirectory?: string; evidenceHrefPrefix?: string; + publicSummaryImageHref?: string; }) { const bundle = path.resolve(input.bundle); const outputDirectory = path.resolve(input.outputDirectory ?? bundle); @@ -146,6 +147,7 @@ export async function regenerateRunnerDashboard(input: { entries, campaign, history, + publicSummaryImageHref: input.publicSummaryImageHref, }); const upgraded = { ...campaign, diff --git a/tests/runner-e2e/dashboard.ts b/tests/runner-e2e/dashboard.ts index 23f80352f7..13a75fe547 100644 --- a/tests/runner-e2e/dashboard.ts +++ b/tests/runner-e2e/dashboard.ts @@ -26,6 +26,7 @@ export interface RunnerDashboardInput { entries: readonly RunnerDashboardEntry[]; campaign?: RunnerE2ECampaign; history?: RunnerE2EHistoryIndex; + publicSummaryImageHref?: string; } interface ResolvedScreenshot { @@ -74,6 +75,17 @@ function safeEvidenceHref(base: string | undefined, relative: string) { return `${cleanBase}/${cleanRelative}`; } +function safePublicAssetHref(relative: string | undefined) { + if (!relative || /^(?:[a-z]+:|\/\/|\/)/i.test(relative)) return null; + const segments = relative.split("/"); + if ( + segments.some((segment) => !segment || segment === "." || segment === "..") + ) { + return null; + } + return segments.map(encodeURIComponent).join("/"); +} + function compactJson(value: unknown) { const serialized = JSON.stringify(value); return serialized && serialized.length > 1_200 @@ -563,6 +575,9 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { ) .join(""); const historySection = renderHistory(input.history); + const publicSummaryImageHref = safePublicAssetHref( + input.publicSummaryImageHref, + ); return ` @@ -572,7 +587,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { - + ${html(input.title)} · Paperclip + + +
PPaperclip
Trusted history publication
+
+

Runner full-stack E2E

+

Campaign summary

+
+
${passed}/${expectedIds.length}Known executions passed
+
${expectedIds.length - passed}Failed or incomplete
+
${html(durationLabel(durationMs))}Total test time
+
+
${rows}
+
Generated from fixed catalog labels and sanitized numeric/status fields. Provider output is never rendered here.
+
+ +`; +} + +export async function writePublicCampaignSummaryImage( + campaign: RunnerE2ECampaign, + output: string, +) { + await mkdir(path.dirname(output), { recursive: true }); + const browser = await chromium.launch({ + headless: true, + env: { + LANG: "C.UTF-8", + PATH: process.env.PATH ?? "", + TMPDIR: process.env.RUNNER_TEMP ?? "/tmp", + }, + }); + try { + const context = await browser.newContext({ + viewport: { width: 1400, height: 900 }, + deviceScaleFactor: 1, + }); + await context.setOffline(true); + await context.route("**/*", (route) => route.abort()); + const page = await context.newPage(); + await page.setContent(renderPublicCampaignSummary(campaign), { + waitUntil: "domcontentloaded", + }); + await page.screenshot({ + path: output, + type: "png", + fullPage: true, + animations: "disabled", + }); + await context.close(); + } finally { + await browser.close(); + } +} diff --git a/tests/runner-e2e/report.test.ts b/tests/runner-e2e/report.test.ts index 13cc9eba01..4a30c9d8fa 100644 --- a/tests/runner-e2e/report.test.ts +++ b/tests/runner-e2e/report.test.ts @@ -196,7 +196,9 @@ describe("runner E2E report aggregation", () => { expect(dashboard).toContain( "Visual evidence is retained in the access-controlled workflow artifact", ); - expect(dashboard).toContain("Public history excludes visual evidence"); + expect(dashboard).toContain( + "Public history excludes provider-produced visual evidence", + ); expect(dashboard).toContain("message_contains"); expect(dashboard).toContain("Matchers and test context"); expect(dashboard).toContain("Campaign billing summary"); @@ -215,8 +217,8 @@ describe("runner E2E report aggregation", () => { expect(dashboard).not.toContain("overflow: auto; max-height: calc(100vh"); expect(dashboard).toContain("@media (max-width: 1180px)"); expect( - await readFile(path.join(output, "assets", "favicon.svg"), "utf8"), - ).toContain(" run.status === "succeeded") + ) { + return false; + } + + return input.interactions.some((interaction) => { + if ( + interaction.kind !== "request_confirmation" || + interaction.status !== "pending" + ) { + return false; + } + const target = record(record(interaction.payload).target); + return !( + target.type === "issue_document" && + target.key === "plan" && + typeof target.revisionId === "string" && + target.revisionId.trim().length > 0 + ); + }); +} + function normalizeMessage(value: string) { return value .replace(/\r\n/g, "\n") diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index de8602e001..e80af814ba 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -11,6 +11,7 @@ import { setupLiveFixtures, type LiveFixtureValues } from "./live-fixtures.js"; import { evaluateMatcher, type MatcherResult } from "./matchers.js"; import { acceptedPlanSessionResetFailures, + hasTerminalMalformedPlanConfirmation, isControlPlaneGovernedResponseWait, isNonExecutingReviewFenceRun, isOpenRouterDeepSeekHelloTerminalVariance, @@ -776,6 +777,26 @@ for (const execution of executions) { }; }; + const rejectPlanConfirmationPoll = (input: { + taskRuns: RunRecord[]; + interactions: InteractionRecord[]; + minimumRunCount: number; + }) => { + const runFailure = definitiveRunFailure(input.taskRuns); + if (runFailure) return runFailure; + if ( + !hasTerminalMalformedPlanConfirmation({ + runs: input.taskRuns, + interactions: input.interactions, + minimumRunCount: input.minimumRunCount, + }) + ) { + return undefined; + } + failureClassOverride = "provider_variance"; + return "succeeded heartbeat run created a pending request_confirmation without a revision-bound Plan target"; + }; + let planLifecycleEvidence: Record | null = null; let questionLifecycleEvidence: Record | null = null; let expectedQuestionResolution: { @@ -798,7 +819,12 @@ for (const execution of executions) { taskRuns.length >= 1 && taskRuns.every((run) => TERMINAL_RUN_STATUSES.has(run.status)) && interactions.some(isPendingPlanConfirmation), - reject: ({ taskRuns }) => definitiveRunFailure(taskRuns), + reject: ({ taskRuns, interactions }) => + rejectPlanConfirmationPoll({ + taskRuns, + interactions, + minimumRunCount: 1, + }), }); const draftInteraction = draftState.interactions.find( isPendingPlanConfirmation, @@ -874,7 +900,12 @@ for (const execution of executions) { isPendingPlanConfirmation(interaction) && interaction.id !== draftInteraction.id, ), - reject: ({ taskRuns }) => definitiveRunFailure(taskRuns), + reject: ({ taskRuns, interactions }) => + rejectPlanConfirmationPoll({ + taskRuns, + interactions, + minimumRunCount: 2, + }), }); const revisedInteraction = revisedState.interactions.find( (interaction) => @@ -1110,7 +1141,12 @@ for (const execution of executions) { taskRuns.length >= 1 && taskRuns.every((run) => TERMINAL_RUN_STATUSES.has(run.status)) && interactions.some(isPendingPlanConfirmation), - reject: ({ taskRuns }) => definitiveRunFailure(taskRuns), + reject: ({ taskRuns, interactions }) => + rejectPlanConfirmationPoll({ + taskRuns, + interactions, + minimumRunCount: 1, + }), }); const interaction = pendingState.interactions.find( isPendingPlanConfirmation, diff --git a/tests/runner-e2e/selectors.ts b/tests/runner-e2e/selectors.ts index 581f25136f..d5bf2cdd5c 100644 --- a/tests/runner-e2e/selectors.ts +++ b/tests/runner-e2e/selectors.ts @@ -196,7 +196,16 @@ export function buildMatrixJobs( credentialName: execution.profile.credential, environmentId: execution.environment.id, caseId: execution.task.id, - timeoutMinutes: execution.environment.id === "daytona" ? 40 : 25, + timeoutMinutes: Math.max( + execution.environment.id === "daytona" ? 40 : 25, + Math.ceil( + (2 * + (execution.task.attemptTimeoutMs[execution.environment.id] + + 90_000) + + 5 * 60_000) / + 60_000, + ), + ), needsDaytona: execution.environment.id === "daytona", })) .sort((left, right) => left.executionId.localeCompare(right.executionId)); diff --git a/tests/runner-e2e/support.test.ts b/tests/runner-e2e/support.test.ts index c1eb609a11..79fbe6d3e8 100644 --- a/tests/runner-e2e/support.test.ts +++ b/tests/runner-e2e/support.test.ts @@ -35,6 +35,7 @@ import { } from "./ports.js"; import { acceptedPlanSessionResetFailures, + hasTerminalMalformedPlanConfirmation, isControlPlaneGovernedResponseWait, isNonExecutingReviewFenceRun, isOpenRouterDeepSeekHelloTerminalVariance, @@ -373,6 +374,45 @@ describe("runner E2E matchers", () => { }); describe("runner E2E run observations", () => { + it("retries only terminal Plan confirmations missing a revision-bound target", () => { + const observation = { + runs: [{ status: "succeeded" }], + interactions: [ + { + kind: "request_confirmation", + status: "pending", + payload: { version: 1, prompt: "Approve the Plan?" }, + }, + ], + minimumRunCount: 1, + }; + + expect(hasTerminalMalformedPlanConfirmation(observation)).toBe(true); + expect( + hasTerminalMalformedPlanConfirmation({ + ...observation, + runs: [{ status: "running" }], + }), + ).toBe(false); + expect( + hasTerminalMalformedPlanConfirmation({ + ...observation, + interactions: [ + { + ...observation.interactions[0], + payload: { + target: { + type: "issue_document", + key: "plan", + revisionId: "revision-1", + }, + }, + }, + ], + }), + ).toBe(false); + }); + it("retries only the zero-marker DeepSeek hello terminal emission variance", () => { const expectedMarker = "PC_H_nonce-1"; const observation = { diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts index 93623e2d24..c0c3b38cc9 100644 --- a/tests/runner-e2e/workflow-security.test.ts +++ b/tests/runner-e2e/workflow-security.test.ts @@ -670,11 +670,21 @@ describe("public repository paid workflow security", () => { expect(publisher).not.toMatch(/AWS_(?:ACCESS|SECRET)_KEY/); expect(publisher).not.toMatch(/aws s3 (?:rm|sync .*--delete)/); expect(workflow).toContain("history_source_ready"); + expect(workflow).toContain("Verify normalized history source report"); + expect(workflow).not.toContain("sanitized_screenshot="); expect(workflow).toContain( - "Verify history source report and private screenshot evidence", + "Publish trusted-summary S3 history and structured Pages bundle", ); - expect(workflow).toContain("private_screenshot="); - expect(workflow).toContain("Publish pruned immutable history"); + expect(workflow).toContain( + "Publish trusted summary image to S3 and prune the Pages bundle", + ); + expect(publisher).toContain( + "pnpm exec playwright install --with-deps --only-shell chromium", + ); + expect(workflow).toContain( + "Package structured-only dashboard for GitHub Pages", + ); + expect(workflow).toContain("path: runner-e2e-merged-report/pages"); expect(workflow).toContain("Publish latest structured dashboard"); expect(workflow).not.toContain("dashboard_ready"); expect(workflow).not.toContain("Publish latest screenshot dashboard");