diff --git a/.github/workflows/runner-protocol-live-evals.yml b/.github/workflows/runner-protocol-live-evals.yml index 060a7309bd..f659f5ec65 100644 --- a/.github/workflows/runner-protocol-live-evals.yml +++ b/.github/workflows/runner-protocol-live-evals.yml @@ -14,7 +14,7 @@ on: type: string required: false rosters: - description: "Comma-separated live roster IDs/files, or all for the complete direct suite" + description: "Comma-separated live roster IDs/files, or all for the maintained enabled direct suite" type: string default: "all" required: false @@ -268,11 +268,13 @@ jobs: --pack-destination "$RUNNER_TEMP/runner-protocol-build/package" package="$(find "$RUNNER_TEMP/runner-protocol-build/package" -maxdepth 1 -type f -name '*.tgz' -print -quit)" test -f "$package" - npm install --prefix "$RUNNER_TEMP/runner-protocol-build/portable" --omit=dev "$package" + pnpm --filter @paperclipai/paperclip-runner deploy --prod \ + "$RUNNER_TEMP/runner-protocol-build/portable" cp "$package" "$RUNNER_TEMP/runner-protocol-build/paperclip-runner.tgz" cp packages/paperclip-runner/runner/target/debug/paperclip-runnerd "$RUNNER_TEMP/runner-protocol-build/paperclip-runnerd" cp -R packages/paperclip-runner/dist-issue-thread "$RUNNER_TEMP/runner-protocol-build/dist-issue-thread" - test -f "$RUNNER_TEMP/runner-protocol-build/portable/node_modules/@paperclipai/paperclip-runner/dist/cli/eval-session.js" + test -f "$RUNNER_TEMP/runner-protocol-build/portable/dist/cli/eval-session.js" + test -d "$RUNNER_TEMP/runner-protocol-build/portable/node_modules/.pnpm" test -x "$RUNNER_TEMP/runner-protocol-build/paperclip-runnerd" tar --create --gzip --file runner-protocol-build.tar.gz -C "$RUNNER_TEMP/runner-protocol-build" . sha256sum runner-protocol-build.tar.gz > runner-protocol-build.tar.gz.sha256 @@ -433,7 +435,7 @@ jobs: python3 .paperclip-evals/evals/paperclip-runner/tools/run_live_roster.py run \ --roster ".paperclip-evals/evals/paperclip-runner/rosters/$ROSTER_FILE" \ --case "$CASE_ID" \ - --runner-cli runner-protocol-build/extracted/portable/node_modules/@paperclipai/paperclip-runner/dist/cli/eval-session.js \ + --runner-cli runner-protocol-build/extracted/portable/dist/cli/eval-session.js \ --runner-package runner-protocol-build/extracted/paperclip-runner.tgz \ --runnerd runner-protocol-build/extracted/paperclip-runnerd \ --runs-root cell-output/runs \ diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index 2cdc74774b..6ed8f6f214 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -6,11 +6,14 @@ separate from both the browser full-stack model E2E and the stress-derived workflow schedule in `runner-workflow-evals.md`. The canonical unit of work is one live roster plus one authored case. A full -campaign selects every `rosters/live-*.json` file at one immutable -`paperclip-evals` commit. That includes the complete 35-case provider rosters -and the smaller ACPX Codex control roster. The native resume reliability gate -is not a normal one-turn roster: it requires its separately governed external -resource campaign and remains opt-in. +campaign selects every enabled lane declared by +`campaigns/live-direct-full.json` at one immutable `paperclip-evals` commit. +That includes the complete 35-case provider rosters and the smaller ACPX Codex +control roster. A disabled roster remains available for an explicit diagnostic +selection, but is never inferred into a paid `all` run from the files present +on disk. The native resume reliability gate is not a normal one-turn roster: +it requires its separately governed external-resource campaign and remains +opt-in. Managed-provider evidence identifies the immutable deployed provider artifact: Claude Managed uses its Agent version, while AgentCore uses its qualification @@ -25,8 +28,11 @@ from the default branch and provide: - `target_branch`: the Paperclip branch to build and test; - `evals_sha`: an exact 40-character commit from `paperclipai/paperclip-evals`; -- `rosters`: `all` for the entire direct suite, or a comma-separated diagnostic - subset; +- `rosters`: `all` for every enabled lane in the canonical + `live-direct-full.json` campaign, or a comma-separated diagnostic subset. + Disabled lanes remain available only through an explicit diagnostic + selection; `all` never spends against a lane that the eval program has + marked disabled; - `max_infrastructure_retries`: zero through three, applied only when an attempt explicitly reports a retryable infrastructure failure. @@ -43,11 +49,26 @@ the native daemon, provider dependencies, and the attempt viewer are built once and reused by every cell. Because the complete suite requires two matrix shards, this workflow accepts a shared concurrency ceiling from 2 through 100. -`all` is intentionally literal. A disabled driver, missing remote profile, or -unavailable provider is retained as an infrastructure result; it is not -silently omitted. In particular, the ACPX Pi roster remains visible while Pi -is disabled in the current Runner. Use a roster subset only for diagnosis, not -to claim the full campaign is green. +The reused provider runtime is created with `pnpm deploy --prod` from the +frozen workspace lock. That preserves the repository's qualified dependency +versions and patched ACP server bytes. Do not replace this step with a fresh +`npm install` of the packed Runner tarball: npm cannot apply the workspace's +`patchedDependencies`, so the resulting ACPX executables no longer match their +qualified digests. + +The direct eval CLI also materializes a minimal immutable native runtime +context in each isolated attempt workspace. This keeps the direct layer on the +same `paperclip.native-execution-input.v3` contract as production, including +the AgentCore HarnessSkill upload path, without borrowing any browser E2E +setup. + +`all` means the maintained enabled campaign, not every matching file in the +roster directory. A missing campaign file fails closed. Within an enabled +lane, a missing remote profile or unavailable provider is retained as an +infrastructure result; it is not silently omitted. The current ACPX Pi roster +is declared disabled because its Runner security profile is not qualified, so +it runs only when selected explicitly for diagnosis. Use a roster subset only +for diagnosis, not to claim the full maintained campaign is green. A provider turn that reaches a durable failed, interrupted, or otherwise non-completed terminal still produces an attempt artifact and is scored as a @@ -70,7 +91,9 @@ Claude Managed also requires the four nonsecret nonsecret `PAPERCLIP_AWS_AGENTCORE_*` profile variables, including `PAPERCLIP_AWS_AGENTCORE_EXECUTION_ROLE_ARN` and the immutable `PAPERCLIP_AWS_AGENTCORE_QUALIFICATION_REVISION`; the eval fails closed when -that deployed revision differs from the pinned roster config. The workflow +that deployed revision differs from the pinned roster config. The currently +qualified context-aware harness revision is +`aws-agentcore-harness-context-v2`. The workflow writes the GitHub OIDC token to a mode-`0600` file and never forwards long-lived AWS access keys. Provision the AgentCore stack with `--github-oidc-provider-arn` so that scoped diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/aws_agentcore_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/src/aws_agentcore_provider.rs index 0536b243a7..68d34bb652 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/aws_agentcore_provider.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/aws_agentcore_provider.rs @@ -50,6 +50,7 @@ const MAX_CONTEXT_UPLOAD_BYTES: usize = 64 * 1024 * 1024; const MAX_MEMORY_HISTORY_PAGES: usize = 1_000; const MAX_MEMORY_HISTORY_EVENTS: usize = 100_000; const MAX_INTERRUPT_DRAIN_EVENTS: usize = 256; +const AGENTCORE_HARNESS_SKILLS_TOOL: &str = "skills"; #[cfg(not(test))] const AGENTCORE_INTERRUPT_USAGE_RECONCILIATION_TIMEOUT: Duration = Duration::from_secs(2); #[cfg(test)] @@ -477,7 +478,7 @@ async fn upload_context_directory( .map_err(|error| { format!( "AgentCore context S3 upload failed: {}", - redact_aws_error(&error.to_string()) + classify_aws_sdk_error(&error) ) })?; } @@ -688,6 +689,7 @@ fn network_loop( let skills = skills.clone(); let system_instructions = system_instructions.clone(); runtime.spawn(async move { + let allowed_tools = invocation_allowed_tools(allowed_tools, !skills.is_empty()); // The qualified Harness version is the immutable model // authority. Supplying a redundant invocation override // changes AgentCore's authorization path and can require @@ -735,11 +737,13 @@ fn network_loop( }; let _ = reply.send(Ok(())); let mut tool_blocks: BTreeMap = BTreeMap::new(); + let mut message_stopped = false; loop { match response.stream.recv().await { Ok(Some(event)) => normalize_stream_event( event, &mut tool_blocks, + &mut message_stopped, &events, &invocation_id, ), @@ -811,6 +815,7 @@ fn network_loop( fn normalize_stream_event( event: InvokeHarnessStreamOutput, tool_blocks: &mut BTreeMap, + message_stopped: &mut bool, events: &SyncSender, invocation_id: &str, ) { @@ -851,6 +856,18 @@ fn normalize_stream_event( if let Some((call_id, remote_name, input)) = tool_blocks.remove(&value.content_block_index()) { + // HarnessSkill contents are loaded on demand by AgentCore's + // built-in `skills` tool. Its trace is surfaced in the same + // ToolUse-shaped stream as client-owned inline functions, but + // AgentCore executes it inside the harness. Do not send it + // across PRP as a Paperclip semantic operation. + if remote_name == AGENTCORE_HARNESS_SKILLS_TOOL { + let _ = events.send(NetworkEvent::new( + invocation_id, + NetworkEventKind::ReasoningProgress, + )); + return; + } let parsed = if input.trim().is_empty() { Ok(json!({})) } else { @@ -897,6 +914,7 @@ fn normalize_stream_event( )); } InvokeHarnessStreamOutput::MessageStop(value) => { + *message_stopped = true; let _ = events.send(NetworkEvent::new( invocation_id, NetworkEventKind::Stop(value.stop_reason().as_str().to_owned()), @@ -904,16 +922,33 @@ fn normalize_stream_event( } InvokeHarnessStreamOutput::MessageStart(_) => {} _ => { - let _ = events.send(NetworkEvent::new( - invocation_id, - NetworkEventKind::Failure( - "AgentCore SDK did not recognize an EventStream record".to_owned(), - ), - )); + // AgentCore may append forward-compatible bookkeeping records + // after the authoritative MessageStop. The pinned SDK exposes + // those records only as `Unknown`, without their union name or + // payload. Once MessageStop has sealed the model outcome, ignore + // such a trailing record; InvocationComplete still requires the + // known stop reason and usage metadata. Unknown records before + // MessageStop remain fatal because they could affect the turn. + normalize_unknown_stream_event(*message_stopped, events, invocation_id); } } } +fn normalize_unknown_stream_event( + message_stopped: bool, + events: &SyncSender, + invocation_id: &str, +) { + if !message_stopped { + let _ = events.send(NetworkEvent::new( + invocation_id, + NetworkEventKind::Failure( + "AgentCore SDK did not recognize an EventStream record".to_owned(), + ), + )); + } +} + fn observe_memory_history_page( pages: &mut usize, events: &mut usize, @@ -1439,6 +1474,69 @@ impl AwsAgentCoreHarnessProvider { params: json!({ "turnId": turn_id, "turn": { "id": turn_id, "status": "interrupted" }, "stopReason": "interrupted" }), }); } + + fn invoke_tool_result_continuation(&mut self) -> Result<(), LocalRunnerError> { + if self.pending.is_empty() || self.delivered_results.len() != self.pending.len() { + return Err(LocalRunnerError::invalid( + "AgentCore tool-result continuation batch is incomplete", + )); + } + + // AgentCore requires every toolUse from the assistant message and all + // matching user toolResults in one continuation. This also guarantees + // that parallel governed mutations cannot advance separate model turns. + let mut assistant_builder = + HarnessMessage::builder().role(HarnessConversationRole::Assistant); + let mut user_builder = HarnessMessage::builder().role(HarnessConversationRole::User); + for (call_id, pending) in &self.pending { + let delivered = self.delivered_results.get(call_id).ok_or_else(|| { + LocalRunnerError::invalid("AgentCore tool-result batch is incomplete") + })?; + let tool_use = HarnessToolUseBlock::builder() + .name(pending.remote_name.clone()) + .tool_use_id(call_id.clone()) + .input(json_to_document(&pending.input)?) + .r#type(HarnessToolUseType::ToolUse) + .build() + .map_err(|_| { + LocalRunnerError::invalid("failed to build AgentCore tool-use continuation") + })?; + let tool_result = HarnessToolResultBlock::builder() + .tool_use_id(call_id.clone()) + // Although the AgentCore data-plane model advertises JSON + // result blocks, the managed Harness runtime currently + // rejects them with `content_type= | unsupported type`. + // Preserve the complete structured result as compact JSON in + // the supported text variant. + .content(encode_tool_result_content(&delivered.result)?) + .status(if delivered.is_error { + HarnessToolUseStatus::Error + } else { + HarnessToolUseStatus::Success + }) + .r#type(HarnessToolUseType::ToolUse) + .build() + .map_err(|_| { + LocalRunnerError::invalid("failed to build AgentCore tool-result continuation") + })?; + assistant_builder = assistant_builder.content(HarnessContentBlock::ToolUse(tool_use)); + user_builder = user_builder.content(HarnessContentBlock::ToolResult(tool_result)); + } + let assistant = assistant_builder.build().map_err(|_| { + LocalRunnerError::invalid("failed to build AgentCore assistant continuation") + })?; + let user = user_builder.build().map_err(|_| { + LocalRunnerError::invalid("failed to build AgentCore user continuation") + })?; + // The durable runner records ToolResult before calling this method. A + // transport ambiguity therefore never repeats the Paperclip mutation; + // it fails closed with the last authoritative Memory cursor preserved + // for the control plane's reconciliation workflow. + self.invoke(vec![assistant, user])?; + self.pending.clear(); + self.delivered_results.clear(); + Ok(()) + } } impl Provider for AwsAgentCoreHarnessProvider { @@ -1709,11 +1807,12 @@ impl Provider for AwsAgentCoreHarnessProvider { ))) } NetworkEventKind::Stop(reason) => { - if self.pending_stop_reason.replace(reason).is_some() { - return Err(LocalRunnerError::invalid( - "AgentCore emitted more than one stop reason for an invocation", - )); - } + // Harness-managed tools such as `skills` can produce several + // complete message boundaries inside one InvokeHarness + // stream (tool_use, tool_result, then the assistant's final + // stop). The final MessageStop before metadata is the + // authoritative outcome for the invocation. + self.pending_stop_reason = Some(reason); Ok(None) } NetworkEventKind::InvocationComplete => { @@ -1734,10 +1833,20 @@ impl Provider for AwsAgentCoreHarnessProvider { } match reason.as_str() { "tool_use" | "tool_result" | "partial_turn" => { - Ok(Some(ProviderEvent::Notification { - method: "provider/waitingForToolResult".to_owned(), - params: json!({ "turnId": self.current_turn_id, "runtimeSessionId": self.session_id }), - })) + if self.pending.is_empty() { + return Err(LocalRunnerError::invalid( + "AgentCore stopped for a tool without a pending inline function", + )); + } + if self.delivered_results.len() == self.pending.len() { + self.invoke_tool_result_continuation()?; + Ok(None) + } else { + Ok(Some(ProviderEvent::Notification { + method: "provider/waitingForToolResult".to_owned(), + params: json!({ "turnId": self.current_turn_id, "runtimeSessionId": self.session_id }), + })) + } } "end_turn" | "stop_sequence" | "interrupted" => { let turn_id = self.current_turn_id.take(); @@ -1783,6 +1892,14 @@ impl Provider for AwsAgentCoreHarnessProvider { } NetworkEventKind::Failure(detail) => { self.active_invocation_id = None; + self.pending_stop_reason = None; + // A transport failure is terminal for the current invocation. + // Results can race the failure across the control-plane and + // provider channels, so discard both sides of the pending + // batch. A late result must fail validation instead of + // starting a continuation after the failed invocation. + self.pending.clear(); + self.delivered_results.clear(); Err(LocalRunnerError::invalid(format!( "AgentCore transport failed: {detail}" ))) @@ -1817,60 +1934,15 @@ impl Provider for AwsAgentCoreHarnessProvider { return Ok(()); } - // AgentCore requires every toolUse from the assistant message and all - // matching user toolResults in one continuation. This also guarantees - // that parallel governed mutations cannot advance separate model turns. - let mut assistant_builder = - HarnessMessage::builder().role(HarnessConversationRole::Assistant); - let mut user_builder = HarnessMessage::builder().role(HarnessConversationRole::User); - for (call_id, pending) in &self.pending { - let delivered = self.delivered_results.get(call_id).ok_or_else(|| { - LocalRunnerError::invalid("AgentCore tool-result batch is incomplete") - })?; - let tool_use = HarnessToolUseBlock::builder() - .name(pending.remote_name.clone()) - .tool_use_id(call_id.clone()) - .input(json_to_document(&pending.input)?) - .r#type(HarnessToolUseType::ToolUse) - .build() - .map_err(|_| { - LocalRunnerError::invalid("failed to build AgentCore tool-use continuation") - })?; - let tool_result = HarnessToolResultBlock::builder() - .tool_use_id(call_id.clone()) - // Although the AgentCore data-plane model advertises JSON - // result blocks, the managed Harness runtime currently - // rejects them with `content_type= | unsupported type`. - // Preserve the complete structured result as compact JSON in - // the supported text variant. - .content(encode_tool_result_content(&delivered.result)?) - .status(if delivered.is_error { - HarnessToolUseStatus::Error - } else { - HarnessToolUseStatus::Success - }) - .r#type(HarnessToolUseType::ToolUse) - .build() - .map_err(|_| { - LocalRunnerError::invalid("failed to build AgentCore tool-result continuation") - })?; - assistant_builder = assistant_builder.content(HarnessContentBlock::ToolUse(tool_use)); - user_builder = user_builder.content(HarnessContentBlock::ToolResult(tool_result)); + // A ToolUse block arrives before the same EventStream's MessageStop + // and usage metadata. The control plane can return its result during + // that interval; retain it durably and let InvocationComplete start + // the continuation after the first invocation is fully sealed. + if self.active_invocation_id.is_some() { + return Ok(()); } - let assistant = assistant_builder.build().map_err(|_| { - LocalRunnerError::invalid("failed to build AgentCore assistant continuation") - })?; - let user = user_builder.build().map_err(|_| { - LocalRunnerError::invalid("failed to build AgentCore user continuation") - })?; - // The durable runner records ToolResult before calling this method. A - // transport ambiguity therefore never repeats the Paperclip mutation; - // it fails closed with the last authoritative Memory cursor preserved - // for the control plane's reconciliation workflow. - self.invoke(vec![assistant, user])?; - self.pending.clear(); - self.delivered_results.clear(); - Ok(()) + + self.invoke_tool_result_continuation() } fn shutdown(&mut self) -> Result<(), LocalRunnerError> { @@ -2006,6 +2078,20 @@ fn encode_tools( Ok((encoded, allowed, reverse, schemas)) } +fn invocation_allowed_tools(mut allowed: Vec, has_skills: bool) -> Vec { + if has_skills + && !allowed + .iter() + .any(|tool| tool == AGENTCORE_HARNESS_SKILLS_TOOL) + { + // HarnessSkill progressive disclosure is performed by AgentCore's + // built-in `skills` tool. It must be admitted alongside the narrowly + // scoped Paperclip inline functions whenever skills are attached. + allowed.push(AGENTCORE_HARNESS_SKILLS_TOOL.to_owned()); + } + allowed +} + fn remote_tool_name(operation_id: &str) -> String { let mut slug = operation_id .chars() @@ -2188,7 +2274,8 @@ mod tests { use super::*; use aws_sdk_bedrockagentcore::types::{ HarnessContentBlockDeltaEvent, HarnessContentBlockStartEvent, HarnessContentBlockStopEvent, - HarnessToolUseBlockDelta, HarnessToolUseBlockStart, + HarnessMessageStopEvent, HarnessStopReason, HarnessToolUseBlockDelta, + HarnessToolUseBlockStart, }; use std::sync::{Arc, Mutex}; @@ -2211,7 +2298,7 @@ mod tests { context_bucket: "paperclip-context-test".to_owned(), context_prefix: "companies/company-test/profiles/profile-test".to_owned(), context_kms_key_arn: "arn:aws:kms:us-east-1:123456789012:key/test".to_owned(), - qualification_revision: "aws-agentcore-harness-v1".to_owned(), + qualification_revision: "aws-agentcore-harness-context-v2".to_owned(), event_expiry_days: 90, max_estimated_session_cost_usd: 1.0, max_iterations: 8, @@ -2468,6 +2555,167 @@ mod tests { )); } + #[test] + fn harness_managed_message_stops_resolve_to_the_final_stop_reason() { + let mut provider = provider_with_events( + vec![ + invocation_event(NetworkEventKind::Stop("tool_use".to_owned())), + invocation_event(NetworkEventKind::Stop("tool_result".to_owned())), + invocation_event(NetworkEventKind::Stop("end_turn".to_owned())), + invocation_event(NetworkEventKind::Usage { + input_tokens: 840, + output_tokens: 62, + cache_read_input_tokens: 0, + cache_write_input_tokens: 0, + latency_ms: 2_646, + }), + invocation_event(NetworkEventKind::InvocationComplete), + ], + restored_usage_snapshot(None).unwrap(), + ); + assert!(provider.poll().unwrap().is_none()); + assert!(provider.poll().unwrap().is_none()); + assert!(provider.poll().unwrap().is_none()); + assert!(matches!( + provider.poll().unwrap().unwrap(), + ProviderEvent::Notification { ref method, .. } + if method == "thread/tokenUsage/updated" + )); + match provider.poll().unwrap().unwrap() { + ProviderEvent::Notification { method, params } => { + assert_eq!(method, "turn/completed"); + assert_eq!(params["stopReason"], "end_turn"); + } + other => panic!("unexpected terminal event: {other:?}"), + } + } + + #[test] + fn early_tool_result_waits_for_invocation_metadata_before_continuing() { + let (command_tx, command_rx) = mpsc::sync_channel(2); + let (event_tx, event_rx) = mpsc::sync_channel(4); + let (captured_tx, captured_rx) = mpsc::sync_channel(1); + let join = thread::spawn(move || { + while let Ok(command) = command_rx.recv() { + match command { + NetworkCommand::Invoke { + messages, reply, .. + } => { + captured_tx.send(messages.len()).unwrap(); + reply.send(Ok(())).unwrap(); + } + NetworkCommand::Shutdown => break, + NetworkCommand::StopRuntime { reply, .. } => { + reply.send(Ok(())).unwrap(); + } + NetworkCommand::DeleteMemory { reply } => { + reply.send(Ok(())).unwrap(); + } + } + } + }); + let mut provider = provider_with_worker( + NetworkWorker { + commands: command_tx, + events: event_rx, + join: Some(join), + }, + restored_usage_snapshot(None).unwrap(), + ); + provider.pending.insert( + "tool-use-1".to_owned(), + RemoteToolUse { + remote_name: "pc_get_task_context_abc123".to_owned(), + operation_id: "get_task_context".to_owned(), + input: json!({}), + }, + ); + provider + .deliver_tool_result(&ToolResult { + call_id: "tool-use-1".to_owned(), + operation_id: "get_task_context".to_owned(), + result: json!({"ok": true}), + is_error: false, + }) + .unwrap(); + assert!(captured_rx.try_recv().is_err()); + assert_eq!(provider.delivered_results.len(), 1); + + for kind in [ + NetworkEventKind::Stop("tool_use".to_owned()), + NetworkEventKind::Usage { + input_tokens: 100, + output_tokens: 10, + cache_read_input_tokens: 0, + cache_write_input_tokens: 0, + latency_ms: 25, + }, + NetworkEventKind::InvocationComplete, + ] { + event_tx.send(invocation_event(kind)).unwrap(); + } + assert!(provider.poll().unwrap().is_none()); + assert!(matches!( + provider.poll().unwrap().unwrap(), + ProviderEvent::Notification { ref method, .. } + if method == "thread/tokenUsage/updated" + )); + assert!(provider.poll().unwrap().is_none()); + assert_eq!(captured_rx.recv_timeout(Duration::from_secs(1)).unwrap(), 2); + assert_eq!(provider.invocation_counter, 2); + assert!(provider.active_invocation_id.is_some()); + assert!(provider.pending.is_empty()); + assert!(provider.delivered_results.is_empty()); + } + + #[test] + fn transport_failure_discards_pending_tool_state_and_rejects_late_result() { + let mut provider = provider_with_events( + vec![invocation_event(NetworkEventKind::Failure( + "connection reset".to_owned(), + ))], + restored_usage_snapshot(None).unwrap(), + ); + provider.pending.insert( + "tool-use-1".to_owned(), + RemoteToolUse { + remote_name: "pc_get_task_context_abc123".to_owned(), + operation_id: "get_task_context".to_owned(), + input: json!({}), + }, + ); + provider.delivered_results.insert( + "tool-use-1".to_owned(), + ToolResult { + call_id: "tool-use-1".to_owned(), + operation_id: "get_task_context".to_owned(), + result: json!({"ok": true}), + is_error: false, + }, + ); + provider.pending_stop_reason = Some("tool_use".to_owned()); + + let error = provider.poll().unwrap_err(); + assert!(error.to_string().contains("AgentCore transport failed")); + assert!(provider.active_invocation_id.is_none()); + assert!(provider.pending_stop_reason.is_none()); + assert!(provider.pending.is_empty()); + assert!(provider.delivered_results.is_empty()); + + let late_result_error = provider + .deliver_tool_result(&ToolResult { + call_id: "tool-use-1".to_owned(), + operation_id: "get_task_context".to_owned(), + result: json!({"ok": true}), + is_error: false, + }) + .unwrap_err(); + assert!(late_result_error + .to_string() + .contains("does not match a pending tool use")); + assert_eq!(provider.invocation_counter, 1); + } + #[test] fn mid_stream_interrupt_waits_for_late_usage_and_suppresses_truncated_completion() { let late_usage = NetworkEventKind::Usage { @@ -3106,6 +3354,20 @@ mod tests { assert!(allowed[0].starts_with("@*/pc_")); } + #[test] + fn harness_skills_tool_is_allowed_only_when_skills_are_attached() { + let inline = vec!["@*/pc_get_task_context_abc123".to_owned()]; + assert_eq!(invocation_allowed_tools(inline.clone(), false), inline); + assert_eq!( + invocation_allowed_tools(inline.clone(), true), + vec![inline[0].clone(), AGENTCORE_HARNESS_SKILLS_TOOL.to_owned()] + ); + assert_eq!( + invocation_allowed_tools(vec![AGENTCORE_HARNESS_SKILLS_TOOL.to_owned()], true), + vec![AGENTCORE_HARNESS_SKILLS_TOOL.to_owned()] + ); + } + #[test] fn rejects_more_than_sixty_four_tools() { let tool = AuthorizedTool { @@ -3178,6 +3440,7 @@ mod tests { fn eventstream_text_delta_is_normalized_without_provider_objects() { let (sender, receiver) = mpsc::sync_channel(4); let mut blocks = BTreeMap::new(); + let mut message_stopped = false; let event = HarnessContentBlockDeltaEvent::builder() .content_block_index(0) .delta(HarnessContentBlockDelta::Text("hello".to_owned())) @@ -3186,6 +3449,7 @@ mod tests { normalize_stream_event( InvokeHarnessStreamOutput::ContentBlockDelta(event), &mut blocks, + &mut message_stopped, &sender, "invocation-1", ); @@ -3205,6 +3469,7 @@ mod tests { fn eventstream_tool_json_is_buffered_until_the_block_is_complete() { let (sender, receiver) = mpsc::sync_channel(8); let mut blocks = BTreeMap::new(); + let mut message_stopped = false; let start = HarnessToolUseBlockStart::builder() .tool_use_id("tool-use-1") .name("pc_get_task_abc123") @@ -3220,6 +3485,7 @@ mod tests { .unwrap(), ), &mut blocks, + &mut message_stopped, &sender, "invocation-1", ); @@ -3238,6 +3504,7 @@ mod tests { .unwrap(), ), &mut blocks, + &mut message_stopped, &sender, "invocation-1", ); @@ -3251,6 +3518,7 @@ mod tests { .unwrap(), ), &mut blocks, + &mut message_stopped, &sender, "invocation-1", ); @@ -3273,10 +3541,57 @@ mod tests { } } + #[test] + fn eventstream_harness_skills_trace_is_not_forwarded_as_an_inline_function() { + let (sender, receiver) = mpsc::sync_channel(4); + let mut blocks = BTreeMap::new(); + let mut message_stopped = false; + let start = HarnessToolUseBlockStart::builder() + .tool_use_id("harness-skill-load-1") + .name(AGENTCORE_HARNESS_SKILLS_TOOL) + .r#type(HarnessToolUseType::ToolUse) + .build() + .unwrap(); + normalize_stream_event( + InvokeHarnessStreamOutput::ContentBlockStart( + HarnessContentBlockStartEvent::builder() + .content_block_index(3) + .start(HarnessContentBlockStart::ToolUse(start)) + .build() + .unwrap(), + ), + &mut blocks, + &mut message_stopped, + &sender, + "invocation-1", + ); + normalize_stream_event( + InvokeHarnessStreamOutput::ContentBlockStop( + HarnessContentBlockStopEvent::builder() + .content_block_index(3) + .build() + .unwrap(), + ), + &mut blocks, + &mut message_stopped, + &sender, + "invocation-1", + ); + match receiver.try_recv().unwrap() { + NetworkEvent { + invocation_id, + kind: NetworkEventKind::ReasoningProgress, + } => assert_eq!(invocation_id, "invocation-1"), + other => panic!("unexpected normalized event: {other:?}"), + } + assert!(receiver.try_recv().is_err()); + } + #[test] fn eventstream_empty_tool_input_is_normalized_to_an_empty_object() { let (sender, receiver) = mpsc::sync_channel(4); let mut blocks = BTreeMap::new(); + let mut message_stopped = false; let start = HarnessToolUseBlockStart::builder() .tool_use_id("tool-use-empty") .name("pc_get_task_context_abc123") @@ -3292,6 +3607,7 @@ mod tests { .unwrap(), ), &mut blocks, + &mut message_stopped, &sender, "invocation-1", ); @@ -3309,6 +3625,7 @@ mod tests { .unwrap(), ), &mut blocks, + &mut message_stopped, &sender, "invocation-1", ); @@ -3320,6 +3637,7 @@ mod tests { .unwrap(), ), &mut blocks, + &mut message_stopped, &sender, "invocation-1", ); @@ -3342,6 +3660,47 @@ mod tests { } } + #[test] + fn unknown_eventstream_record_before_message_stop_fails_closed() { + let (sender, receiver) = mpsc::sync_channel(4); + normalize_unknown_stream_event(false, &sender, "invocation-1"); + match receiver.try_recv().unwrap() { + NetworkEvent { + kind: NetworkEventKind::Failure(detail), + .. + } => assert_eq!( + detail, + "AgentCore SDK did not recognize an EventStream record" + ), + other => panic!("unexpected normalized event: {other:?}"), + } + } + + #[test] + fn trailing_unknown_eventstream_record_cannot_overturn_message_stop() { + let (sender, receiver) = mpsc::sync_channel(4); + let mut blocks = BTreeMap::new(); + let mut message_stopped = false; + normalize_stream_event( + InvokeHarnessStreamOutput::MessageStop( + HarnessMessageStopEvent::builder() + .stop_reason(HarnessStopReason::EndTurn) + .build() + .unwrap(), + ), + &mut blocks, + &mut message_stopped, + &sender, + "invocation-1", + ); + assert!(matches!( + receiver.try_recv().unwrap().kind, + NetworkEventKind::Stop(reason) if reason == "end_turn" + )); + normalize_unknown_stream_event(message_stopped, &sender, "invocation-1"); + assert!(receiver.try_recv().is_err()); + } + #[test] fn tool_results_use_the_harness_supported_text_content_variant() { let value = json!({"ok": true, "nested": {"value": 7}}); diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs index 0370c7bb90..2b8212c794 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs @@ -43,7 +43,7 @@ const MAX_INSTRUCTIONS_BYTES: usize = 1024 * 1024; const QUALIFIED_CLAUDE_MODEL: &str = "claude-sonnet-5"; const QUALIFIED_CLAUDE_BETA: &str = "managed-agents-2026-04-01"; const QUALIFIED_AGENTCORE_MODEL: &str = "global.anthropic.claude-sonnet-4-6"; -const QUALIFIED_AGENTCORE_REVISION: &str = "aws-agentcore-harness-v1"; +const QUALIFIED_AGENTCORE_REVISION: &str = "aws-agentcore-harness-context-v2"; fn initial_event_sequence() -> u64 { 1 diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs index 101e049f6d..ba36834ab7 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs @@ -283,7 +283,7 @@ fn managed_prepare_payload(kind: &str) -> Value { "contextBucket": "context-bucket", "contextPrefix": "companies/company/profiles/profile", "contextKmsKeyArn": "arn:aws:kms:us-east-1:123456789012:key/test", - "qualificationRevision": "aws-agentcore-harness-v1", + "qualificationRevision": "aws-agentcore-harness-context-v2", "eventExpiryDays": 90, "maxEstimatedSessionCostUsd": 1.0, "maxIterations": 8, diff --git a/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs index 7314b63748..13ccfbb108 100644 --- a/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs +++ b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs @@ -77,6 +77,31 @@ function parseRosterSelection(value) { return new Set(selected); } +async function maintainedRosterSelection(programRoot) { + const campaignPath = resolve(programRoot, "campaigns/live-direct-full.json"); + const campaign = await loadObject(campaignPath); + if ( + campaign.schema !== "paperclip-runner/live-campaign/v1" || + !Array.isArray(campaign.lanes) + ) { + throw new Error(`Unsupported live campaign schema in ${campaignPath}`); + } + const selected = campaign.lanes + .filter((lane) => lane.executionClass !== "disabled") + .map((lane) => { + const rosterPath = inside( + programRoot, + resolve(dirname(campaignPath), String(lane.roster ?? "")), + "Campaign roster", + ); + return basename(rosterPath); + }); + if (selected.length === 0 || new Set(selected).size !== selected.length) { + throw new Error("Maintained live campaign must contain unique enabled rosters"); + } + return new Set(selected); +} + export async function buildProtocolEvalCatalog({ evalsRoot, rosterSelection = "all", @@ -94,7 +119,9 @@ export async function buildProtocolEvalCatalog({ } const programRoot = resolve(evalsRoot, "evals/paperclip-runner"); const rosterRoot = resolve(programRoot, "rosters"); - const selected = parseRosterSelection(rosterSelection); + const requested = parseRosterSelection(rosterSelection); + const selected = + requested ?? (await maintainedRosterSelection(programRoot)); const rosterFiles = (await readdir(rosterRoot, { withFileTypes: true })) .filter( (entry) => diff --git a/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.test.mjs b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.test.mjs index 474f89971b..0703d3de7d 100644 --- a/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.test.mjs +++ b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.test.mjs @@ -26,6 +26,7 @@ async function fixture() { mkdir(join(program, "rosters"), { recursive: true }), mkdir(join(program, "configs"), { recursive: true }), mkdir(join(program, "cases"), { recursive: true }), + mkdir(join(program, "campaigns"), { recursive: true }), ]); const config = { schema: "paperclip-runner/eval-config/v1", @@ -65,6 +66,19 @@ async function fixture() { join(program, "rosters/live-opencode-model.json"), JSON.stringify(roster), ), + writeFile( + join(program, "campaigns/live-direct-full.json"), + JSON.stringify({ + schema: "paperclip-runner/live-campaign/v1", + lanes: [ + { + id: "opencode-model", + executionClass: "default", + roster: "../rosters/live-opencode-model.json", + }, + ], + }), + ), ]); return { root, program, config, evalCase, roster }; } @@ -119,6 +133,83 @@ test("catalogs roster plus case cells and emits bounded balanced shards", async ); }); +test("all selects the maintained enabled campaign and explicit diagnostics can select disabled rosters", async () => { + const { root, program, config, evalCase } = await fixture(); + const disabledRoster = { + schema: "paperclip-runner/live-roster/v1", + id: "protocol-live-disabled-model", + model: config.model, + config: "../configs/live-opencode-model.json", + cases: [evalCase.id], + }; + await Promise.all([ + writeFile( + join(program, "rosters/live-disabled-model.json"), + JSON.stringify(disabledRoster), + ), + writeFile( + join(program, "campaigns/live-direct-full.json"), + JSON.stringify({ + schema: "paperclip-runner/live-campaign/v1", + lanes: [ + { + id: "opencode-model", + executionClass: "default", + roster: "../rosters/live-opencode-model.json", + }, + { + id: "disabled-model", + executionClass: "disabled", + roster: "../rosters/live-disabled-model.json", + }, + ], + }), + ), + ]); + + const maintained = await buildProtocolEvalCatalog({ + evalsRoot: root, + campaignId: "gha-42-1", + }); + assert.deepEqual( + maintained.rosters.map((roster) => roster.rosterId), + ["protocol-live-opencode-model"], + ); + + const diagnostic = await buildProtocolEvalCatalog({ + evalsRoot: root, + campaignId: "gha-42-2", + rosterSelection: "protocol-live-disabled-model", + }); + assert.deepEqual( + diagnostic.rosters.map((roster) => roster.rosterId), + ["protocol-live-disabled-model"], + ); +}); + +test("all fails closed when the maintained campaign is missing", async () => { + const { root, program } = await fixture(); + await rm(join(program, "campaigns/live-direct-full.json")); + + await assert.rejects( + buildProtocolEvalCatalog({ + evalsRoot: root, + campaignId: "gha-42-1", + }), + /ENOENT/, + ); + + const diagnostic = await buildProtocolEvalCatalog({ + evalsRoot: root, + campaignId: "gha-42-2", + rosterSelection: "protocol-live-opencode-model", + }); + assert.deepEqual( + diagnostic.rosters.map((roster) => roster.rosterId), + ["protocol-live-opencode-model"], + ); +}); + test("aggregates retained attempts and synthesizes missing cells as infrastructure", async () => { const { root, config, evalCase } = await fixture(); const catalog = await buildProtocolEvalCatalog({ diff --git a/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs b/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs index cc1be0437d..0569facb0a 100644 --- a/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs +++ b/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs @@ -105,6 +105,15 @@ test("resolves both repositories immutably and bounds total matrix concurrency", } assert.match(workflow, /matrix_0/u); assert.match(workflow, /matrix_1/u); + assert.match( + workflow, + /pnpm --filter @paperclipai\/paperclip-runner deploy --prod/u, + ); + assert.match( + workflow, + /--runner-cli runner-protocol-build\/extracted\/portable\/dist\/cli\/eval-session\.js/u, + ); + assert.doesNotMatch(workflow, /npm install --prefix/u); }); test("publishes only the separately sanitized Evalbook through trusted OIDC code", async () => { diff --git a/packages/paperclip-runner/src/backends/native-backend-factory.test.ts b/packages/paperclip-runner/src/backends/native-backend-factory.test.ts index 185805b736..545777babc 100644 --- a/packages/paperclip-runner/src/backends/native-backend-factory.test.ts +++ b/packages/paperclip-runner/src/backends/native-backend-factory.test.ts @@ -209,7 +209,7 @@ function managedExecution( contextBucket: "context-bucket", contextPrefix: "companies/company/profiles/profile", contextKmsKeyArn: "arn:aws:kms:us-east-1:123456789012:key/test", - qualificationRevision: "aws-agentcore-harness-v1", + qualificationRevision: "aws-agentcore-harness-context-v2", eventExpiryDays: 90, }, }, @@ -388,7 +388,7 @@ describe("native backend factory", () => { [ "aws_agentcore" as const, "aws_agentcore_harness_api", - "aws-agentcore-harness-v1", + "aws-agentcore-harness-context-v2", ], ])("routes %s through runnerd", async (kind, name, version) => { const backend = createNativeSessionBackend(managedExecution(kind), { diff --git a/packages/paperclip-runner/src/cli/eval-session-contract.test.ts b/packages/paperclip-runner/src/cli/eval-session-contract.test.ts index e2765f0307..322a4fbf70 100644 --- a/packages/paperclip-runner/src/cli/eval-session-contract.test.ts +++ b/packages/paperclip-runner/src/cli/eval-session-contract.test.ts @@ -1,5 +1,10 @@ +import { chmod, mkdtemp, readFile, rm, stat } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + import { describe, expect, it } from "vitest"; +import { parseNativeRuntimeContext } from "../contracts/runtime-context.js"; import type { CapabilityLiveSessionSnapshot } from "../live/live-session.js"; import { evalSessionUsage, @@ -7,7 +12,9 @@ import { } from "./eval-session-contract.js"; import { boundedEvalSessionUsage, + evalRuntimeSystemInstructions, evalSessionProviderVersion, + prepareEvalRuntimeContext, } from "./eval-session.js"; function request(overrides: Record = {}): unknown { @@ -55,6 +62,35 @@ function agentCoreProfile(overrides: Record = {}) { } describe("eval-session request contract", () => { + it("materializes a production-v3 runtime context for direct live providers", async () => { + const workspace = await mkdtemp(join(tmpdir(), "paperclip-eval-context-")); + let instructionRoot: string | null = null; + try { + const context = await prepareEvalRuntimeContext(workspace); + instructionRoot = context.instructions.bundle.rootPath; + expect(parseNativeRuntimeContext(context)).toEqual(context); + expect(context.skills).toEqual([]); + expect(context.mcp.assignmentSetId).toBe("paperclip-runner-direct-eval-v1"); + expect(context.instructions.entryPath).toBe("AGENTS.md"); + expect(await readFile( + join(context.instructions.bundle.rootPath, "AGENTS.md"), + "utf8", + )).toContain("Paperclip direct live evaluation"); + const systemInstructions = evalRuntimeSystemInstructions(context); + expect(systemInstructions).toContain("Paperclip direct live evaluation"); + expect(systemInstructions).toContain( + `Read-only instruction sibling root: ${context.instructions.bundle.rootPath}`, + ); + expect((await stat(context.instructions.bundle.rootPath)).mode & 0o777) + .toBe(0o555); + } finally { + if (instructionRoot !== null) { + await chmod(instructionRoot, 0o700).catch(() => undefined); + } + await rm(workspace, { recursive: true, force: true }); + } + }); + it("normalizes the current local live-session provider contract", () => { expect(parseEvalSessionRequest(request())).toMatchObject({ provider: "codex", diff --git a/packages/paperclip-runner/src/cli/eval-session.ts b/packages/paperclip-runner/src/cli/eval-session.ts index 609ec73090..220aee7adb 100644 --- a/packages/paperclip-runner/src/cli/eval-session.ts +++ b/packages/paperclip-runner/src/cli/eval-session.ts @@ -1,9 +1,19 @@ #!/usr/bin/env node import { createHash } from "node:crypto"; -import { readFile, writeFile } from "node:fs/promises"; +import { chmod, mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { + NATIVE_RUNTIME_ASSET_SCHEMA, + PAPERCLIP_EXECUTION_PROMPT, + PAPERCLIP_EXECUTION_PROMPT_REVISION, + canonicalNativeRuntimeContextDigest, + composeNativeSystemInstructions, + nativeRuntimePromptDigest, + parseNativeRuntimeContext, + type NativeRuntimeContextSnapshot, +} from "../contracts/runtime-context.js"; import { projectCapabilityDevtools } from "../devtools/index.js"; import { resolveQualifiedAcpxProfile } from "../drivers/acpx/qualified-profiles.js"; import { PAPERCLIP_RUNNER_BUILD_METADATA } from "../evals/build-metadata.js"; @@ -53,6 +63,94 @@ async function sha256(path: string): Promise { return createHash("sha256").update(await readFile(path)).digest("hex"); } +const EVAL_RUNTIME_INSTRUCTIONS = [ + "# Paperclip direct live evaluation", + "", + "Use the provided Paperclip semantic tools to inspect and act on the assigned task.", + "Treat the seeded control-plane state as authoritative and keep every action within the requested scope.", + "", +].join("\n"); + +/** Materializes the minimal immutable v3 context used by production-native providers. */ +export async function prepareEvalRuntimeContext( + workingDirectory: string, +): Promise { + const contextRoot = await mkdtemp( + resolve(workingDirectory, ".paperclip-eval-runtime-context-"), + ); + const instructionRoot = resolve(contextRoot, "instructions"); + const entryPath = "AGENTS.md"; + const entry = Buffer.from(EVAL_RUNTIME_INSTRUCTIONS); + const entryDigest = createHash("sha256").update(entry).digest("hex"); + const manifestFiles = [{ + path: entryPath, + sha256: entryDigest, + mode: 0o444, + size: entry.byteLength, + }]; + const assetDigest = createHash("sha256") + .update(JSON.stringify(manifestFiles)) + .digest("hex"); + const manifestText = `${JSON.stringify({ + schema: "paperclip.runtime-asset-manifest.v1", + digest: assetDigest, + fileCount: manifestFiles.length, + totalBytes: entry.byteLength, + files: manifestFiles, + })}\n`; + const manifestDigest = createHash("sha256") + .update(manifestText) + .digest("hex"); + await mkdir(instructionRoot, { recursive: true, mode: 0o700 }); + const entryFile = resolve(instructionRoot, entryPath); + await writeFile(entryFile, entry, { flag: "wx", mode: 0o444 }); + await chmod(entryFile, 0o444); + await chmod(instructionRoot, 0o555); + + const semanticCatalogDigest = + PAPERCLIP_RUNNER_BUILD_METADATA.semanticCatalog.sha256.replace( + /^sha256:/, + "", + ); + const context = { + prompt: { + revision: PAPERCLIP_EXECUTION_PROMPT_REVISION, + text: PAPERCLIP_EXECUTION_PROMPT, + digest: nativeRuntimePromptDigest(), + }, + instructions: { + entryPath, + bundle: { + schema: NATIVE_RUNTIME_ASSET_SCHEMA, + digest: assetDigest, + manifestDigest, + rootPath: instructionRoot, + fileCount: 1, + totalBytes: entry.byteLength, + }, + }, + skills: [], + mcp: { + assignmentSetId: "paperclip-runner-direct-eval-v1", + digest: semanticCatalogDigest, + bindingId: null, + }, + } satisfies Omit; + return parseNativeRuntimeContext({ + ...context, + aggregateDigest: canonicalNativeRuntimeContextDigest(context), + }); +} + +export function evalRuntimeSystemInstructions( + runtimeContext: NativeRuntimeContextSnapshot, +): string { + return composeNativeSystemInstructions( + runtimeContext, + EVAL_RUNTIME_INSTRUCTIONS, + ); +} + export function evalSessionProviderVersion( request: EvalSessionRequest, ): string | null { @@ -189,10 +287,15 @@ export async function runEvalSessionCli( const requestedDriver = request.driver ?? expectedEvalSessionDriver(requestedProvider); const requestedProviderVersion = evalSessionProviderVersion(request); + const runtimeContext = await prepareEvalRuntimeContext( + resolve(request.session.workingDirectory ?? process.cwd()), + ); const service = options.serviceFactory?.(runnerdPath) ?? new CapabilityLiveSessionService({ transportOptions: { runnerBinary: runnerdPath, + runtimeContext, + baseInstructions: evalRuntimeSystemInstructions(runtimeContext), // The transport performs the provider-specific allowlisting. Supplying // the source environment here is still required: without it the // isolated Codex home has no credential source and runnerd receives no diff --git a/packages/paperclip-runner/src/contracts/native-execution.test.ts b/packages/paperclip-runner/src/contracts/native-execution.test.ts index a45c6a664c..5e0c90a266 100644 --- a/packages/paperclip-runner/src/contracts/native-execution.test.ts +++ b/packages/paperclip-runner/src/contracts/native-execution.test.ts @@ -235,7 +235,7 @@ describe("NativeExecutionInputV1", () => { contextBucket: "paperclip-agentcore-context", contextPrefix: "paperclip/runtime", contextKmsKeyArn: "arn:aws:kms:us-east-1:123456789012:key/test", - qualificationRevision: "aws-agentcore-harness-v1", + qualificationRevision: "aws-agentcore-harness-context-v2", eventExpiryDays: 90, }, maxEstimatedSessionCostUsd: 1, diff --git a/packages/paperclip-runner/src/live/live-session.test.ts b/packages/paperclip-runner/src/live/live-session.test.ts index b569868318..da740a6d1a 100644 --- a/packages/paperclip-runner/src/live/live-session.test.ts +++ b/packages/paperclip-runner/src/live/live-session.test.ts @@ -662,6 +662,27 @@ describe("Capability live runnerd and Codex session", () => { await service.shutdown(session.id); }); + it("passes caller-supplied native system instructions to the provider", async () => { + const state = providerState(); + const service = new CapabilityLiveSessionService({ + transportFactory: fakeTransportFactory(state), + transportOptions: { + baseInstructions: + "Native instructions\n\nRead-only instruction sibling root: /runtime/instructions", + }, + }); + const session = await service.create(); + + expect( + state.transports[0]?.requests.find( + (request) => request.method === "thread/start", + )?.params.baseInstructions, + ).toBe( + "Native instructions\n\nRead-only instruction sibling root: /runtime/instructions", + ); + await service.shutdown(session.id); + }); + it("attributes Claude Managed sessions to the pinned immutable Agent version", async () => { const state = providerState(); const managedProfiles: Array | undefined> = []; diff --git a/packages/paperclip-runner/src/live/live-session.ts b/packages/paperclip-runner/src/live/live-session.ts index 4f7b8c0f6b..88ded8caf3 100644 --- a/packages/paperclip-runner/src/live/live-session.ts +++ b/packages/paperclip-runner/src/live/live-session.ts @@ -2382,7 +2382,8 @@ export class CapabilityLiveSession { config: createSkilllessCodexThreadConfig(this.#config.workingDirectory), permissions: CODEX_PERMISSION_PROFILE, runtimeWorkspaceRoots: [this.#config.workingDirectory], - baseInstructions: LIVE_BASE_INSTRUCTIONS, + baseInstructions: + this.#transportOptions.baseInstructions ?? LIVE_BASE_INSTRUCTIONS, persistExtendedHistory: true, }); const resumedThread = record(resumed.thread); @@ -2407,7 +2408,8 @@ export class CapabilityLiveSession { permissions: CODEX_PERMISSION_PROFILE, runtimeWorkspaceRoots: [this.#config.workingDirectory], approvalPolicy: "never", - baseInstructions: LIVE_BASE_INSTRUCTIONS, + baseInstructions: + this.#transportOptions.baseInstructions ?? LIVE_BASE_INSTRUCTIONS, completionContract: LIVE_COMPLETION_CONTRACT, dynamicTools: [ ...tools.map(dynamicToolSpec), diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index 00ba33f701..eb07c5bfa4 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -634,6 +634,66 @@ it("derives the ACPX package authority only from the verified dist/cli layout", ).toThrow("ACPX sidecar must use the provider package dist/cli layout"); }); +it("keeps a self-rooted pnpm deployment inside its dependency authority", async () => { + const deploymentRoot = await mkdtemp( + join(tmpdir(), "paperclip-deployed-provider-root-"), + ); + const deployedPackageRoot = deploymentRoot; + await mkdir(join(deployedPackageRoot, "dist", "cli"), { recursive: true }); + await mkdir(join(deploymentRoot, "node_modules", ".pnpm"), { + recursive: true, + }); + try { + expect( + runnerdLaunchProfileInternals.acpxProviderPackageAuthority( + join( + deployedPackageRoot, + "dist", + "cli", + "acpx-runtime-sidecar.cjs", + ), + deployedPackageRoot, + ), + ).toEqual({ + root: deploymentRoot, + manifest: join(deployedPackageRoot, "package.json"), + }); + } finally { + await rm(deploymentRoot, { recursive: true, force: true }); + } +}); + +it("keeps a scoped npm-installed package inside its portable dependency root", async () => { + const deploymentRoot = await mkdtemp( + join(tmpdir(), "paperclip-npm-provider-root-"), + ); + const deployedPackageRoot = join( + deploymentRoot, + "node_modules", + "@paperclipai", + "paperclip-runner", + ); + await mkdir(join(deployedPackageRoot, "dist", "cli"), { recursive: true }); + try { + expect( + runnerdLaunchProfileInternals.acpxProviderPackageAuthority( + join( + deployedPackageRoot, + "dist", + "cli", + "acpx-runtime-sidecar.cjs", + ), + deployedPackageRoot, + ), + ).toEqual({ + root: deploymentRoot, + manifest: join(deployedPackageRoot, "package.json"), + }); + } finally { + await rm(deploymentRoot, { recursive: true, force: true }); + } +}); + it("requires a provider-pack authority for remote ACPX artifact hashes", () => { expect(() => runnerdLaunchProfileInternals.acpxRunnerLaunchProfile( diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index 497185158c..a28105aa37 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -937,6 +937,8 @@ export interface CapabilityRunnerdCodexTransportOptions { opencodeProxySha256?: string; opencodeRuntimeDirectory?: string; environment?: NodeJS.ProcessEnv; + /** Provider system instructions supplied by a native execution caller. */ + baseInstructions?: string; closeGraceMs?: number; onDiagnostic?: (message: string) => void; onEvidence?: (evidence: Readonly) => void; @@ -1549,7 +1551,10 @@ function resolveBuildOwnedCliArtifact( ); } -function acpxProviderPackageAuthority(sidecarScript: string): { +function acpxProviderPackageAuthority( + sidecarScript: string, + ownerPackageRoot = packageRoot, +): { root: string; manifest: string; } { @@ -1564,12 +1569,25 @@ function acpxProviderPackageAuthority(sidecarScript: string): { ); } const sidecarPackageRoot = resolve(cliDirectory, "../.."); - // A local source build consumes pnpm's workspace-owned node_modules tree. - // A deployed provider pack owns a closed node_modules tree at its own root. - return sidecarPackageRoot === packageRoot + // A local source build lives at /packages/paperclip-runner and + // resolves dependencies from /node_modules. `pnpm deploy` makes + // the package itself the deployment root and owns /node_modules/.pnpm. + // The older npm-installed portable shape nests the scoped package at + // /node_modules/@paperclipai/paperclip-runner. The verifier always + // receives the directory that owns node_modules, regardless of which + // portable shape launched the already-authenticated sidecar. + const sourceDependencyRoot = resolve(ownerPackageRoot, "../.."); + const localDependencyRoot = existsSync( + resolve(ownerPackageRoot, "node_modules", ".pnpm"), + ) + ? ownerPackageRoot + : basename(sourceDependencyRoot) === "node_modules" + ? resolve(sourceDependencyRoot, "..") + : sourceDependencyRoot; + return sidecarPackageRoot === ownerPackageRoot ? { - root: resolve(packageRoot, "../.."), - manifest: resolve(packageRoot, "package.json"), + root: localDependencyRoot, + manifest: resolve(ownerPackageRoot, "package.json"), } : { root: sidecarPackageRoot, diff --git a/server/src/__tests__/managed-agent-profile-routes-authz.test.ts b/server/src/__tests__/managed-agent-profile-routes-authz.test.ts index 540f3f5b16..a57c7182d9 100644 --- a/server/src/__tests__/managed-agent-profile-routes-authz.test.ts +++ b/server/src/__tests__/managed-agent-profile-routes-authz.test.ts @@ -123,7 +123,7 @@ describe("managed provider profile route authorization", () => { contextBucket: "paperclip-runner-context", contextPrefix: "profiles/example", contextKmsKeyArn: "arn:aws:kms:us-east-1:123456789012:key/example", - qualificationRevision: "aws-agentcore-harness-v1", + qualificationRevision: "aws-agentcore-harness-context-v2", defaultModel: "global.anthropic.claude-sonnet-4-6", eventExpiryDays: 90, }, diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.ts index dda1c36904..56da28aca7 100644 --- a/server/src/routes/openapi.ts +++ b/server/src/routes/openapi.ts @@ -4626,7 +4626,7 @@ registry.registerPath({ configuration: z.record(z.string(), z.unknown()), enabled: z.boolean().optional(), retentionAcknowledged: z.boolean().optional(), - qualification: z.object({ suite: z.literal("aws-agentcore-harness-v1") }).strict().optional(), + qualification: z.object({ suite: z.literal("aws-agentcore-harness-context-v2") }).strict().optional(), })), }, responses: { diff --git a/server/src/services/heartbeat-runner-provider-config.test.ts b/server/src/services/heartbeat-runner-provider-config.test.ts index 344ca5c91b..72b194cac7 100644 --- a/server/src/services/heartbeat-runner-provider-config.test.ts +++ b/server/src/services/heartbeat-runner-provider-config.test.ts @@ -87,7 +87,7 @@ describe("Paperclip Runner native provider configuration", () => { contextBucket: "paperclip-context", contextPrefix: "runner/", contextKmsKeyArn: "arn:aws:kms:us-east-1:123456789012:key/key-1", - qualificationRevision: "aws-agentcore-harness-v1", + qualificationRevision: "aws-agentcore-harness-context-v2", eventExpiryDays: 90, }; const stored = { @@ -224,7 +224,7 @@ describe("Paperclip Runner native provider configuration", () => { contextBucket: "paperclip-context", contextPrefix: "runner/", contextKmsKeyArn: "arn:aws:kms:us-east-1:123456789012:key/key-1", - qualificationRevision: "aws-agentcore-harness-v1", + qualificationRevision: "aws-agentcore-harness-context-v2", defaultModel: "global.anthropic.claude-sonnet-4-6", eventExpiryDays: 90, defaultMaxEstimatedSessionCostUsd: 1.25, @@ -278,7 +278,7 @@ describe("Paperclip Runner native provider configuration", () => { contextBucket: "paperclip-context", contextPrefix: "runner/", contextKmsKeyArn: "arn:aws:kms:us-east-1:123456789012:key/key-1", - qualificationRevision: "aws-agentcore-harness-v1", + qualificationRevision: "aws-agentcore-harness-context-v2", defaultModel: "global.anthropic.claude-sonnet-4-6", eventExpiryDays: 90, defaultMaxEstimatedSessionCostUsd: 1.25, @@ -352,7 +352,7 @@ describe("Paperclip Runner native provider configuration", () => { contextBucket: "paperclip-context", contextPrefix: "runner/", contextKmsKeyArn: "arn:aws:kms:us-east-1:123456789012:key/key-1", - qualificationRevision: "aws-agentcore-harness-v1", + qualificationRevision: "aws-agentcore-harness-context-v2", defaultModel: "global.anthropic.claude-sonnet-4-6", eventExpiryDays: 90, defaultMaxEstimatedSessionCostUsd: 1.25, diff --git a/server/src/services/native-runtime/native-session-resume.test.ts b/server/src/services/native-runtime/native-session-resume.test.ts index 31aee96413..6e4c4c5bb2 100644 --- a/server/src/services/native-runtime/native-session-resume.test.ts +++ b/server/src/services/native-runtime/native-session-resume.test.ts @@ -301,7 +301,7 @@ describe("buildNativeExecutionInput wake projection", () => { contextBucket: "paperclip-context", contextPrefix: "runner/", contextKmsKeyArn: "arn:aws:kms:us-east-1:123456789012:key/key-1", - qualificationRevision: "aws-agentcore-harness-v1", + qualificationRevision: "aws-agentcore-harness-context-v2", eventExpiryDays: 90, }, maxEstimatedSessionCostUsd: 1.25, diff --git a/server/src/services/provider-profile-qualification.ts b/server/src/services/provider-profile-qualification.ts index e26d52d285..3a691e7024 100644 --- a/server/src/services/provider-profile-qualification.ts +++ b/server/src/services/provider-profile-qualification.ts @@ -11,7 +11,8 @@ export const CLAUDE_MANAGED_QUALIFICATION = { } as const; export const CLAUDE_MANAGED_QUALIFIED_MODEL = "claude-sonnet-5" as const; -export const AGENTCORE_QUALIFICATION_SUITE = "aws-agentcore-harness-v1" as const; +export const AGENTCORE_QUALIFICATION_SUITE = + "aws-agentcore-harness-context-v2" as const; export const AGENTCORE_QUALIFIED_MODEL = "global.anthropic.claude-sonnet-4-6" as const; const REVISION_PREFIX = "sha256:"; diff --git a/server/src/services/remote-agent-profiles.test.ts b/server/src/services/remote-agent-profiles.test.ts index 3dcd77a129..b2841709b8 100644 --- a/server/src/services/remote-agent-profiles.test.ts +++ b/server/src/services/remote-agent-profiles.test.ts @@ -28,7 +28,7 @@ const AWS_CONFIGURATION = { contextBucket: "paperclip-runner-context", contextPrefix: "profiles/example", contextKmsKeyArn: "arn:aws:kms:us-east-1:123456789012:key/example", - qualificationRevision: "aws-agentcore-harness-v1", + qualificationRevision: "aws-agentcore-harness-context-v2", defaultModel: "global.anthropic.claude-sonnet-4-6", eventExpiryDays: 90, defaultMaxEstimatedSessionCostUsd: 1, @@ -44,7 +44,7 @@ function remoteInput( configuration: { ...AWS_CONFIGURATION }, enabled: false, retentionAcknowledged: false, - qualification: { suite: "aws-agentcore-harness-v1" }, + qualification: { suite: "aws-agentcore-harness-context-v2" }, ...overrides, }; } @@ -362,7 +362,7 @@ describe("remote agent profile metadata validation", () => { }); it("does not allow a qualified AgentCore profile key to be repointed", async () => { - const qualification = { suite: "aws-agentcore-harness-v1" }; + const qualification = { suite: "aws-agentcore-harness-context-v2" }; const existing = { id: "30000000-0000-4000-8000-000000000003", companyId: COMPANY_ID, @@ -506,7 +506,7 @@ describe("remote agent profile metadata validation", () => { qualifiedRevision: existingManaged.qualifiedRevision, }); - const awsQualification = { suite: "aws-agentcore-harness-v1" }; + const awsQualification = { suite: "aws-agentcore-harness-context-v2" }; const existingRemote = { id: "30000000-0000-4000-8000-000000000005", companyId: COMPANY_ID, @@ -547,7 +547,7 @@ describe("remote agent profile metadata validation", () => { }); it("rejects runtime use when stored identity drifts from the qualified revision", async () => { - const qualification = { suite: "aws-agentcore-harness-v1" }; + const qualification = { suite: "aws-agentcore-harness-context-v2" }; const qualifiedRevision = computeRemoteAgentProfileRevision({ service: "aws_bedrock_agentcore_harness", configuration: AWS_CONFIGURATION,