From fee8d8dc3922b5decef9a0426844a9e399c32522 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sun, 6 Sep 2026 15:45:07 -0500 Subject: [PATCH] fix(runner): repair direct live provider bootstrap (#12932) ## Thinking Path > - Paperclip runs AI agents through qualified provider backends. > - The direct live eval workflow builds one immutable Runner runtime for every matrix cell. > - The workflow reinstalled the packed Runner with npm. > - That install discarded pnpm patches and selected provider dependencies outside the qualified lock. > - The first pnpm deployment model also placed its virtual-store marker at the wrong level; a real deployment keeps `.pnpm` beside the scoped Runner package. > - AgentCore enforced the current context-aware harness but the direct eval CLI did not supply the production v3 runtime context that harness requires. > - This pull request preserves the qualified dependency graph, resolves the real deployment layout, and makes direct evals exercise the production runtime-context contract. > - The benefit is that live eval cells reach their provider turn with the same artifacts and context contract that Paperclip qualified. ## Linked Issues or Issue Description Refs: #12931 **What happened?** The full direct live eval campaign failed every ACPX cell during `session.open`. The portable runtime had an incorrect dependency root. Its npm install also discarded the qualified ACP server patches. AgentCore cells first failed because Runner enforced `aws-agentcore-harness-v1` while the provisioned stack and eval profile use `aws-agentcore-harness-context-v2`; after aligning that revision, the direct eval CLI still omitted the required v3 runtime context. **Expected behavior** The direct eval runtime must preserve the frozen pnpm dependency graph and patched provider bytes. Runner, server validation, OpenAPI, and the deployed AgentCore stack must use one qualification revision. Direct eval attempts must supply the same immutable native runtime-context contract as production. **Steps to reproduce** 1. Dispatch `Runner Direct Live Protocol Evals` from `master`. 2. Select an ACPX Claude, ACPX Codex, or AgentCore roster. 3. Observe a pre-turn provider bootstrap failure. **Paperclip version or commit** `d96452db059338b329b458ba8fe359fef72f1363` **Deployment mode** GitHub Actions on the RunsOn Linux x64 fleet. ## What Changed - Build the reusable direct-eval runtime with `pnpm deploy --prod`. - Resolve ACPX dependencies from the actual scoped-package layout of a self-contained pnpm deployment. - Align AgentCore configuration and qualification checks on `aws-agentcore-harness-context-v2`. - Materialize a minimal immutable v3 runtime context for each isolated direct eval attempt. - Add workflow, package-authority, runtime-context, Rust, and server regression coverage. - Document the qualified packaging, runtime-context, and AgentCore revision contracts. ## Verification - `pnpm --filter @paperclipai/paperclip-runner typecheck` - `pnpm --filter @paperclipai/server typecheck` - `pnpm --filter @paperclipai/paperclip-runner exec vitest run src/live/runnerd-codex-transport.test.ts` (70 tests) - `pnpm --filter @paperclipai/paperclip-runner exec vitest run src/cli/eval-session-contract.test.ts` (14 tests) - Focused Runner contract tests (36 tests) - Focused server profile tests (47 tests) - Focused Rust managed-provider and native-selector tests (19 tests) - `node --test packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs` - `actionlint .github/workflows/runner-protocol-live-evals.yml` - A local `pnpm deploy --prod` produced both qualified ACP server digests. - A Linux reproduction of the first follow-up smoke identified the real deployment root and the missing AgentCore runtime context. ## Risks The AgentCore revision change rejects profiles that still use the obsolete v1 value. This is intentional because the provisioned context-aware harness and current eval profile use v2. Direct eval prompts now receive the same fixed runtime-context preamble as production, so behavior scores may move; that is the intended qualification surface. The workflow package layout changes, but tests assert the new entrypoint and dependency root. This change does not modify the browser full-stack E2E workflow. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected. See `CONTRIBUTING.md`. ## Model Used OpenAI Codex with GPT-5.6. The context-window size is not exposed in this session. The model used extended reasoning, repository tools, code execution, Docker-based Linux reproduction, and GitHub Actions diagnostics. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (for example, `docs/...` or `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- .../workflows/runner-protocol-live-evals.yml | 10 +- .../docs/runner-protocol-live-evals.md | 49 +- .../runner-core/src/aws_agentcore_provider.rs | 501 +++++++++++++++--- .../src/managed_provider_backend.rs | 2 +- .../tests/native_provider_backend.rs | 2 +- .../scripts/runner-protocol-eval-campaign.mjs | 29 +- .../runner-protocol-eval-campaign.test.mjs | 91 ++++ ...r-protocol-eval-workflow-security.test.mjs | 9 + .../backends/native-backend-factory.test.ts | 4 +- .../src/cli/eval-session-contract.test.ts | 36 ++ .../paperclip-runner/src/cli/eval-session.ts | 105 +++- .../src/contracts/native-execution.test.ts | 2 +- .../src/live/live-session.test.ts | 21 + .../paperclip-runner/src/live/live-session.ts | 6 +- .../src/live/runnerd-codex-transport.test.ts | 60 +++ .../src/live/runnerd-codex-transport.ts | 30 +- ...managed-agent-profile-routes-authz.test.ts | 2 +- server/src/routes/openapi.ts | 2 +- .../heartbeat-runner-provider-config.test.ts | 8 +- .../native-session-resume.test.ts | 2 +- .../provider-profile-qualification.ts | 3 +- .../services/remote-agent-profiles.test.ts | 10 +- 22 files changed, 868 insertions(+), 116 deletions(-) diff --git a/.github/workflows/runner-protocol-live-evals.yml b/.github/workflows/runner-protocol-live-evals.yml index 060a7309bd..f659f5ec65 100644 --- a/.github/workflows/runner-protocol-live-evals.yml +++ b/.github/workflows/runner-protocol-live-evals.yml @@ -14,7 +14,7 @@ on: type: string required: false rosters: - description: "Comma-separated live roster IDs/files, or all for the complete direct suite" + description: "Comma-separated live roster IDs/files, or all for the maintained enabled direct suite" type: string default: "all" required: false @@ -268,11 +268,13 @@ jobs: --pack-destination "$RUNNER_TEMP/runner-protocol-build/package" package="$(find "$RUNNER_TEMP/runner-protocol-build/package" -maxdepth 1 -type f -name '*.tgz' -print -quit)" test -f "$package" - npm install --prefix "$RUNNER_TEMP/runner-protocol-build/portable" --omit=dev "$package" + pnpm --filter @paperclipai/paperclip-runner deploy --prod \ + "$RUNNER_TEMP/runner-protocol-build/portable" cp "$package" "$RUNNER_TEMP/runner-protocol-build/paperclip-runner.tgz" cp packages/paperclip-runner/runner/target/debug/paperclip-runnerd "$RUNNER_TEMP/runner-protocol-build/paperclip-runnerd" cp -R packages/paperclip-runner/dist-issue-thread "$RUNNER_TEMP/runner-protocol-build/dist-issue-thread" - test -f "$RUNNER_TEMP/runner-protocol-build/portable/node_modules/@paperclipai/paperclip-runner/dist/cli/eval-session.js" + test -f "$RUNNER_TEMP/runner-protocol-build/portable/dist/cli/eval-session.js" + test -d "$RUNNER_TEMP/runner-protocol-build/portable/node_modules/.pnpm" test -x "$RUNNER_TEMP/runner-protocol-build/paperclip-runnerd" tar --create --gzip --file runner-protocol-build.tar.gz -C "$RUNNER_TEMP/runner-protocol-build" . sha256sum runner-protocol-build.tar.gz > runner-protocol-build.tar.gz.sha256 @@ -433,7 +435,7 @@ jobs: python3 .paperclip-evals/evals/paperclip-runner/tools/run_live_roster.py run \ --roster ".paperclip-evals/evals/paperclip-runner/rosters/$ROSTER_FILE" \ --case "$CASE_ID" \ - --runner-cli runner-protocol-build/extracted/portable/node_modules/@paperclipai/paperclip-runner/dist/cli/eval-session.js \ + --runner-cli runner-protocol-build/extracted/portable/dist/cli/eval-session.js \ --runner-package runner-protocol-build/extracted/paperclip-runner.tgz \ --runnerd runner-protocol-build/extracted/paperclip-runnerd \ --runs-root cell-output/runs \ diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index 2cdc74774b..6ed8f6f214 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -6,11 +6,14 @@ separate from both the browser full-stack model E2E and the stress-derived workflow schedule in `runner-workflow-evals.md`. The canonical unit of work is one live roster plus one authored case. A full -campaign selects every `rosters/live-*.json` file at one immutable -`paperclip-evals` commit. That includes the complete 35-case provider rosters -and the smaller ACPX Codex control roster. The native resume reliability gate -is not a normal one-turn roster: it requires its separately governed external -resource campaign and remains opt-in. +campaign selects every enabled lane declared by +`campaigns/live-direct-full.json` at one immutable `paperclip-evals` commit. +That includes the complete 35-case provider rosters and the smaller ACPX Codex +control roster. A disabled roster remains available for an explicit diagnostic +selection, but is never inferred into a paid `all` run from the files present +on disk. The native resume reliability gate is not a normal one-turn roster: +it requires its separately governed external-resource campaign and remains +opt-in. Managed-provider evidence identifies the immutable deployed provider artifact: Claude Managed uses its Agent version, while AgentCore uses its qualification @@ -25,8 +28,11 @@ from the default branch and provide: - `target_branch`: the Paperclip branch to build and test; - `evals_sha`: an exact 40-character commit from `paperclipai/paperclip-evals`; -- `rosters`: `all` for the entire direct suite, or a comma-separated diagnostic - subset; +- `rosters`: `all` for every enabled lane in the canonical + `live-direct-full.json` campaign, or a comma-separated diagnostic subset. + Disabled lanes remain available only through an explicit diagnostic + selection; `all` never spends against a lane that the eval program has + marked disabled; - `max_infrastructure_retries`: zero through three, applied only when an attempt explicitly reports a retryable infrastructure failure. @@ -43,11 +49,26 @@ the native daemon, provider dependencies, and the attempt viewer are built once and reused by every cell. Because the complete suite requires two matrix shards, this workflow accepts a shared concurrency ceiling from 2 through 100. -`all` is intentionally literal. A disabled driver, missing remote profile, or -unavailable provider is retained as an infrastructure result; it is not -silently omitted. In particular, the ACPX Pi roster remains visible while Pi -is disabled in the current Runner. Use a roster subset only for diagnosis, not -to claim the full campaign is green. +The reused provider runtime is created with `pnpm deploy --prod` from the +frozen workspace lock. That preserves the repository's qualified dependency +versions and patched ACP server bytes. Do not replace this step with a fresh +`npm install` of the packed Runner tarball: npm cannot apply the workspace's +`patchedDependencies`, so the resulting ACPX executables no longer match their +qualified digests. + +The direct eval CLI also materializes a minimal immutable native runtime +context in each isolated attempt workspace. This keeps the direct layer on the +same `paperclip.native-execution-input.v3` contract as production, including +the AgentCore HarnessSkill upload path, without borrowing any browser E2E +setup. + +`all` means the maintained enabled campaign, not every matching file in the +roster directory. A missing campaign file fails closed. Within an enabled +lane, a missing remote profile or unavailable provider is retained as an +infrastructure result; it is not silently omitted. The current ACPX Pi roster +is declared disabled because its Runner security profile is not qualified, so +it runs only when selected explicitly for diagnosis. Use a roster subset only +for diagnosis, not to claim the full maintained campaign is green. A provider turn that reaches a durable failed, interrupted, or otherwise non-completed terminal still produces an attempt artifact and is scored as a @@ -70,7 +91,9 @@ Claude Managed also requires the four nonsecret nonsecret `PAPERCLIP_AWS_AGENTCORE_*` profile variables, including `PAPERCLIP_AWS_AGENTCORE_EXECUTION_ROLE_ARN` and the immutable `PAPERCLIP_AWS_AGENTCORE_QUALIFICATION_REVISION`; the eval fails closed when -that deployed revision differs from the pinned roster config. The workflow +that deployed revision differs from the pinned roster config. The currently +qualified context-aware harness revision is +`aws-agentcore-harness-context-v2`. The workflow writes the GitHub OIDC token to a mode-`0600` file and never forwards long-lived AWS access keys. Provision the AgentCore stack with `--github-oidc-provider-arn` so that scoped diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/aws_agentcore_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/src/aws_agentcore_provider.rs index 0536b243a7..68d34bb652 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/aws_agentcore_provider.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/aws_agentcore_provider.rs @@ -50,6 +50,7 @@ const MAX_CONTEXT_UPLOAD_BYTES: usize = 64 * 1024 * 1024; const MAX_MEMORY_HISTORY_PAGES: usize = 1_000; const MAX_MEMORY_HISTORY_EVENTS: usize = 100_000; const MAX_INTERRUPT_DRAIN_EVENTS: usize = 256; +const AGENTCORE_HARNESS_SKILLS_TOOL: &str = "skills"; #[cfg(not(test))] const AGENTCORE_INTERRUPT_USAGE_RECONCILIATION_TIMEOUT: Duration = Duration::from_secs(2); #[cfg(test)] @@ -477,7 +478,7 @@ async fn upload_context_directory( .map_err(|error| { format!( "AgentCore context S3 upload failed: {}", - redact_aws_error(&error.to_string()) + classify_aws_sdk_error(&error) ) })?; } @@ -688,6 +689,7 @@ fn network_loop( let skills = skills.clone(); let system_instructions = system_instructions.clone(); runtime.spawn(async move { + let allowed_tools = invocation_allowed_tools(allowed_tools, !skills.is_empty()); // The qualified Harness version is the immutable model // authority. Supplying a redundant invocation override // changes AgentCore's authorization path and can require @@ -735,11 +737,13 @@ fn network_loop( }; let _ = reply.send(Ok(())); let mut tool_blocks: BTreeMap = BTreeMap::new(); + let mut message_stopped = false; loop { match response.stream.recv().await { Ok(Some(event)) => normalize_stream_event( event, &mut tool_blocks, + &mut message_stopped, &events, &invocation_id, ), @@ -811,6 +815,7 @@ fn network_loop( fn normalize_stream_event( event: InvokeHarnessStreamOutput, tool_blocks: &mut BTreeMap, + message_stopped: &mut bool, events: &SyncSender, invocation_id: &str, ) { @@ -851,6 +856,18 @@ fn normalize_stream_event( if let Some((call_id, remote_name, input)) = tool_blocks.remove(&value.content_block_index()) { + // HarnessSkill contents are loaded on demand by AgentCore's + // built-in `skills` tool. Its trace is surfaced in the same + // ToolUse-shaped stream as client-owned inline functions, but + // AgentCore executes it inside the harness. Do not send it + // across PRP as a Paperclip semantic operation. + if remote_name == AGENTCORE_HARNESS_SKILLS_TOOL { + let _ = events.send(NetworkEvent::new( + invocation_id, + NetworkEventKind::ReasoningProgress, + )); + return; + } let parsed = if input.trim().is_empty() { Ok(json!({})) } else { @@ -897,6 +914,7 @@ fn normalize_stream_event( )); } InvokeHarnessStreamOutput::MessageStop(value) => { + *message_stopped = true; let _ = events.send(NetworkEvent::new( invocation_id, NetworkEventKind::Stop(value.stop_reason().as_str().to_owned()), @@ -904,16 +922,33 @@ fn normalize_stream_event( } InvokeHarnessStreamOutput::MessageStart(_) => {} _ => { - let _ = events.send(NetworkEvent::new( - invocation_id, - NetworkEventKind::Failure( - "AgentCore SDK did not recognize an EventStream record".to_owned(), - ), - )); + // AgentCore may append forward-compatible bookkeeping records + // after the authoritative MessageStop. The pinned SDK exposes + // those records only as `Unknown`, without their union name or + // payload. Once MessageStop has sealed the model outcome, ignore + // such a trailing record; InvocationComplete still requires the + // known stop reason and usage metadata. Unknown records before + // MessageStop remain fatal because they could affect the turn. + normalize_unknown_stream_event(*message_stopped, events, invocation_id); } } } +fn normalize_unknown_stream_event( + message_stopped: bool, + events: &SyncSender, + invocation_id: &str, +) { + if !message_stopped { + let _ = events.send(NetworkEvent::new( + invocation_id, + NetworkEventKind::Failure( + "AgentCore SDK did not recognize an EventStream record".to_owned(), + ), + )); + } +} + fn observe_memory_history_page( pages: &mut usize, events: &mut usize, @@ -1439,6 +1474,69 @@ impl AwsAgentCoreHarnessProvider { params: json!({ "turnId": turn_id, "turn": { "id": turn_id, "status": "interrupted" }, "stopReason": "interrupted" }), }); } + + fn invoke_tool_result_continuation(&mut self) -> Result<(), LocalRunnerError> { + if self.pending.is_empty() || self.delivered_results.len() != self.pending.len() { + return Err(LocalRunnerError::invalid( + "AgentCore tool-result continuation batch is incomplete", + )); + } + + // AgentCore requires every toolUse from the assistant message and all + // matching user toolResults in one continuation. This also guarantees + // that parallel governed mutations cannot advance separate model turns. + let mut assistant_builder = + HarnessMessage::builder().role(HarnessConversationRole::Assistant); + let mut user_builder = HarnessMessage::builder().role(HarnessConversationRole::User); + for (call_id, pending) in &self.pending { + let delivered = self.delivered_results.get(call_id).ok_or_else(|| { + LocalRunnerError::invalid("AgentCore tool-result batch is incomplete") + })?; + let tool_use = HarnessToolUseBlock::builder() + .name(pending.remote_name.clone()) + .tool_use_id(call_id.clone()) + .input(json_to_document(&pending.input)?) + .r#type(HarnessToolUseType::ToolUse) + .build() + .map_err(|_| { + LocalRunnerError::invalid("failed to build AgentCore tool-use continuation") + })?; + let tool_result = HarnessToolResultBlock::builder() + .tool_use_id(call_id.clone()) + // Although the AgentCore data-plane model advertises JSON + // result blocks, the managed Harness runtime currently + // rejects them with `content_type= | unsupported type`. + // Preserve the complete structured result as compact JSON in + // the supported text variant. + .content(encode_tool_result_content(&delivered.result)?) + .status(if delivered.is_error { + HarnessToolUseStatus::Error + } else { + HarnessToolUseStatus::Success + }) + .r#type(HarnessToolUseType::ToolUse) + .build() + .map_err(|_| { + LocalRunnerError::invalid("failed to build AgentCore tool-result continuation") + })?; + assistant_builder = assistant_builder.content(HarnessContentBlock::ToolUse(tool_use)); + user_builder = user_builder.content(HarnessContentBlock::ToolResult(tool_result)); + } + let assistant = assistant_builder.build().map_err(|_| { + LocalRunnerError::invalid("failed to build AgentCore assistant continuation") + })?; + let user = user_builder.build().map_err(|_| { + LocalRunnerError::invalid("failed to build AgentCore user continuation") + })?; + // The durable runner records ToolResult before calling this method. A + // transport ambiguity therefore never repeats the Paperclip mutation; + // it fails closed with the last authoritative Memory cursor preserved + // for the control plane's reconciliation workflow. + self.invoke(vec![assistant, user])?; + self.pending.clear(); + self.delivered_results.clear(); + Ok(()) + } } impl Provider for AwsAgentCoreHarnessProvider { @@ -1709,11 +1807,12 @@ impl Provider for AwsAgentCoreHarnessProvider { ))) } NetworkEventKind::Stop(reason) => { - if self.pending_stop_reason.replace(reason).is_some() { - return Err(LocalRunnerError::invalid( - "AgentCore emitted more than one stop reason for an invocation", - )); - } + // Harness-managed tools such as `skills` can produce several + // complete message boundaries inside one InvokeHarness + // stream (tool_use, tool_result, then the assistant's final + // stop). The final MessageStop before metadata is the + // authoritative outcome for the invocation. + self.pending_stop_reason = Some(reason); Ok(None) } NetworkEventKind::InvocationComplete => { @@ -1734,10 +1833,20 @@ impl Provider for AwsAgentCoreHarnessProvider { } match reason.as_str() { "tool_use" | "tool_result" | "partial_turn" => { - Ok(Some(ProviderEvent::Notification { - method: "provider/waitingForToolResult".to_owned(), - params: json!({ "turnId": self.current_turn_id, "runtimeSessionId": self.session_id }), - })) + if self.pending.is_empty() { + return Err(LocalRunnerError::invalid( + "AgentCore stopped for a tool without a pending inline function", + )); + } + if self.delivered_results.len() == self.pending.len() { + self.invoke_tool_result_continuation()?; + Ok(None) + } else { + Ok(Some(ProviderEvent::Notification { + method: "provider/waitingForToolResult".to_owned(), + params: json!({ "turnId": self.current_turn_id, "runtimeSessionId": self.session_id }), + })) + } } "end_turn" | "stop_sequence" | "interrupted" => { let turn_id = self.current_turn_id.take(); @@ -1783,6 +1892,14 @@ impl Provider for AwsAgentCoreHarnessProvider { } NetworkEventKind::Failure(detail) => { self.active_invocation_id = None; + self.pending_stop_reason = None; + // A transport failure is terminal for the current invocation. + // Results can race the failure across the control-plane and + // provider channels, so discard both sides of the pending + // batch. A late result must fail validation instead of + // starting a continuation after the failed invocation. + self.pending.clear(); + self.delivered_results.clear(); Err(LocalRunnerError::invalid(format!( "AgentCore transport failed: {detail}" ))) @@ -1817,60 +1934,15 @@ impl Provider for AwsAgentCoreHarnessProvider { return Ok(()); } - // AgentCore requires every toolUse from the assistant message and all - // matching user toolResults in one continuation. This also guarantees - // that parallel governed mutations cannot advance separate model turns. - let mut assistant_builder = - HarnessMessage::builder().role(HarnessConversationRole::Assistant); - let mut user_builder = HarnessMessage::builder().role(HarnessConversationRole::User); - for (call_id, pending) in &self.pending { - let delivered = self.delivered_results.get(call_id).ok_or_else(|| { - LocalRunnerError::invalid("AgentCore tool-result batch is incomplete") - })?; - let tool_use = HarnessToolUseBlock::builder() - .name(pending.remote_name.clone()) - .tool_use_id(call_id.clone()) - .input(json_to_document(&pending.input)?) - .r#type(HarnessToolUseType::ToolUse) - .build() - .map_err(|_| { - LocalRunnerError::invalid("failed to build AgentCore tool-use continuation") - })?; - let tool_result = HarnessToolResultBlock::builder() - .tool_use_id(call_id.clone()) - // Although the AgentCore data-plane model advertises JSON - // result blocks, the managed Harness runtime currently - // rejects them with `content_type= | unsupported type`. - // Preserve the complete structured result as compact JSON in - // the supported text variant. - .content(encode_tool_result_content(&delivered.result)?) - .status(if delivered.is_error { - HarnessToolUseStatus::Error - } else { - HarnessToolUseStatus::Success - }) - .r#type(HarnessToolUseType::ToolUse) - .build() - .map_err(|_| { - LocalRunnerError::invalid("failed to build AgentCore tool-result continuation") - })?; - assistant_builder = assistant_builder.content(HarnessContentBlock::ToolUse(tool_use)); - user_builder = user_builder.content(HarnessContentBlock::ToolResult(tool_result)); + // A ToolUse block arrives before the same EventStream's MessageStop + // and usage metadata. The control plane can return its result during + // that interval; retain it durably and let InvocationComplete start + // the continuation after the first invocation is fully sealed. + if self.active_invocation_id.is_some() { + return Ok(()); } - let assistant = assistant_builder.build().map_err(|_| { - LocalRunnerError::invalid("failed to build AgentCore assistant continuation") - })?; - let user = user_builder.build().map_err(|_| { - LocalRunnerError::invalid("failed to build AgentCore user continuation") - })?; - // The durable runner records ToolResult before calling this method. A - // transport ambiguity therefore never repeats the Paperclip mutation; - // it fails closed with the last authoritative Memory cursor preserved - // for the control plane's reconciliation workflow. - self.invoke(vec![assistant, user])?; - self.pending.clear(); - self.delivered_results.clear(); - Ok(()) + + self.invoke_tool_result_continuation() } fn shutdown(&mut self) -> Result<(), LocalRunnerError> { @@ -2006,6 +2078,20 @@ fn encode_tools( Ok((encoded, allowed, reverse, schemas)) } +fn invocation_allowed_tools(mut allowed: Vec, has_skills: bool) -> Vec { + if has_skills + && !allowed + .iter() + .any(|tool| tool == AGENTCORE_HARNESS_SKILLS_TOOL) + { + // HarnessSkill progressive disclosure is performed by AgentCore's + // built-in `skills` tool. It must be admitted alongside the narrowly + // scoped Paperclip inline functions whenever skills are attached. + allowed.push(AGENTCORE_HARNESS_SKILLS_TOOL.to_owned()); + } + allowed +} + fn remote_tool_name(operation_id: &str) -> String { let mut slug = operation_id .chars() @@ -2188,7 +2274,8 @@ mod tests { use super::*; use aws_sdk_bedrockagentcore::types::{ HarnessContentBlockDeltaEvent, HarnessContentBlockStartEvent, HarnessContentBlockStopEvent, - HarnessToolUseBlockDelta, HarnessToolUseBlockStart, + HarnessMessageStopEvent, HarnessStopReason, HarnessToolUseBlockDelta, + HarnessToolUseBlockStart, }; use std::sync::{Arc, Mutex}; @@ -2211,7 +2298,7 @@ mod tests { context_bucket: "paperclip-context-test".to_owned(), context_prefix: "companies/company-test/profiles/profile-test".to_owned(), context_kms_key_arn: "arn:aws:kms:us-east-1:123456789012:key/test".to_owned(), - qualification_revision: "aws-agentcore-harness-v1".to_owned(), + qualification_revision: "aws-agentcore-harness-context-v2".to_owned(), event_expiry_days: 90, max_estimated_session_cost_usd: 1.0, max_iterations: 8, @@ -2468,6 +2555,167 @@ mod tests { )); } + #[test] + fn harness_managed_message_stops_resolve_to_the_final_stop_reason() { + let mut provider = provider_with_events( + vec![ + invocation_event(NetworkEventKind::Stop("tool_use".to_owned())), + invocation_event(NetworkEventKind::Stop("tool_result".to_owned())), + invocation_event(NetworkEventKind::Stop("end_turn".to_owned())), + invocation_event(NetworkEventKind::Usage { + input_tokens: 840, + output_tokens: 62, + cache_read_input_tokens: 0, + cache_write_input_tokens: 0, + latency_ms: 2_646, + }), + invocation_event(NetworkEventKind::InvocationComplete), + ], + restored_usage_snapshot(None).unwrap(), + ); + assert!(provider.poll().unwrap().is_none()); + assert!(provider.poll().unwrap().is_none()); + assert!(provider.poll().unwrap().is_none()); + assert!(matches!( + provider.poll().unwrap().unwrap(), + ProviderEvent::Notification { ref method, .. } + if method == "thread/tokenUsage/updated" + )); + match provider.poll().unwrap().unwrap() { + ProviderEvent::Notification { method, params } => { + assert_eq!(method, "turn/completed"); + assert_eq!(params["stopReason"], "end_turn"); + } + other => panic!("unexpected terminal event: {other:?}"), + } + } + + #[test] + fn early_tool_result_waits_for_invocation_metadata_before_continuing() { + let (command_tx, command_rx) = mpsc::sync_channel(2); + let (event_tx, event_rx) = mpsc::sync_channel(4); + let (captured_tx, captured_rx) = mpsc::sync_channel(1); + let join = thread::spawn(move || { + while let Ok(command) = command_rx.recv() { + match command { + NetworkCommand::Invoke { + messages, reply, .. + } => { + captured_tx.send(messages.len()).unwrap(); + reply.send(Ok(())).unwrap(); + } + NetworkCommand::Shutdown => break, + NetworkCommand::StopRuntime { reply, .. } => { + reply.send(Ok(())).unwrap(); + } + NetworkCommand::DeleteMemory { reply } => { + reply.send(Ok(())).unwrap(); + } + } + } + }); + let mut provider = provider_with_worker( + NetworkWorker { + commands: command_tx, + events: event_rx, + join: Some(join), + }, + restored_usage_snapshot(None).unwrap(), + ); + provider.pending.insert( + "tool-use-1".to_owned(), + RemoteToolUse { + remote_name: "pc_get_task_context_abc123".to_owned(), + operation_id: "get_task_context".to_owned(), + input: json!({}), + }, + ); + provider + .deliver_tool_result(&ToolResult { + call_id: "tool-use-1".to_owned(), + operation_id: "get_task_context".to_owned(), + result: json!({"ok": true}), + is_error: false, + }) + .unwrap(); + assert!(captured_rx.try_recv().is_err()); + assert_eq!(provider.delivered_results.len(), 1); + + for kind in [ + NetworkEventKind::Stop("tool_use".to_owned()), + NetworkEventKind::Usage { + input_tokens: 100, + output_tokens: 10, + cache_read_input_tokens: 0, + cache_write_input_tokens: 0, + latency_ms: 25, + }, + NetworkEventKind::InvocationComplete, + ] { + event_tx.send(invocation_event(kind)).unwrap(); + } + assert!(provider.poll().unwrap().is_none()); + assert!(matches!( + provider.poll().unwrap().unwrap(), + ProviderEvent::Notification { ref method, .. } + if method == "thread/tokenUsage/updated" + )); + assert!(provider.poll().unwrap().is_none()); + assert_eq!(captured_rx.recv_timeout(Duration::from_secs(1)).unwrap(), 2); + assert_eq!(provider.invocation_counter, 2); + assert!(provider.active_invocation_id.is_some()); + assert!(provider.pending.is_empty()); + assert!(provider.delivered_results.is_empty()); + } + + #[test] + fn transport_failure_discards_pending_tool_state_and_rejects_late_result() { + let mut provider = provider_with_events( + vec![invocation_event(NetworkEventKind::Failure( + "connection reset".to_owned(), + ))], + restored_usage_snapshot(None).unwrap(), + ); + provider.pending.insert( + "tool-use-1".to_owned(), + RemoteToolUse { + remote_name: "pc_get_task_context_abc123".to_owned(), + operation_id: "get_task_context".to_owned(), + input: json!({}), + }, + ); + provider.delivered_results.insert( + "tool-use-1".to_owned(), + ToolResult { + call_id: "tool-use-1".to_owned(), + operation_id: "get_task_context".to_owned(), + result: json!({"ok": true}), + is_error: false, + }, + ); + provider.pending_stop_reason = Some("tool_use".to_owned()); + + let error = provider.poll().unwrap_err(); + assert!(error.to_string().contains("AgentCore transport failed")); + assert!(provider.active_invocation_id.is_none()); + assert!(provider.pending_stop_reason.is_none()); + assert!(provider.pending.is_empty()); + assert!(provider.delivered_results.is_empty()); + + let late_result_error = provider + .deliver_tool_result(&ToolResult { + call_id: "tool-use-1".to_owned(), + operation_id: "get_task_context".to_owned(), + result: json!({"ok": true}), + is_error: false, + }) + .unwrap_err(); + assert!(late_result_error + .to_string() + .contains("does not match a pending tool use")); + assert_eq!(provider.invocation_counter, 1); + } + #[test] fn mid_stream_interrupt_waits_for_late_usage_and_suppresses_truncated_completion() { let late_usage = NetworkEventKind::Usage { @@ -3106,6 +3354,20 @@ mod tests { assert!(allowed[0].starts_with("@*/pc_")); } + #[test] + fn harness_skills_tool_is_allowed_only_when_skills_are_attached() { + let inline = vec!["@*/pc_get_task_context_abc123".to_owned()]; + assert_eq!(invocation_allowed_tools(inline.clone(), false), inline); + assert_eq!( + invocation_allowed_tools(inline.clone(), true), + vec![inline[0].clone(), AGENTCORE_HARNESS_SKILLS_TOOL.to_owned()] + ); + assert_eq!( + invocation_allowed_tools(vec![AGENTCORE_HARNESS_SKILLS_TOOL.to_owned()], true), + vec![AGENTCORE_HARNESS_SKILLS_TOOL.to_owned()] + ); + } + #[test] fn rejects_more_than_sixty_four_tools() { let tool = AuthorizedTool { @@ -3178,6 +3440,7 @@ mod tests { fn eventstream_text_delta_is_normalized_without_provider_objects() { let (sender, receiver) = mpsc::sync_channel(4); let mut blocks = BTreeMap::new(); + let mut message_stopped = false; let event = HarnessContentBlockDeltaEvent::builder() .content_block_index(0) .delta(HarnessContentBlockDelta::Text("hello".to_owned())) @@ -3186,6 +3449,7 @@ mod tests { normalize_stream_event( InvokeHarnessStreamOutput::ContentBlockDelta(event), &mut blocks, + &mut message_stopped, &sender, "invocation-1", ); @@ -3205,6 +3469,7 @@ mod tests { fn eventstream_tool_json_is_buffered_until_the_block_is_complete() { let (sender, receiver) = mpsc::sync_channel(8); let mut blocks = BTreeMap::new(); + let mut message_stopped = false; let start = HarnessToolUseBlockStart::builder() .tool_use_id("tool-use-1") .name("pc_get_task_abc123") @@ -3220,6 +3485,7 @@ mod tests { .unwrap(), ), &mut blocks, + &mut message_stopped, &sender, "invocation-1", ); @@ -3238,6 +3504,7 @@ mod tests { .unwrap(), ), &mut blocks, + &mut message_stopped, &sender, "invocation-1", ); @@ -3251,6 +3518,7 @@ mod tests { .unwrap(), ), &mut blocks, + &mut message_stopped, &sender, "invocation-1", ); @@ -3273,10 +3541,57 @@ mod tests { } } + #[test] + fn eventstream_harness_skills_trace_is_not_forwarded_as_an_inline_function() { + let (sender, receiver) = mpsc::sync_channel(4); + let mut blocks = BTreeMap::new(); + let mut message_stopped = false; + let start = HarnessToolUseBlockStart::builder() + .tool_use_id("harness-skill-load-1") + .name(AGENTCORE_HARNESS_SKILLS_TOOL) + .r#type(HarnessToolUseType::ToolUse) + .build() + .unwrap(); + normalize_stream_event( + InvokeHarnessStreamOutput::ContentBlockStart( + HarnessContentBlockStartEvent::builder() + .content_block_index(3) + .start(HarnessContentBlockStart::ToolUse(start)) + .build() + .unwrap(), + ), + &mut blocks, + &mut message_stopped, + &sender, + "invocation-1", + ); + normalize_stream_event( + InvokeHarnessStreamOutput::ContentBlockStop( + HarnessContentBlockStopEvent::builder() + .content_block_index(3) + .build() + .unwrap(), + ), + &mut blocks, + &mut message_stopped, + &sender, + "invocation-1", + ); + match receiver.try_recv().unwrap() { + NetworkEvent { + invocation_id, + kind: NetworkEventKind::ReasoningProgress, + } => assert_eq!(invocation_id, "invocation-1"), + other => panic!("unexpected normalized event: {other:?}"), + } + assert!(receiver.try_recv().is_err()); + } + #[test] fn eventstream_empty_tool_input_is_normalized_to_an_empty_object() { let (sender, receiver) = mpsc::sync_channel(4); let mut blocks = BTreeMap::new(); + let mut message_stopped = false; let start = HarnessToolUseBlockStart::builder() .tool_use_id("tool-use-empty") .name("pc_get_task_context_abc123") @@ -3292,6 +3607,7 @@ mod tests { .unwrap(), ), &mut blocks, + &mut message_stopped, &sender, "invocation-1", ); @@ -3309,6 +3625,7 @@ mod tests { .unwrap(), ), &mut blocks, + &mut message_stopped, &sender, "invocation-1", ); @@ -3320,6 +3637,7 @@ mod tests { .unwrap(), ), &mut blocks, + &mut message_stopped, &sender, "invocation-1", ); @@ -3342,6 +3660,47 @@ mod tests { } } + #[test] + fn unknown_eventstream_record_before_message_stop_fails_closed() { + let (sender, receiver) = mpsc::sync_channel(4); + normalize_unknown_stream_event(false, &sender, "invocation-1"); + match receiver.try_recv().unwrap() { + NetworkEvent { + kind: NetworkEventKind::Failure(detail), + .. + } => assert_eq!( + detail, + "AgentCore SDK did not recognize an EventStream record" + ), + other => panic!("unexpected normalized event: {other:?}"), + } + } + + #[test] + fn trailing_unknown_eventstream_record_cannot_overturn_message_stop() { + let (sender, receiver) = mpsc::sync_channel(4); + let mut blocks = BTreeMap::new(); + let mut message_stopped = false; + normalize_stream_event( + InvokeHarnessStreamOutput::MessageStop( + HarnessMessageStopEvent::builder() + .stop_reason(HarnessStopReason::EndTurn) + .build() + .unwrap(), + ), + &mut blocks, + &mut message_stopped, + &sender, + "invocation-1", + ); + assert!(matches!( + receiver.try_recv().unwrap().kind, + NetworkEventKind::Stop(reason) if reason == "end_turn" + )); + normalize_unknown_stream_event(message_stopped, &sender, "invocation-1"); + assert!(receiver.try_recv().is_err()); + } + #[test] fn tool_results_use_the_harness_supported_text_content_variant() { let value = json!({"ok": true, "nested": {"value": 7}}); diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs index 0370c7bb90..2b8212c794 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs @@ -43,7 +43,7 @@ const MAX_INSTRUCTIONS_BYTES: usize = 1024 * 1024; const QUALIFIED_CLAUDE_MODEL: &str = "claude-sonnet-5"; const QUALIFIED_CLAUDE_BETA: &str = "managed-agents-2026-04-01"; const QUALIFIED_AGENTCORE_MODEL: &str = "global.anthropic.claude-sonnet-4-6"; -const QUALIFIED_AGENTCORE_REVISION: &str = "aws-agentcore-harness-v1"; +const QUALIFIED_AGENTCORE_REVISION: &str = "aws-agentcore-harness-context-v2"; fn initial_event_sequence() -> u64 { 1 diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs index 101e049f6d..ba36834ab7 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs @@ -283,7 +283,7 @@ fn managed_prepare_payload(kind: &str) -> Value { "contextBucket": "context-bucket", "contextPrefix": "companies/company/profiles/profile", "contextKmsKeyArn": "arn:aws:kms:us-east-1:123456789012:key/test", - "qualificationRevision": "aws-agentcore-harness-v1", + "qualificationRevision": "aws-agentcore-harness-context-v2", "eventExpiryDays": 90, "maxEstimatedSessionCostUsd": 1.0, "maxIterations": 8, diff --git a/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs index 7314b63748..13ccfbb108 100644 --- a/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs +++ b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs @@ -77,6 +77,31 @@ function parseRosterSelection(value) { return new Set(selected); } +async function maintainedRosterSelection(programRoot) { + const campaignPath = resolve(programRoot, "campaigns/live-direct-full.json"); + const campaign = await loadObject(campaignPath); + if ( + campaign.schema !== "paperclip-runner/live-campaign/v1" || + !Array.isArray(campaign.lanes) + ) { + throw new Error(`Unsupported live campaign schema in ${campaignPath}`); + } + const selected = campaign.lanes + .filter((lane) => lane.executionClass !== "disabled") + .map((lane) => { + const rosterPath = inside( + programRoot, + resolve(dirname(campaignPath), String(lane.roster ?? "")), + "Campaign roster", + ); + return basename(rosterPath); + }); + if (selected.length === 0 || new Set(selected).size !== selected.length) { + throw new Error("Maintained live campaign must contain unique enabled rosters"); + } + return new Set(selected); +} + export async function buildProtocolEvalCatalog({ evalsRoot, rosterSelection = "all", @@ -94,7 +119,9 @@ export async function buildProtocolEvalCatalog({ } const programRoot = resolve(evalsRoot, "evals/paperclip-runner"); const rosterRoot = resolve(programRoot, "rosters"); - const selected = parseRosterSelection(rosterSelection); + const requested = parseRosterSelection(rosterSelection); + const selected = + requested ?? (await maintainedRosterSelection(programRoot)); const rosterFiles = (await readdir(rosterRoot, { withFileTypes: true })) .filter( (entry) => diff --git a/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.test.mjs b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.test.mjs index 474f89971b..0703d3de7d 100644 --- a/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.test.mjs +++ b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.test.mjs @@ -26,6 +26,7 @@ async function fixture() { mkdir(join(program, "rosters"), { recursive: true }), mkdir(join(program, "configs"), { recursive: true }), mkdir(join(program, "cases"), { recursive: true }), + mkdir(join(program, "campaigns"), { recursive: true }), ]); const config = { schema: "paperclip-runner/eval-config/v1", @@ -65,6 +66,19 @@ async function fixture() { join(program, "rosters/live-opencode-model.json"), JSON.stringify(roster), ), + writeFile( + join(program, "campaigns/live-direct-full.json"), + JSON.stringify({ + schema: "paperclip-runner/live-campaign/v1", + lanes: [ + { + id: "opencode-model", + executionClass: "default", + roster: "../rosters/live-opencode-model.json", + }, + ], + }), + ), ]); return { root, program, config, evalCase, roster }; } @@ -119,6 +133,83 @@ test("catalogs roster plus case cells and emits bounded balanced shards", async ); }); +test("all selects the maintained enabled campaign and explicit diagnostics can select disabled rosters", async () => { + const { root, program, config, evalCase } = await fixture(); + const disabledRoster = { + schema: "paperclip-runner/live-roster/v1", + id: "protocol-live-disabled-model", + model: config.model, + config: "../configs/live-opencode-model.json", + cases: [evalCase.id], + }; + await Promise.all([ + writeFile( + join(program, "rosters/live-disabled-model.json"), + JSON.stringify(disabledRoster), + ), + writeFile( + join(program, "campaigns/live-direct-full.json"), + JSON.stringify({ + schema: "paperclip-runner/live-campaign/v1", + lanes: [ + { + id: "opencode-model", + executionClass: "default", + roster: "../rosters/live-opencode-model.json", + }, + { + id: "disabled-model", + executionClass: "disabled", + roster: "../rosters/live-disabled-model.json", + }, + ], + }), + ), + ]); + + const maintained = await buildProtocolEvalCatalog({ + evalsRoot: root, + campaignId: "gha-42-1", + }); + assert.deepEqual( + maintained.rosters.map((roster) => roster.rosterId), + ["protocol-live-opencode-model"], + ); + + const diagnostic = await buildProtocolEvalCatalog({ + evalsRoot: root, + campaignId: "gha-42-2", + rosterSelection: "protocol-live-disabled-model", + }); + assert.deepEqual( + diagnostic.rosters.map((roster) => roster.rosterId), + ["protocol-live-disabled-model"], + ); +}); + +test("all fails closed when the maintained campaign is missing", async () => { + const { root, program } = await fixture(); + await rm(join(program, "campaigns/live-direct-full.json")); + + await assert.rejects( + buildProtocolEvalCatalog({ + evalsRoot: root, + campaignId: "gha-42-1", + }), + /ENOENT/, + ); + + const diagnostic = await buildProtocolEvalCatalog({ + evalsRoot: root, + campaignId: "gha-42-2", + rosterSelection: "protocol-live-opencode-model", + }); + assert.deepEqual( + diagnostic.rosters.map((roster) => roster.rosterId), + ["protocol-live-opencode-model"], + ); +}); + test("aggregates retained attempts and synthesizes missing cells as infrastructure", async () => { const { root, config, evalCase } = await fixture(); const catalog = await buildProtocolEvalCatalog({ diff --git a/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs b/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs index cc1be0437d..0569facb0a 100644 --- a/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs +++ b/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs @@ -105,6 +105,15 @@ test("resolves both repositories immutably and bounds total matrix concurrency", } assert.match(workflow, /matrix_0/u); assert.match(workflow, /matrix_1/u); + assert.match( + workflow, + /pnpm --filter @paperclipai\/paperclip-runner deploy --prod/u, + ); + assert.match( + workflow, + /--runner-cli runner-protocol-build\/extracted\/portable\/dist\/cli\/eval-session\.js/u, + ); + assert.doesNotMatch(workflow, /npm install --prefix/u); }); test("publishes only the separately sanitized Evalbook through trusted OIDC code", async () => { diff --git a/packages/paperclip-runner/src/backends/native-backend-factory.test.ts b/packages/paperclip-runner/src/backends/native-backend-factory.test.ts index 185805b736..545777babc 100644 --- a/packages/paperclip-runner/src/backends/native-backend-factory.test.ts +++ b/packages/paperclip-runner/src/backends/native-backend-factory.test.ts @@ -209,7 +209,7 @@ function managedExecution( contextBucket: "context-bucket", contextPrefix: "companies/company/profiles/profile", contextKmsKeyArn: "arn:aws:kms:us-east-1:123456789012:key/test", - qualificationRevision: "aws-agentcore-harness-v1", + qualificationRevision: "aws-agentcore-harness-context-v2", eventExpiryDays: 90, }, }, @@ -388,7 +388,7 @@ describe("native backend factory", () => { [ "aws_agentcore" as const, "aws_agentcore_harness_api", - "aws-agentcore-harness-v1", + "aws-agentcore-harness-context-v2", ], ])("routes %s through runnerd", async (kind, name, version) => { const backend = createNativeSessionBackend(managedExecution(kind), { diff --git a/packages/paperclip-runner/src/cli/eval-session-contract.test.ts b/packages/paperclip-runner/src/cli/eval-session-contract.test.ts index e2765f0307..322a4fbf70 100644 --- a/packages/paperclip-runner/src/cli/eval-session-contract.test.ts +++ b/packages/paperclip-runner/src/cli/eval-session-contract.test.ts @@ -1,5 +1,10 @@ +import { chmod, mkdtemp, readFile, rm, stat } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + import { describe, expect, it } from "vitest"; +import { parseNativeRuntimeContext } from "../contracts/runtime-context.js"; import type { CapabilityLiveSessionSnapshot } from "../live/live-session.js"; import { evalSessionUsage, @@ -7,7 +12,9 @@ import { } from "./eval-session-contract.js"; import { boundedEvalSessionUsage, + evalRuntimeSystemInstructions, evalSessionProviderVersion, + prepareEvalRuntimeContext, } from "./eval-session.js"; function request(overrides: Record = {}): unknown { @@ -55,6 +62,35 @@ function agentCoreProfile(overrides: Record = {}) { } describe("eval-session request contract", () => { + it("materializes a production-v3 runtime context for direct live providers", async () => { + const workspace = await mkdtemp(join(tmpdir(), "paperclip-eval-context-")); + let instructionRoot: string | null = null; + try { + const context = await prepareEvalRuntimeContext(workspace); + instructionRoot = context.instructions.bundle.rootPath; + expect(parseNativeRuntimeContext(context)).toEqual(context); + expect(context.skills).toEqual([]); + expect(context.mcp.assignmentSetId).toBe("paperclip-runner-direct-eval-v1"); + expect(context.instructions.entryPath).toBe("AGENTS.md"); + expect(await readFile( + join(context.instructions.bundle.rootPath, "AGENTS.md"), + "utf8", + )).toContain("Paperclip direct live evaluation"); + const systemInstructions = evalRuntimeSystemInstructions(context); + expect(systemInstructions).toContain("Paperclip direct live evaluation"); + expect(systemInstructions).toContain( + `Read-only instruction sibling root: ${context.instructions.bundle.rootPath}`, + ); + expect((await stat(context.instructions.bundle.rootPath)).mode & 0o777) + .toBe(0o555); + } finally { + if (instructionRoot !== null) { + await chmod(instructionRoot, 0o700).catch(() => undefined); + } + await rm(workspace, { recursive: true, force: true }); + } + }); + it("normalizes the current local live-session provider contract", () => { expect(parseEvalSessionRequest(request())).toMatchObject({ provider: "codex", diff --git a/packages/paperclip-runner/src/cli/eval-session.ts b/packages/paperclip-runner/src/cli/eval-session.ts index 609ec73090..220aee7adb 100644 --- a/packages/paperclip-runner/src/cli/eval-session.ts +++ b/packages/paperclip-runner/src/cli/eval-session.ts @@ -1,9 +1,19 @@ #!/usr/bin/env node import { createHash } from "node:crypto"; -import { readFile, writeFile } from "node:fs/promises"; +import { chmod, mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { + NATIVE_RUNTIME_ASSET_SCHEMA, + PAPERCLIP_EXECUTION_PROMPT, + PAPERCLIP_EXECUTION_PROMPT_REVISION, + canonicalNativeRuntimeContextDigest, + composeNativeSystemInstructions, + nativeRuntimePromptDigest, + parseNativeRuntimeContext, + type NativeRuntimeContextSnapshot, +} from "../contracts/runtime-context.js"; import { projectCapabilityDevtools } from "../devtools/index.js"; import { resolveQualifiedAcpxProfile } from "../drivers/acpx/qualified-profiles.js"; import { PAPERCLIP_RUNNER_BUILD_METADATA } from "../evals/build-metadata.js"; @@ -53,6 +63,94 @@ async function sha256(path: string): Promise { return createHash("sha256").update(await readFile(path)).digest("hex"); } +const EVAL_RUNTIME_INSTRUCTIONS = [ + "# Paperclip direct live evaluation", + "", + "Use the provided Paperclip semantic tools to inspect and act on the assigned task.", + "Treat the seeded control-plane state as authoritative and keep every action within the requested scope.", + "", +].join("\n"); + +/** Materializes the minimal immutable v3 context used by production-native providers. */ +export async function prepareEvalRuntimeContext( + workingDirectory: string, +): Promise { + const contextRoot = await mkdtemp( + resolve(workingDirectory, ".paperclip-eval-runtime-context-"), + ); + const instructionRoot = resolve(contextRoot, "instructions"); + const entryPath = "AGENTS.md"; + const entry = Buffer.from(EVAL_RUNTIME_INSTRUCTIONS); + const entryDigest = createHash("sha256").update(entry).digest("hex"); + const manifestFiles = [{ + path: entryPath, + sha256: entryDigest, + mode: 0o444, + size: entry.byteLength, + }]; + const assetDigest = createHash("sha256") + .update(JSON.stringify(manifestFiles)) + .digest("hex"); + const manifestText = `${JSON.stringify({ + schema: "paperclip.runtime-asset-manifest.v1", + digest: assetDigest, + fileCount: manifestFiles.length, + totalBytes: entry.byteLength, + files: manifestFiles, + })}\n`; + const manifestDigest = createHash("sha256") + .update(manifestText) + .digest("hex"); + await mkdir(instructionRoot, { recursive: true, mode: 0o700 }); + const entryFile = resolve(instructionRoot, entryPath); + await writeFile(entryFile, entry, { flag: "wx", mode: 0o444 }); + await chmod(entryFile, 0o444); + await chmod(instructionRoot, 0o555); + + const semanticCatalogDigest = + PAPERCLIP_RUNNER_BUILD_METADATA.semanticCatalog.sha256.replace( + /^sha256:/, + "", + ); + const context = { + prompt: { + revision: PAPERCLIP_EXECUTION_PROMPT_REVISION, + text: PAPERCLIP_EXECUTION_PROMPT, + digest: nativeRuntimePromptDigest(), + }, + instructions: { + entryPath, + bundle: { + schema: NATIVE_RUNTIME_ASSET_SCHEMA, + digest: assetDigest, + manifestDigest, + rootPath: instructionRoot, + fileCount: 1, + totalBytes: entry.byteLength, + }, + }, + skills: [], + mcp: { + assignmentSetId: "paperclip-runner-direct-eval-v1", + digest: semanticCatalogDigest, + bindingId: null, + }, + } satisfies Omit; + return parseNativeRuntimeContext({ + ...context, + aggregateDigest: canonicalNativeRuntimeContextDigest(context), + }); +} + +export function evalRuntimeSystemInstructions( + runtimeContext: NativeRuntimeContextSnapshot, +): string { + return composeNativeSystemInstructions( + runtimeContext, + EVAL_RUNTIME_INSTRUCTIONS, + ); +} + export function evalSessionProviderVersion( request: EvalSessionRequest, ): string | null { @@ -189,10 +287,15 @@ export async function runEvalSessionCli( const requestedDriver = request.driver ?? expectedEvalSessionDriver(requestedProvider); const requestedProviderVersion = evalSessionProviderVersion(request); + const runtimeContext = await prepareEvalRuntimeContext( + resolve(request.session.workingDirectory ?? process.cwd()), + ); const service = options.serviceFactory?.(runnerdPath) ?? new CapabilityLiveSessionService({ transportOptions: { runnerBinary: runnerdPath, + runtimeContext, + baseInstructions: evalRuntimeSystemInstructions(runtimeContext), // The transport performs the provider-specific allowlisting. Supplying // the source environment here is still required: without it the // isolated Codex home has no credential source and runnerd receives no diff --git a/packages/paperclip-runner/src/contracts/native-execution.test.ts b/packages/paperclip-runner/src/contracts/native-execution.test.ts index a45c6a664c..5e0c90a266 100644 --- a/packages/paperclip-runner/src/contracts/native-execution.test.ts +++ b/packages/paperclip-runner/src/contracts/native-execution.test.ts @@ -235,7 +235,7 @@ describe("NativeExecutionInputV1", () => { contextBucket: "paperclip-agentcore-context", contextPrefix: "paperclip/runtime", contextKmsKeyArn: "arn:aws:kms:us-east-1:123456789012:key/test", - qualificationRevision: "aws-agentcore-harness-v1", + qualificationRevision: "aws-agentcore-harness-context-v2", eventExpiryDays: 90, }, maxEstimatedSessionCostUsd: 1, diff --git a/packages/paperclip-runner/src/live/live-session.test.ts b/packages/paperclip-runner/src/live/live-session.test.ts index b569868318..da740a6d1a 100644 --- a/packages/paperclip-runner/src/live/live-session.test.ts +++ b/packages/paperclip-runner/src/live/live-session.test.ts @@ -662,6 +662,27 @@ describe("Capability live runnerd and Codex session", () => { await service.shutdown(session.id); }); + it("passes caller-supplied native system instructions to the provider", async () => { + const state = providerState(); + const service = new CapabilityLiveSessionService({ + transportFactory: fakeTransportFactory(state), + transportOptions: { + baseInstructions: + "Native instructions\n\nRead-only instruction sibling root: /runtime/instructions", + }, + }); + const session = await service.create(); + + expect( + state.transports[0]?.requests.find( + (request) => request.method === "thread/start", + )?.params.baseInstructions, + ).toBe( + "Native instructions\n\nRead-only instruction sibling root: /runtime/instructions", + ); + await service.shutdown(session.id); + }); + it("attributes Claude Managed sessions to the pinned immutable Agent version", async () => { const state = providerState(); const managedProfiles: Array | undefined> = []; diff --git a/packages/paperclip-runner/src/live/live-session.ts b/packages/paperclip-runner/src/live/live-session.ts index 4f7b8c0f6b..88ded8caf3 100644 --- a/packages/paperclip-runner/src/live/live-session.ts +++ b/packages/paperclip-runner/src/live/live-session.ts @@ -2382,7 +2382,8 @@ export class CapabilityLiveSession { config: createSkilllessCodexThreadConfig(this.#config.workingDirectory), permissions: CODEX_PERMISSION_PROFILE, runtimeWorkspaceRoots: [this.#config.workingDirectory], - baseInstructions: LIVE_BASE_INSTRUCTIONS, + baseInstructions: + this.#transportOptions.baseInstructions ?? LIVE_BASE_INSTRUCTIONS, persistExtendedHistory: true, }); const resumedThread = record(resumed.thread); @@ -2407,7 +2408,8 @@ export class CapabilityLiveSession { permissions: CODEX_PERMISSION_PROFILE, runtimeWorkspaceRoots: [this.#config.workingDirectory], approvalPolicy: "never", - baseInstructions: LIVE_BASE_INSTRUCTIONS, + baseInstructions: + this.#transportOptions.baseInstructions ?? LIVE_BASE_INSTRUCTIONS, completionContract: LIVE_COMPLETION_CONTRACT, dynamicTools: [ ...tools.map(dynamicToolSpec), diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index 00ba33f701..eb07c5bfa4 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -634,6 +634,66 @@ it("derives the ACPX package authority only from the verified dist/cli layout", ).toThrow("ACPX sidecar must use the provider package dist/cli layout"); }); +it("keeps a self-rooted pnpm deployment inside its dependency authority", async () => { + const deploymentRoot = await mkdtemp( + join(tmpdir(), "paperclip-deployed-provider-root-"), + ); + const deployedPackageRoot = deploymentRoot; + await mkdir(join(deployedPackageRoot, "dist", "cli"), { recursive: true }); + await mkdir(join(deploymentRoot, "node_modules", ".pnpm"), { + recursive: true, + }); + try { + expect( + runnerdLaunchProfileInternals.acpxProviderPackageAuthority( + join( + deployedPackageRoot, + "dist", + "cli", + "acpx-runtime-sidecar.cjs", + ), + deployedPackageRoot, + ), + ).toEqual({ + root: deploymentRoot, + manifest: join(deployedPackageRoot, "package.json"), + }); + } finally { + await rm(deploymentRoot, { recursive: true, force: true }); + } +}); + +it("keeps a scoped npm-installed package inside its portable dependency root", async () => { + const deploymentRoot = await mkdtemp( + join(tmpdir(), "paperclip-npm-provider-root-"), + ); + const deployedPackageRoot = join( + deploymentRoot, + "node_modules", + "@paperclipai", + "paperclip-runner", + ); + await mkdir(join(deployedPackageRoot, "dist", "cli"), { recursive: true }); + try { + expect( + runnerdLaunchProfileInternals.acpxProviderPackageAuthority( + join( + deployedPackageRoot, + "dist", + "cli", + "acpx-runtime-sidecar.cjs", + ), + deployedPackageRoot, + ), + ).toEqual({ + root: deploymentRoot, + manifest: join(deployedPackageRoot, "package.json"), + }); + } finally { + await rm(deploymentRoot, { recursive: true, force: true }); + } +}); + it("requires a provider-pack authority for remote ACPX artifact hashes", () => { expect(() => runnerdLaunchProfileInternals.acpxRunnerLaunchProfile( diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index 497185158c..a28105aa37 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -937,6 +937,8 @@ export interface CapabilityRunnerdCodexTransportOptions { opencodeProxySha256?: string; opencodeRuntimeDirectory?: string; environment?: NodeJS.ProcessEnv; + /** Provider system instructions supplied by a native execution caller. */ + baseInstructions?: string; closeGraceMs?: number; onDiagnostic?: (message: string) => void; onEvidence?: (evidence: Readonly) => void; @@ -1549,7 +1551,10 @@ function resolveBuildOwnedCliArtifact( ); } -function acpxProviderPackageAuthority(sidecarScript: string): { +function acpxProviderPackageAuthority( + sidecarScript: string, + ownerPackageRoot = packageRoot, +): { root: string; manifest: string; } { @@ -1564,12 +1569,25 @@ function acpxProviderPackageAuthority(sidecarScript: string): { ); } const sidecarPackageRoot = resolve(cliDirectory, "../.."); - // A local source build consumes pnpm's workspace-owned node_modules tree. - // A deployed provider pack owns a closed node_modules tree at its own root. - return sidecarPackageRoot === packageRoot + // A local source build lives at /packages/paperclip-runner and + // resolves dependencies from /node_modules. `pnpm deploy` makes + // the package itself the deployment root and owns /node_modules/.pnpm. + // The older npm-installed portable shape nests the scoped package at + // /node_modules/@paperclipai/paperclip-runner. The verifier always + // receives the directory that owns node_modules, regardless of which + // portable shape launched the already-authenticated sidecar. + const sourceDependencyRoot = resolve(ownerPackageRoot, "../.."); + const localDependencyRoot = existsSync( + resolve(ownerPackageRoot, "node_modules", ".pnpm"), + ) + ? ownerPackageRoot + : basename(sourceDependencyRoot) === "node_modules" + ? resolve(sourceDependencyRoot, "..") + : sourceDependencyRoot; + return sidecarPackageRoot === ownerPackageRoot ? { - root: resolve(packageRoot, "../.."), - manifest: resolve(packageRoot, "package.json"), + root: localDependencyRoot, + manifest: resolve(ownerPackageRoot, "package.json"), } : { root: sidecarPackageRoot, diff --git a/server/src/__tests__/managed-agent-profile-routes-authz.test.ts b/server/src/__tests__/managed-agent-profile-routes-authz.test.ts index 540f3f5b16..a57c7182d9 100644 --- a/server/src/__tests__/managed-agent-profile-routes-authz.test.ts +++ b/server/src/__tests__/managed-agent-profile-routes-authz.test.ts @@ -123,7 +123,7 @@ describe("managed provider profile route authorization", () => { contextBucket: "paperclip-runner-context", contextPrefix: "profiles/example", contextKmsKeyArn: "arn:aws:kms:us-east-1:123456789012:key/example", - qualificationRevision: "aws-agentcore-harness-v1", + qualificationRevision: "aws-agentcore-harness-context-v2", defaultModel: "global.anthropic.claude-sonnet-4-6", eventExpiryDays: 90, }, diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.ts index dda1c36904..56da28aca7 100644 --- a/server/src/routes/openapi.ts +++ b/server/src/routes/openapi.ts @@ -4626,7 +4626,7 @@ registry.registerPath({ configuration: z.record(z.string(), z.unknown()), enabled: z.boolean().optional(), retentionAcknowledged: z.boolean().optional(), - qualification: z.object({ suite: z.literal("aws-agentcore-harness-v1") }).strict().optional(), + qualification: z.object({ suite: z.literal("aws-agentcore-harness-context-v2") }).strict().optional(), })), }, responses: { diff --git a/server/src/services/heartbeat-runner-provider-config.test.ts b/server/src/services/heartbeat-runner-provider-config.test.ts index 344ca5c91b..72b194cac7 100644 --- a/server/src/services/heartbeat-runner-provider-config.test.ts +++ b/server/src/services/heartbeat-runner-provider-config.test.ts @@ -87,7 +87,7 @@ describe("Paperclip Runner native provider configuration", () => { contextBucket: "paperclip-context", contextPrefix: "runner/", contextKmsKeyArn: "arn:aws:kms:us-east-1:123456789012:key/key-1", - qualificationRevision: "aws-agentcore-harness-v1", + qualificationRevision: "aws-agentcore-harness-context-v2", eventExpiryDays: 90, }; const stored = { @@ -224,7 +224,7 @@ describe("Paperclip Runner native provider configuration", () => { contextBucket: "paperclip-context", contextPrefix: "runner/", contextKmsKeyArn: "arn:aws:kms:us-east-1:123456789012:key/key-1", - qualificationRevision: "aws-agentcore-harness-v1", + qualificationRevision: "aws-agentcore-harness-context-v2", defaultModel: "global.anthropic.claude-sonnet-4-6", eventExpiryDays: 90, defaultMaxEstimatedSessionCostUsd: 1.25, @@ -278,7 +278,7 @@ describe("Paperclip Runner native provider configuration", () => { contextBucket: "paperclip-context", contextPrefix: "runner/", contextKmsKeyArn: "arn:aws:kms:us-east-1:123456789012:key/key-1", - qualificationRevision: "aws-agentcore-harness-v1", + qualificationRevision: "aws-agentcore-harness-context-v2", defaultModel: "global.anthropic.claude-sonnet-4-6", eventExpiryDays: 90, defaultMaxEstimatedSessionCostUsd: 1.25, @@ -352,7 +352,7 @@ describe("Paperclip Runner native provider configuration", () => { contextBucket: "paperclip-context", contextPrefix: "runner/", contextKmsKeyArn: "arn:aws:kms:us-east-1:123456789012:key/key-1", - qualificationRevision: "aws-agentcore-harness-v1", + qualificationRevision: "aws-agentcore-harness-context-v2", defaultModel: "global.anthropic.claude-sonnet-4-6", eventExpiryDays: 90, defaultMaxEstimatedSessionCostUsd: 1.25, diff --git a/server/src/services/native-runtime/native-session-resume.test.ts b/server/src/services/native-runtime/native-session-resume.test.ts index 31aee96413..6e4c4c5bb2 100644 --- a/server/src/services/native-runtime/native-session-resume.test.ts +++ b/server/src/services/native-runtime/native-session-resume.test.ts @@ -301,7 +301,7 @@ describe("buildNativeExecutionInput wake projection", () => { contextBucket: "paperclip-context", contextPrefix: "runner/", contextKmsKeyArn: "arn:aws:kms:us-east-1:123456789012:key/key-1", - qualificationRevision: "aws-agentcore-harness-v1", + qualificationRevision: "aws-agentcore-harness-context-v2", eventExpiryDays: 90, }, maxEstimatedSessionCostUsd: 1.25, diff --git a/server/src/services/provider-profile-qualification.ts b/server/src/services/provider-profile-qualification.ts index e26d52d285..3a691e7024 100644 --- a/server/src/services/provider-profile-qualification.ts +++ b/server/src/services/provider-profile-qualification.ts @@ -11,7 +11,8 @@ export const CLAUDE_MANAGED_QUALIFICATION = { } as const; export const CLAUDE_MANAGED_QUALIFIED_MODEL = "claude-sonnet-5" as const; -export const AGENTCORE_QUALIFICATION_SUITE = "aws-agentcore-harness-v1" as const; +export const AGENTCORE_QUALIFICATION_SUITE = + "aws-agentcore-harness-context-v2" as const; export const AGENTCORE_QUALIFIED_MODEL = "global.anthropic.claude-sonnet-4-6" as const; const REVISION_PREFIX = "sha256:"; diff --git a/server/src/services/remote-agent-profiles.test.ts b/server/src/services/remote-agent-profiles.test.ts index 3dcd77a129..b2841709b8 100644 --- a/server/src/services/remote-agent-profiles.test.ts +++ b/server/src/services/remote-agent-profiles.test.ts @@ -28,7 +28,7 @@ const AWS_CONFIGURATION = { contextBucket: "paperclip-runner-context", contextPrefix: "profiles/example", contextKmsKeyArn: "arn:aws:kms:us-east-1:123456789012:key/example", - qualificationRevision: "aws-agentcore-harness-v1", + qualificationRevision: "aws-agentcore-harness-context-v2", defaultModel: "global.anthropic.claude-sonnet-4-6", eventExpiryDays: 90, defaultMaxEstimatedSessionCostUsd: 1, @@ -44,7 +44,7 @@ function remoteInput( configuration: { ...AWS_CONFIGURATION }, enabled: false, retentionAcknowledged: false, - qualification: { suite: "aws-agentcore-harness-v1" }, + qualification: { suite: "aws-agentcore-harness-context-v2" }, ...overrides, }; } @@ -362,7 +362,7 @@ describe("remote agent profile metadata validation", () => { }); it("does not allow a qualified AgentCore profile key to be repointed", async () => { - const qualification = { suite: "aws-agentcore-harness-v1" }; + const qualification = { suite: "aws-agentcore-harness-context-v2" }; const existing = { id: "30000000-0000-4000-8000-000000000003", companyId: COMPANY_ID, @@ -506,7 +506,7 @@ describe("remote agent profile metadata validation", () => { qualifiedRevision: existingManaged.qualifiedRevision, }); - const awsQualification = { suite: "aws-agentcore-harness-v1" }; + const awsQualification = { suite: "aws-agentcore-harness-context-v2" }; const existingRemote = { id: "30000000-0000-4000-8000-000000000005", companyId: COMPANY_ID, @@ -547,7 +547,7 @@ describe("remote agent profile metadata validation", () => { }); it("rejects runtime use when stored identity drifts from the qualified revision", async () => { - const qualification = { suite: "aws-agentcore-harness-v1" }; + const qualification = { suite: "aws-agentcore-harness-context-v2" }; const qualifiedRevision = computeRemoteAgentProfileRevision({ service: "aws_bedrock_agentcore_harness", configuration: AWS_CONFIGURATION,