From ef53c45b29a37a34e30a28b4cc67cc668e1f1bb6 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 6 Sep 2026 10:29:14 -0500 Subject: [PATCH] fix(runner): repair direct live provider bootstrap Co-Authored-By: Paperclip --- .../workflows/runner-protocol-live-evals.yml | 8 ++++-- .../docs/runner-protocol-live-evals.md | 11 +++++++- .../runner-core/src/aws_agentcore_provider.rs | 2 +- .../src/managed_provider_backend.rs | 2 +- .../tests/native_provider_backend.rs | 2 +- ...r-protocol-eval-workflow-security.test.mjs | 9 ++++++ .../backends/native-backend-factory.test.ts | 4 +-- .../src/contracts/native-execution.test.ts | 2 +- .../src/live/runnerd-codex-transport.test.ts | 28 +++++++++++++++++++ .../src/live/runnerd-codex-transport.ts | 20 ++++++++++--- ...managed-agent-profile-routes-authz.test.ts | 2 +- server/src/routes/openapi.ts | 2 +- .../heartbeat-runner-provider-config.test.ts | 8 +++--- .../native-session-resume.test.ts | 2 +- .../provider-profile-qualification.ts | 3 +- .../services/remote-agent-profiles.test.ts | 10 +++---- 16 files changed, 88 insertions(+), 27 deletions(-) diff --git a/.github/workflows/runner-protocol-live-evals.yml b/.github/workflows/runner-protocol-live-evals.yml index 060a7309bd..f396e8a975 100644 --- a/.github/workflows/runner-protocol-live-evals.yml +++ b/.github/workflows/runner-protocol-live-evals.yml @@ -268,11 +268,13 @@ jobs: --pack-destination "$RUNNER_TEMP/runner-protocol-build/package" package="$(find "$RUNNER_TEMP/runner-protocol-build/package" -maxdepth 1 -type f -name '*.tgz' -print -quit)" test -f "$package" - npm install --prefix "$RUNNER_TEMP/runner-protocol-build/portable" --omit=dev "$package" + pnpm --filter @paperclipai/paperclip-runner deploy --prod \ + "$RUNNER_TEMP/runner-protocol-build/portable" cp "$package" "$RUNNER_TEMP/runner-protocol-build/paperclip-runner.tgz" cp packages/paperclip-runner/runner/target/debug/paperclip-runnerd "$RUNNER_TEMP/runner-protocol-build/paperclip-runnerd" cp -R packages/paperclip-runner/dist-issue-thread "$RUNNER_TEMP/runner-protocol-build/dist-issue-thread" - test -f "$RUNNER_TEMP/runner-protocol-build/portable/node_modules/@paperclipai/paperclip-runner/dist/cli/eval-session.js" + test -f "$RUNNER_TEMP/runner-protocol-build/portable/dist/cli/eval-session.js" + test -d "$RUNNER_TEMP/runner-protocol-build/portable/node_modules/.pnpm" test -x "$RUNNER_TEMP/runner-protocol-build/paperclip-runnerd" tar --create --gzip --file runner-protocol-build.tar.gz -C "$RUNNER_TEMP/runner-protocol-build" . sha256sum runner-protocol-build.tar.gz > runner-protocol-build.tar.gz.sha256 @@ -433,7 +435,7 @@ jobs: python3 .paperclip-evals/evals/paperclip-runner/tools/run_live_roster.py run \ --roster ".paperclip-evals/evals/paperclip-runner/rosters/$ROSTER_FILE" \ --case "$CASE_ID" \ - --runner-cli runner-protocol-build/extracted/portable/node_modules/@paperclipai/paperclip-runner/dist/cli/eval-session.js \ + --runner-cli runner-protocol-build/extracted/portable/dist/cli/eval-session.js \ --runner-package runner-protocol-build/extracted/paperclip-runner.tgz \ --runnerd runner-protocol-build/extracted/paperclip-runnerd \ --runs-root cell-output/runs \ diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index 2cdc74774b..8c4761e980 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -43,6 +43,13 @@ the native daemon, provider dependencies, and the attempt viewer are built once and reused by every cell. Because the complete suite requires two matrix shards, this workflow accepts a shared concurrency ceiling from 2 through 100. +The reused provider runtime is created with `pnpm deploy --prod` from the +frozen workspace lock. That preserves the repository's qualified dependency +versions and patched ACP server bytes. Do not replace this step with a fresh +`npm install` of the packed Runner tarball: npm cannot apply the workspace's +`patchedDependencies`, so the resulting ACPX executables no longer match their +qualified digests. + `all` is intentionally literal. A disabled driver, missing remote profile, or unavailable provider is retained as an infrastructure result; it is not silently omitted. In particular, the ACPX Pi roster remains visible while Pi @@ -70,7 +77,9 @@ Claude Managed also requires the four nonsecret nonsecret `PAPERCLIP_AWS_AGENTCORE_*` profile variables, including `PAPERCLIP_AWS_AGENTCORE_EXECUTION_ROLE_ARN` and the immutable `PAPERCLIP_AWS_AGENTCORE_QUALIFICATION_REVISION`; the eval fails closed when -that deployed revision differs from the pinned roster config. The workflow +that deployed revision differs from the pinned roster config. The currently +qualified context-aware harness revision is +`aws-agentcore-harness-context-v2`. The workflow writes the GitHub OIDC token to a mode-`0600` file and never forwards long-lived AWS access keys. Provision the AgentCore stack with `--github-oidc-provider-arn` so that scoped diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/aws_agentcore_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/src/aws_agentcore_provider.rs index 0536b243a7..a1127f5b5c 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/aws_agentcore_provider.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/aws_agentcore_provider.rs @@ -2211,7 +2211,7 @@ mod tests { context_bucket: "paperclip-context-test".to_owned(), context_prefix: "companies/company-test/profiles/profile-test".to_owned(), context_kms_key_arn: "arn:aws:kms:us-east-1:123456789012:key/test".to_owned(), - qualification_revision: "aws-agentcore-harness-v1".to_owned(), + qualification_revision: "aws-agentcore-harness-context-v2".to_owned(), event_expiry_days: 90, max_estimated_session_cost_usd: 1.0, max_iterations: 8, diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs index 0370c7bb90..2b8212c794 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs @@ -43,7 +43,7 @@ const MAX_INSTRUCTIONS_BYTES: usize = 1024 * 1024; const QUALIFIED_CLAUDE_MODEL: &str = "claude-sonnet-5"; const QUALIFIED_CLAUDE_BETA: &str = "managed-agents-2026-04-01"; const QUALIFIED_AGENTCORE_MODEL: &str = "global.anthropic.claude-sonnet-4-6"; -const QUALIFIED_AGENTCORE_REVISION: &str = "aws-agentcore-harness-v1"; +const QUALIFIED_AGENTCORE_REVISION: &str = "aws-agentcore-harness-context-v2"; fn initial_event_sequence() -> u64 { 1 diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs index 101e049f6d..ba36834ab7 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs @@ -283,7 +283,7 @@ fn managed_prepare_payload(kind: &str) -> Value { "contextBucket": "context-bucket", "contextPrefix": "companies/company/profiles/profile", "contextKmsKeyArn": "arn:aws:kms:us-east-1:123456789012:key/test", - "qualificationRevision": "aws-agentcore-harness-v1", + "qualificationRevision": "aws-agentcore-harness-context-v2", "eventExpiryDays": 90, "maxEstimatedSessionCostUsd": 1.0, "maxIterations": 8, diff --git a/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs b/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs index cc1be0437d..0569facb0a 100644 --- a/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs +++ b/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs @@ -105,6 +105,15 @@ test("resolves both repositories immutably and bounds total matrix concurrency", } assert.match(workflow, /matrix_0/u); assert.match(workflow, /matrix_1/u); + assert.match( + workflow, + /pnpm --filter @paperclipai\/paperclip-runner deploy --prod/u, + ); + assert.match( + workflow, + /--runner-cli runner-protocol-build\/extracted\/portable\/dist\/cli\/eval-session\.js/u, + ); + assert.doesNotMatch(workflow, /npm install --prefix/u); }); test("publishes only the separately sanitized Evalbook through trusted OIDC code", async () => { diff --git a/packages/paperclip-runner/src/backends/native-backend-factory.test.ts b/packages/paperclip-runner/src/backends/native-backend-factory.test.ts index 185805b736..545777babc 100644 --- a/packages/paperclip-runner/src/backends/native-backend-factory.test.ts +++ b/packages/paperclip-runner/src/backends/native-backend-factory.test.ts @@ -209,7 +209,7 @@ function managedExecution( contextBucket: "context-bucket", contextPrefix: "companies/company/profiles/profile", contextKmsKeyArn: "arn:aws:kms:us-east-1:123456789012:key/test", - qualificationRevision: "aws-agentcore-harness-v1", + qualificationRevision: "aws-agentcore-harness-context-v2", eventExpiryDays: 90, }, }, @@ -388,7 +388,7 @@ describe("native backend factory", () => { [ "aws_agentcore" as const, "aws_agentcore_harness_api", - "aws-agentcore-harness-v1", + "aws-agentcore-harness-context-v2", ], ])("routes %s through runnerd", async (kind, name, version) => { const backend = createNativeSessionBackend(managedExecution(kind), { diff --git a/packages/paperclip-runner/src/contracts/native-execution.test.ts b/packages/paperclip-runner/src/contracts/native-execution.test.ts index a45c6a664c..5e0c90a266 100644 --- a/packages/paperclip-runner/src/contracts/native-execution.test.ts +++ b/packages/paperclip-runner/src/contracts/native-execution.test.ts @@ -235,7 +235,7 @@ describe("NativeExecutionInputV1", () => { contextBucket: "paperclip-agentcore-context", contextPrefix: "paperclip/runtime", contextKmsKeyArn: "arn:aws:kms:us-east-1:123456789012:key/test", - qualificationRevision: "aws-agentcore-harness-v1", + qualificationRevision: "aws-agentcore-harness-context-v2", eventExpiryDays: 90, }, maxEstimatedSessionCostUsd: 1, diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index 00ba33f701..c0d62a561d 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -634,6 +634,34 @@ it("derives the ACPX package authority only from the verified dist/cli layout", ).toThrow("ACPX sidecar must use the provider package dist/cli layout"); }); +it("keeps a deployed ACPX package inside its pnpm-owned dependency root", async () => { + const deployedPackageRoot = await mkdtemp( + join(tmpdir(), "paperclip-deployed-provider-package-"), + ); + await mkdir(join(deployedPackageRoot, "dist", "cli"), { recursive: true }); + await mkdir(join(deployedPackageRoot, "node_modules", ".pnpm"), { + recursive: true, + }); + try { + expect( + runnerdLaunchProfileInternals.acpxProviderPackageAuthority( + join( + deployedPackageRoot, + "dist", + "cli", + "acpx-runtime-sidecar.cjs", + ), + deployedPackageRoot, + ), + ).toEqual({ + root: deployedPackageRoot, + manifest: join(deployedPackageRoot, "package.json"), + }); + } finally { + await rm(deployedPackageRoot, { recursive: true, force: true }); + } +}); + it("requires a provider-pack authority for remote ACPX artifact hashes", () => { expect(() => runnerdLaunchProfileInternals.acpxRunnerLaunchProfile( diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index 497185158c..ec86cb6166 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -1549,7 +1549,10 @@ function resolveBuildOwnedCliArtifact( ); } -function acpxProviderPackageAuthority(sidecarScript: string): { +function acpxProviderPackageAuthority( + sidecarScript: string, + ownerPackageRoot = packageRoot, +): { root: string; manifest: string; } { @@ -1566,10 +1569,19 @@ function acpxProviderPackageAuthority(sidecarScript: string): { const sidecarPackageRoot = resolve(cliDirectory, "../.."); // A local source build consumes pnpm's workspace-owned node_modules tree. // A deployed provider pack owns a closed node_modules tree at its own root. - return sidecarPackageRoot === packageRoot + // Source builds resolve provider dependencies from the monorepo root. A + // `pnpm deploy`, however, owns a complete virtual store below the deployed + // package root and must not escape to its caller's filesystem. This marker + // is generated by pnpm itself and keeps the two layouts unambiguous. + const localDependencyRoot = existsSync( + resolve(ownerPackageRoot, "node_modules", ".pnpm"), + ) + ? ownerPackageRoot + : resolve(ownerPackageRoot, "../.."); + return sidecarPackageRoot === ownerPackageRoot ? { - root: resolve(packageRoot, "../.."), - manifest: resolve(packageRoot, "package.json"), + root: localDependencyRoot, + manifest: resolve(ownerPackageRoot, "package.json"), } : { root: sidecarPackageRoot, diff --git a/server/src/__tests__/managed-agent-profile-routes-authz.test.ts b/server/src/__tests__/managed-agent-profile-routes-authz.test.ts index 540f3f5b16..a57c7182d9 100644 --- a/server/src/__tests__/managed-agent-profile-routes-authz.test.ts +++ b/server/src/__tests__/managed-agent-profile-routes-authz.test.ts @@ -123,7 +123,7 @@ describe("managed provider profile route authorization", () => { contextBucket: "paperclip-runner-context", contextPrefix: "profiles/example", contextKmsKeyArn: "arn:aws:kms:us-east-1:123456789012:key/example", - qualificationRevision: "aws-agentcore-harness-v1", + qualificationRevision: "aws-agentcore-harness-context-v2", defaultModel: "global.anthropic.claude-sonnet-4-6", eventExpiryDays: 90, }, diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.ts index dda1c36904..56da28aca7 100644 --- a/server/src/routes/openapi.ts +++ b/server/src/routes/openapi.ts @@ -4626,7 +4626,7 @@ registry.registerPath({ configuration: z.record(z.string(), z.unknown()), enabled: z.boolean().optional(), retentionAcknowledged: z.boolean().optional(), - qualification: z.object({ suite: z.literal("aws-agentcore-harness-v1") }).strict().optional(), + qualification: z.object({ suite: z.literal("aws-agentcore-harness-context-v2") }).strict().optional(), })), }, responses: { diff --git a/server/src/services/heartbeat-runner-provider-config.test.ts b/server/src/services/heartbeat-runner-provider-config.test.ts index 344ca5c91b..72b194cac7 100644 --- a/server/src/services/heartbeat-runner-provider-config.test.ts +++ b/server/src/services/heartbeat-runner-provider-config.test.ts @@ -87,7 +87,7 @@ describe("Paperclip Runner native provider configuration", () => { contextBucket: "paperclip-context", contextPrefix: "runner/", contextKmsKeyArn: "arn:aws:kms:us-east-1:123456789012:key/key-1", - qualificationRevision: "aws-agentcore-harness-v1", + qualificationRevision: "aws-agentcore-harness-context-v2", eventExpiryDays: 90, }; const stored = { @@ -224,7 +224,7 @@ describe("Paperclip Runner native provider configuration", () => { contextBucket: "paperclip-context", contextPrefix: "runner/", contextKmsKeyArn: "arn:aws:kms:us-east-1:123456789012:key/key-1", - qualificationRevision: "aws-agentcore-harness-v1", + qualificationRevision: "aws-agentcore-harness-context-v2", defaultModel: "global.anthropic.claude-sonnet-4-6", eventExpiryDays: 90, defaultMaxEstimatedSessionCostUsd: 1.25, @@ -278,7 +278,7 @@ describe("Paperclip Runner native provider configuration", () => { contextBucket: "paperclip-context", contextPrefix: "runner/", contextKmsKeyArn: "arn:aws:kms:us-east-1:123456789012:key/key-1", - qualificationRevision: "aws-agentcore-harness-v1", + qualificationRevision: "aws-agentcore-harness-context-v2", defaultModel: "global.anthropic.claude-sonnet-4-6", eventExpiryDays: 90, defaultMaxEstimatedSessionCostUsd: 1.25, @@ -352,7 +352,7 @@ describe("Paperclip Runner native provider configuration", () => { contextBucket: "paperclip-context", contextPrefix: "runner/", contextKmsKeyArn: "arn:aws:kms:us-east-1:123456789012:key/key-1", - qualificationRevision: "aws-agentcore-harness-v1", + qualificationRevision: "aws-agentcore-harness-context-v2", defaultModel: "global.anthropic.claude-sonnet-4-6", eventExpiryDays: 90, defaultMaxEstimatedSessionCostUsd: 1.25, diff --git a/server/src/services/native-runtime/native-session-resume.test.ts b/server/src/services/native-runtime/native-session-resume.test.ts index 31aee96413..6e4c4c5bb2 100644 --- a/server/src/services/native-runtime/native-session-resume.test.ts +++ b/server/src/services/native-runtime/native-session-resume.test.ts @@ -301,7 +301,7 @@ describe("buildNativeExecutionInput wake projection", () => { contextBucket: "paperclip-context", contextPrefix: "runner/", contextKmsKeyArn: "arn:aws:kms:us-east-1:123456789012:key/key-1", - qualificationRevision: "aws-agentcore-harness-v1", + qualificationRevision: "aws-agentcore-harness-context-v2", eventExpiryDays: 90, }, maxEstimatedSessionCostUsd: 1.25, diff --git a/server/src/services/provider-profile-qualification.ts b/server/src/services/provider-profile-qualification.ts index e26d52d285..3a691e7024 100644 --- a/server/src/services/provider-profile-qualification.ts +++ b/server/src/services/provider-profile-qualification.ts @@ -11,7 +11,8 @@ export const CLAUDE_MANAGED_QUALIFICATION = { } as const; export const CLAUDE_MANAGED_QUALIFIED_MODEL = "claude-sonnet-5" as const; -export const AGENTCORE_QUALIFICATION_SUITE = "aws-agentcore-harness-v1" as const; +export const AGENTCORE_QUALIFICATION_SUITE = + "aws-agentcore-harness-context-v2" as const; export const AGENTCORE_QUALIFIED_MODEL = "global.anthropic.claude-sonnet-4-6" as const; const REVISION_PREFIX = "sha256:"; diff --git a/server/src/services/remote-agent-profiles.test.ts b/server/src/services/remote-agent-profiles.test.ts index 3dcd77a129..b2841709b8 100644 --- a/server/src/services/remote-agent-profiles.test.ts +++ b/server/src/services/remote-agent-profiles.test.ts @@ -28,7 +28,7 @@ const AWS_CONFIGURATION = { contextBucket: "paperclip-runner-context", contextPrefix: "profiles/example", contextKmsKeyArn: "arn:aws:kms:us-east-1:123456789012:key/example", - qualificationRevision: "aws-agentcore-harness-v1", + qualificationRevision: "aws-agentcore-harness-context-v2", defaultModel: "global.anthropic.claude-sonnet-4-6", eventExpiryDays: 90, defaultMaxEstimatedSessionCostUsd: 1, @@ -44,7 +44,7 @@ function remoteInput( configuration: { ...AWS_CONFIGURATION }, enabled: false, retentionAcknowledged: false, - qualification: { suite: "aws-agentcore-harness-v1" }, + qualification: { suite: "aws-agentcore-harness-context-v2" }, ...overrides, }; } @@ -362,7 +362,7 @@ describe("remote agent profile metadata validation", () => { }); it("does not allow a qualified AgentCore profile key to be repointed", async () => { - const qualification = { suite: "aws-agentcore-harness-v1" }; + const qualification = { suite: "aws-agentcore-harness-context-v2" }; const existing = { id: "30000000-0000-4000-8000-000000000003", companyId: COMPANY_ID, @@ -506,7 +506,7 @@ describe("remote agent profile metadata validation", () => { qualifiedRevision: existingManaged.qualifiedRevision, }); - const awsQualification = { suite: "aws-agentcore-harness-v1" }; + const awsQualification = { suite: "aws-agentcore-harness-context-v2" }; const existingRemote = { id: "30000000-0000-4000-8000-000000000005", companyId: COMPANY_ID, @@ -547,7 +547,7 @@ describe("remote agent profile metadata validation", () => { }); it("rejects runtime use when stored identity drifts from the qualified revision", async () => { - const qualification = { suite: "aws-agentcore-harness-v1" }; + const qualification = { suite: "aws-agentcore-harness-context-v2" }; const qualifiedRevision = computeRemoteAgentProfileRevision({ service: "aws_bedrock_agentcore_harness", configuration: AWS_CONFIGURATION,