From 865b4854fb44d3689f1c0ff17e3e715d52aaea73 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Thu, 3 Sep 2026 13:12:58 -0500 Subject: [PATCH] ci(runner): build paid artifacts once per campaign (#12777) ## Thinking Path > - Paid cells repeated the same TypeScript and Rust builds even when one campaign selected dozens of cells. > - The trusted workflow can compile once without provider credentials and distribute run-scoped, digest-verified artifacts. > - The paid cell can then disable install lifecycle scripts, verify each artifact before extraction, and expose provider credentials only to the final test step. > - Local JS-backed providers also need the setup-node interpreter permission-qualified before Rust verifies the launch artifact. ## Linked Issues or Issue Description Run 33786122875 proved target-lock setup and catalog selection, then failed before provider creation because trusted master did not yet qualify the setup-node interpreter. The same workflow also rebuilt TypeScript and Rust inside every matrix cell. ## What Changed - Build runner TypeScript and native binaries once per campaign in a credential-free job. - Build the remote provider pack once only when selected Daytona cells require it. - Upload run-scoped bundles with SHA-256 manifests and verify before extraction in each paid cell. - Remove repeated TypeScript, provider-pack, and Rust builds from paid cells. - Qualify the local provider Node interpreter before verified launch. - Propagate the resolved target lockfile through all five target-code jobs. - Keep local-only selection off Daytona and exclude Xiaomi from the 67-cell catalog. ## Risks A shared build artifact could fan out a bad payload to many cells. The producing jobs receive no provider credentials, use the exact authorized target SHA and resolved lockfile, and publish run-scoped artifacts. Every consuming job verifies SHA-256 before extraction. Paid dependency setup keeps lifecycle scripts disabled and provider credentials remain scoped to the final test step. ## Verification - Focused runner workflow-security, catalog, and Daytona-image tests: 25/25 passed. - Prettier passed. - Actionlint passed with only the two pre-existing SC2129 style notices ignored. - Git diff check passed. ## Model Used OpenAI Codex, GPT-5. ## Checklist - [x] Build jobs are credential-free. - [x] Paid installs disable lifecycle scripts. - [x] Artifacts are run-scoped and digest-verified before extraction. - [x] Trusted report and history jobs remain isolated from target artifacts. - [x] No Daytona or Xiaomi paid run was started for this change. --- .github/workflows/runner-full-stack-e2e.yml | 336 ++++++++++++++++++-- tests/runner-e2e/README.md | 42 ++- tests/runner-e2e/SECURITY.md | 20 +- tests/runner-e2e/catalog.test.ts | 34 +- tests/runner-e2e/catalog.ts | 115 ++++--- tests/runner-e2e/daytona-image.test.ts | 2 +- tests/runner-e2e/history.test.ts | 12 +- tests/runner-e2e/types.ts | 1 + tests/runner-e2e/workflow-security.test.ts | 96 +++++- 9 files changed, 542 insertions(+), 116 deletions(-) diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.yml index 376bc16658..31719b0d3f 100644 --- a/.github/workflows/runner-full-stack-e2e.yml +++ b/.github/workflows/runner-full-stack-e2e.yml @@ -198,6 +198,9 @@ jobs: outputs: matrix: ${{ steps.catalog.outputs.matrix }} needs_daytona: ${{ steps.catalog.outputs.needs_daytona }} + needs_runner_typescript: ${{ steps.catalog.outputs.needs_runner_typescript }} + needs_native_binaries: ${{ steps.catalog.outputs.needs_native_binaries }} + needs_remote_provider_pack: ${{ steps.catalog.outputs.needs_remote_provider_pack }} execution_ids: ${{ steps.catalog.outputs.execution_ids }} max_parallel: ${{ steps.catalog.outputs.max_parallel }} daytona_image_content_id: ${{ steps.daytona_image_content.outputs.content_id }} @@ -301,9 +304,14 @@ jobs: args+=(--all) fi catalog_json="$(pnpm --silent test:e2e:runner -- "${args[@]}")" - echo "matrix=$(jq -c '{include: .include}' <<< "$catalog_json")" >> "$GITHUB_OUTPUT" - echo "needs_daytona=$(jq -r '.needsDaytona' <<< "$catalog_json")" >> "$GITHUB_OUTPUT" - echo "execution_ids=$(jq -c '.executionIds' <<< "$catalog_json")" >> "$GITHUB_OUTPUT" + { + echo "matrix=$(jq -c '{include: .include}' <<< "$catalog_json")" + echo "needs_daytona=$(jq -r '.needsDaytona' <<< "$catalog_json")" + echo "needs_runner_typescript=$(jq -r '[.include[] | select((.profileId == "runner-opencode") or (.profileId | startswith("runner-acpx-")) or (.suiteId == "openrouter-model-breadth"))] | length > 0' <<< "$catalog_json")" + echo "needs_native_binaries=$(jq -r '[.include[] | select((.profileId | startswith("runner-")) or (.suiteId == "openrouter-model-breadth"))] | length > 0' <<< "$catalog_json")" + echo "needs_remote_provider_pack=$(jq -r '[.include[] | select((.environmentId == "daytona") and ((.profileId == "runner-opencode") or (.profileId | startswith("runner-acpx-"))))] | length > 0' <<< "$catalog_json")" + echo "execution_ids=$(jq -c '.executionIds' <<< "$catalog_json")" + } >> "$GITHUB_OUTPUT" if ! [[ "$MAX_PARALLEL_LIMIT" =~ ^[1-9][0-9]*$ ]] || [ "$MAX_PARALLEL_LIMIT" -gt 100 ]; then echo "Runner selection emitted an invalid max-parallel limit." >&2 exit 1 @@ -440,9 +448,253 @@ jobs: echo "source_revision=$source_revision" >> "$GITHUB_OUTPUT" echo "content_id=$published_content_id" >> "$GITHUB_OUTPUT" + build_runner_artifacts: + name: Build reusable runner campaign artifacts + needs: [authorize, target_lock, catalog] + if: github.event_name != 'schedule' || vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' + # Compile native binaries on the same reviewed image used to execute them, + # avoiding libc/architecture drift between GitHub-hosted and AWS lanes. + runs-on: ${{ needs.authorize.outputs.test_runner }} + timeout-minutes: 20 + permissions: + contents: read + outputs: + build_artifact_name: ${{ steps.build_artifact_name.outputs.name }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ needs.authorize.outputs.target_sha }} + persist-credentials: false + + - name: Download resolved target lockfile + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + artifact-ids: ${{ needs.target_lock.outputs.artifact_id }} + path: ${{ runner.temp }}/runner-e2e-target-lock + + - name: Restore resolved target lockfile + env: + TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} + EXPECTED_LOCK_SHA256: ${{ needs.target_lock.outputs.lock_sha256 }} + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$TARGET_SHA" + lock="$RUNNER_TEMP/runner-e2e-target-lock/pnpm-lock.yaml" + test -f "$lock" + test "$(find "$(dirname "$lock")" -type f | wc -l | tr -d ' ')" = 1 + test "$(sha256sum "$lock" | cut -d ' ' -f 1)" = "$EXPECTED_LOCK_SHA256" + cp "$lock" pnpm-lock.yaml + test "$(sha256sum pnpm-lock.yaml | cut -d ' ' -f 1)" = "$EXPECTED_LOCK_SHA256" + + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 + with: + version: 9.15.4 + + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + cache: pnpm + + - run: pnpm install --frozen-lockfile --ignore-scripts + + # build:typescript also builds the eval-kernel dependency, so the two + # TypeScript trees are compiled at most once in this campaign. + - name: Build shared TypeScript and native runner outputs + env: + NEEDS_RUNNER_TYPESCRIPT: ${{ needs.catalog.outputs.needs_runner_typescript }} + NEEDS_NATIVE_BINARIES: ${{ needs.catalog.outputs.needs_native_binaries }} + run: | + set -euo pipefail + if [ "$NEEDS_RUNNER_TYPESCRIPT" = true ]; then + pnpm --filter @paperclipai/paperclip-runner build:typescript + else + pnpm --filter @paperclipai/paperclip-eval-kernel build + fi + if [ "$NEEDS_NATIVE_BINARIES" = true ]; then + pnpm --filter @paperclipai/paperclip-runner build:runner-binaries + fi + + - name: Package immutable campaign outputs + env: + NEEDS_RUNNER_TYPESCRIPT: ${{ needs.catalog.outputs.needs_runner_typescript }} + NEEDS_NATIVE_BINARIES: ${{ needs.catalog.outputs.needs_native_binaries }} + run: | + set -euo pipefail + binary_root="packages/paperclip-runner/runner/target/debug" + binaries=( + conformance-tracer + paperclip-runnerd + fake-harness + fake-codex-app-server + fake-acpx-sidecar + ) + archive_paths=( + packages/paperclip-eval-kernel/dist + ) + if [ "$NEEDS_RUNNER_TYPESCRIPT" = true ]; then + test -d packages/paperclip-runner/dist + archive_paths+=(packages/paperclip-runner/dist) + fi + if [ "$NEEDS_NATIVE_BINARIES" = true ]; then + for binary in "${binaries[@]}"; do + test -x "$binary_root/$binary" + archive_paths+=("$binary_root/$binary") + done + fi + tar --create --gzip \ + --file runner-e2e-build-bundle.tar.gz \ + "${archive_paths[@]}" + sha256sum runner-e2e-build-bundle.tar.gz > runner-e2e-build-bundle.tar.gz.sha256 + + - name: Name immutable shared campaign outputs + id: build_artifact_name + run: echo "name=runner-e2e-build-${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT" + + - name: Upload immutable shared campaign outputs + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ steps.build_artifact_name.outputs.name }} + path: | + runner-e2e-build-bundle.tar.gz + runner-e2e-build-bundle.tar.gz.sha256 + retention-days: 1 + compression-level: 0 + if-no-files-found: error + + build_remote_provider_pack: + name: Build reusable remote provider pack + needs: + [authorize, target_lock, catalog, daytona_image, build_runner_artifacts] + if: github.event_name != 'schedule' || vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + outputs: + provider_pack_artifact_name: ${{ steps.provider_pack_artifact_name.outputs.name }} + steps: + - name: No remote provider pack needed + if: needs.catalog.outputs.needs_remote_provider_pack != 'true' + run: echo "Selected cells do not require a remote provider pack." + + - if: needs.catalog.outputs.needs_remote_provider_pack == 'true' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ needs.authorize.outputs.target_sha }} + persist-credentials: false + + - name: Download resolved target lockfile + if: needs.catalog.outputs.needs_remote_provider_pack == 'true' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + artifact-ids: ${{ needs.target_lock.outputs.artifact_id }} + path: ${{ runner.temp }}/runner-e2e-target-lock + + - name: Restore resolved target lockfile + if: needs.catalog.outputs.needs_remote_provider_pack == 'true' + env: + TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} + EXPECTED_LOCK_SHA256: ${{ needs.target_lock.outputs.lock_sha256 }} + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$TARGET_SHA" + lock="$RUNNER_TEMP/runner-e2e-target-lock/pnpm-lock.yaml" + test -f "$lock" + test "$(find "$(dirname "$lock")" -type f | wc -l | tr -d ' ')" = 1 + test "$(sha256sum "$lock" | cut -d ' ' -f 1)" = "$EXPECTED_LOCK_SHA256" + cp "$lock" pnpm-lock.yaml + test "$(sha256sum pnpm-lock.yaml | cut -d ' ' -f 1)" = "$EXPECTED_LOCK_SHA256" + + - if: needs.catalog.outputs.needs_remote_provider_pack == 'true' + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 + with: + version: 9.15.4 + + - if: needs.catalog.outputs.needs_remote_provider_pack == 'true' + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + cache: pnpm + + - if: needs.catalog.outputs.needs_remote_provider_pack == 'true' + run: pnpm install --frozen-lockfile --ignore-scripts + + - name: Download immutable shared campaign outputs + if: needs.catalog.outputs.needs_remote_provider_pack == 'true' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: ${{ needs.build_runner_artifacts.outputs.build_artifact_name }} + path: runner-e2e-build + + - name: Verify and restore shared TypeScript outputs + if: needs.catalog.outputs.needs_remote_provider_pack == 'true' + run: | + set -euo pipefail + ( + cd runner-e2e-build + sha256sum --check runner-e2e-build-bundle.tar.gz.sha256 + ) + tar --extract --gzip \ + --file runner-e2e-build/runner-e2e-build-bundle.tar.gz \ + --directory "$GITHUB_WORKSPACE" + test -d packages/paperclip-eval-kernel/dist + test -d packages/paperclip-runner/dist + + - name: Assemble native remote provider pack + if: needs.catalog.outputs.needs_remote_provider_pack == 'true' + env: + # A reused image can have an older source revision with the same + # content ID. Matching that revision lets remote execution reuse the + # verified pack already installed in the immutable image. + PAPERCLIP_RUNNER_SOURCE_REVISION: ${{ needs.daytona_image.outputs.source_revision }} + run: node packages/paperclip-runner/scripts/build-provider-pack.mjs packages/paperclip-runner/provider-pack + + - name: Package verified remote provider pack + if: needs.catalog.outputs.needs_remote_provider_pack == 'true' + env: + IMAGE_SOURCE_REVISION: ${{ needs.daytona_image.outputs.source_revision }} + run: | + set -euo pipefail + test -f packages/paperclip-runner/provider-pack/provider-pack.json + jq -e \ + --arg revision "$IMAGE_SOURCE_REVISION" \ + '.schema == "paperclip-runner/remote-provider-pack/v1" and + .payload.runnerSourceRevision == $revision and + (.digest | test("^sha256:[0-9a-f]{64}$"))' \ + packages/paperclip-runner/provider-pack/provider-pack.json >/dev/null + tar --create --gzip \ + --file runner-e2e-provider-pack.tar.gz \ + packages/paperclip-runner/provider-pack + sha256sum runner-e2e-provider-pack.tar.gz > runner-e2e-provider-pack.tar.gz.sha256 + + - name: Name immutable remote provider pack + id: provider_pack_artifact_name + if: needs.catalog.outputs.needs_remote_provider_pack == 'true' + run: echo "name=runner-e2e-provider-pack-${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT" + + - name: Upload immutable remote provider pack + if: needs.catalog.outputs.needs_remote_provider_pack == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ steps.provider_pack_artifact_name.outputs.name }} + path: | + runner-e2e-provider-pack.tar.gz + runner-e2e-provider-pack.tar.gz.sha256 + retention-days: 1 + compression-level: 0 + if-no-files-found: error + test: name: ${{ matrix.executionId }} - needs: [authorize, target_lock, catalog, daytona_image] + needs: + [ + authorize, + target_lock, + catalog, + daytona_image, + build_runner_artifacts, + build_remote_provider_pack, + ] # The authorize job selects only one of two literal, reviewed runner labels; # no dispatch input or repository variable can inject an arbitrary label. runs-on: ${{ needs.authorize.outputs.test_runner }} @@ -510,30 +762,76 @@ jobs: # the protected environment during setup. - run: pnpm install --frozen-lockfile --ignore-scripts - - name: Build runner TypeScript prerequisites - run: pnpm --filter @paperclipai/paperclip-eval-kernel build + - name: Download immutable campaign outputs + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: ${{ needs.build_runner_artifacts.outputs.build_artifact_name }} + path: runner-e2e-build - - name: Build local JS-backed provider artifacts - if: matrix.environmentId == 'local' && (matrix.profileId == 'runner-opencode' || startsWith(matrix.profileId, 'runner-acpx-')) - run: pnpm --filter @paperclipai/paperclip-runner build:typescript + - name: Download immutable remote provider pack + if: matrix.environmentId == 'daytona' && (matrix.profileId == 'runner-opencode' || startsWith(matrix.profileId, 'runner-acpx-')) + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: ${{ needs.build_remote_provider_pack.outputs.provider_pack_artifact_name }} + path: runner-e2e-provider-pack - - name: Build native remote provider pack + - name: Verify and restore campaign outputs + env: + NEEDS_RUNNER_TYPESCRIPT: ${{ matrix.profileId == 'runner-opencode' || startsWith(matrix.profileId, 'runner-acpx-') || matrix.suiteId == 'openrouter-model-breadth' }} + NEEDS_NATIVE_BINARY: ${{ startsWith(matrix.profileId, 'runner-') || matrix.suiteId == 'openrouter-model-breadth' }} + run: | + set -euo pipefail + ( + cd runner-e2e-build + sha256sum --check runner-e2e-build-bundle.tar.gz.sha256 + ) + tar --extract --gzip \ + --file runner-e2e-build/runner-e2e-build-bundle.tar.gz \ + --directory "$GITHUB_WORKSPACE" + test -d packages/paperclip-eval-kernel/dist + if [ "$NEEDS_RUNNER_TYPESCRIPT" = true ]; then + test -d packages/paperclip-runner/dist + fi + if [ "$NEEDS_NATIVE_BINARY" = true ]; then + test -x packages/paperclip-runner/runner/target/debug/paperclip-runnerd + fi + + - name: Verify and restore remote provider pack if: matrix.environmentId == 'daytona' && (matrix.profileId == 'runner-opencode' || startsWith(matrix.profileId, 'runner-acpx-')) env: - # Reused images retain the source revision that was embedded in their - # provider pack. Matching it here lets the server reuse that exact - # preinstalled pack instead of uploading a duplicate to the lease. - PAPERCLIP_RUNNER_SOURCE_REVISION: ${{ needs.daytona_image.outputs.source_revision }} - run: pnpm --filter @paperclipai/paperclip-runner build:provider-pack + IMAGE_SOURCE_REVISION: ${{ needs.daytona_image.outputs.source_revision }} + run: | + set -euo pipefail + ( + cd runner-e2e-provider-pack + sha256sum --check runner-e2e-provider-pack.tar.gz.sha256 + ) + tar --extract --gzip \ + --file runner-e2e-provider-pack/runner-e2e-provider-pack.tar.gz \ + --directory "$GITHUB_WORKSPACE" + jq -e \ + --arg revision "$IMAGE_SOURCE_REVISION" \ + '.schema == "paperclip-runner/remote-provider-pack/v1" and + .payload.runnerSourceRevision == $revision and + (.digest | test("^sha256:[0-9a-f]{64}$"))' \ + packages/paperclip-runner/provider-pack/provider-pack.json >/dev/null + + - name: Qualify local provider Node interpreter + if: matrix.environmentId == 'local' && (matrix.profileId == 'runner-opencode' || startsWith(matrix.profileId, 'runner-acpx-') || matrix.suiteId == 'openrouter-model-breadth') + run: | + node <<'NODE' + const fs = require("node:fs"); + const mode = fs.statSync(process.execPath).mode & 0o777; + fs.chmodSync(process.execPath, mode & ~0o022); + if ((fs.statSync(process.execPath).mode & 0o022) !== 0) { + throw new Error("provider Node interpreter remains group- or world-writable"); + } + NODE - name: Install pinned legacy Claude CLI if: matrix.profileId == 'legacy-claude' run: npm install --global --omit=dev @anthropic-ai/claude-code@2.1.19 - - name: Build native runner binaries - if: startsWith(matrix.profileId, 'runner-') || matrix.suiteId == 'openrouter-model-breadth' - run: pnpm --filter @paperclipai/paperclip-runner build:runner-binaries - - name: Install Chromium run: pnpm exec playwright install --with-deps chromium diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 24aa2f8f9a..be091a341a 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -80,9 +80,14 @@ workflows: 42 cells. Its cases are: same Plan, browser acceptance of the new revision, and verified execution; - `ask-question`: a direct answer from a task created in Ask mode. -`openrouter-model-breadth` (**OpenRouter Model Breadth**) is five models from -the tracked weekly tool-capable ranking snapshot × native OpenCode × local × -three workflows: 15 cells. Its cases are: +`openrouter-model-breadth` (**OpenRouter Model Breadth**) is four qualified +models from the tracked weekly tool-capable ranking snapshot × native OpenCode +× local, with 11 supported model/workflow cells. Xiaomi MiMo V2.5 remains +recorded in the immutable ranking snapshot but is excluded from paid +qualification because its latency repeatedly exhausts the cell deadline. +DeepSeek V4 Flash remains qualified for hello and question/resume, but its Plan +cell is excluded after three successful semantic completions consistently +ignored the required exact final response. Its cases are: - `hello-complete`: a basic nonce response and explicit Done transition; - `question-resume-complete`: one structured question, browser selection of @@ -98,7 +103,7 @@ duplicating the final response. The second workflow restarts the isolated Paperclip server while the interaction is waiting, reloads that state, and then resumes it. The suite has no Daytona cells. -The complete catalog is 71 cells and 123 expected paid agent turns. Follow-up +The complete catalog is 67 cells and 116 expected paid agent turns. Follow-up steps remain ordered within their cell; all other cells are independent. Narrow selectors are strongly recommended while developing fixtures. @@ -272,18 +277,22 @@ any branch in `paperclipai/paperclip`. The authorization job resolves that branch to one immutable commit before any checkout. A separate credential-free job checks out the resolved commit and regenerates `pnpm-lock.yaml` once with `--ignore-scripts --no-frozen-lockfile --lockfile-only`. It uploads that exact -lockfile under a run-attempt-scoped artifact ID and records its SHA-256. Catalog, -image, and paid test jobs download the artifact by ID, verify its digest, and -restore it before setup or a frozen install. The paid test job disables +lockfile under a run-attempt-scoped artifact ID and records its SHA-256. +Catalog, image, shared-build, provider-pack, and paid test jobs download the +artifact by ID, verify its digest, and restore it before setup or a frozen +install. The shared-build, provider-pack, and paid test jobs all disable dependency lifecycle scripts, and provider secrets are introduced only in the final test step. This permits an authorized target branch to exercise an intentionally uncommitted workspace patch while keeping every target job on one -identical dependency resolution. Report sanitization and AWS history -publication do not consume the target lockfile; they explicitly check out and -install from the trusted workflow commit. The workflow definition, runner-group -permission, and protected-environment deployment still come from the default -branch. Do not select the target branch in GitHub's **Use workflow from** -control. +identical dependency resolution. The shared-build job compiles the selected +campaign's TypeScript outputs and native binaries once, then each paid cell +verifies and extracts the immutable bundle. Remote native cells similarly reuse +one verified provider pack. Report +sanitization and AWS history publication do not consume the target lockfile; +they explicitly check out and install from the trusted workflow commit. The +workflow definition, runner-group permission, and protected-environment +deployment still come from the default branch. Do not select the target branch +in GitHub's **Use workflow from** control. Because this repository is public, manual campaigns fail before checkout unless the trusted workflow runs from the default branch and both the original actor @@ -319,10 +328,9 @@ by the repository variable `RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED=true`. Set it only after the live acceptance ladder in the architecture plan is green. Set `RUNNER_E2E_AWS_ENABLED=true` to route paid cells to the repository-scoped ephemeral AWS RunsOn fleet selected by -`runs-on/fleet=paperclip-public-pr-x64/env=public-ci`. Any other value retains -the standard GitHub-hosted `ubuntu-latest` target. Set -`RUNNER_E2E_MAX_PARALLEL` to an integer from 1–100 on AWS (default 100); use at -least 71 to run the current +`runs-on/fleet=paperclip-public-pr-x64/env=public-ci`. Any other value uses the +proven GitHub-hosted `ubuntu-latest` target. Set `RUNNER_E2E_MAX_PARALLEL` to an +integer from 1–100 on AWS (default 100); use at least 67 to run the current complete catalog in one wave. The fallback runner retains its 1–57 limit and default of 32. Multi-turn steps are sequential inside their cell while independent cells overlap. Artifacts and merged HTML/JUnit/normalized reports diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md index d730ff4770..cba5da13d6 100644 --- a/tests/runner-e2e/SECURITY.md +++ b/tests/runner-e2e/SECURITY.md @@ -24,14 +24,18 @@ that branch through the GitHub API and passes only its immutable commit SHA to a credential-free target-lock job. That job checks out the commit, regenerates `pnpm-lock.yaml` once with lifecycle scripts disabled and lockfile-only mode, then uploads the file under a run-attempt-scoped artifact ID. Catalog, image, -and paid test jobs download that exact artifact by ID, verify its recorded -SHA-256, and restore it before setup or a frozen dependency install. The lock -resolver receives no provider credentials and must never run repository -lifecycle scripts. The paid test job also installs with lifecycle scripts -disabled, and provider secrets are scoped only to its final test step rather -than dependency setup. Report sanitization and AWS history publication -explicitly use the trusted workflow commit and do not consume the target -lockfile. Never run the workflow definition from the target branch. +shared-build, provider-pack, and paid test jobs download that exact artifact by +ID, verify its recorded SHA-256, and restore it before setup or a frozen +dependency install. The lock resolver receives no provider credentials and +must never run repository lifecycle scripts. The shared-build and provider-pack +jobs also receive no provider credentials and disable dependency lifecycle +scripts; they package outputs with SHA-256 sidecars that consumers verify +before extraction. The paid test job installs with lifecycle scripts disabled, +and provider secrets are scoped only to its final test step rather than +dependency setup. Report sanitization and AWS +history publication explicitly use the trusted workflow commit and do not +consume the target lockfile. Never run the workflow definition from the target +branch. The workflows verify both the original actor and triggering actor for every scheduled or manual attempt, including human reruns. Every diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index f75281cdbf..05498dedf6 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -2,6 +2,8 @@ import { describe, expect, it } from "vitest"; import { runnerEnvironments, runnerMatrix, + openRouterBreadthExcludedExecutionIds, + openRouterBreadthExcludedModelIds, openRouterBreadthProfiles, openRouterBreadthTasks, localIntegrityTasks, @@ -21,16 +23,16 @@ import { describe("runner E2E catalog", () => { it("validates the core, local-integrity, and breadth suites", () => { expect(runnerProfiles).toHaveLength(7); - expect(openRouterBreadthProfiles).toHaveLength(5); + expect(openRouterBreadthProfiles).toHaveLength(4); expect(runnerEnvironments).toHaveLength(2); expect(runnerTasks).toHaveLength(3); expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 42, 14, 15, + 42, 14, 11, ]); - expect(validateRunnerCatalog()).toHaveLength(71); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(71); + expect(validateRunnerCatalog()).toHaveLength(67); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(67); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(42); @@ -43,19 +45,29 @@ describe("runner E2E catalog", () => { runnerMatrix.filter( (entry) => entry.suite.id === "openrouter-model-breadth", ), - ).toHaveLength(15); + ).toHaveLength(11); expect( runnerMatrix.reduce( (total, execution) => total + execution.task.expectedRunCount, 0, ), - ).toBe(123); + ).toBe(116); }); - it("derives five local native OpenCode profiles from the ranked snapshot", () => { + it("derives the qualified local native OpenCode profiles from the ranked snapshot", () => { + expect(openRouterBreadthExcludedModelIds).toEqual(["xiaomi/mimo-v2.5"]); + expect(openRouterBreadthExcludedExecutionIds).toEqual([ + "openrouter-model-breadth.openrouter-deepseek-deepseek-v4-flash-0731.local.plan-approve-complete", + ]); + expect( + runnerMatrix.some( + (execution) => + execution.id === openRouterBreadthExcludedExecutionIds[0], + ), + ).toBe(false); expect( openRouterBreadthProfiles.map((profile) => profile.ranking?.rank), - ).toEqual([1, 2, 3, 4, 5]); + ).toEqual([1, 3, 4, 5]); expect( openRouterBreadthProfiles.every( (profile) => @@ -257,7 +269,7 @@ describe("runner E2E selectors", () => { const selected = selectRunnerExecutions( parseRunnerSelectors(["--suite", "openrouter-model-breadth"]), ); - expect(selected).toHaveLength(15); + expect(selected).toHaveLength(11); expect( selected.every( (entry) => @@ -294,9 +306,9 @@ describe("runner E2E selectors", () => { const jobs = buildMatrixJobs( selectRunnerExecutions(parseRunnerSelectors(["--all"])), ); - expect(jobs).toHaveLength(71); + expect(jobs).toHaveLength(67); expect(jobs.filter((job) => job.needsDaytona)).toHaveLength(21); - expect(new Set(jobs.map((job) => job.executionId)).size).toBe(71); + expect(new Set(jobs.map((job) => job.executionId)).size).toBe(67); expect( jobs.every((job) => runnerMatrix.some( diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 5fd997ea5f..f1feae7f6a 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -262,28 +262,40 @@ export const runnerProfiles: readonly RunnerProfileFixture[] = [ }), ] as const; +export const openRouterBreadthExcludedModelIds = ["xiaomi/mimo-v2.5"] as const; +export const openRouterBreadthExcludedExecutionIds = [ + "openrouter-model-breadth.openrouter-deepseek-deepseek-v4-flash-0731.local.plan-approve-complete", +] as const; +const openRouterBreadthExcludedModelIdSet = new Set( + openRouterBreadthExcludedModelIds, +); + export const openRouterBreadthProfiles: readonly RunnerProfileFixture[] = - openRouterRankingSnapshot.models.map((rankedModel) => - nativeProfile({ - id: openRouterProfileId(rankedModel.id), - label: `#${rankedModel.rank} ${rankedModel.name}`, - provider: "opencode", - model: `openrouter/${rankedModel.id}`, - credential: "OPENROUTER_API_KEY", - supportedEnvironments: ["local"], - modelQualification: { - source: "openrouter_rankings_snapshot", - qualificationId: `${openRouterRankingSnapshot.snapshotId}:${rankedModel.rank}`, - }, - ranking: { - rank: rankedModel.rank, - canonicalModelId: rankedModel.id, - snapshotId: openRouterRankingSnapshot.snapshotId, - capturedAt: openRouterRankingSnapshot.capturedAt, - sourceUrl: openRouterRankingSnapshot.sourceUrl, - }, - }), - ); + openRouterRankingSnapshot.models + .filter( + (rankedModel) => !openRouterBreadthExcludedModelIdSet.has(rankedModel.id), + ) + .map((rankedModel) => + nativeProfile({ + id: openRouterProfileId(rankedModel.id), + label: `#${rankedModel.rank} ${rankedModel.name}`, + provider: "opencode", + model: `openrouter/${rankedModel.id}`, + credential: "OPENROUTER_API_KEY", + supportedEnvironments: ["local"], + modelQualification: { + source: "openrouter_rankings_snapshot", + qualificationId: `${openRouterRankingSnapshot.snapshotId}:${rankedModel.rank}`, + }, + ranking: { + rank: rankedModel.rank, + canonicalModelId: rankedModel.id, + snapshotId: openRouterRankingSnapshot.snapshotId, + capturedAt: openRouterRankingSnapshot.capturedAt, + sourceUrl: openRouterRankingSnapshot.sourceUrl, + }, + }), + ); function requiredDaytonaSecret(input: EnvironmentFixtureBuildInput) { const apiKey = input.secretRefs.DAYTONA_API_KEY; @@ -731,12 +743,15 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ profiles: openRouterBreadthProfiles, environments: [localEnvironment], tasks: openRouterBreadthTasks, - expectedMatrixSize: 15, + excludedExecutionIds: openRouterBreadthExcludedExecutionIds, + expectedMatrixSize: 11, definitionMetadata: { rankingSnapshotId: openRouterRankingSnapshot.snapshotId, rankingContentHash: openRouterRankingSnapshot.contentHash, rankingCapturedAt: openRouterRankingSnapshot.capturedAt, rankingSourceUrl: openRouterRankingSnapshot.sourceUrl, + excludedModelIds: openRouterBreadthExcludedModelIds, + excludedExecutionIds: openRouterBreadthExcludedExecutionIds, }, }, ] as const; @@ -759,6 +774,7 @@ export function suiteDefinitionHash(suite: RunnerSuiteFixture) { restartServerBeforeQuestionAnswer: task.restartServerBeforeQuestionAnswer ?? false, })), + excludedExecutionIds: [...(suite.excludedExecutionIds ?? [])].sort(), metadata: suite.definitionMetadata ?? null, }), ) @@ -768,36 +784,39 @@ export function suiteDefinitionHash(suite: RunnerSuiteFixture) { export function buildRunnerMatrix( suites: readonly RunnerSuiteFixture[] = runnerSuites, ): MatrixExecution[] { - return suites.flatMap((suite) => - suite.profiles.flatMap((profile) => + return suites.flatMap((suite) => { + const excludedExecutionIds = new Set(suite.excludedExecutionIds ?? []); + return suite.profiles.flatMap((profile) => suite.environments .filter((environment) => profile.supportedEnvironments.includes(environment.id), ) .flatMap((environment) => - suite.tasks.map((task) => ({ - id: `${suite.id}.${profile.id}.${environment.id}.${task.id}`, - suite, - suiteDefinitionHash: suiteDefinitionHash(suite), - profile, - environment, - task, - groups: [ - ...new Set([ - ...suite.groups, - ...profile.groups, - ...environment.groups, - ...task.groups, - ]), - ], - requiredCredentials: [ - profile.credential, - ...(environment.credential ? [environment.credential] : []), - ], - })), + suite.tasks + .map((task) => ({ + id: `${suite.id}.${profile.id}.${environment.id}.${task.id}`, + suite, + suiteDefinitionHash: suiteDefinitionHash(suite), + profile, + environment, + task, + groups: [ + ...new Set([ + ...suite.groups, + ...profile.groups, + ...environment.groups, + ...task.groups, + ]), + ], + requiredCredentials: [ + profile.credential, + ...(environment.credential ? [environment.credential] : []), + ], + })) + .filter((execution) => !excludedExecutionIds.has(execution.id)), ), - ), - ); + ); + }); } function duplicateIds(values: readonly { id: string }[]) { @@ -941,8 +960,8 @@ export function validateRunnerCatalog(): MatrixExecution[] { ); } } - if (matrix.length !== 71) - throw new Error(`Expected 71 runner executions; received ${matrix.length}`); + if (matrix.length !== 67) + throw new Error(`Expected 67 runner executions; received ${matrix.length}`); return matrix; } diff --git a/tests/runner-e2e/daytona-image.test.ts b/tests/runner-e2e/daytona-image.test.ts index 9aefe4163d..19211d331d 100644 --- a/tests/runner-e2e/daytona-image.test.ts +++ b/tests/runner-e2e/daytona-image.test.ts @@ -75,7 +75,7 @@ describe("runner E2E Daytona image contract", () => { expect(workflow).toContain('.Config.User == "daytona"'); expect(workflow).toContain("PAPERCLIP_RUNNER_PROVIDER_PACK_ROOT="); expect(workflow).toContain( - "pnpm --filter @paperclipai/paperclip-runner build:provider-pack", + "node packages/paperclip-runner/scripts/build-provider-pack.mjs packages/paperclip-runner/provider-pack", ); expect(workflow).toContain( "PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH: ${{ github.workspace }}/packages/paperclip-runner/provider-pack", diff --git a/tests/runner-e2e/history.test.ts b/tests/runner-e2e/history.test.ts index 6748b76ae5..8a6cec4c7c 100644 --- a/tests/runner-e2e/history.test.ts +++ b/tests/runner-e2e/history.test.ts @@ -68,16 +68,16 @@ describe("runner E2E campaign history", () => { expected: breadth.map((execution) => execution.id), results: breadth.map((execution) => result(execution, "passed")), }); - expect(campaign).toMatchObject({ complete: false, passed: 15, failed: 0 }); + expect(campaign).toMatchObject({ complete: false, passed: 11, failed: 0 }); expect(campaign.suites[0]).toMatchObject({ suiteId: "openrouter-model-breadth", complete: true, - selected: 15, + selected: 11, }); expect(campaign.billing).toMatchObject({ - reportedLlmCostUsd: 0.15, - llm: { inputTokens: 1_500, outputTokens: 375 }, + llm: { inputTokens: 1_100, outputTokens: 275 }, }); + expect(campaign.billing.reportedLlmCostUsd).toBeCloseTo(0.11, 10); const history = mergeRunnerHistory( emptyRunnerHistory(), campaignHistoryRecord(campaign, "https://history.example/runner-e2e"), @@ -151,8 +151,8 @@ describe("runner E2E campaign history", () => { expect(index).toContain("Runner E2E campaigns"); expect(index).toContain("complete-green"); expect(index).toContain("complete-red"); - expect(index).toContain("71/71 passed"); - expect(index).toContain("70/71 passed"); + expect(index).toContain("67/67 passed"); + expect(index).toContain("66/67 passed"); expect(index).toContain("Open report →"); expect(index).toContain( "Visual evidence remains in access-controlled workflow artifacts", diff --git a/tests/runner-e2e/types.ts b/tests/runner-e2e/types.ts index bac6edc49b..ffc3e671bd 100644 --- a/tests/runner-e2e/types.ts +++ b/tests/runner-e2e/types.ts @@ -156,6 +156,7 @@ export interface RunnerSuiteFixture { profiles: readonly RunnerProfileFixture[]; environments: readonly EnvironmentFixture[]; tasks: readonly RunnerTaskFixture[]; + excludedExecutionIds?: readonly string[]; expectedMatrixSize: number; definitionMetadata?: Readonly>; } diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts index d9f23b786c..1393260125 100644 --- a/tests/runner-e2e/workflow-security.test.ts +++ b/tests/runner-e2e/workflow-security.test.ts @@ -3,6 +3,12 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; const repositoryRoot = path.resolve(import.meta.dirname, "../.."); +const fullStackTestNeeds = + /needs:\s*\[\s*authorize,\s*target_lock,\s*catalog,\s*daytona_image,\s*build_runner_artifacts,\s*build_remote_provider_pack,?\s*\]/u; +const buildRunnerNeeds = + /needs:\s*\[\s*authorize,\s*target_lock,\s*catalog,?\s*\]/u; +const buildRemoteProviderPackNeeds = + /needs:\s*\[\s*authorize,\s*target_lock,\s*catalog,\s*daytona_image,\s*build_runner_artifacts,?\s*\]/u; describe("public repository paid workflow security", () => { it("gates every provider-secret job with stable actor IDs", async () => { @@ -111,9 +117,7 @@ describe("public repository paid workflow security", () => { expect(paidJob).toContain( "runs-on: ${{ needs.authorize.outputs.test_runner }}", ); - expect(paidJob).toContain( - "needs: [authorize, target_lock, catalog, daytona_image]", - ); + expect(paidJob).toMatch(fullStackTestNeeds); expect(paidJob).toContain("name: runner-e2e-paid"); expect(paidJob).toMatch( /Reauthorize paid execution before provider access[\s\S]*actions\/checkout@[0-9a-f]{40}[\s\S]*persist-credentials: false[\s\S]*Download resolved target lockfile/, @@ -145,6 +149,14 @@ describe("public repository paid workflow security", () => { ), fullStack.slice( fullStack.indexOf(" daytona_image:"), + fullStack.indexOf(" build_runner_artifacts:"), + ), + fullStack.slice( + fullStack.indexOf(" build_runner_artifacts:"), + fullStack.indexOf(" build_remote_provider_pack:"), + ), + fullStack.slice( + fullStack.indexOf(" build_remote_provider_pack:"), fullStack.indexOf(" test:"), ), paidJob, @@ -176,13 +188,18 @@ describe("public repository paid workflow security", () => { ); } expect(fullStack.match(/Download resolved target lockfile/g)).toHaveLength( - 3, + 5, ); expect(fullStack.match(/Restore resolved target lockfile/g)).toHaveLength( - 3, + 5, ); + expect( + fullStack.match( + /ref: \$\{\{ needs\.authorize\.outputs\.target_sha \}\}/g, + ), + ).toHaveLength(6); expect(fullStack.match(/ref: \$\{\{ github\.sha \}\}/g)).toHaveLength(2); - expect(fullStack.match(/persist-credentials: false/g)).toHaveLength(6); + expect(fullStack.match(/persist-credentials: false/g)).toHaveLength(8); expect(fullStack).not.toContain("ref: ${{ inputs.target_branch }}"); expect(fullStack).toContain( "PAPERCLIP_RUNNER_SOURCE_REVISION=${TARGET_SHA}", @@ -260,6 +277,73 @@ describe("public repository paid workflow security", () => { } }); + it("builds runner outputs once without provider credentials and verifies them in every paid cell", async () => { + const workflow = await readFile( + path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"), + "utf8", + ); + const buildJobStart = workflow.indexOf(" build_runner_artifacts:"); + const testJobStart = workflow.indexOf(" test:", buildJobStart); + const reportJobStart = workflow.indexOf(" report:", testJobStart); + const buildJob = workflow.slice(buildJobStart, testJobStart); + const testJob = workflow.slice(testJobStart, reportJobStart); + + expect(buildJobStart).toBeGreaterThan(0); + expect(testJobStart).toBeGreaterThan(buildJobStart); + expect(buildJob).toMatch(buildRunnerNeeds); + expect(buildJob).toMatch(buildRemoteProviderPackNeeds); + expect(buildJob).not.toContain("environment:"); + expect(buildJob).not.toContain("secrets."); + expect( + buildJob.match(/pnpm install --frozen-lockfile --ignore-scripts/g), + ).toHaveLength(2); + expect(buildJob).toContain( + "pnpm --filter @paperclipai/paperclip-runner build:typescript", + ); + expect(buildJob).toContain( + "pnpm --filter @paperclipai/paperclip-runner build:runner-binaries", + ); + expect(buildJob).toContain( + "node packages/paperclip-runner/scripts/build-provider-pack.mjs", + ); + expect(buildJob).toContain("runner-e2e-build-bundle.tar.gz.sha256"); + expect(buildJob).toContain("runner-e2e-provider-pack.tar.gz.sha256"); + expect(buildJob).toContain( + "build_artifact_name: ${{ steps.build_artifact_name.outputs.name }}", + ); + expect(buildJob).toContain( + "needs.build_runner_artifacts.outputs.build_artifact_name", + ); + expect(buildJob).toContain( + "provider_pack_artifact_name: ${{ steps.provider_pack_artifact_name.outputs.name }}", + ); + expect(workflow).toContain("needs_runner_typescript="); + expect(workflow).toContain("needs_native_binaries="); + expect(workflow).toContain("needs_remote_provider_pack="); + + expect(testJob).toMatch(fullStackTestNeeds); + expect(testJob).toContain("Download immutable campaign outputs"); + expect(testJob).toContain("Download immutable remote provider pack"); + expect(testJob).toContain( + "needs.build_runner_artifacts.outputs.build_artifact_name", + ); + expect(testJob).toContain( + "needs.build_remote_provider_pack.outputs.provider_pack_artifact_name", + ); + expect(testJob).toContain("sha256sum --check"); + expect(testJob.indexOf("sha256sum --check")).toBeLessThan( + testJob.indexOf("tar --extract"), + ); + expect(testJob).toContain( + "test -x packages/paperclip-runner/runner/target/debug/paperclip-runnerd", + ); + expect(testJob).toContain(".payload.runnerSourceRevision == $revision"); + expect(workflow).toContain("Qualify local provider Node interpreter"); + expect(testJob).not.toContain("build:typescript"); + expect(testJob).not.toContain("build:runner-binaries"); + expect(testJob).not.toContain("build-provider-pack.mjs"); + }); + it("uses environment-scoped OIDC for a no-delete history publisher", async () => { const workflow = await readFile( path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"),