From 6e50ca9d0ae5936d4966acde271b791fbf198f6e Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Thu, 3 Sep 2026 12:39:18 -0500 Subject: [PATCH] ci(runner): prepare target lockfile once for paid validation (#12774) ## Thinking Path > - The trusted target-branch runner workflow checks out PR code before paid tests. > - PR policy intentionally forbids manual lockfile commits. > - Some runner changes legitimately alter pnpm patch hashes. > - Frozen installs therefore fail before test selection. > - Resolve one script-disabled lockfile from the authorized immutable target SHA and distribute it by exact artifact ID and digest. > - Keep provider credentials and trusted reporting outside this resolution job. ## Linked Issues or Issue Description Target-branch paid runner campaigns currently fail frozen install when a PR changes pnpm patch content, even though ordinary PR CI regenerates the lockfile. ## What Changed - Added one credential-free target-lock job that resolves the authorized immutable target SHA with lifecycle scripts disabled. - Uploaded the resolved lockfile with its SHA-256 and restored it by exact artifact ID before every target-code frozen install. - Left trusted reporting and history jobs on the workflow SHA. - Changed the disabled-AWS fallback from unavailable ubuntu-latest-m to ubuntu-latest. ## Risks The workflow evaluates pnpm lockfile resolution from authorized target code. That job receives no provider credentials, disables lifecycle scripts, rejects unrelated workspace mutations, and exposes only a digest-verified lockfile artifact. Paid-secret jobs consume only that lockfile after exact artifact-ID and SHA-256 validation. ## Verification - Runner workflow-security focused tests pass. - actionlint passes. - Prettier and git diff checks pass. ## Model Used OpenAI Codex, GPT-5. ## Checklist - [x] Change is narrowly scoped to paid runner orchestration. - [x] Target lock resolution has no provider credentials and disables lifecycle scripts. - [x] Downloaded artifacts are selected by exact artifact ID and verified by SHA-256. - [x] Trusted reporting and history jobs remain on the workflow SHA. --- .github/workflows/runner-full-stack-e2e.yml | 120 +++++++++++++++++++- tests/runner-e2e/README.md | 26 +++-- tests/runner-e2e/SECURITY.md | 27 +++-- tests/runner-e2e/workflow-security.test.ts | 93 +++++++++++++-- 4 files changed, 236 insertions(+), 30 deletions(-) diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.yml index 19bf0fda71..376bc16658 100644 --- a/.github/workflows/runner-full-stack-e2e.yml +++ b/.github/workflows/runner-full-stack-e2e.yml @@ -121,7 +121,7 @@ jobs: AWS_PAID_RUNNER_ENABLED: ${{ vars.RUNNER_E2E_AWS_ENABLED }} run: | set -euo pipefail - github_runner='ubuntu-latest-m' + github_runner='ubuntu-latest' aws_runner='runs-on/fleet=paperclip-public-pr-x64/env=public-ci' if [ "$AWS_PAID_RUNNER_ENABLED" = true ]; then @@ -140,9 +140,56 @@ jobs: echo '::notice title=Paid runner routing::RUNNER_E2E_AWS_ENABLED is not true; using the existing paid runner' fi + target_lock: + name: Resolve target pnpm lockfile + needs: authorize + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + outputs: + artifact_id: ${{ steps.upload.outputs.artifact-id }} + lock_sha256: ${{ steps.lock.outputs.sha256 }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ needs.authorize.outputs.target_sha }} + persist-credentials: false + + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 + with: + version: 9.15.4 + + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + + - name: Resolve target lockfile without lifecycle scripts + id: lock + run: | + set -euo pipefail + pnpm install --ignore-scripts --no-frozen-lockfile --lockfile-only + test -s pnpm-lock.yaml + unexpected="$(git status --short | awk '$2 != "pnpm-lock.yaml" { print }')" + if [ -n "$unexpected" ]; then + echo "Lockfile resolution changed files other than pnpm-lock.yaml:" >&2 + echo "$unexpected" >&2 + exit 1 + fi + echo "sha256=$(sha256sum pnpm-lock.yaml | cut -d ' ' -f 1)" >> "$GITHUB_OUTPUT" + + - name: Upload resolved target lockfile + id: upload + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: runner-e2e-target-pnpm-lock-${{ github.run_id }}-${{ github.run_attempt }} + path: pnpm-lock.yaml + retention-days: 30 + if-no-files-found: error + catalog: name: Validate catalog and select cells - needs: authorize + needs: [authorize, target_lock] if: github.event_name != 'schedule' || vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' runs-on: ubuntu-latest timeout-minutes: 10 @@ -160,6 +207,26 @@ jobs: ref: ${{ needs.authorize.outputs.target_sha }} persist-credentials: false + - name: Download resolved target lockfile + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + artifact-ids: ${{ needs.target_lock.outputs.artifact_id }} + path: ${{ runner.temp }}/runner-e2e-target-lock + + - name: Restore resolved target lockfile + env: + TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} + EXPECTED_LOCK_SHA256: ${{ needs.target_lock.outputs.lock_sha256 }} + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$TARGET_SHA" + lock="$RUNNER_TEMP/runner-e2e-target-lock/pnpm-lock.yaml" + test -f "$lock" + test "$(find "$(dirname "$lock")" -type f | wc -l | tr -d ' ')" = 1 + test "$(sha256sum "$lock" | cut -d ' ' -f 1)" = "$EXPECTED_LOCK_SHA256" + cp "$lock" pnpm-lock.yaml + test "$(sha256sum pnpm-lock.yaml | cut -d ' ' -f 1)" = "$EXPECTED_LOCK_SHA256" + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 with: version: 9.15.4 @@ -249,7 +316,7 @@ jobs: daytona_image: name: Publish verified Daytona image - needs: [authorize, catalog] + needs: [authorize, target_lock, catalog] runs-on: ubuntu-latest timeout-minutes: 45 permissions: @@ -266,6 +333,26 @@ jobs: ref: ${{ needs.authorize.outputs.target_sha }} persist-credentials: false + - name: Download resolved target lockfile + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + artifact-ids: ${{ needs.target_lock.outputs.artifact_id }} + path: ${{ runner.temp }}/runner-e2e-target-lock + + - name: Restore resolved target lockfile + env: + TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} + EXPECTED_LOCK_SHA256: ${{ needs.target_lock.outputs.lock_sha256 }} + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$TARGET_SHA" + lock="$RUNNER_TEMP/runner-e2e-target-lock/pnpm-lock.yaml" + test -f "$lock" + test "$(find "$(dirname "$lock")" -type f | wc -l | tr -d ' ')" = 1 + test "$(sha256sum "$lock" | cut -d ' ' -f 1)" = "$EXPECTED_LOCK_SHA256" + cp "$lock" pnpm-lock.yaml + test "$(sha256sum pnpm-lock.yaml | cut -d ' ' -f 1)" = "$EXPECTED_LOCK_SHA256" + - name: No Daytona image needed id: local_only if: needs.catalog.outputs.needs_daytona != 'true' @@ -355,7 +442,7 @@ jobs: test: name: ${{ matrix.executionId }} - needs: [authorize, catalog, daytona_image] + needs: [authorize, target_lock, catalog, daytona_image] # The authorize job selects only one of two literal, reviewed runner labels; # no dispatch input or repository variable can inject an arbitrary label. runs-on: ${{ needs.authorize.outputs.test_runner }} @@ -390,6 +477,26 @@ jobs: ref: ${{ needs.authorize.outputs.target_sha }} persist-credentials: false + - name: Download resolved target lockfile + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + artifact-ids: ${{ needs.target_lock.outputs.artifact_id }} + path: ${{ runner.temp }}/runner-e2e-target-lock + + - name: Restore resolved target lockfile + env: + TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} + EXPECTED_LOCK_SHA256: ${{ needs.target_lock.outputs.lock_sha256 }} + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$TARGET_SHA" + lock="$RUNNER_TEMP/runner-e2e-target-lock/pnpm-lock.yaml" + test -f "$lock" + test "$(find "$(dirname "$lock")" -type f | wc -l | tr -d ' ')" = 1 + test "$(sha256sum "$lock" | cut -d ' ' -f 1)" = "$EXPECTED_LOCK_SHA256" + cp "$lock" pnpm-lock.yaml + test "$(sha256sum pnpm-lock.yaml | cut -d ' ' -f 1)" = "$EXPECTED_LOCK_SHA256" + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 with: version: 9.15.4 @@ -398,7 +505,10 @@ jobs: with: node-version: 24 - - run: pnpm install --frozen-lockfile + # This job receives provider credentials only in the final paid-test + # step. Keep target-selected dependency lifecycle code from running in + # the protected environment during setup. + - run: pnpm install --frozen-lockfile --ignore-scripts - name: Build runner TypeScript prerequisites run: pnpm --filter @paperclipai/paperclip-eval-kernel build diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 29f8179757..24aa2f8f9a 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -269,12 +269,21 @@ auto-stop/archive/delete values remain as cancellation backstops. never runs for a pull request or ordinary push. Start the trusted workflow from the default branch. A CODEOWNER can set the optional `target_branch` input to any branch in `paperclipai/paperclip`. The authorization job resolves that -branch to one immutable commit before any checkout. Catalog, image, and paid -test jobs check out that exact commit. Report sanitization and AWS history -publication explicitly check out the trusted workflow commit. The workflow -definition, runner-group permission, and protected-environment deployment still -come from the default branch. Do not select the target branch in GitHub's **Use -workflow from** control. +branch to one immutable commit before any checkout. A separate credential-free +job checks out the resolved commit and regenerates `pnpm-lock.yaml` once with +`--ignore-scripts --no-frozen-lockfile --lockfile-only`. It uploads that exact +lockfile under a run-attempt-scoped artifact ID and records its SHA-256. Catalog, +image, and paid test jobs download the artifact by ID, verify its digest, and +restore it before setup or a frozen install. The paid test job disables +dependency lifecycle scripts, and provider secrets are introduced only in the +final test step. This permits an authorized target branch to exercise an +intentionally uncommitted workspace patch while keeping every target job on one +identical dependency resolution. Report sanitization and AWS history +publication do not consume the target lockfile; they explicitly check out and +install from the trusted workflow commit. The workflow definition, runner-group +permission, and protected-environment deployment still come from the default +branch. Do not select the target branch in GitHub's **Use workflow from** +control. Because this repository is public, manual campaigns fail before checkout unless the trusted workflow runs from the default branch and both the original actor @@ -311,8 +320,9 @@ only after the live acceptance ladder in the architecture plan is green. Set `RUNNER_E2E_AWS_ENABLED=true` to route paid cells to the repository-scoped ephemeral AWS RunsOn fleet selected by `runs-on/fleet=paperclip-public-pr-x64/env=public-ci`. Any other value retains -the existing `ubuntu-latest-m` target. Set `RUNNER_E2E_MAX_PARALLEL` to an -integer from 1–100 on AWS (default 100); use at least 71 to run the current +the standard GitHub-hosted `ubuntu-latest` target. Set +`RUNNER_E2E_MAX_PARALLEL` to an integer from 1–100 on AWS (default 100); use at +least 71 to run the current complete catalog in one wave. The fallback runner retains its 1–57 limit and default of 32. Multi-turn steps are sequential inside their cell while independent cells overlap. Artifacts and merged HTML/JUnit/normalized reports diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md index d99294444c..d730ff4770 100644 --- a/tests/runner-e2e/SECURITY.md +++ b/tests/runner-e2e/SECURITY.md @@ -20,10 +20,18 @@ gh api users/LOGIN --jq '{login,id}' The paid workflows reject manual dispatches when the workflow definition does not come from the default branch. A trusted dispatcher may name any branch in `paperclipai/paperclip` as the code under test. The authorization job resolves -that branch through the GitHub API and passes only its immutable commit SHA to -the catalog, image, and paid test checkouts. Report sanitization and AWS history -publication explicitly use the trusted workflow commit. Never run the workflow -definition from the target branch. +that branch through the GitHub API and passes only its immutable commit SHA to a +credential-free target-lock job. That job checks out the commit, regenerates +`pnpm-lock.yaml` once with lifecycle scripts disabled and lockfile-only mode, +then uploads the file under a run-attempt-scoped artifact ID. Catalog, image, +and paid test jobs download that exact artifact by ID, verify its recorded +SHA-256, and restore it before setup or a frozen dependency install. The lock +resolver receives no provider credentials and must never run repository +lifecycle scripts. The paid test job also installs with lifecycle scripts +disabled, and provider secrets are scoped only to its final test step rather +than dependency setup. Report sanitization and AWS history publication +explicitly use the trusted workflow commit and do not consume the target +lockfile. Never run the workflow definition from the target branch. The workflows verify both the original actor and triggering actor for every scheduled or manual attempt, including human reruns. Every @@ -56,8 +64,8 @@ only `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `OPENROUTER_API_KEY`, and organization-level Actions secrets: environment scoping is the boundary that prevents branch or pull-request jobs from requesting them. Require approval from an account in `RUNNER_E2E_ALLOWED_ACTOR_IDS` for this environment and -disable administrator bypass. The authorize, -catalog, image, report, history, and Pages jobs receive none of these secrets. +disable administrator bypass. The authorize, target-lock, catalog, image, +report, history, and Pages jobs receive none of these secrets. Each full-stack matrix cell receives only its selected profile credential, plus Daytona only for Daytona cells. Secret-bearing and OIDC jobs use frozen installs without a shared dependency cache. @@ -75,9 +83,10 @@ the actor gate, environment branch restriction, and protected default branch. When `RUNNER_E2E_AWS_ENABLED=true`, paid matrix cells use the exact RunsOn fleet selector `runs-on/fleet=paperclip-public-pr-x64/env=public-ci`, matching the AWS fleet selected by `pr-trusted.yml` only after its stable numeric-ID trust gate. -Any other or missing toggle value falls back to the existing `ubuntu-latest-m` -paid runner and its lower concurrency ceiling. The workflow chooses between -those two reviewed literal labels; it never evaluates a configured runner label. +Any other or missing toggle value falls back to the standard GitHub-hosted +`ubuntu-latest` runner and its lower concurrency ceiling. The workflow chooses +between those two reviewed literal labels; it never evaluates a configured +runner label. Keep both runner targets restricted to `paperclipai/paperclip` and workflows that independently authorize trusted source revisions. Never let a fork or diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts index eb0745fb12..d9f23b786c 100644 --- a/tests/runner-e2e/workflow-security.test.ts +++ b/tests/runner-e2e/workflow-security.test.ts @@ -66,12 +66,16 @@ describe("public repository paid workflow security", () => { ); const authorizeJob = fullStack.slice( fullStack.indexOf(" authorize:"), + fullStack.indexOf(" target_lock:"), + ); + const targetLockJob = fullStack.slice( + fullStack.indexOf(" target_lock:"), fullStack.indexOf(" catalog:"), ); expect(authorizeJob).toContain( "aws_runner='runs-on/fleet=paperclip-public-pr-x64/env=public-ci'", ); - expect(authorizeJob).toContain("github_runner='ubuntu-latest-m'"); + expect(authorizeJob).toContain("github_runner='ubuntu-latest'"); expect(authorizeJob).toContain( "AWS_PAID_RUNNER_ENABLED: ${{ vars.RUNNER_E2E_AWS_ENABLED }}", ); @@ -82,13 +86,46 @@ describe("public repository paid workflow security", () => { "repos/$REPOSITORY/branches/$encoded_branch", ); expect(authorizeJob).toContain('echo "sha=$target_sha"'); + expect(authorizeJob).not.toContain("actions/checkout@"); + expect(authorizeJob).not.toContain("pnpm install"); + expect(targetLockJob).toContain("name: Resolve target pnpm lockfile"); + expect(targetLockJob).toContain("needs: authorize"); + expect(targetLockJob).toContain( + "ref: ${{ needs.authorize.outputs.target_sha }}", + ); + expect(targetLockJob).toContain("persist-credentials: false"); + expect(targetLockJob).toContain( + "pnpm install --ignore-scripts --no-frozen-lockfile --lockfile-only", + ); + expect(targetLockJob).toContain( + "artifact_id: ${{ steps.upload.outputs.artifact-id }}", + ); + expect(targetLockJob).toContain("lock_sha256:"); + expect(targetLockJob).toContain( + "runner-e2e-target-pnpm-lock-${{ github.run_id }}-${{ github.run_attempt }}", + ); + expect(targetLockJob).not.toContain("name: runner-e2e-paid"); + expect(targetLockJob).not.toMatch( + /(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA)_API_KEY/, + ); expect(paidJob).toContain( "runs-on: ${{ needs.authorize.outputs.test_runner }}", ); - expect(paidJob).toContain("needs: [authorize, catalog, daytona_image]"); + expect(paidJob).toContain( + "needs: [authorize, target_lock, catalog, daytona_image]", + ); expect(paidJob).toContain("name: runner-e2e-paid"); expect(paidJob).toMatch( - /Reauthorize paid execution before provider access[\s\S]*actions\/checkout@[0-9a-f]{40}[\s\S]*persist-credentials: false/, + /Reauthorize paid execution before provider access[\s\S]*actions\/checkout@[0-9a-f]{40}[\s\S]*persist-credentials: false[\s\S]*Download resolved target lockfile/, + ); + const paidInstall = paidJob.indexOf( + "pnpm install --frozen-lockfile --ignore-scripts", + ); + const paidExecution = paidJob.indexOf("- name: Run paid cell"); + expect(paidInstall).toBeGreaterThan(0); + expect(paidExecution).toBeGreaterThan(paidInstall); + expect(paidJob.slice(0, paidExecution)).not.toMatch( + /secrets\.(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA)_API_KEY/, ); expect(authorizeJob).toContain('echo "max_parallel_limit=100"'); expect(fullStack).toContain('[ "$MAX_PARALLEL_LIMIT" -gt 100 ]'); @@ -101,13 +138,51 @@ describe("public repository paid workflow security", () => { expect(fullStack).toContain( "cancel-in-progress: ${{ github.event_name == 'workflow_dispatch' && inputs.target_branch != '' && inputs.target_branch != github.event.repository.default_branch }}", ); - expect( - fullStack.match( - /ref: \$\{\{ needs\.authorize\.outputs\.target_sha \}\}/g, + const targetCodeJobs = [ + fullStack.slice( + fullStack.indexOf(" catalog:"), + fullStack.indexOf(" daytona_image:"), ), - ).toHaveLength(3); + fullStack.slice( + fullStack.indexOf(" daytona_image:"), + fullStack.indexOf(" test:"), + ), + paidJob, + ]; + for (const targetCodeJob of targetCodeJobs) { + const checkout = targetCodeJob.indexOf("actions/checkout@"); + const downloadLock = targetCodeJob.indexOf( + "Download resolved target lockfile", + ); + const restoreLock = targetCodeJob.indexOf( + "Restore resolved target lockfile", + ); + const setupNode = targetCodeJob.indexOf("actions/setup-node@"); + const install = targetCodeJob.indexOf("pnpm install --frozen-lockfile"); + expect(checkout).toBeGreaterThan(0); + expect(downloadLock).toBeGreaterThan(checkout); + expect(restoreLock).toBeGreaterThan(downloadLock); + if (setupNode >= 0) { + expect(setupNode).toBeGreaterThan(restoreLock); + } + if (install >= 0) { + expect(install).toBeGreaterThan(restoreLock); + } + expect(targetCodeJob).toContain( + "artifact-ids: ${{ needs.target_lock.outputs.artifact_id }}", + ); + expect(targetCodeJob).toContain( + "EXPECTED_LOCK_SHA256: ${{ needs.target_lock.outputs.lock_sha256 }}", + ); + } + expect(fullStack.match(/Download resolved target lockfile/g)).toHaveLength( + 3, + ); + expect(fullStack.match(/Restore resolved target lockfile/g)).toHaveLength( + 3, + ); expect(fullStack.match(/ref: \$\{\{ github\.sha \}\}/g)).toHaveLength(2); - expect(fullStack.match(/persist-credentials: false/g)).toHaveLength(5); + expect(fullStack.match(/persist-credentials: false/g)).toHaveLength(6); expect(fullStack).not.toContain("ref: ${{ inputs.target_branch }}"); expect(fullStack).toContain( "PAPERCLIP_RUNNER_SOURCE_REVISION=${TARGET_SHA}", @@ -121,10 +196,12 @@ describe("public repository paid workflow security", () => { expect(reportJob).not.toContain( "ref: ${{ needs.authorize.outputs.target_sha }}", ); + expect(reportJob).not.toContain("Download resolved target lockfile"); expect(historyJob).toContain("ref: ${{ github.sha }}"); expect(historyJob).not.toContain( "ref: ${{ needs.authorize.outputs.target_sha }}", ); + expect(historyJob).not.toContain("Download resolved target lockfile"); for (const [secret, condition] of Object.entries({ OPENAI_API_KEY: "matrix.credentialName == 'OPENAI_API_KEY'", ANTHROPIC_API_KEY: "matrix.credentialName == 'ANTHROPIC_API_KEY'",