diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.yml index 102fe0b642..8883b360a7 100644 --- a/.github/workflows/runner-full-stack-e2e.yml +++ b/.github/workflows/runner-full-stack-e2e.yml @@ -39,7 +39,9 @@ permissions: concurrency: group: runner-full-stack-e2e-${{ github.ref }} - cancel-in-progress: false + # Development-only validation refs supersede older runs on the same ref. + # Preserve every protected default-branch campaign for its paid audit trail. + cancel-in-progress: ${{ github.ref != format('refs/heads/{0}', github.event.repository.default_branch) }} jobs: authorize: @@ -49,6 +51,10 @@ jobs: timeout-minutes: 5 permissions: contents: read + outputs: + test_runner: ${{ steps.runner.outputs.runner }} + max_parallel_default: ${{ steps.runner.outputs.max_parallel_default }} + max_parallel_limit: ${{ steps.runner.outputs.max_parallel_limit }} steps: - name: Require default branch and allowlisted numeric actor IDs env: @@ -83,6 +89,31 @@ jobs: fi done + - name: Select paid test runner + id: runner + env: + AWS_PAID_RUNNER_ENABLED: ${{ vars.RUNNER_E2E_AWS_ENABLED }} + run: | + set -euo pipefail + github_runner='ubuntu-latest-m' + aws_runner='runs-on/fleet=paperclip-public-pr-x64/env=public-ci' + + if [ "$AWS_PAID_RUNNER_ENABLED" = true ]; then + { + echo "runner=$aws_runner" + echo "max_parallel_default=100" + echo "max_parallel_limit=100" + } >> "$GITHUB_OUTPUT" + echo '::notice title=Paid runner routing::Using an ephemeral RunsOn Fleet runner' + else + { + echo "runner=$github_runner" + echo "max_parallel_default=32" + echo "max_parallel_limit=57" + } >> "$GITHUB_OUTPUT" + echo '::notice title=Paid runner routing::RUNNER_E2E_AWS_ENABLED is not true; using the existing paid runner' + fi + catalog: name: Validate catalog and select cells needs: authorize @@ -128,7 +159,8 @@ jobs: SELECT_ENVIRONMENT: ${{ inputs.environment }} SELECT_CASE: ${{ inputs.case }} SELECT_ID: ${{ inputs.id }} - MAX_PARALLEL: ${{ vars.RUNNER_E2E_MAX_PARALLEL || '32' }} + MAX_PARALLEL: ${{ vars.RUNNER_E2E_MAX_PARALLEL || needs.authorize.outputs.max_parallel_default }} + MAX_PARALLEL_LIMIT: ${{ needs.authorize.outputs.max_parallel_limit }} run: | set -euo pipefail args=(--matrix-json) @@ -176,8 +208,12 @@ jobs: echo "matrix=$(jq -c '{include: .include}' <<< "$catalog_json")" >> "$GITHUB_OUTPUT" echo "needs_daytona=$(jq -r '.needsDaytona' <<< "$catalog_json")" >> "$GITHUB_OUTPUT" echo "execution_ids=$(jq -c '.executionIds' <<< "$catalog_json")" >> "$GITHUB_OUTPUT" - if ! [[ "$MAX_PARALLEL" =~ ^[1-9][0-9]*$ ]] || [ "$MAX_PARALLEL" -gt 57 ]; then - echo "RUNNER_E2E_MAX_PARALLEL must be an integer from 1 through 57." >&2 + if ! [[ "$MAX_PARALLEL_LIMIT" =~ ^[1-9][0-9]*$ ]] || [ "$MAX_PARALLEL_LIMIT" -gt 100 ]; then + echo "Runner selection emitted an invalid max-parallel limit." >&2 + exit 1 + fi + if ! [[ "$MAX_PARALLEL" =~ ^[1-9][0-9]*$ ]] || [ "$MAX_PARALLEL" -gt "$MAX_PARALLEL_LIMIT" ]; then + echo "RUNNER_E2E_MAX_PARALLEL must be an integer from 1 through $MAX_PARALLEL_LIMIT for the selected runner." >&2 exit 1 fi echo "max_parallel=$MAX_PARALLEL" >> "$GITHUB_OUTPUT" @@ -286,8 +322,10 @@ jobs: test: name: ${{ matrix.executionId }} - needs: [catalog, daytona_image] - runs-on: ubuntu-latest-m + needs: [authorize, catalog, daytona_image] + # The authorize job selects only one of two literal, reviewed runner labels; + # no dispatch input or repository variable can inject an arbitrary label. + runs-on: ${{ needs.authorize.outputs.test_runner }} timeout-minutes: ${{ matrix.timeoutMinutes }} permissions: contents: read @@ -315,6 +353,8 @@ jobs: jq -e --argjson candidate "$ACTOR_ID" 'index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 with: diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 4c4d42e43f..c7d9cb1320 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -286,17 +286,27 @@ accept comma-separated values for repeatable dimensions. The nightly cron is `08:47 UTC`, but scheduled execution is intentionally gated by the repository variable `RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED=true`. Set it only after the live acceptance ladder in the architecture plan is green. -Set `RUNNER_E2E_MAX_PARALLEL` to an integer from 1–57 (default 32). Paid cells -run on `ubuntu-latest-m`; multi-turn steps are sequential inside their cell -while independent cells overlap. Artifacts and merged HTML/JUnit/normalized -reports are retained for 30 days. +Set `RUNNER_E2E_AWS_ENABLED=true` to route paid cells to the repository-scoped +ephemeral AWS RunsOn fleet selected by +`runs-on/fleet=paperclip-public-pr-x64/env=public-ci`. Any other value retains +the existing `ubuntu-latest-m` target. Set `RUNNER_E2E_MAX_PARALLEL` to an +integer from 1–100 on AWS (default 100); use at least 71 to run the current +complete catalog in one wave. The fallback runner retains its 1–57 limit and +default of 32. Multi-turn steps are sequential inside their cell while +independent cells overlap. Artifacts and merged HTML/JUnit/normalized reports +are retained for 30 days. -Restrict the `ubuntu-latest-m` runner group to this workflow and the selected -repository. Do not let pull-request or fork-triggered workflows target that -group, do not mix it with untrusted workloads, and use ephemeral/reimaged -runners so one paid cell cannot leave state for the next. These runner-group -controls are external GitHub settings and are as important as the workflow -checks in a public repository. +Restrict the RunsOn fleet to this repository and independently trusted +workflows. Do not let untrusted pull-request or fork-triggered workflows target +it, and require a fresh ephemeral instance for each job so one paid cell cannot +leave state for the next. Provider secrets remain protected by the stable-ID +authorization checks and the default-branch-only `runner-e2e-paid` environment; +the fleet itself is not an authorization boundary. These external fleet controls +are as important as the workflow checks in a public repository. + +Non-default validation runs share a concurrency key per ref and cancel an older +run when a replacement is dispatched. Protected default-branch paid campaigns +are retained and are never auto-cancelled, preserving their audit trail. GitHub Actions artifacts are access-controlled 30-day operational copies, not the permanent public history. They retain packaged PNG/WebM and generated diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md index 6a833e2fa6..19f407bb76 100644 --- a/tests/runner-e2e/SECURITY.md +++ b/tests/runner-e2e/SECURITY.md @@ -62,15 +62,29 @@ job. It contains no long-lived AWS key. Required reviewers may be added when a human approval on every nightly publication is acceptable; otherwise rely on the actor gate, environment branch restriction, and protected default branch. -## Runner group isolation +## Runner fleet isolation -Restrict the `ubuntu-latest-m` runner group to `paperclipai/paperclip` and, when -the GitHub plan supports selected-workflow restrictions, to -`.github/workflows/runner-full-stack-e2e.yml` on the default branch. Never let -fork or pull-request workflows target the group. Use ephemeral runners, or -guaranteed reimaging between jobs, and do not share this group with untrusted -workloads. Disable interactive SSH/debug access for paid jobs unless a separate -incident procedure explicitly authorizes it. +When `RUNNER_E2E_AWS_ENABLED=true`, paid matrix cells use the exact RunsOn fleet +selector `runs-on/fleet=paperclip-public-pr-x64/env=public-ci`, matching the AWS +fleet selected by `pr-trusted.yml` only after its stable numeric-ID trust gate. +Any other or missing toggle value falls back to the existing `ubuntu-latest-m` +paid runner and its lower concurrency ceiling. The workflow chooses between +those two reviewed literal labels; it never evaluates a configured runner label. + +Keep both runner targets restricted to `paperclipai/paperclip` and workflows +that independently authorize trusted source revisions. Never let a fork or +untrusted pull-request workflow target them. The RunsOn fleet must launch a +fresh ephemeral instance for every job, prohibit persistent runner reuse, and +disable interactive SSH/debug access unless a separate incident procedure +explicitly authorizes it. + +Changing the runner does not widen secret access. The paid workflow still has +only schedule and manual triggers, requires the protected default branch and +allowlisted stable actor IDs before checkout, repeats that authorization as the +first matrix step, and receives provider credentials only from the protected +`runner-e2e-paid` environment. The fleet selector is an exact workflow literal; +the only repository-controlled input is its boolean rollout switch, so +configuration cannot redirect a secret-bearing job to an arbitrary runner. ## AWS OIDC and S3 diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts index fb7479725e..8b45eee73e 100644 --- a/tests/runner-e2e/workflow-security.test.ts +++ b/tests/runner-e2e/workflow-security.test.ts @@ -25,9 +25,10 @@ describe("public repository paid workflow security", () => { const providerAccess = contents.search( /(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA)_API_KEY:\s*\$\{\{\s*[^}]*secrets\./, ); - expect(authorize, `${name} must have an authorization job`).toBeGreaterThan( - 0, - ); + expect( + authorize, + `${name} must have an authorization job`, + ).toBeGreaterThan(0); expect( reauthorize, `${name} must reauthorize partial job reruns`, @@ -43,7 +44,7 @@ describe("public repository paid workflow security", () => { expect(contents).toContain("RUNNER_E2E_ALLOWED_ACTOR_IDS"); expect(contents).toContain("github.actor_id"); expect(contents).toContain("github.triggering_actor"); - expect(contents).toContain('refs/heads/$DEFAULT_BRANCH'); + expect(contents).toContain("refs/heads/$DEFAULT_BRANCH"); expect(contents).toContain("needs: authorize"); expect(contents).toContain("name: runner-e2e-paid"); expect(contents).not.toMatch( @@ -59,6 +60,37 @@ describe("public repository paid workflow security", () => { } const fullStack = workflows[0]!.contents; + const paidJob = fullStack.slice( + fullStack.indexOf(" test:"), + fullStack.indexOf(" report:"), + ); + const authorizeJob = fullStack.slice( + fullStack.indexOf(" authorize:"), + fullStack.indexOf(" catalog:"), + ); + expect(authorizeJob).toContain( + "aws_runner='runs-on/fleet=paperclip-public-pr-x64/env=public-ci'", + ); + expect(authorizeJob).toContain("github_runner='ubuntu-latest-m'"); + expect(authorizeJob).toContain( + "AWS_PAID_RUNNER_ENABLED: ${{ vars.RUNNER_E2E_AWS_ENABLED }}", + ); + expect(paidJob).toContain( + "runs-on: ${{ needs.authorize.outputs.test_runner }}", + ); + expect(paidJob).toContain("needs: [authorize, catalog, daytona_image]"); + expect(paidJob).toContain("name: runner-e2e-paid"); + expect(paidJob).toMatch( + /Reauthorize paid execution before provider access[\s\S]*actions\/checkout@[0-9a-f]{40}[\s\S]*persist-credentials: false/, + ); + expect(authorizeJob).toContain('echo "max_parallel_limit=100"'); + expect(fullStack).toContain('[ "$MAX_PARALLEL_LIMIT" -gt 100 ]'); + expect(fullStack).toContain( + '[ "$MAX_PARALLEL" -gt "$MAX_PARALLEL_LIMIT" ]', + ); + expect(fullStack).toContain( + "cancel-in-progress: ${{ github.ref != format('refs/heads/{0}', github.event.repository.default_branch) }}", + ); for (const [secret, condition] of Object.entries({ OPENAI_API_KEY: "matrix.credentialName == 'OPENAI_API_KEY'", ANTHROPIC_API_KEY: "matrix.credentialName == 'ANTHROPIC_API_KEY'", @@ -83,7 +115,10 @@ describe("public repository paid workflow security", () => { ); for (const name of names) { - const contents = await readFile(path.join(workflowDirectory, name), "utf8"); + const contents = await readFile( + path.join(workflowDirectory, name), + "utf8", + ); const providerSecretReferences = [ ...contents.matchAll( /secrets(?:\.(?:OPENAI_API_KEY|ANTHROPIC_API_KEY|OPENROUTER_API_KEY|DAYTONA_API_KEY)\b|\[['"](?:OPENAI_API_KEY|ANTHROPIC_API_KEY|OPENROUTER_API_KEY|DAYTONA_API_KEY)['"]\])/g, @@ -101,10 +136,7 @@ describe("public repository paid workflow security", () => { it("runs paid scheduled campaigns only on Sundays", async () => { const workflows = await Promise.all( ["runner-full-stack-e2e.yml", "runner-live-evals.yml"].map((name) => - readFile( - path.join(repositoryRoot, ".github/workflows", name), - "utf8", - ), + readFile(path.join(repositoryRoot, ".github/workflows", name), "utf8"), ), ); for (const workflow of workflows) {