RomanNum3ral
/
paperclip
mirror of https://github.com/paperclipai/paperclip.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ci(runner): route paid matrix to AWS fleet

This commit is contained in:
Dotta 2026-09-03 08:02:25 -05:00
parent 77fa7d19b4
commit e97ccd6f70
4 changed files with 137 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
.github/workflows/runner-full-stack-e2e.yml vendored
View File

@ -39,7 +39,9 @@ permissions:
concurrency:
group: runner-full-stack-e2e-${{ github.ref }}
cancel-in-progress: false
# Development-only validation refs supersede older runs on the same ref.
# Preserve every protected default-branch campaign for its paid audit trail.
cancel-in-progress: ${{ github.ref != format('refs/heads/{0}', github.event.repository.default_branch) }}
jobs:
authorize:
@ -49,6 +51,10 @@ jobs:
timeout-minutes: 5
permissions:
contents: read
outputs:
test_runner: ${{ steps.runner.outputs.runner }}
max_parallel_default: ${{ steps.runner.outputs.max_parallel_default }}
max_parallel_limit: ${{ steps.runner.outputs.max_parallel_limit }}
steps:
- name: Require default branch and allowlisted numeric actor IDs
env:
@ -83,6 +89,31 @@ jobs:
fi
done
- name: Select paid test runner
id: runner
env:
AWS_PAID_RUNNER_ENABLED: ${{ vars.RUNNER_E2E_AWS_ENABLED }}
run: |
set -euo pipefail
github_runner='ubuntu-latest-m'
aws_runner='runs-on/fleet=paperclip-public-pr-x64/env=public-ci'
if [ "$AWS_PAID_RUNNER_ENABLED" = true ]; then
{
echo "runner=$aws_runner"
echo "max_parallel_default=100"
echo "max_parallel_limit=100"
} >> "$GITHUB_OUTPUT"
echo '::notice title=Paid runner routing::Using an ephemeral RunsOn Fleet runner'
else
{
echo "runner=$github_runner"
echo "max_parallel_default=32"
echo "max_parallel_limit=57"
} >> "$GITHUB_OUTPUT"
echo '::notice title=Paid runner routing::RUNNER_E2E_AWS_ENABLED is not true; using the existing paid runner'
fi
catalog:
name: Validate catalog and select cells
needs: authorize
@ -131,7 +162,8 @@ jobs:
SELECT_ENVIRONMENT: ${{ inputs.environment }}
SELECT_CASE: ${{ inputs.case }}
SELECT_ID: ${{ inputs.id }}
MAX_PARALLEL: ${{ vars.RUNNER_E2E_MAX_PARALLEL || '32' }}
MAX_PARALLEL: ${{ vars.RUNNER_E2E_MAX_PARALLEL || needs.authorize.outputs.max_parallel_default }}
MAX_PARALLEL_LIMIT: ${{ needs.authorize.outputs.max_parallel_limit }}
run: |
set -euo pipefail
args=(--matrix-json)
@ -184,8 +216,12 @@ jobs:
echo "needs_remote_provider_pack=$(jq -r '[.include[] | select((.environmentId == "daytona") and ((.profileId == "runner-opencode") or (.profileId | startswith("runner-acpx-"))))] | length > 0' <<< "$catalog_json")"
echo "execution_ids=$(jq -c '.executionIds' <<< "$catalog_json")"
} >> "$GITHUB_OUTPUT"
if ! [[ "$MAX_PARALLEL" =~ ^[1-9][0-9]*$ ]] || [ "$MAX_PARALLEL" -gt 57 ]; then
echo "RUNNER_E2E_MAX_PARALLEL must be an integer from 1 through 57." >&2
if ! [[ "$MAX_PARALLEL_LIMIT" =~ ^[1-9][0-9]*$ ]] || [ "$MAX_PARALLEL_LIMIT" -gt 100 ]; then
echo "Runner selection emitted an invalid max-parallel limit." >&2
exit 1
fi
if ! [[ "$MAX_PARALLEL" =~ ^[1-9][0-9]*$ ]] || [ "$MAX_PARALLEL" -gt "$MAX_PARALLEL_LIMIT" ]; then
echo "RUNNER_E2E_MAX_PARALLEL must be an integer from 1 through $MAX_PARALLEL_LIMIT for the selected runner." >&2
exit 1
fi
echo "max_parallel=$MAX_PARALLEL" >> "$GITHUB_OUTPUT"
@ -296,12 +332,16 @@ jobs:
name: Build reusable runner campaign artifacts
needs: [authorize, catalog]
if: github.event_name != 'schedule' || vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true'
runs-on: ubuntu-latest
# Compile native binaries on the same reviewed image used to execute them,
# avoiding libc/architecture drift between GitHub-hosted and AWS lanes.
runs-on: ${{ needs.authorize.outputs.test_runner }}
timeout-minutes: 20
permissions:
contents: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
with:
@ -466,8 +506,10 @@ jobs:
test:
name: ${{ matrix.executionId }}
needs: [catalog, daytona_image, build_runner_artifacts, build_remote_provider_pack]
runs-on: ubuntu-latest-m
needs: [authorize, catalog, daytona_image, build_runner_artifacts, build_remote_provider_pack]
# The authorize job selects only one of two literal, reviewed runner labels;
# no dispatch input or repository variable can inject an arbitrary label.
runs-on: ${{ needs.authorize.outputs.test_runner }}
timeout-minutes: ${{ matrix.timeoutMinutes }}
permissions:
contents: read
@ -495,6 +537,8 @@ jobs:
jq -e --argjson candidate "$ACTOR_ID" 'index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
with:

30
tests/runner-e2e/README.md
View File

@ -286,17 +286,27 @@ accept comma-separated values for repeatable dimensions.
The nightly cron is `08:47 UTC`, but scheduled execution is intentionally gated
by the repository variable `RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED=true`. Set it
only after the live acceptance ladder in the architecture plan is green.
Set `RUNNER_E2E_MAX_PARALLEL` to an integer from 1–57 (default 32). Paid cells
run on `ubuntu-latest-m`; multi-turn steps are sequential inside their cell
while independent cells overlap. Artifacts and merged HTML/JUnit/normalized
reports are retained for 30 days.
Set `RUNNER_E2E_AWS_ENABLED=true` to route paid cells to the repository-scoped
ephemeral AWS RunsOn fleet selected by
`runs-on/fleet=paperclip-public-pr-x64/env=public-ci`. Any other value retains
the existing `ubuntu-latest-m` target. Set `RUNNER_E2E_MAX_PARALLEL` to an
integer from 1–100 on AWS (default 100); use at least 71 to run the current
complete catalog in one wave. The fallback runner retains its 1–57 limit and
default of 32. Multi-turn steps are sequential inside their cell while
independent cells overlap. Artifacts and merged HTML/JUnit/normalized reports
are retained for 30 days.
Restrict the `ubuntu-latest-m` runner group to this workflow and the selected
repository. Do not let pull-request or fork-triggered workflows target that
group, do not mix it with untrusted workloads, and use ephemeral/reimaged
runners so one paid cell cannot leave state for the next. These runner-group
controls are external GitHub settings and are as important as the workflow
checks in a public repository.
Restrict the RunsOn fleet to this repository and independently trusted
workflows. Do not let untrusted pull-request or fork-triggered workflows target
it, and require a fresh ephemeral instance for each job so one paid cell cannot
leave state for the next. Provider secrets remain protected by the stable-ID
authorization checks and the default-branch-only `runner-e2e-paid` environment;
the fleet itself is not an authorization boundary. These external fleet controls
are as important as the workflow checks in a public repository.
Non-default validation runs share a concurrency key per ref and cancel an older
run when a replacement is dispatched. Protected default-branch paid campaigns
are retained and are never auto-cancelled, preserving their audit trail.
GitHub Actions artifacts are access-controlled 30-day operational copies, not
the permanent public history. They retain packaged PNG/WebM and generated

30
tests/runner-e2e/SECURITY.md
View File

@ -62,15 +62,29 @@ job. It contains no long-lived AWS key. Required reviewers may be added when a
human approval on every nightly publication is acceptable; otherwise rely on
the actor gate, environment branch restriction, and protected default branch.
## Runner group isolation
## Runner fleet isolation
Restrict the `ubuntu-latest-m` runner group to `paperclipai/paperclip` and, when
the GitHub plan supports selected-workflow restrictions, to
`.github/workflows/runner-full-stack-e2e.yml` on the default branch. Never let
fork or pull-request workflows target the group. Use ephemeral runners, or
guaranteed reimaging between jobs, and do not share this group with untrusted
workloads. Disable interactive SSH/debug access for paid jobs unless a separate
incident procedure explicitly authorizes it.
When `RUNNER_E2E_AWS_ENABLED=true`, paid matrix cells use the exact RunsOn fleet
selector `runs-on/fleet=paperclip-public-pr-x64/env=public-ci`, matching the AWS
fleet selected by `pr-trusted.yml` only after its stable numeric-ID trust gate.
Any other or missing toggle value falls back to the existing `ubuntu-latest-m`
paid runner and its lower concurrency ceiling. The workflow chooses between
those two reviewed literal labels; it never evaluates a configured runner label.
Keep both runner targets restricted to `paperclipai/paperclip` and workflows
that independently authorize trusted source revisions. Never let a fork or
untrusted pull-request workflow target them. The RunsOn fleet must launch a
fresh ephemeral instance for every job, prohibit persistent runner reuse, and
disable interactive SSH/debug access unless a separate incident procedure
explicitly authorizes it.
Changing the runner does not widen secret access. The paid workflow still has
only schedule and manual triggers, requires the protected default branch and
allowlisted stable actor IDs before checkout, repeats that authorization as the
first matrix step, and receives provider credentials only from the protected
`runner-e2e-paid` environment. The fleet selector is an exact workflow literal;
the only repository-controlled input is its boolean rollout switch, so
configuration cannot redirect a secret-bearing job to an arbitrary runner.
## AWS OIDC and S3

54
tests/runner-e2e/workflow-security.test.ts
View File

@ -25,9 +25,10 @@ describe("public repository paid workflow security", () => {
const providerAccess = contents.search(
/(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA)_API_KEY:\s*\$\{\{\s*[^}]*secrets\./,
);
expect(authorize, `${name} must have an authorization job`).toBeGreaterThan(
0,
);
expect(
authorize,
`${name} must have an authorization job`,
).toBeGreaterThan(0);
expect(
reauthorize,
`${name} must reauthorize partial job reruns`,
@ -43,7 +44,7 @@ describe("public repository paid workflow security", () => {
expect(contents).toContain("RUNNER_E2E_ALLOWED_ACTOR_IDS");
expect(contents).toContain("github.actor_id");
expect(contents).toContain("github.triggering_actor");
expect(contents).toContain('refs/heads/$DEFAULT_BRANCH');
expect(contents).toContain("refs/heads/$DEFAULT_BRANCH");
expect(contents).toContain("needs: authorize");
expect(contents).toContain("name: runner-e2e-paid");
expect(contents).not.toMatch(
@ -59,6 +60,39 @@ describe("public repository paid workflow security", () => {
}
const fullStack = workflows[0]!.contents;
const paidJob = fullStack.slice(
fullStack.indexOf(" test:"),
fullStack.indexOf(" report:"),
);
const authorizeJob = fullStack.slice(
fullStack.indexOf(" authorize:"),
fullStack.indexOf(" catalog:"),
);
expect(authorizeJob).toContain(
"aws_runner='runs-on/fleet=paperclip-public-pr-x64/env=public-ci'",
);
expect(authorizeJob).toContain("github_runner='ubuntu-latest-m'");
expect(authorizeJob).toContain(
"AWS_PAID_RUNNER_ENABLED: ${{ vars.RUNNER_E2E_AWS_ENABLED }}",
);
expect(paidJob).toContain(
"runs-on: ${{ needs.authorize.outputs.test_runner }}",
);
expect(paidJob).toContain(
"needs: [authorize, catalog, daytona_image, build_runner_artifacts, build_remote_provider_pack]",
);
expect(paidJob).toContain("name: runner-e2e-paid");
expect(paidJob).toMatch(
/Reauthorize paid execution before provider access[\s\S]*actions\/checkout@[0-9a-f]{40}[\s\S]*persist-credentials: false/,
);
expect(authorizeJob).toContain('echo "max_parallel_limit=100"');
expect(fullStack).toContain('[ "$MAX_PARALLEL_LIMIT" -gt 100 ]');
expect(fullStack).toContain(
'[ "$MAX_PARALLEL" -gt "$MAX_PARALLEL_LIMIT" ]',
);
expect(fullStack).toContain(
"cancel-in-progress: ${{ github.ref != format('refs/heads/{0}', github.event.repository.default_branch) }}",
);
for (const [secret, condition] of Object.entries({
OPENAI_API_KEY: "matrix.credentialName == 'OPENAI_API_KEY'",
ANTHROPIC_API_KEY: "matrix.credentialName == 'ANTHROPIC_API_KEY'",
@ -83,7 +117,10 @@ describe("public repository paid workflow security", () => {
);
for (const name of names) {
const contents = await readFile(path.join(workflowDirectory, name), "utf8");
const contents = await readFile(
path.join(workflowDirectory, name),
"utf8",
);
const providerSecretReferences = [
...contents.matchAll(
/secrets(?:\.(?:OPENAI_API_KEY|ANTHROPIC_API_KEY|OPENROUTER_API_KEY|DAYTONA_API_KEY)\b|\[['"](?:OPENAI_API_KEY|ANTHROPIC_API_KEY|OPENROUTER_API_KEY|DAYTONA_API_KEY)['"]\])/g,
@ -101,10 +138,7 @@ describe("public repository paid workflow security", () => {
it("runs paid scheduled campaigns only on Sundays", async () => {
const workflows = await Promise.all(
["runner-full-stack-e2e.yml", "runner-live-evals.yml"].map((name) =>
readFile(
path.join(repositoryRoot, ".github/workflows", name),
"utf8",
),
readFile(path.join(repositoryRoot, ".github/workflows", name), "utf8"),
),
);
for (const workflow of workflows) {
@ -155,7 +189,7 @@ describe("public repository paid workflow security", () => {
expect(workflow).toContain("needs_remote_provider_pack=");
expect(testJob).toContain(
"needs: [catalog, daytona_image, build_runner_artifacts, build_remote_provider_pack]",
"needs: [authorize, catalog, daytona_image, build_runner_artifacts, build_remote_provider_pack]",
);
expect(testJob).toContain("Download immutable campaign outputs");
expect(testJob).toContain("Download immutable remote provider pack");