name: Runner Live Evals on: schedule: - cron: "17 6 * * 0" workflow_dispatch: inputs: candidate: description: "Comma-separated live candidate IDs (for example codex-luna)" type: string required: false case: description: "Comma-separated workflow case IDs" type: string required: false limit: description: "Maximum executions after candidate/case filtering" type: string required: false concurrency: group: runner-live-evals-${{ github.ref }} cancel-in-progress: true jobs: authorize: name: Authorize paid campaign if: github.event_name != 'schedule' || vars.RUNNER_LIVE_EVALS_NIGHTLY_ENABLED == 'true' runs-on: ubuntu-latest timeout-minutes: 5 permissions: contents: read outputs: eval_runner: ${{ steps.runner.outputs.runner }} steps: - name: Require default branch and allowlisted numeric actor IDs env: GH_TOKEN: ${{ github.token }} REF: ${{ github.ref }} DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} ACTOR: ${{ github.actor }} ACTOR_ID: ${{ github.actor_id }} TRIGGERING_ACTOR: ${{ github.triggering_actor }} ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} run: | set -euo pipefail if [ "$REF" != "refs/heads/$DEFAULT_BRANCH" ]; then echo "Paid runner live evals may run only from the default branch." >&2 exit 1 fi if ! jq -e 'type == "array" and length > 0 and all(.[]; type == "number" and . > 0 and floor == .)' <<< "${ALLOWED_ACTOR_IDS:-}" >/dev/null; then echo "RUNNER_E2E_ALLOWED_ACTOR_IDS must be a non-empty JSON array of numeric GitHub user IDs." >&2 exit 1 fi triggering_actor_id="$(gh api "users/$TRIGGERING_ACTOR" --jq .id)" if [ "$triggering_actor_id" != "$ACTOR_ID" ] && [ "$TRIGGERING_ACTOR" = "$ACTOR" ]; then echo "GitHub actor identity contexts disagree; refusing the paid run." >&2 exit 1 fi candidates=("$triggering_actor_id" "$ACTOR_ID") for candidate in "${candidates[@]}"; do if ! jq -e --argjson candidate "$candidate" 'index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null; then echo "The initiating GitHub account is not authorized to run paid runner live evals." >&2 exit 1 fi done - name: Select paid eval runner id: runner env: AWS_PAID_RUNNER_ENABLED: ${{ vars.RUNNER_E2E_AWS_ENABLED }} run: | set -euo pipefail github_runner='ubuntu-latest' aws_runner='runs-on/fleet=paperclip-public-pr-x64/env=public-ci' if [ "$AWS_PAID_RUNNER_ENABLED" = true ]; then echo "runner=$aws_runner" >> "$GITHUB_OUTPUT" echo '::notice title=Paid eval routing::Using an ephemeral RunsOn Fleet runner' else echo "runner=$github_runner" >> "$GITHUB_OUTPUT" echo '::notice title=Paid eval routing::RUNNER_E2E_AWS_ENABLED is not true; using the proven GitHub-hosted runner' fi live_matrix: name: Balanced provider/model matrix needs: authorize if: github.event_name != 'schedule' || vars.RUNNER_LIVE_EVALS_NIGHTLY_ENABLED == 'true' # The authorize job selects one of two literal, reviewed runner labels; # dispatch inputs and repository variables cannot inject an arbitrary label. runs-on: ${{ needs.authorize.outputs.eval_runner }} timeout-minutes: 180 permissions: contents: read environment: name: runner-e2e-paid steps: - name: Reauthorize paid execution before provider access env: GH_TOKEN: ${{ github.token }} REF: ${{ github.ref }} DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} ACTOR_ID: ${{ github.actor_id }} TRIGGERING_ACTOR: ${{ github.triggering_actor }} ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} run: | set -euo pipefail test "$REF" = "refs/heads/$DEFAULT_BRANCH" jq -e 'type == "array" and length > 0 and all(.[]; type == "number" and . > 0 and floor == .)' <<< "${ALLOWED_ACTOR_IDS:-}" >/dev/null triggering_actor_id="$(gh api "users/$TRIGGERING_ACTOR" --jq .id)" jq -e --argjson candidate "$triggering_actor_id" 'index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null jq -e --argjson candidate "$ACTOR_ID" 'index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Generate private eval-repository token id: evals_token env: COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }} GH_REPO: ${{ github.repository }} run: | set -euo pipefail token="$(node .github/scripts/get-bot-token.mjs)" echo "::add-mask::$token" echo "value=$token" >> "$GITHUB_OUTPUT" - name: Checkout canonical Evalbook reporter uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: repository: paperclipai/paperclip-evals ref: 0c8dfea0ef71a73e909b59a5c0484554cbee199b path: .paperclip-evals token: ${{ steps.evals_token.outputs.value }} persist-credentials: false - name: Setup pnpm uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 with: version: 9.15.4 - name: Setup Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 with: node-version: 24 - name: Install dependencies run: pnpm install --frozen-lockfile - name: Restore compatible weekly baseline uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5 with: path: packages/paperclip-runner/.paperclip-local/evals/workflows/history key: runner-live-eval-history-${{ github.ref_name }}-${{ github.run_id }} restore-keys: | runner-live-eval-history-${{ github.ref_name }}- - name: Build provider-neutral eval kernel run: pnpm --filter @paperclipai/paperclip-eval-kernel build - name: Run trend-only live matrix env: OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} PAPERCLIP_EVAL_BASELINE_READY: "true" PAPERCLIP_EVAL_RUNNER_BUILD: ${{ github.sha }} PAPERCLIP_EVAL_MAX_CAMPAIGN_COST_USD: "12" PAPERCLIP_EVAL_SCHEDULE_SEED: runner-live-seven-week-v1 PAPERCLIP_EVAL_CANDIDATE: ${{ inputs.candidate }} PAPERCLIP_EVAL_CASE: ${{ inputs.case }} PAPERCLIP_EVAL_LIMIT: ${{ inputs.limit }} PAPERCLIP_EVALBOOK_PROGRAM: ${{ github.workspace }}/.paperclip-evals/evals/paperclip-runner/tools/eval_program.py run: pnpm --filter @paperclipai/paperclip-runner report:runner-live-evals - name: Publish job summary if: always() run: | summary=packages/paperclip-runner/.paperclip-local/evals/workflows/github-live-summary.md if [ -f "$summary" ]; then cat "$summary" >> "$GITHUB_STEP_SUMMARY" fi - name: Upload safe live eval bundle if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: runner-live-evals-${{ github.run_id }} path: packages/paperclip-runner/.paperclip-local/evals/workflows/ retention-days: 30 if-no-files-found: error