193 lines
7.7 KiB
YAML
193 lines
7.7 KiB
YAML
name: Runner Live Evals
|
|
|
|
on:
|
|
schedule:
|
|
- cron: "17 6 * * 0"
|
|
workflow_dispatch:
|
|
inputs:
|
|
candidate:
|
|
description: "Comma-separated live candidate IDs (for example codex-luna)"
|
|
type: string
|
|
required: false
|
|
case:
|
|
description: "Comma-separated workflow case IDs"
|
|
type: string
|
|
required: false
|
|
limit:
|
|
description: "Maximum executions after candidate/case filtering"
|
|
type: string
|
|
required: false
|
|
|
|
concurrency:
|
|
group: runner-live-evals-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
authorize:
|
|
name: Authorize paid campaign
|
|
if: github.event_name != 'schedule' || vars.RUNNER_LIVE_EVALS_NIGHTLY_ENABLED == 'true'
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 5
|
|
permissions:
|
|
contents: read
|
|
outputs:
|
|
eval_runner: ${{ steps.runner.outputs.runner }}
|
|
steps:
|
|
- name: Require default branch and allowlisted numeric actor IDs
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
REF: ${{ github.ref }}
|
|
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
|
ACTOR: ${{ github.actor }}
|
|
ACTOR_ID: ${{ github.actor_id }}
|
|
TRIGGERING_ACTOR: ${{ github.triggering_actor }}
|
|
ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
|
|
run: |
|
|
set -euo pipefail
|
|
if [ "$REF" != "refs/heads/$DEFAULT_BRANCH" ]; then
|
|
echo "Paid runner live evals may run only from the default branch." >&2
|
|
exit 1
|
|
fi
|
|
if ! jq -e 'type == "array" and length > 0 and all(.[]; type == "number" and . > 0 and floor == .)' <<< "${ALLOWED_ACTOR_IDS:-}" >/dev/null; then
|
|
echo "RUNNER_E2E_ALLOWED_ACTOR_IDS must be a non-empty JSON array of numeric GitHub user IDs." >&2
|
|
exit 1
|
|
fi
|
|
triggering_actor_id="$(gh api "users/$TRIGGERING_ACTOR" --jq .id)"
|
|
if [ "$triggering_actor_id" != "$ACTOR_ID" ] && [ "$TRIGGERING_ACTOR" = "$ACTOR" ]; then
|
|
echo "GitHub actor identity contexts disagree; refusing the paid run." >&2
|
|
exit 1
|
|
fi
|
|
candidates=("$triggering_actor_id" "$ACTOR_ID")
|
|
for candidate in "${candidates[@]}"; do
|
|
if ! jq -e --argjson candidate "$candidate" 'index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null; then
|
|
echo "The initiating GitHub account is not authorized to run paid runner live evals." >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
- name: Select paid eval runner
|
|
id: runner
|
|
env:
|
|
AWS_PAID_RUNNER_ENABLED: ${{ vars.RUNNER_E2E_AWS_ENABLED }}
|
|
run: |
|
|
set -euo pipefail
|
|
github_runner='ubuntu-latest'
|
|
aws_runner='runs-on/fleet=paperclip-public-pr-x64/env=public-ci'
|
|
|
|
if [ "$AWS_PAID_RUNNER_ENABLED" = true ]; then
|
|
echo "runner=$aws_runner" >> "$GITHUB_OUTPUT"
|
|
echo '::notice title=Paid eval routing::Using an ephemeral RunsOn Fleet runner'
|
|
else
|
|
echo "runner=$github_runner" >> "$GITHUB_OUTPUT"
|
|
echo '::notice title=Paid eval routing::RUNNER_E2E_AWS_ENABLED is not true; using the proven GitHub-hosted runner'
|
|
fi
|
|
|
|
live_matrix:
|
|
name: Balanced provider/model matrix
|
|
needs: authorize
|
|
if: github.event_name != 'schedule' || vars.RUNNER_LIVE_EVALS_NIGHTLY_ENABLED == 'true'
|
|
# The authorize job selects one of two literal, reviewed runner labels;
|
|
# dispatch inputs and repository variables cannot inject an arbitrary label.
|
|
runs-on: ${{ needs.authorize.outputs.eval_runner }}
|
|
timeout-minutes: 180
|
|
permissions:
|
|
contents: read
|
|
environment:
|
|
name: runner-e2e-paid
|
|
|
|
steps:
|
|
- name: Reauthorize paid execution before provider access
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
REF: ${{ github.ref }}
|
|
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
|
ACTOR_ID: ${{ github.actor_id }}
|
|
TRIGGERING_ACTOR: ${{ github.triggering_actor }}
|
|
ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
|
|
run: |
|
|
set -euo pipefail
|
|
test "$REF" = "refs/heads/$DEFAULT_BRANCH"
|
|
jq -e 'type == "array" and length > 0 and all(.[]; type == "number" and . > 0 and floor == .)' <<< "${ALLOWED_ACTOR_IDS:-}" >/dev/null
|
|
triggering_actor_id="$(gh api "users/$TRIGGERING_ACTOR" --jq .id)"
|
|
jq -e --argjson candidate "$triggering_actor_id" 'index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null
|
|
jq -e --argjson candidate "$ACTOR_ID" 'index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null
|
|
|
|
- name: Checkout repository
|
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
|
|
- name: Generate private eval-repository token
|
|
id: evals_token
|
|
env:
|
|
COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }}
|
|
GH_REPO: ${{ github.repository }}
|
|
run: |
|
|
set -euo pipefail
|
|
token="$(node .github/scripts/get-bot-token.mjs)"
|
|
echo "::add-mask::$token"
|
|
echo "value=$token" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Checkout canonical Evalbook reporter
|
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
with:
|
|
repository: paperclipai/paperclip-evals
|
|
ref: 0c8dfea0ef71a73e909b59a5c0484554cbee199b
|
|
path: .paperclip-evals
|
|
token: ${{ steps.evals_token.outputs.value }}
|
|
persist-credentials: false
|
|
|
|
- name: Setup pnpm
|
|
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
|
|
with:
|
|
version: 9.15.4
|
|
|
|
- name: Setup Node.js
|
|
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
|
with:
|
|
node-version: 24
|
|
|
|
- name: Install dependencies
|
|
run: pnpm install --frozen-lockfile
|
|
|
|
- name: Restore compatible weekly baseline
|
|
uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5
|
|
with:
|
|
path: packages/paperclip-runner/.paperclip-local/evals/workflows/history
|
|
key: runner-live-eval-history-${{ github.ref_name }}-${{ github.run_id }}
|
|
restore-keys: |
|
|
runner-live-eval-history-${{ github.ref_name }}-
|
|
|
|
- name: Build provider-neutral eval kernel
|
|
run: pnpm --filter @paperclipai/paperclip-eval-kernel build
|
|
|
|
- name: Run trend-only live matrix
|
|
env:
|
|
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
|
|
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
PAPERCLIP_EVAL_BASELINE_READY: "true"
|
|
PAPERCLIP_EVAL_RUNNER_BUILD: ${{ github.sha }}
|
|
PAPERCLIP_EVAL_MAX_CAMPAIGN_COST_USD: "12"
|
|
PAPERCLIP_EVAL_SCHEDULE_SEED: runner-live-seven-week-v1
|
|
PAPERCLIP_EVAL_CANDIDATE: ${{ inputs.candidate }}
|
|
PAPERCLIP_EVAL_CASE: ${{ inputs.case }}
|
|
PAPERCLIP_EVAL_LIMIT: ${{ inputs.limit }}
|
|
PAPERCLIP_EVALBOOK_PROGRAM: ${{ github.workspace }}/.paperclip-evals/evals/paperclip-runner/tools/eval_program.py
|
|
run: pnpm --filter @paperclipai/paperclip-runner report:runner-live-evals
|
|
|
|
- name: Publish job summary
|
|
if: always()
|
|
run: |
|
|
summary=packages/paperclip-runner/.paperclip-local/evals/workflows/github-live-summary.md
|
|
if [ -f "$summary" ]; then
|
|
cat "$summary" >> "$GITHUB_STEP_SUMMARY"
|
|
fi
|
|
|
|
- name: Upload safe live eval bundle
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: runner-live-evals-${{ github.run_id }}
|
|
path: packages/paperclip-runner/.paperclip-local/evals/workflows/
|
|
retention-days: 30
|
|
if-no-files-found: error
|