diff --git a/.github/workflows/runner-live-evals.yml b/.github/workflows/runner-live-evals.yml index 978bdca02e..9f797ab89d 100644 --- a/.github/workflows/runner-live-evals.yml +++ b/.github/workflows/runner-live-evals.yml @@ -4,6 +4,19 @@ on: schedule: - cron: "17 6 * * 0" workflow_dispatch: + inputs: + candidate: + description: "Comma-separated live candidate IDs (for example codex-luna)" + type: string + required: false + case: + description: "Comma-separated workflow case IDs" + type: string + required: false + limit: + description: "Maximum executions after candidate/case filtering" + type: string + required: false concurrency: group: runner-live-evals-${{ github.ref }} @@ -17,6 +30,8 @@ jobs: timeout-minutes: 5 permissions: contents: read + outputs: + eval_runner: ${{ steps.runner.outputs.runner }} steps: - name: Require default branch and allowlisted numeric actor IDs env: @@ -50,11 +65,30 @@ jobs: fi done + - name: Select paid eval runner + id: runner + env: + AWS_PAID_RUNNER_ENABLED: ${{ vars.RUNNER_E2E_AWS_ENABLED }} + run: | + set -euo pipefail + github_runner='ubuntu-latest' + aws_runner='runs-on/fleet=paperclip-public-pr-x64/env=public-ci' + + if [ "$AWS_PAID_RUNNER_ENABLED" = true ]; then + echo "runner=$aws_runner" >> "$GITHUB_OUTPUT" + echo '::notice title=Paid eval routing::Using an ephemeral RunsOn Fleet runner' + else + echo "runner=$github_runner" >> "$GITHUB_OUTPUT" + echo '::notice title=Paid eval routing::RUNNER_E2E_AWS_ENABLED is not true; using the proven GitHub-hosted runner' + fi + live_matrix: name: Balanced provider/model matrix needs: authorize if: github.event_name != 'schedule' || vars.RUNNER_LIVE_EVALS_NIGHTLY_ENABLED == 'true' - runs-on: ubuntu-latest + # The authorize job selects one of two literal, reviewed runner labels; + # dispatch inputs and repository variables cannot inject an arbitrary label. + runs-on: ${{ needs.authorize.outputs.eval_runner }} timeout-minutes: 180 permissions: contents: read @@ -81,6 +115,26 @@ jobs: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - name: Generate private eval-repository token + id: evals_token + env: + COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }} + GH_REPO: ${{ github.repository }} + run: | + set -euo pipefail + token="$(node .github/scripts/get-bot-token.mjs)" + echo "::add-mask::$token" + echo "value=$token" >> "$GITHUB_OUTPUT" + + - name: Checkout canonical Evalbook reporter + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + repository: paperclipai/paperclip-evals + ref: 0c8dfea0ef71a73e909b59a5c0484554cbee199b + path: .paperclip-evals + token: ${{ steps.evals_token.outputs.value }} + persist-credentials: false + - name: Setup pnpm uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 with: @@ -114,6 +168,10 @@ jobs: PAPERCLIP_EVAL_RUNNER_BUILD: ${{ github.sha }} PAPERCLIP_EVAL_MAX_CAMPAIGN_COST_USD: "12" PAPERCLIP_EVAL_SCHEDULE_SEED: runner-live-seven-week-v1 + PAPERCLIP_EVAL_CANDIDATE: ${{ inputs.candidate }} + PAPERCLIP_EVAL_CASE: ${{ inputs.case }} + PAPERCLIP_EVAL_LIMIT: ${{ inputs.limit }} + PAPERCLIP_EVALBOOK_PROGRAM: ${{ github.workspace }}/.paperclip-evals/evals/paperclip-runner/tools/eval_program.py run: pnpm --filter @paperclipai/paperclip-runner report:runner-live-evals - name: Publish job summary diff --git a/.github/workflows/runner-protocol-live-evals.yml b/.github/workflows/runner-protocol-live-evals.yml new file mode 100644 index 0000000000..9f6352b033 --- /dev/null +++ b/.github/workflows/runner-protocol-live-evals.yml @@ -0,0 +1,679 @@ +name: Runner Direct Live Protocol Evals + +on: + schedule: + - cron: "23 9 * * 0" + workflow_dispatch: + inputs: + target_branch: + description: "Branch in paperclipai/paperclip to evaluate; trusted orchestration still runs from master" + type: string + required: false + evals_sha: + description: "Exact 40-character paperclipai/paperclip-evals commit to execute" + type: string + required: false + rosters: + description: "Comma-separated live roster IDs/files, or all for the complete direct suite" + type: string + default: "all" + required: false + max_infrastructure_retries: + description: "Automatic retries only for explicitly retryable infrastructure failures (0-3)" + type: number + default: 1 + +permissions: + contents: read + +concurrency: + group: runner-protocol-live-evals-${{ github.event_name == 'workflow_dispatch' && inputs.target_branch != '' && inputs.target_branch != github.event.repository.default_branch && format('development-{0}', inputs.target_branch) || format('protected-{0}', github.run_id) }} + cancel-in-progress: ${{ github.event_name == 'workflow_dispatch' && inputs.target_branch != '' && inputs.target_branch != github.event.repository.default_branch }} + +jobs: + authorize: + name: Authorize paid direct eval campaign + if: github.event_name != 'schedule' || vars.RUNNER_PROTOCOL_EVAL_NIGHTLY_ENABLED == 'true' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + outputs: + test_runner: ${{ steps.runner.outputs.runner }} + max_parallel_default: ${{ steps.runner.outputs.max_parallel_default }} + max_parallel_limit: ${{ steps.runner.outputs.max_parallel_limit }} + target_sha: ${{ steps.target.outputs.sha }} + target_ref: ${{ steps.target.outputs.ref }} + evals_sha: ${{ steps.evals.outputs.sha }} + steps: + - name: Require default branch and allowlisted numeric actor IDs + env: + GH_TOKEN: ${{ github.token }} + REPOSITORY: ${{ github.repository }} + REF: ${{ github.ref }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + ACTOR: ${{ github.actor }} + ACTOR_ID: ${{ github.actor_id }} + TRIGGERING_ACTOR: ${{ github.triggering_actor }} + ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} + run: | + set -euo pipefail + if [ "$REF" != "refs/heads/$DEFAULT_BRANCH" ]; then + echo "Paid direct Runner eval campaigns may run only from the default branch." >&2 + exit 1 + fi + if ! jq -e 'type == "array" and length > 0 and all(.[]; type == "number" and . > 0 and floor == .)' <<< "${ALLOWED_ACTOR_IDS:-}" >/dev/null; then + echo "RUNNER_E2E_ALLOWED_ACTOR_IDS must be a non-empty JSON array of numeric GitHub user IDs." >&2 + exit 1 + fi + triggering_actor_id="$(gh api "users/$TRIGGERING_ACTOR" --jq .id)" + if [ "$triggering_actor_id" != "$ACTOR_ID" ] && [ "$TRIGGERING_ACTOR" = "$ACTOR" ]; then + echo "GitHub actor identity contexts disagree; refusing the paid run." >&2 + exit 1 + fi + for candidate in "$triggering_actor_id" "$ACTOR_ID"; do + if ! jq -e --argjson candidate "$candidate" 'index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null; then + echo "The initiating GitHub account is not authorized to run paid Runner eval campaigns." >&2 + exit 1 + fi + done + + - name: Resolve requested Paperclip branch to an immutable commit + id: target + env: + GH_TOKEN: ${{ github.token }} + REPOSITORY: ${{ github.repository }} + TARGET_BRANCH: ${{ inputs.target_branch || github.event.repository.default_branch }} + run: | + set -euo pipefail + if [ -z "$TARGET_BRANCH" ] || [[ "$TARGET_BRANCH" == refs/* ]]; then + echo "target_branch must name a branch in this repository without a refs/ prefix." >&2 + exit 1 + fi + encoded_branch="$(jq -rn --arg branch "$TARGET_BRANCH" '$branch | @uri')" + target_sha="$(gh api -X GET "repos/$REPOSITORY/branches/$encoded_branch" --jq .commit.sha)" + [[ "$target_sha" =~ ^[0-9a-f]{40}$ ]] + echo "sha=$target_sha" >> "$GITHUB_OUTPUT" + echo "ref=refs/heads/$TARGET_BRANCH" >> "$GITHUB_OUTPUT" + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Generate private eval-repository token + id: evals_token + env: + COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }} + GH_REPO: ${{ github.repository }} + run: | + set -euo pipefail + token="$(node .github/scripts/get-bot-token.mjs)" + echo "::add-mask::$token" + echo "value=$token" >> "$GITHUB_OUTPUT" + + - name: Verify the private eval program is pinned to an exact commit + id: evals + env: + GH_TOKEN: ${{ steps.evals_token.outputs.value }} + EVALS_SHA: ${{ inputs.evals_sha || vars.RUNNER_PROTOCOL_EVALS_SHA }} + run: | + set -euo pipefail + if ! [[ "$EVALS_SHA" =~ ^[0-9a-f]{40}$ ]]; then + echo "evals_sha (or RUNNER_PROTOCOL_EVALS_SHA for schedules) must be an exact 40-character commit." >&2 + exit 1 + fi + resolved="$(gh api -X GET "repos/paperclipai/paperclip-evals/commits/$EVALS_SHA" --jq .sha)" + test "$resolved" = "$EVALS_SHA" + echo "sha=$resolved" >> "$GITHUB_OUTPUT" + + - name: Validate retry envelope + env: + RETRIES: ${{ github.event_name == 'schedule' && 1 || inputs.max_infrastructure_retries }} + run: | + set -euo pipefail + [[ "$RETRIES" =~ ^[0-3]$ ]] + + - name: Select paid test runner + id: runner + env: + AWS_PAID_RUNNER_ENABLED: ${{ vars.RUNNER_E2E_AWS_ENABLED }} + run: | + set -euo pipefail + if [ "$AWS_PAID_RUNNER_ENABLED" = true ]; then + { + echo 'runner=runs-on/fleet=paperclip-public-pr-x64/env=public-ci' + echo 'max_parallel_default=100' + echo 'max_parallel_limit=100' + } >> "$GITHUB_OUTPUT" + else + { + echo 'runner=ubuntu-latest' + echo 'max_parallel_default=32' + echo 'max_parallel_limit=57' + } >> "$GITHUB_OUTPUT" + fi + + catalog: + name: Pin and fan out the direct Evalbook roster + needs: authorize + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + outputs: + matrix_0: ${{ steps.catalog.outputs.matrix_0 }} + matrix_1: ${{ steps.catalog.outputs.matrix_1 }} + matrix_1_present: ${{ steps.catalog.outputs.matrix_1_present }} + max_parallel_per_shard: ${{ steps.catalog.outputs.max_parallel_per_shard }} + selected: ${{ steps.catalog.outputs.selected }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Generate private eval-repository token + id: evals_token + env: + COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }} + GH_REPO: ${{ github.repository }} + run: | + set -euo pipefail + token="$(node .github/scripts/get-bot-token.mjs)" + echo "::add-mask::$token" + echo "value=$token" >> "$GITHUB_OUTPUT" + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + repository: paperclipai/paperclip-evals + ref: ${{ needs.authorize.outputs.evals_sha }} + path: .paperclip-evals + token: ${{ steps.evals_token.outputs.value }} + persist-credentials: false + + - name: Build the two bounded roster-plus-case matrices + id: catalog + env: + PAPERCLIP_PROTOCOL_EVAL_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} + PAPERCLIP_PROTOCOL_EVALS_SHA: ${{ needs.authorize.outputs.evals_sha }} + MAX_PARALLEL: ${{ vars.RUNNER_E2E_MAX_PARALLEL || needs.authorize.outputs.max_parallel_default }} + MAX_PARALLEL_LIMIT: ${{ needs.authorize.outputs.max_parallel_limit }} + ROSTERS: ${{ inputs.rosters || 'all' }} + run: | + set -euo pipefail + if ! [[ "$MAX_PARALLEL" =~ ^[1-9][0-9]*$ ]] || [ "$MAX_PARALLEL" -lt 2 ] || [ "$MAX_PARALLEL" -gt "$MAX_PARALLEL_LIMIT" ]; then + echo "RUNNER_E2E_MAX_PARALLEL must be an integer from 2 through $MAX_PARALLEL_LIMIT for the two-shard direct suite." >&2 + exit 1 + fi + node packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs catalog \ + --evals-root .paperclip-evals \ + --rosters "$ROSTERS" \ + --campaign-id "gha-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ + --max-parallel "$MAX_PARALLEL" \ + --output runner-protocol-eval-catalog.json + + - name: Upload immutable campaign catalog + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: runner-protocol-eval-catalog-${{ github.run_id }}-${{ github.run_attempt }} + path: runner-protocol-eval-catalog.json + retention-days: 30 + if-no-files-found: error + + build_runner: + name: Build portable direct-eval runner once + needs: [authorize, catalog] + runs-on: ${{ needs.authorize.outputs.test_runner }} + timeout-minutes: 30 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ needs.authorize.outputs.target_sha }} + persist-credentials: false + + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 + env: + NPM_CONFIG_AUDIT: "false" + NPM_CONFIG_FUND: "false" + NPM_CONFIG_UPDATE_NOTIFIER: "false" + with: + version: 9.15.4 + + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + cache: pnpm + + - run: pnpm install --frozen-lockfile --ignore-scripts + + - name: Build runner CLI, daemon, and canonical attempt viewer + run: | + set -euo pipefail + pnpm --filter @paperclipai/paperclip-runner build:typescript + pnpm --filter @paperclipai/paperclip-runner build:runner-binaries + pnpm --filter @paperclipai/paperclip-runner build:issue-thread + + - name: Package a portable provider runtime + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/runner-protocol-build/package" "$RUNNER_TEMP/runner-protocol-build/portable" + pnpm --dir packages/paperclip-runner pack \ + --pack-destination "$RUNNER_TEMP/runner-protocol-build/package" + package="$(find "$RUNNER_TEMP/runner-protocol-build/package" -maxdepth 1 -type f -name '*.tgz' -print -quit)" + test -f "$package" + npm install --prefix "$RUNNER_TEMP/runner-protocol-build/portable" --omit=dev "$package" + cp "$package" "$RUNNER_TEMP/runner-protocol-build/paperclip-runner.tgz" + cp packages/paperclip-runner/runner/target/debug/paperclip-runnerd "$RUNNER_TEMP/runner-protocol-build/paperclip-runnerd" + cp -R packages/paperclip-runner/dist-issue-thread "$RUNNER_TEMP/runner-protocol-build/dist-issue-thread" + test -f "$RUNNER_TEMP/runner-protocol-build/portable/node_modules/@paperclipai/paperclip-runner/dist/cli/eval-session.js" + test -x "$RUNNER_TEMP/runner-protocol-build/paperclip-runnerd" + tar --create --gzip --file runner-protocol-build.tar.gz -C "$RUNNER_TEMP/runner-protocol-build" . + sha256sum runner-protocol-build.tar.gz > runner-protocol-build.tar.gz.sha256 + + - name: Upload immutable portable runner + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: runner-protocol-build-${{ needs.authorize.outputs.target_sha }}-${{ github.run_id }}-${{ github.run_attempt }} + path: | + runner-protocol-build.tar.gz + runner-protocol-build.tar.gz.sha256 + retention-days: 1 + compression-level: 0 + if-no-files-found: error + + eval_shard_0: + name: Direct eval ${{ matrix.rosterId }} / ${{ matrix.caseId }} + needs: [authorize, catalog, build_runner] + runs-on: ${{ needs.authorize.outputs.test_runner }} + timeout-minutes: 18 + permissions: + contents: read + id-token: write + environment: + name: runner-e2e-paid + strategy: + fail-fast: false + max-parallel: ${{ fromJSON(needs.catalog.outputs.max_parallel_per_shard) }} + matrix: ${{ fromJSON(needs.catalog.outputs.matrix_0) }} + steps: &direct_eval_steps + - name: Reauthorize paid execution before provider access + env: + GH_TOKEN: ${{ github.token }} + REF: ${{ github.ref }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + ACTOR: ${{ github.actor }} + ACTOR_ID: ${{ github.actor_id }} + TRIGGERING_ACTOR: ${{ github.triggering_actor }} + ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} + run: | + set -euo pipefail + test "$REF" = "refs/heads/$DEFAULT_BRANCH" + triggering_actor_id="$(gh api "users/$TRIGGERING_ACTOR" --jq .id)" + test "$triggering_actor_id" = "$ACTOR_ID" || test "$TRIGGERING_ACTOR" != "$ACTOR" + for candidate in "$triggering_actor_id" "$ACTOR_ID"; do + jq -e --argjson candidate "$candidate" 'type == "array" and index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null + done + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Generate private eval-repository token + id: evals_token + env: + COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }} + GH_REPO: ${{ github.repository }} + run: | + set -euo pipefail + token="$(node .github/scripts/get-bot-token.mjs)" + echo "::add-mask::$token" + echo "value=$token" >> "$GITHUB_OUTPUT" + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + repository: paperclipai/paperclip-evals + ref: ${{ needs.authorize.outputs.evals_sha }} + path: .paperclip-evals + token: ${{ steps.evals_token.outputs.value }} + persist-credentials: false + + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + + - name: Download portable runner + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: runner-protocol-build-${{ needs.authorize.outputs.target_sha }}-${{ github.run_id }}-${{ github.run_attempt }} + path: runner-protocol-build + + - name: Verify and extract portable runner + run: | + set -euo pipefail + cd runner-protocol-build + sha256sum --check runner-protocol-build.tar.gz.sha256 + mkdir extracted + tar --extract --gzip --file runner-protocol-build.tar.gz --directory extracted + test -x extracted/paperclip-runnerd + + - name: Prepare short-lived AgentCore web identity + if: matrix.credentialName == 'AWS_AGENTCORE_OIDC' + env: + AGENTCORE_ROLE_ARN: ${{ vars.PAPERCLIP_AWS_AGENTCORE_EXECUTION_ROLE_ARN }} + run: | + set -euo pipefail + test -n "$AGENTCORE_ROLE_ARN" + token="$(curl --fail --silent --show-error \ + -H "Authorization: Bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ + "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=sts.amazonaws.com" | jq -r .value)" + test -n "$token" + echo "::add-mask::$token" + token_file="$RUNNER_TEMP/runner-protocol-agentcore-token" + printf '%s' "$token" > "$token_file" + chmod 600 "$token_file" + { + echo "AWS_WEB_IDENTITY_TOKEN_FILE=$token_file" + echo "AWS_ROLE_ARN=$AGENTCORE_ROLE_ARN" + echo "AWS_ROLE_SESSION_NAME=runner-protocol-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + } >> "$GITHUB_ENV" + + - name: Run one immutable direct protocol cell + id: direct_eval + env: + CELL_ID: ${{ matrix.cellId }} + ROSTER_FILE: ${{ matrix.rosterFile }} + CASE_ID: ${{ matrix.caseId }} + CREDENTIAL_NAME: ${{ matrix.credentialName }} + PROVIDER: ${{ matrix.provider }} + MAX_INFRASTRUCTURE_RETRIES: ${{ github.event_name == 'schedule' && 1 || inputs.max_infrastructure_retries }} + OPENAI_API_KEY: ${{ matrix.credentialName == 'OPENAI_API_KEY' && secrets.OPENAI_API_KEY || '' }} + ANTHROPIC_API_KEY: ${{ matrix.credentialName == 'ANTHROPIC_API_KEY' && secrets.ANTHROPIC_API_KEY || '' }} + OPENROUTER_API_KEY: ${{ matrix.credentialName == 'OPENROUTER_API_KEY' && secrets.OPENROUTER_API_KEY || '' }} + PAPERCLIP_CLAUDE_MANAGED_PROFILE_ID: ${{ vars.PAPERCLIP_CLAUDE_MANAGED_PROFILE_ID }} + PAPERCLIP_CLAUDE_MANAGED_AGENT_ID: ${{ vars.PAPERCLIP_CLAUDE_MANAGED_AGENT_ID }} + PAPERCLIP_CLAUDE_MANAGED_AGENT_VERSION: ${{ vars.PAPERCLIP_CLAUDE_MANAGED_AGENT_VERSION }} + PAPERCLIP_CLAUDE_MANAGED_ENVIRONMENT_ID: ${{ vars.PAPERCLIP_CLAUDE_MANAGED_ENVIRONMENT_ID }} + AWS_REGION: ${{ vars.PAPERCLIP_AWS_AGENTCORE_REGION }} + AWS_DEFAULT_REGION: ${{ vars.PAPERCLIP_AWS_AGENTCORE_REGION }} + PAPERCLIP_AWS_AGENTCORE_PROFILE_ID: ${{ vars.PAPERCLIP_AWS_AGENTCORE_PROFILE_ID }} + PAPERCLIP_AWS_AGENTCORE_ACCOUNT_ID: ${{ vars.PAPERCLIP_AWS_AGENTCORE_ACCOUNT_ID }} + PAPERCLIP_AWS_AGENTCORE_HARNESS_ARN: ${{ vars.PAPERCLIP_AWS_AGENTCORE_HARNESS_ARN }} + PAPERCLIP_AWS_AGENTCORE_HARNESS_VERSION: ${{ vars.PAPERCLIP_AWS_AGENTCORE_HARNESS_VERSION }} + PAPERCLIP_AWS_AGENTCORE_ENDPOINT_ARN: ${{ vars.PAPERCLIP_AWS_AGENTCORE_ENDPOINT_ARN }} + PAPERCLIP_AWS_AGENTCORE_ENDPOINT_QUALIFIER: ${{ vars.PAPERCLIP_AWS_AGENTCORE_ENDPOINT_QUALIFIER }} + PAPERCLIP_AWS_AGENTCORE_RUNTIME_ARN: ${{ vars.PAPERCLIP_AWS_AGENTCORE_RUNTIME_ARN }} + PAPERCLIP_AWS_AGENTCORE_MEMORY_ARN: ${{ vars.PAPERCLIP_AWS_AGENTCORE_MEMORY_ARN }} + PAPERCLIP_AWS_AGENTCORE_MEMORY_ID: ${{ vars.PAPERCLIP_AWS_AGENTCORE_MEMORY_ID }} + PAPERCLIP_AWS_AGENTCORE_INVOCATION_ROLE_ARN: ${{ vars.PAPERCLIP_AWS_AGENTCORE_INVOCATION_ROLE_ARN }} + PAPERCLIP_AWS_AGENTCORE_CONTEXT_BUCKET: ${{ vars.PAPERCLIP_AWS_AGENTCORE_CONTEXT_BUCKET }} + PAPERCLIP_AWS_AGENTCORE_CONTEXT_PREFIX: ${{ vars.PAPERCLIP_AWS_AGENTCORE_CONTEXT_PREFIX }} + PAPERCLIP_AWS_AGENTCORE_CONTEXT_KMS_KEY_ARN: ${{ vars.PAPERCLIP_AWS_AGENTCORE_CONTEXT_KMS_KEY_ARN }} + PAPERCLIP_AWS_AGENTCORE_QUALIFICATION_REVISION: ${{ vars.PAPERCLIP_AWS_AGENTCORE_QUALIFICATION_REVISION }} + run: | + set -euo pipefail + mkdir -p cell-output/runs + if [ "$CREDENTIAL_NAME" != "AWS_AGENTCORE_OIDC" ]; then + test -n "${!CREDENTIAL_NAME:-}" + fi + if [ "$PROVIDER" = "claude_managed" ]; then + test -n "$PAPERCLIP_CLAUDE_MANAGED_PROFILE_ID" + test -n "$PAPERCLIP_CLAUDE_MANAGED_AGENT_ID" + test -n "$PAPERCLIP_CLAUDE_MANAGED_AGENT_VERSION" + test -n "$PAPERCLIP_CLAUDE_MANAGED_ENVIRONMENT_ID" + fi + set +e + python3 .paperclip-evals/evals/paperclip-runner/tools/run_live_roster.py run \ + --roster ".paperclip-evals/evals/paperclip-runner/rosters/$ROSTER_FILE" \ + --case "$CASE_ID" \ + --runner-cli runner-protocol-build/extracted/portable/node_modules/@paperclipai/paperclip-runner/dist/cli/eval-session.js \ + --runner-package runner-protocol-build/extracted/paperclip-runner.tgz \ + --runnerd runner-protocol-build/extracted/paperclip-runnerd \ + --runs-root cell-output/runs \ + --max-infrastructure-retries "$MAX_INFRASTRUCTURE_RETRIES" \ + --run-id "gha-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${CELL_ID}" + status=$? + set -e + CELL_EXIT_CODE="$status" node --input-type=module <<'NODE' + import { writeFileSync } from "node:fs"; + writeFileSync("cell-output/cell.json", `${JSON.stringify({ + schema: "paperclip.runner-protocol-eval.cell/v1", + cellId: process.env.CELL_ID, + rosterFile: process.env.ROSTER_FILE, + caseId: process.env.CASE_ID, + exitCode: Number(process.env.CELL_EXIT_CODE), + }, null, 2)}\n`, { mode: 0o600 }); + NODE + exit "$status" + + - name: Upload access-controlled cell attempt + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: runner-protocol-eval-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.cellId }} + path: cell-output/ + retention-days: 30 + if-no-files-found: error + + eval_shard_1: + name: Direct eval ${{ matrix.rosterId }} / ${{ matrix.caseId }} + if: needs.catalog.outputs.matrix_1_present == 'true' + needs: [authorize, catalog, build_runner] + runs-on: ${{ needs.authorize.outputs.test_runner }} + timeout-minutes: 18 + permissions: + contents: read + id-token: write + environment: + name: runner-e2e-paid + strategy: + fail-fast: false + max-parallel: ${{ fromJSON(needs.catalog.outputs.max_parallel_per_shard) }} + matrix: ${{ fromJSON(needs.catalog.outputs.matrix_1) }} + steps: *direct_eval_steps + + report: + name: Merge attempts and render canonical Evalbook + if: always() && !cancelled() && needs.catalog.result == 'success' && needs.build_runner.result == 'success' + needs: [authorize, catalog, build_runner, eval_shard_0, eval_shard_1] + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + actions: read + contents: read + outputs: + public_report_ready: ${{ steps.public_report.outputs.ready }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Generate private eval-repository token + id: evals_token + env: + COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }} + GH_REPO: ${{ github.repository }} + run: | + set -euo pipefail + token="$(node .github/scripts/get-bot-token.mjs)" + echo "::add-mask::$token" + echo "value=$token" >> "$GITHUB_OUTPUT" + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + repository: paperclipai/paperclip-evals + ref: ${{ needs.authorize.outputs.evals_sha }} + path: .paperclip-evals + token: ${{ steps.evals_token.outputs.value }} + persist-credentials: false + + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + + - name: Download immutable campaign catalog + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: runner-protocol-eval-catalog-${{ github.run_id }}-${{ github.run_attempt }} + path: runner-protocol-catalog + + - name: Download portable runner and viewer + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: runner-protocol-build-${{ needs.authorize.outputs.target_sha }}-${{ github.run_id }}-${{ github.run_attempt }} + path: runner-protocol-build + + - name: Download every access-controlled cell + id: download_cells + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + pattern: runner-protocol-eval-${{ github.run_id }}-${{ github.run_attempt }}-* + path: downloaded-runner-protocol-evals + merge-multiple: false + + - name: Retry cell download after artifact transport failure + if: steps.download_cells.outcome == 'failure' + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + pattern: runner-protocol-eval-${{ github.run_id }}-${{ github.run_attempt }}-* + path: downloaded-runner-protocol-evals + merge-multiple: false + + - name: Materialize an empty download root when every cell failed early + run: mkdir -p downloaded-runner-protocol-evals + + - name: Verify portable viewer + run: | + set -euo pipefail + cd runner-protocol-build + sha256sum --check runner-protocol-build.tar.gz.sha256 + mkdir extracted + tar --extract --gzip --file runner-protocol-build.tar.gz --directory extracted + test -f extracted/dist-issue-thread/index.html + + - name: Aggregate every expected cell, including missing infrastructure cells + env: + PAPERCLIP_PROTOCOL_EVAL_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} + PAPERCLIP_PROTOCOL_EVAL_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }} + PAPERCLIP_PROTOCOL_EVALS_SHA: ${{ needs.authorize.outputs.evals_sha }} + PAPERCLIP_PROTOCOL_EVAL_WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + node packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs aggregate \ + --catalog runner-protocol-catalog/runner-protocol-eval-catalog.json \ + --downloads downloaded-runner-protocol-evals \ + --evals-root .paperclip-evals \ + --runs-out runner-protocol-merged/runs \ + --campaign-out runner-protocol-merged/campaign.json + + - name: Render the access-controlled canonical Evalbook report + run: | + python3 .paperclip-evals/evals/paperclip-runner/tools/eval_program.py report \ + --runs-root runner-protocol-merged/runs \ + --output runner-protocol-merged/report \ + --viewer-root runner-protocol-build/extracted/dist-issue-thread \ + --inventory .paperclip-evals/evals/paperclip-runner/inventory.json \ + --coverage-matrix .paperclip-evals/evals/paperclip-runner/coverage-matrix.json + cp runner-protocol-merged/campaign.json runner-protocol-merged/report/campaign.json + + - name: Render the same canonical grid from a public-safe evidence projection + run: | + node packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs sanitize \ + --runs-root runner-protocol-merged/runs \ + --output runner-protocol-merged/public-runs + python3 .paperclip-evals/evals/paperclip-runner/tools/eval_program.py report \ + --runs-root runner-protocol-merged/public-runs \ + --output runner-protocol-merged/public-report \ + --inventory .paperclip-evals/evals/paperclip-runner/inventory.json \ + --coverage-matrix .paperclip-evals/evals/paperclip-runner/coverage-matrix.json + cp runner-protocol-merged/campaign.json runner-protocol-merged/public-report/campaign.json + + - name: Enforce the static public allowlist + id: public_report + run: | + node --input-type=module -e 'import { validatePublicProtocolEvalReport } from "./packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.mjs"; await validatePublicProtocolEvalReport("runner-protocol-merged/public-report");' + echo "ready=true" >> "$GITHUB_OUTPUT" + + - name: Add campaign result to the workflow summary + run: | + { + echo '## Runner direct live protocol evals' + echo + jq -r '"- Cells: \(.totals.passed)/\(.totals.selected) passed\n- Behavior failures: \(.totals.behaviorFailures)\n- Infrastructure failures: \(.totals.infrastructureFailures)\n- Paperclip: `\(.source.paperclip.sha)`\n- Evals: `\(.source.evals.sha)`"' runner-protocol-merged/campaign.json + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload access-controlled canonical Evalbook and raw attempts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: runner-protocol-eval-report-${{ github.run_id }}-${{ github.run_attempt }} + path: runner-protocol-merged/ + retention-days: 30 + if-no-files-found: error + + - name: Upload publisher-only sanitized Evalbook + if: steps.public_report.outputs.ready == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: runner-protocol-eval-public-${{ github.run_id }}-${{ github.run_attempt }} + path: runner-protocol-merged/public-report/ + retention-days: 1 + if-no-files-found: error + + - name: Enforce complete green campaign + if: always() + run: jq -e '.complete == true and .allPassed == true' runner-protocol-merged/campaign.json >/dev/null + + publish_history: + name: Publish immutable Evalbook and mutable campaign index + needs: [authorize, catalog, report] + if: always() && needs.report.outputs.public_report_ready == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + concurrency: + group: runner-protocol-eval-history-publish + cancel-in-progress: false + permissions: + contents: read + id-token: write + environment: + name: runner-e2e-history + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + # AWS credentials can execute only the publisher from the trusted workflow revision. + ref: ${{ github.sha }} + persist-credentials: false + + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + + - name: Download only the sanitized canonical Evalbook + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: runner-protocol-eval-public-${{ github.run_id }}-${{ github.run_attempt }} + path: runner-protocol-public-report + + - name: Exchange GitHub OIDC identity for scoped AWS credentials + uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6 + with: + role-to-assume: ${{ vars.RUNNER_PROTOCOL_EVAL_HISTORY_AWS_ROLE_ARN || vars.RUNNER_E2E_HISTORY_AWS_ROLE_ARN }} + aws-region: ${{ vars.RUNNER_PROTOCOL_EVAL_HISTORY_AWS_REGION || vars.RUNNER_E2E_HISTORY_AWS_REGION }} + + - name: Publish versioned report and refresh the root index + env: + PAPERCLIP_RUNNER_PROTOCOL_EVAL_PUBLIC_REPORT_DIR: ${{ github.workspace }}/runner-protocol-public-report + RUNNER_PROTOCOL_EVAL_HISTORY_S3_BUCKET: ${{ vars.RUNNER_PROTOCOL_EVAL_HISTORY_S3_BUCKET || vars.RUNNER_E2E_HISTORY_S3_BUCKET }} + RUNNER_PROTOCOL_EVAL_HISTORY_PREFIX: ${{ vars.RUNNER_PROTOCOL_EVAL_HISTORY_PREFIX || 'runner-protocol-evals' }} + RUNNER_PROTOCOL_EVAL_HISTORY_PUBLIC_BASE_URL: ${{ vars.RUNNER_PROTOCOL_EVAL_HISTORY_PUBLIC_BASE_URL || vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL }} + run: node packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.mjs diff --git a/packages/paperclip-runner/README.md b/packages/paperclip-runner/README.md index e0560b1dde..a628694615 100644 --- a/packages/paperclip-runner/README.md +++ b/packages/paperclip-runner/README.md @@ -190,7 +190,25 @@ Live console provider-backed routes are loopback-only and reject wildcard/LAN binds. Browser mutations require same-origin Fetch Metadata, matching Origin, and JSON content; see the protocol-server tutorial for direct `curl` examples. -## Live, chaos, and AWS AgentCore operations +## Direct live protocol qualification + +The canonical direct live protocol suite lives in the separate +`paperclip-evals` repository under `evals/paperclip-runner/`. Its +`live-mini.json` roster is the complete 35-case Codex qualification lane. Build +this package's TypeScript output, release `paperclip-runnerd`, package tarball, +and `dist-issue-thread` viewer, then use the roster runner documented in that +repository. The package ships the required orchestration entry point as +`paperclip-runner-eval-session` (`dist/cli/eval-session.js`). Evalbook owns the +consistent HTML matrix and read-only attempt drill-down pages. + +The hosted full-campaign workflow, parallel matrix, credential boundaries, +canonical report merge, and versioned S3 index are documented in +[`docs/runner-protocol-live-evals.md`](docs/runner-protocol-live-evals.md). + +This direct protocol qualification is separate from the stress-derived Runner +workflow schedule below and from the full-stack browser model E2E suite. + +## Stress-derived workflow, chaos, and AWS AgentCore operations The deterministic workflow scorer and the chaos schedule do not require provider credentials: @@ -209,13 +227,22 @@ additional scheduling after the observed campaign total reaches that value: ```sh PAPERCLIP_EVAL_MAX_CAMPAIGN_COST_USD=12 \ + PAPERCLIP_EVALS_ROOT=/path/to/paperclip-evals \ pnpm --filter @paperclipai/paperclip-runner report:runner-live-evals + +# Run two scheduled native Codex executions only. +PAPERCLIP_EVALS_ROOT=/path/to/paperclip-evals \ + pnpm --filter @paperclipai/paperclip-runner report:runner-live-evals -- \ + --candidate codex-luna --limit 2 ``` GitHub-hosted live campaigns additionally require the default branch, an allowlisted numeric actor ID, the protected `runner-e2e-paid` environment, and -an explicit repository variable before scheduled runs are enabled. Uploaded -reports contain redacted observations and trace digests, not raw provider +an explicit repository variable before scheduled runs are enabled. Manual +dispatches accept the same candidate, case, and execution-limit selectors. The +paid job uses the reviewed RunsOn Fleet label when `RUNNER_E2E_AWS_ENABLED=true` +and otherwise stays on `ubuntu-latest`. Uploaded reports contain redacted +observations and trace digests, not raw provider frames, prompts, credentials, tool arguments, or hidden reasoning. The AgentCore proof-of-concept uses an AWS CLI v2 profile to provision a @@ -234,6 +261,21 @@ pnpm --filter @paperclipai/paperclip-runner smoke:capability:aws-agentcore pnpm --filter @paperclipai/paperclip-runner aws-agentcore:destroy -- --yes ``` +To admit the hosted direct-eval workflow, provision with the account-local +GitHub Actions OIDC provider and keep the default exact repository and protected +environment binding: + +```sh +pnpm --filter @paperclipai/paperclip-runner aws-agentcore:provision -- \ + --aws-profile paperclip-dev \ + --github-oidc-provider-arn arn:aws:iam:::oidc-provider/token.actions.githubusercontent.com +``` + +This adds only `repo:paperclipai/paperclip:environment:runner-e2e-paid` as a +web-identity subject on the scoped invocation role. The generated nonsecret +profile records that role as both the local invocation role and the hosted +execution role. + Provisioning can incur Bedrock, AgentCore Runtime/Memory, storage, and private networking charges. Provisioning refuses to modify a colliding stack unless its Paperclip ownership tags and template description match. A verified @@ -256,8 +298,8 @@ recorded lab unless `--force` is also supplied. | `check:clean-consumers` | Pack the runner and install its root, evals, and testing exports in a clean consumer. | | `test:eval-slice` | Run the credential-free eval bundle, scoring, and behavior/fault slice. | | `test:runner-workflow-evals` | Run the deterministic provider-neutral workflow matrix. | -| `report:runner-workflow-evals` | Validate deterministic results and write local reports only when every scoreable result passes. | -| `report:runner-live-evals` | Execute the paid forty-execution provider schedule with qualification and campaign-cost guards. | +| `report:runner-workflow-evals` | Validate deterministic fail-closed results and write JSON, Markdown, JUnit, and GitHub-safe reports. | +| `report:runner-live-evals` | Execute the paid provider schedule and render its immutable attempts with the canonical `paperclip-evals` HTML grid. | | `report:runner-chaos-evals` | Write the credential-free eight-scenario chaos schedule. | | `test:aws-agentcore-provisioning` | Validate the AgentCore template and wrapper safety contracts without provisioning. | | `aws-agentcore:provision` / `probe` / `lab` / `destroy` | Manage the scoped AgentCore proof-of-concept lifecycle. | diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md new file mode 100644 index 0000000000..2cdc74774b --- /dev/null +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -0,0 +1,148 @@ +# Direct live Runner protocol evals + +This is the provider-backed, one-turn protocol qualification layer in +`paperclipai/paperclip-evals/evals/paperclip-runner`. It is intentionally +separate from both the browser full-stack model E2E and the stress-derived +workflow schedule in `runner-workflow-evals.md`. + +The canonical unit of work is one live roster plus one authored case. A full +campaign selects every `rosters/live-*.json` file at one immutable +`paperclip-evals` commit. That includes the complete 35-case provider rosters +and the smaller ACPX Codex control roster. The native resume reliability gate +is not a normal one-turn roster: it requires its separately governed external +resource campaign and remains opt-in. + +Managed-provider evidence identifies the immutable deployed provider artifact: +Claude Managed uses its Agent version, while AgentCore uses its qualification +revision. Transport API or beta versions remain separate protocol metadata and +must not replace that runtime identity in a report. + +## Hosted campaign + +Use the `Runner Direct Live Protocol Evals` workflow. Dispatch the workflow +from the default branch and provide: + +- `target_branch`: the Paperclip branch to build and test; +- `evals_sha`: an exact 40-character commit from + `paperclipai/paperclip-evals`; +- `rosters`: `all` for the entire direct suite, or a comma-separated diagnostic + subset; +- `max_infrastructure_retries`: zero through three, applied only when an + attempt explicitly reports a retryable infrastructure failure. + +The authorization job resolves the Paperclip branch to a commit and verifies +the supplied eval commit before any checkout. A short-lived bot token generated +from `COMMITPERCLIP_KEY` authorizes each checkout of the private eval repository; +the token is masked and is never forwarded to a provider process. The workflow +uses the same numeric actor +allowlist, protected `runner-e2e-paid` environment, RunsOn fleet selector, and +`RUNNER_E2E_MAX_PARALLEL` ceiling as the full-stack E2E workflow. Two balanced +GitHub matrices keep each matrix below GitHub's 256-job limit while keeping +their combined concurrency at or below that shared ceiling. Runner TypeScript, +the native daemon, provider dependencies, and the attempt viewer are built +once and reused by every cell. Because the complete suite requires two matrix +shards, this workflow accepts a shared concurrency ceiling from 2 through 100. + +`all` is intentionally literal. A disabled driver, missing remote profile, or +unavailable provider is retained as an infrastructure result; it is not +silently omitted. In particular, the ACPX Pi roster remains visible while Pi +is disabled in the current Runner. Use a roster subset only for diagnosis, not +to claim the full campaign is green. + +A provider turn that reaches a durable failed, interrupted, or otherwise +non-completed terminal still produces an attempt artifact and is scored as a +behavior result. It is an infrastructure failure only when the harness cannot +produce usable evidence, such as an unavailable provider, invalid profile, or +transport failure. Automatic retries therefore never rerun a measured behavior +failure merely to improve its score. + +## Required protected configuration + +The paid jobs read only the credential selected for each roster: + +- `OPENAI_API_KEY` for native Codex and ACPX Codex; +- `ANTHROPIC_API_KEY` for ACPX Claude and Claude Managed; +- `OPENROUTER_API_KEY` for native OpenCode and ACPX Pi; +- short-lived GitHub OIDC workload identity for AWS AgentCore. + +Claude Managed also requires the four nonsecret +`PAPERCLIP_CLAUDE_MANAGED_*` profile variables. AgentCore requires the +nonsecret `PAPERCLIP_AWS_AGENTCORE_*` profile variables, including +`PAPERCLIP_AWS_AGENTCORE_EXECUTION_ROLE_ARN` and the immutable +`PAPERCLIP_AWS_AGENTCORE_QUALIFICATION_REVISION`; the eval fails closed when +that deployed revision differs from the pinned roster config. The workflow +writes the GitHub OIDC token to a mode-`0600` file and never forwards long-lived +AWS access keys. +Provision the AgentCore stack with `--github-oidc-provider-arn` so that scoped +role admits only the `paperclipai/paperclip` repository's protected +`runner-e2e-paid` environment as its web-identity subject. +Scheduled runs additionally require `RUNNER_PROTOCOL_EVAL_NIGHTLY_ENABLED=true` +and the pinned `RUNNER_PROTOCOL_EVALS_SHA` repository variable. + +## Reports and history + +Each cell uploads its immutable run directory to an access-controlled Actions +artifact. The trusted report job merges all expected cells, represents missing +cell artifacts as infrastructure failures, and invokes the report program from +the pinned eval commit. The full artifact contains the canonical Evalbook grid, +read-only Runner issue-thread attempt pages, and raw immutable run records. + +Public publishing uses a separate projection and a separate trusted OIDC job. +The projection retains model/config identity, status, usage totals, and check +outcomes but removes provider session identifiers, transcripts, semantic-tool +payloads, state revisions, traces, remote profile identities, and raw failure +text. The same Evalbook `report` command renders that projection, so the public +grid and test pages have the standard Evalbook layout. The publisher rejects +scripts, remote resources, symlinks, unknown paths, broken links, raw session +fields, and credential-shaped values. + +S3 publication is additive: + +```text +runner-protocol-evals/ + index.html + history.json + latest.json + latest-green.json + campaigns/ + gha--/ + index.html + latest.html + inventory.html + tests/*.html + attempts/*.html + campaign.json + bundle-manifest.json +``` + +Campaign files use immutable cache headers and a digest manifest. Reusing a +campaign ID with different bytes fails closed. Only the root history and +pointer files are mutable, and the publisher never deletes objects. The root +history retains at most 200 records, reserving one record for the latest green +campaign when it would otherwise fall outside that window so its pointer stays +valid. + +The publishing job uses dedicated `RUNNER_PROTOCOL_EVAL_HISTORY_*` variables +when present and falls back to the existing Runner E2E history role, region, +bucket, and public base URL. Its default top-level prefix is +`runner-protocol-evals`, distinct from `runner-e2e`. The AWS role must allow +additive writes and reads for that prefix. + +## Local publisher checks + +These tests make no provider or AWS calls: + +```sh +pnpm --filter @paperclipai/paperclip-runner test:runner-protocol-eval-publish +``` + +To inspect the catalog without executing it, point the command at a local +evals checkout: + +```sh +pnpm --filter @paperclipai/paperclip-runner \ + report:runner-protocol-eval:catalog -- \ + --evals-root /path/to/paperclip-evals \ + --campaign-id gha-1-1 \ + --output /tmp/runner-protocol-eval-catalog.json +``` diff --git a/packages/paperclip-runner/docs/runner-workflow-evals.md b/packages/paperclip-runner/docs/runner-workflow-evals.md index 19b5357329..ceca8f0387 100644 --- a/packages/paperclip-runner/docs/runner-workflow-evals.md +++ b/packages/paperclip-runner/docs/runner-workflow-evals.md @@ -6,7 +6,9 @@ capability inventory, capability cases, and existing scoring/report readers. The workspace-private `@paperclipai/paperclip-eval-kernel` package owns only structural scenario-by-candidate orchestration. Runner-specific cases, -observations, scoring, traceability, and report rendering remain package-local. +observations, scoring, and traceability remain package-local. The HTML matrix is +rendered by the canonical `paperclip-evals` report program so live results use +the same grid and drill-down pages as the direct Runner eval suite. ## Lanes @@ -14,9 +16,9 @@ observations, scoring, traceability, and report rendering remain package-local. runs the credential-free PR gate over sanitized Codex, OpenCode, and ACPX normalization fixtures. - `pnpm --filter @paperclipai/paperclip-runner report:runner-workflow-evals` - validates the deterministic report and writes JSON, Markdown, JUnit, and - GitHub-safe artifacts under `.paperclip-local/evals/workflows/` only when all - scoreable fixture results pass. It makes no network requests. + validates the deterministic fail-closed fixture matrix and writes JSON, + Markdown, JUnit, and GitHub-safe artifacts under + `.paperclip-local/evals/workflows/`. It makes no network requests. - `pnpm --filter @paperclipai/paperclip-runner report:runner-live-evals` runs the balanced forty-execution schedule against real provider sessions. Live candidate failures are trend-only; missing credentials, qualification @@ -30,10 +32,38 @@ qualification variable names, and budgets. Credentials remain in the environment. `PAPERCLIP_EVAL_MAX_CAMPAIGN_COST_USD` must be a positive finite number and defaults to 12 USD for scheduled runs. +Live executions export one immutable Evalbook attempt per workflow/candidate to +`.paperclip-local/evals/workflows/evalbook-runs/`, then invoke +`evals/paperclip-runner/tools/eval_program.py report` from a `paperclip-evals` +checkout. That program writes the canonical matrix to +`.paperclip-local/evals/workflows/index.html`, plus `latest.html`, test pages, +and attempt pages. Set `PAPERCLIP_EVALBOOK_PROGRAM` to the program's absolute +path or `PAPERCLIP_EVALS_ROOT` to its repository root. Conventional sibling +worktree locations are discovered automatically. GitHub Actions checks out a +pinned `paperclip-evals` revision, so every hosted run uses the same reviewed +report implementation rather than a copied or package-local renderer. +Filtered reports with only a few candidates expand their result columns to the +available viewport, keeping the PASS, FAIL, and INFRA labels visible without a +horizontal scroll. Full matrices retain the canonical scrollable grid and +sticky test-name column. + +The exported artifact contains only the safe workflow observation and +scorecard. Prompts, credentials, raw provider frames, tool arguments, and +reasoning remain excluded. `evalbook-manifest.json` records the generator path +and SHA-256 digest used for the render. + +Local and manual GitHub runs can bound paid execution with comma-separated +`--candidate` and `--case` selectors plus `--limit`. For example, +`report:runner-live-evals -- --candidate codex-luna --limit 2` executes only +the first two Codex entries in that week's validated schedule. Subsets receive +a distinct bundle identity and do not contaminate full-campaign trend history. + The hosted live workflow is default-branch-only and requires an allowlisted numeric actor plus the protected `runner-e2e-paid` environment. Scheduled runs also remain disabled until `RUNNER_LIVE_EVALS_NIGHTLY_ENABLED` is explicitly -set to `true`. +set to `true`. The paid job uses the full-stack workflow's literal runner +selection: `RUNNER_E2E_AWS_ENABLED=true` routes it to the RunsOn Fleet, and any +other value uses `ubuntu-latest`. ## Trace and reasoning safety diff --git a/packages/paperclip-runner/infra/aws-agentcore-paperclip.yaml b/packages/paperclip-runner/infra/aws-agentcore-paperclip.yaml index a7cbd72a1c..afa68ed17f 100644 --- a/packages/paperclip-runner/infra/aws-agentcore-paperclip.yaml +++ b/packages/paperclip-runner/infra/aws-agentcore-paperclip.yaml @@ -14,6 +14,19 @@ Parameters: Type: String Description: Stable IAM user or role ARN allowed to assume the Paperclip invocation role. AllowedPattern: "^arn:aws(-[^:]+)?:iam::[0-9]{12}:(role|user)/.+$" + GitHubOidcProviderArn: + Type: String + Default: "" + Description: Optional account-local GitHub Actions OIDC provider for hosted Runner evals. + AllowedPattern: "^$|^arn:aws(-[^:]+)?:iam::[0-9]{12}:oidc-provider/token[.]actions[.]githubusercontent[.]com$" + GitHubRepository: + Type: String + Default: paperclipai/paperclip + AllowedPattern: "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$" + GitHubEnvironment: + Type: String + Default: runner-e2e-paid + AllowedPattern: "^[A-Za-z0-9_.-]+$" BedrockModelId: Type: String Default: global.anthropic.claude-sonnet-4-6 @@ -53,6 +66,7 @@ Parameters: Conditions: IsPrivate: !Equals [!Ref DeploymentMode, private] + HasGitHubOidcTrust: !Not [!Equals [!Ref GitHubOidcProviderArn, ""]] Resources: ContextEncryptionKey: @@ -304,6 +318,17 @@ Resources: Principal: AWS: !Ref TrustedRunnerPrincipalArn Action: sts:AssumeRole + - !If + - HasGitHubOidcTrust + - Effect: Allow + Principal: + Federated: !Ref GitHubOidcProviderArn + Action: sts:AssumeRoleWithWebIdentity + Condition: + StringEquals: + "token.actions.githubusercontent.com:aud": sts.amazonaws.com + "token.actions.githubusercontent.com:sub": !Sub repo:${GitHubRepository}:environment:${GitHubEnvironment} + - !Ref AWS::NoValue Policies: - PolicyName: InvokeAndOperatePinnedHarness PolicyDocument: diff --git a/packages/paperclip-runner/package.json b/packages/paperclip-runner/package.json index d69f6d9739..c6d02304fc 100644 --- a/packages/paperclip-runner/package.json +++ b/packages/paperclip-runner/package.json @@ -114,12 +114,15 @@ "test:capability-inventory": "node scripts/check-capability-inventory.test.mjs", "test:capability-evals": "vitest run src/conformance/capability-eval-suite.test.ts", "test:eval-slice": "pnpm run ensure:eval-build-deps && vitest run src/eval", - "test:runner-workflow-evals": "pnpm run ensure:eval-build-deps && vitest run src/eval/workflow-evals.test.ts src/eval/live-workflow-executor.test.ts", + "test:runner-workflow-evals": "pnpm run ensure:eval-build-deps && node --test scripts/render-runner-workflow-evalbook.test.mjs && vitest run src/eval/workflow-evals.test.ts src/eval/live-workflow-executor.test.ts", + "test:runner-protocol-eval-publish": "node --test scripts/runner-protocol-eval-campaign.test.mjs scripts/publish-runner-protocol-eval-history.test.mjs scripts/runner-protocol-eval-workflow-security.test.mjs", "check:runner-workflow-traceability": "pnpm run build:typescript && node scripts/check-runner-workflow-traceability.mjs", "report:capability-evals": "pnpm run build:typescript && node scripts/run-capability-eval-suite.mjs", "report:capability-live-evals": "pnpm run build:typescript && cargo build --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --bin paperclip-runnerd && node scripts/run-capability-live-eval-matrix.mjs", "report:runner-workflow-evals": "pnpm run build:typescript && node scripts/run-runner-workflow-evals.mjs", "report:runner-live-evals": "pnpm run build:typescript && pnpm run build:runner-binaries && node scripts/run-runner-live-eval-schedule.mjs --mode nightly --execute", + "report:runner-protocol-eval:catalog": "node scripts/runner-protocol-eval-campaign.mjs catalog", + "report:runner-protocol-eval:publish": "node scripts/publish-runner-protocol-eval-history.mjs", "report:runner-chaos-evals": "pnpm run build:typescript && node scripts/run-runner-live-eval-schedule.mjs --mode chaos", "check:conformance-parity": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core runs_the_mock_core_path_with_stable_output", "check:replay-parity": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core replay_fixture_parity", diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs index 6c77d15241..9bb1784012 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs @@ -31,6 +31,48 @@ fn send(value: Value) -> io::Result<()> { stdout.flush() } +fn send_split_event_burst(state: &FakeState) -> io::Result<()> { + let turn_id = state.active_turn_id.as_deref().unwrap_or("provider-turn-1"); + for index in 0..96 { + send(json!({ + "method": "item/agentMessage/delta", + "params": { + "threadId": state.thread_id, + "turnId": turn_id, + "itemId": "split-burst-message", + "delta": format!("first-{index} "), + } + }))?; + } + send(json!({ + "id": "split-burst-tool", + "method": "item/tool/call", + "params": { + "threadId": state.thread_id, + "turnId": turn_id, + "callId": "split-burst-semantic-call", + "tool": "get_task_context", + "arguments": {} + } + })) +} + +fn finish_split_event_burst(state: &FakeState) -> io::Result<()> { + let turn_id = state.active_turn_id.as_deref().unwrap_or("provider-turn-1"); + for index in 0..48 { + send(json!({ + "method": "item/agentMessage/delta", + "params": { + "threadId": state.thread_id, + "turnId": turn_id, + "itemId": "split-burst-message", + "delta": format!("second-{index} "), + } + }))?; + } + Ok(()) +} + fn load_state(path: &Path) -> FakeState { fs::read(path) .ok() @@ -483,6 +525,7 @@ fn run() -> Result<(), Box> { .any(|value| value == "--opencode-proxy-runtime-question"); let emit_runtime_elicitation = args.iter().any(|value| value == "--runtime-elicitation"); let emit_structured_activity = args.iter().any(|value| value == "--structured-activity"); + let emit_split_event_burst = args.iter().any(|value| value == "--split-event-burst"); let require_skill_instructions = args .iter() .any(|value| value == "--include-skill-instructions"); @@ -703,6 +746,15 @@ fn run() -> Result<(), Box> { finish_turn(&state_path, &mut state, "completed")?; continue; } + if message.get("method").is_none() && message.get("id") == Some(&json!("split-burst-tool")) + { + if message.pointer("/result/success") != Some(&json!(true)) { + return Err("split event burst semantic tool failed".into()); + } + finish_split_event_burst(&state)?; + finish_turn(&state_path, &mut state, "completed")?; + continue; + } if message.get("method").is_none() && message.get("id") == Some(&json!("tool-request-1")) { if message.pointer("/result/success") == Some(&json!(false)) { log_call(call_log.as_deref(), "tool-response:failure")?; @@ -1110,6 +1162,8 @@ fn run() -> Result<(), Box> { } else if emit_structured_activity { send_structured_activity(&state)?; finish_turn(&state_path, &mut state, "completed")?; + } else if emit_split_event_burst { + send_split_event_burst(&state)?; } else if emit_question { send_question(&state)?; } else if !hold_turn { diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs index dcdfb5ed0e..0370c7bb90 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs @@ -140,7 +140,7 @@ impl ManagedProviderDescriptor { fn version(&self) -> &str { match self { - Self::ClaudeManaged(config) => &config.beta_version, + Self::ClaudeManaged(config) => &config.agent_version, Self::AwsAgentcore(config) => &config.qualification_revision, } } @@ -2477,6 +2477,35 @@ mod tests { ); } + #[test] + fn claude_runtime_identity_uses_the_pinned_agent_version() { + let descriptor = ManagedProviderDescriptor::ClaudeManaged(ClaudeManagedProviderConfig { + model: QUALIFIED_CLAUDE_MODEL.to_owned(), + profile_id: "profile-1".to_owned(), + anthropic_agent_id: "agent-1".to_owned(), + agent_version: "17".to_owned(), + environment_id: "environment-1".to_owned(), + beta_version: QUALIFIED_CLAUDE_BETA.to_owned(), + max_session_list_cost_usd: 1.0, + instructions: "Complete the supplied task.".to_owned(), + runtime_context: None, + }); + + assert_eq!(descriptor.version(), "17"); + assert_eq!( + session_event_payload( + &descriptor, + &ProviderRuntimeIdentity::RemoteService { + service: "anthropic_managed_agents".to_owned(), + provider_session_id: "session-17".to_owned(), + process_id: None, + }, + ) + .pointer("/providerDescriptor/providerVersion"), + Some(&json!("17")) + ); + } + #[test] fn agentcore_usage_snapshot_accepts_only_bounded_reconciliation_states() { let usage = json!({ diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/process_supervisor.rs b/packages/paperclip-runner/runner/crates/runner-core/src/process_supervisor.rs index 6ac380da5d..21eca672f7 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/process_supervisor.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/process_supervisor.rs @@ -261,7 +261,7 @@ impl VerifiedProcessLaunch { } #[cfg(target_os = "macos")] { - let executable = materialize_executable(artifact)?; + let executable = materialize_bound_executable(artifact)?; args.push(executable.path.to_string_lossy().into_owned()); temporary_executables.push(executable); } @@ -290,6 +290,7 @@ struct InheritedCommand { #[cfg(target_os = "macos")] struct TemporaryExecutable { path: PathBuf, + cleanup_directory: Option, _file: File, } @@ -297,13 +298,16 @@ struct TemporaryExecutable { impl Drop for TemporaryExecutable { fn drop(&mut self) { let _ = fs::remove_file(&self.path); + if let Some(directory) = &self.cleanup_directory { + let _ = fs::remove_dir(directory); + } } } #[cfg(target_os = "macos")] -fn materialize_executable( +fn verified_executable_parent( artifact: &VerifiedProcessArtifact, -) -> Result { +) -> Result<&Path, LocalRunnerError> { let directory = artifact.display_path.parent().ok_or_else(|| { LocalRunnerError::invalid(format!( "verified process artifact {} has no parent directory", @@ -318,10 +322,15 @@ fn materialize_executable( directory.display() ))); } - let path = directory.join(format!( - ".paperclip-verified-executable-{}", - Uuid::new_v4().simple() - )); + Ok(directory) +} + +#[cfg(target_os = "macos")] +fn materialize_executable_at( + artifact: &VerifiedProcessArtifact, + path: PathBuf, + cleanup_directory: Option, +) -> Result { let mut writable = OpenOptions::new() .read(true) .write(true) @@ -375,6 +384,7 @@ fn materialize_executable( drop(writable); Ok(TemporaryExecutable { path: path.clone(), + cleanup_directory, _file: file, }) })(); @@ -384,6 +394,40 @@ fn materialize_executable( result } +#[cfg(target_os = "macos")] +fn materialize_executable( + artifact: &VerifiedProcessArtifact, +) -> Result { + let directory = verified_executable_parent(artifact)?; + let path = directory.join(format!( + ".paperclip-verified-executable-{}", + Uuid::new_v4().simple() + )); + materialize_executable_at(artifact, path, None) +} + +#[cfg(target_os = "macos")] +fn materialize_bound_executable( + artifact: &VerifiedProcessArtifact, +) -> Result { + let parent = verified_executable_parent(artifact)?; + let directory = parent.join(format!( + ".paperclip-verified-executable-{}", + Uuid::new_v4().simple() + )); + fs::create_dir(&directory).map_err(|error| snapshot_error(&artifact.display_path, error))?; + if let Err(error) = fs::set_permissions(&directory, fs::Permissions::from_mode(0o700)) { + let _ = fs::remove_dir(&directory); + return Err(snapshot_error(&artifact.display_path, error)); + } + let result = + materialize_executable_at(artifact, directory.join("launch"), Some(directory.clone())); + if result.is_err() { + let _ = fs::remove_dir(directory); + } + result +} + #[cfg(any(target_os = "linux", target_os = "macos"))] fn inherited_artifact( artifact: &VerifiedProcessArtifact, @@ -941,4 +985,50 @@ mod tests { assert!(inherited.args[2].starts_with("/dev/fd/")); fs::remove_dir_all(directory).unwrap(); } + + #[cfg(target_os = "macos")] + #[test] + fn executable_arguments_use_the_private_nested_launch_contract() { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = std::env::temp_dir().join(format!( + "paperclip-executable-argument-launch-{}-{nonce}", + std::process::id() + )); + fs::create_dir_all(&directory).unwrap(); + let program = verified_artifact(&directory.join("node"), b"node"); + let executable = verified_artifact(&directory.join("opencode"), b"opencode"); + let launch = VerifiedProcessLaunch::new( + program, + vec![VerifiedProcessArgument::ExecutableArtifact(executable)], + ); + + let inherited = launch.inherited_command().unwrap(); + let command = PathBuf::from(&inherited.args[0]); + let private_directory = command.parent().unwrap().to_path_buf(); + assert_eq!(command.file_name().unwrap(), "launch"); + assert!(private_directory + .file_name() + .unwrap() + .to_string_lossy() + .starts_with(".paperclip-verified-executable-")); + assert_eq!( + fs::symlink_metadata(&private_directory) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o700 + ); + assert_eq!( + fs::symlink_metadata(&command).unwrap().permissions().mode() & 0o777, + 0o500 + ); + + drop(inherited); + assert!(!private_directory.exists()); + fs::remove_dir_all(directory).unwrap(); + } } diff --git a/packages/paperclip-runner/scripts/aws-agentcore-provisioning.test.mjs b/packages/paperclip-runner/scripts/aws-agentcore-provisioning.test.mjs index eae5284f90..2bfdbe6a0b 100644 --- a/packages/paperclip-runner/scripts/aws-agentcore-provisioning.test.mjs +++ b/packages/paperclip-runner/scripts/aws-agentcore-provisioning.test.mjs @@ -82,6 +82,10 @@ test("AgentCore template has closed development/private resources and explicit c assert.match(source, /Sid: DecryptPinnedRuntimeContext[\s\S]*Action: kms:Decrypt[\s\S]*Resource: !GetAtt ContextEncryptionKey\.Arn/); const invocationRole = source.slice(source.indexOf("RunnerInvocationRole:"), source.indexOf("PrivateVpc:")); assert.doesNotMatch(invocationRole, /Action:\s+(?:-\s+)?(?:s3|kms):\*/); + assert.match(invocationRole, /Action: sts:AssumeRoleWithWebIdentity/); + assert.match(invocationRole, /token\.actions\.githubusercontent\.com:aud["']?: sts\.amazonaws\.com/); + assert.match(invocationRole, /token\.actions\.githubusercontent\.com:sub["']?: !Sub repo:\$\{GitHubRepository\}:environment:\$\{GitHubEnvironment\}/); + assert.doesNotMatch(invocationRole, /repo:\*|environment:\*/); assert.match(source, /\$\{AgentHarness\.Arn\}\/harness-endpoint\/\$\{HarnessEndpointName\}/); assert.match(source, /\$\{AgentHarness\.Arn\}\/runtime-endpoint\/\$\{HarnessEndpointName\}/); assert.match(source, /BedrockMarketplaceProductId/); @@ -136,10 +140,13 @@ test("AgentCore wrapper is valid shell and writes only nonsecret profile metadat assert.match(source, /--query endpoint\.status/); assert.match(source, /o\.endpoint\?\.arn/); assert.match(source, /--marketplace-product-id/); + assert.match(source, /--github-oidc-provider-arn/); + assert.match(source, /GitHubOidcProviderArn=\$GITHUB_OIDC_PROVIDER_ARN/); assert.match(source, /BedrockMarketplaceProductId=\$MARKETPLACE_PRODUCT_ID/); assert.match(generatedBlock, /PAPERCLIP_AWS_AGENTCORE_CONTEXT_BUCKET=\$context_bucket/); assert.match(generatedBlock, /PAPERCLIP_AWS_AGENTCORE_CONTEXT_PREFIX=\$context_prefix/); assert.match(generatedBlock, /PAPERCLIP_AWS_AGENTCORE_CONTEXT_KMS_KEY_ARN=\$context_kms_key_arn/); + assert.match(generatedBlock, /PAPERCLIP_AWS_AGENTCORE_EXECUTION_ROLE_ARN=\$role_arn/); assert.match(source, /ContextPrefix=\$CONTEXT_PREFIX/); assert.match(source, /s3 rm "s3:\/\/\$context_bucket\/\$context_prefix\/assets\/" --recursive/); assert.ok(source.indexOf("delete-harness-endpoint") < source.lastIndexOf("cloudformation delete-stack")); diff --git a/packages/paperclip-runner/scripts/aws-agentcore.sh b/packages/paperclip-runner/scripts/aws-agentcore.sh index 26f4e493d8..c782a22e18 100644 --- a/packages/paperclip-runner/scripts/aws-agentcore.sh +++ b/packages/paperclip-runner/scripts/aws-agentcore.sh @@ -22,6 +22,9 @@ MODEL_ID="global.anthropic.claude-sonnet-4-6" MARKETPLACE_PRODUCT_ID="prod-ffvjxvh4ltq64" MARKETPLACE_PRODUCT_ID_EXPLICIT=false TRUSTED_PRINCIPAL="" +GITHUB_OIDC_PROVIDER_ARN="${PAPERCLIP_AWS_AGENTCORE_GITHUB_OIDC_PROVIDER_ARN:-}" +GITHUB_REPOSITORY="paperclipai/paperclip" +GITHUB_ENVIRONMENT="runner-e2e-paid" CONTEXT_PREFIX="" DRY_RUN=false FORCE=false @@ -38,6 +41,9 @@ usage() { " --model MODEL_ID Bedrock-native model ID" \ " --marketplace-product-id ID exact AWS Marketplace product ID for the model" \ " --principal ARN stable IAM role/user trusted to assume runner role" \ + " --github-oidc-provider-arn ARN optional account-local GitHub Actions OIDC provider" \ + " --github-repository OWNER/REPO exact repository admitted by OIDC trust (default: $GITHUB_REPOSITORY)" \ + " --github-environment NAME exact protected environment admitted by OIDC trust (default: $GITHUB_ENVIRONMENT)" \ " --context-prefix PFX S3 prefix dedicated to this qualified profile" \ " --dry-run validate and print changes without deployment" \ " --replace-failed-stack explicitly replace an owned ROLLBACK_COMPLETE stack" \ @@ -55,6 +61,9 @@ while [[ $# -gt 0 ]]; do --model) MODEL_ID="$2"; shift 2 ;; --marketplace-product-id) MARKETPLACE_PRODUCT_ID="$2"; MARKETPLACE_PRODUCT_ID_EXPLICIT=true; shift 2 ;; --principal) TRUSTED_PRINCIPAL="$2"; shift 2 ;; + --github-oidc-provider-arn) GITHUB_OIDC_PROVIDER_ARN="$2"; shift 2 ;; + --github-repository) GITHUB_REPOSITORY="$2"; shift 2 ;; + --github-environment) GITHUB_ENVIRONMENT="$2"; shift 2 ;; --context-prefix) CONTEXT_PREFIX="$2"; shift 2 ;; --dry-run) DRY_RUN=true; shift ;; --replace-failed-stack) REPLACE_FAILED_STACK=true; shift ;; @@ -93,6 +102,14 @@ if [[ ! "$MARKETPLACE_PRODUCT_ID" =~ ^prod-[a-z0-9]+$ ]]; then printf 'Marketplace product ID must use the AWS prod-* form.\n' >&2 exit 2 fi +if [[ -n "$GITHUB_OIDC_PROVIDER_ARN" && ! "$GITHUB_OIDC_PROVIDER_ARN" =~ ^arn:aws(-[^:]+)?:iam::[0-9]{12}:oidc-provider/token\.actions\.githubusercontent\.com$ ]]; then + printf 'GitHub OIDC provider must be the account-local token.actions.githubusercontent.com provider ARN.\n' >&2 + exit 2 +fi +if [[ ! "$GITHUB_REPOSITORY" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ || ! "$GITHUB_ENVIRONMENT" =~ ^[A-Za-z0-9_.-]+$ ]]; then + printf 'GitHub OIDC trust requires exact OWNER/REPO and environment names.\n' >&2 + exit 2 +fi if [[ -z "$CONTEXT_PREFIX" ]]; then CONTEXT_PREFIX="paperclip/agentcore/$STACK_NAME"; fi if [[ ! "$CONTEXT_PREFIX" =~ ^[a-z0-9][a-z0-9/_-]{1,127}$ || "$CONTEXT_PREFIX" == /* || "$CONTEXT_PREFIX" == */ || "$CONTEXT_PREFIX" == *//* || "/$CONTEXT_PREFIX/" == */./* || "/$CONTEXT_PREFIX/" == */../* ]]; then printf 'Context prefix must be a safe, relative S3 key prefix.\n' >&2 @@ -279,6 +296,14 @@ provision() { caller_arn="$(printf '%s' "$identity" | json_field Arn)" account_id="$(printf '%s' "$identity" | json_field Account)" partition="$(printf '%s' "$caller_arn" | cut -d: -f2)" + if [[ -n "$GITHUB_OIDC_PROVIDER_ARN" ]]; then + local expected_github_oidc_provider_arn="arn:$partition:iam::$account_id:oidc-provider/token.actions.githubusercontent.com" + if [[ "$GITHUB_OIDC_PROVIDER_ARN" != "$expected_github_oidc_provider_arn" ]]; then + printf 'GitHub OIDC provider must belong to the current AWS account and partition.\n' >&2 + exit 1 + fi + aws_cli iam get-open-id-connect-provider --open-id-connect-provider-arn "$GITHUB_OIDC_PROVIDER_ARN" >/dev/null + fi model_resource_arn="arn:$partition:bedrock:$AWS_REGION_NAME:$account_id:inference-profile/$MODEL_ID" foundation_model_id="$(printf '%s' "$MODEL_ID" | sed -E 's/^(global|us|eu|apac)\.//')" foundation_model_resource_arn="arn:$partition:bedrock:*::foundation-model/$foundation_model_id" @@ -342,7 +367,11 @@ provision() { "EnvironmentName=development" "DeploymentMode=$DEPLOYMENT_MODE" \ "HarnessEndpointName=$ENDPOINT_NAME" \ "ContextPrefix=$CONTEXT_PREFIX" \ - "TrustedRunnerPrincipalArn=$TRUSTED_PRINCIPAL" "BedrockModelId=$MODEL_ID" \ + "TrustedRunnerPrincipalArn=$TRUSTED_PRINCIPAL" \ + "GitHubOidcProviderArn=$GITHUB_OIDC_PROVIDER_ARN" \ + "GitHubRepository=$GITHUB_REPOSITORY" \ + "GitHubEnvironment=$GITHUB_ENVIRONMENT" \ + "BedrockModelId=$MODEL_ID" \ "BedrockModelResourceArn=$model_resource_arn" \ "BedrockFoundationModelResourceArn=$foundation_model_resource_arn" \ "BedrockMarketplaceProductId=$MARKETPLACE_PRODUCT_ID" \ @@ -400,6 +429,7 @@ provision() { "PAPERCLIP_AWS_AGENTCORE_MEMORY_ARN=$memory_arn" \ "PAPERCLIP_AWS_AGENTCORE_MEMORY_ID=$memory_id" \ "PAPERCLIP_AWS_AGENTCORE_INVOCATION_ROLE_ARN=$role_arn" \ + "PAPERCLIP_AWS_AGENTCORE_EXECUTION_ROLE_ARN=$role_arn" \ "PAPERCLIP_AWS_AGENTCORE_CONTEXT_BUCKET=$context_bucket" \ "PAPERCLIP_AWS_AGENTCORE_CONTEXT_PREFIX=$context_prefix" \ "PAPERCLIP_AWS_AGENTCORE_CONTEXT_KMS_KEY_ARN=$context_kms_key_arn" \ diff --git a/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.mjs b/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.mjs new file mode 100644 index 0000000000..b014d76686 --- /dev/null +++ b/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.mjs @@ -0,0 +1,547 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { execFile } from "node:child_process"; +import { + lstat, + mkdtemp, + mkdir, + readFile, + readdir, + stat, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { extname, join, relative, resolve, sep } from "node:path"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); +const SAFE_CAMPAIGN = /^gha-[1-9][0-9]*-[1-9][0-9]*$/; +const SAFE_REPORT_PATHS = [ + /^(?:index|latest|inventory|real-server)\.html$/, + /^tests\/[A-Za-z0-9][A-Za-z0-9._-]{0,199}\.html$/, + /^attempts\/[A-Za-z0-9][A-Za-z0-9._-]{0,199}\.html$/, + /^campaign\.json$/, +]; +const CREDENTIAL_PATTERNS = [ + /\bAKIA[0-9A-Z]{16}\b/u, + /\bsk-[A-Za-z0-9_-]{20,}\b/u, + /\bBearer\s+[A-Za-z0-9._~-]{16,}\b/iu, + /["'](?:providerSessionId|sessionId)["']\s*:/u, +]; +const ACTIVE_HTML_PATTERNS = [ + /", ">") + .replaceAll('"', """) + .replaceAll("'", "'"); +} + +async function loadObject(path) { + const value = JSON.parse(await readFile(path, "utf8")); + if (value === null || Array.isArray(value) || typeof value !== "object") { + throw new Error(`Expected a JSON object: ${path}`); + } + return value; +} + +export function validateProtocolEvalHistoryDestination({ + bucket, + prefix, + publicBaseUrl, +}) { + if (!/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(bucket)) { + throw new Error( + "RUNNER_PROTOCOL_EVAL_HISTORY_S3_BUCKET is not a valid bucket name", + ); + } + const normalizedPrefix = String(prefix ?? "").replace(/^\/+|\/+$/g, ""); + if ( + !normalizedPrefix || + normalizedPrefix + .split("/") + .some((segment) => !segment || segment === "." || segment === "..") + ) { + throw new Error( + "RUNNER_PROTOCOL_EVAL_HISTORY_PREFIX must be a safe non-empty key prefix", + ); + } + const url = new URL(publicBaseUrl); + if ( + url.protocol !== "https:" || + url.username || + url.password || + url.search || + url.hash + ) { + throw new Error( + "RUNNER_PROTOCOL_EVAL_HISTORY_PUBLIC_BASE_URL must be a credential-free HTTPS URL", + ); + } + return { + bucket, + prefix: normalizedPrefix, + publicBaseUrl: url.href.replace(/\/$/, ""), + }; +} + +export function isPublicProtocolEvalPath(relativePath) { + if ( + relativePath.includes("\\") || + relativePath.startsWith("/") || + relativePath + .split("/") + .some((segment) => !segment || segment === "." || segment === "..") + ) { + return false; + } + return SAFE_REPORT_PATHS.some((pattern) => pattern.test(relativePath)); +} + +async function relativeFiles(root, current = root) { + const entries = await readdir(current, { withFileTypes: true }); + const files = []; + for (const entry of entries) { + const absolute = join(current, entry.name); + if (entry.isSymbolicLink()) + throw new Error(`Refusing public report symlink ${absolute}`); + if (entry.isDirectory()) + files.push(...(await relativeFiles(root, absolute))); + else if (entry.isFile()) + files.push(relative(root, absolute).split(sep).join("/")); + else throw new Error(`Refusing unusual public report path ${absolute}`); + } + return files.sort(); +} + +function internalHtmlHrefs(content) { + return [...content.matchAll(/href\s*=\s*["']([^"']+)["']/giu)] + .map((match) => match[1]) + .filter((href) => href && !href.startsWith("#")); +} + +export async function validatePublicProtocolEvalReport(reportRoot) { + const root = resolve(reportRoot); + const files = await relativeFiles(root); + if (!files.includes("index.html") || !files.includes("campaign.json")) { + throw new Error( + "Public protocol eval report requires index.html and campaign.json", + ); + } + for (const file of files) { + if (!isPublicProtocolEvalPath(file)) { + throw new Error( + `Refusing non-allowlisted public protocol eval path ${file}`, + ); + } + const absolute = resolve(root, ...file.split("/")); + const metadata = await stat(absolute); + if (metadata.size === 0 || metadata.size > 12 * 1024 * 1024) { + throw new Error( + `Public protocol eval file exceeds its size boundary: ${file}`, + ); + } + const content = await readFile(absolute, "utf8"); + for (const pattern of CREDENTIAL_PATTERNS) { + if (pattern.test(content)) + throw new Error( + `Public report contains credential/session material: ${file}`, + ); + } + if (extname(file) !== ".html") continue; + for (const pattern of ACTIVE_HTML_PATTERNS) { + if (pattern.test(content)) + throw new Error( + `Public report contains active or remote HTML: ${file}`, + ); + } + for (const href of internalHtmlHrefs(content)) { + const clean = href.split("#", 1)[0].split("?", 1)[0]; + const target = resolve( + root, + ...file.split("/").slice(0, -1), + ...clean.split("/"), + ); + const rel = relative(root, target).split(sep).join("/"); + if ( + !isPublicProtocolEvalPath(rel) || + !(await lstat(target).catch(() => null))?.isFile() + ) { + throw new Error( + `Public report contains a broken or unsafe link in ${file}: ${href}`, + ); + } + } + } + const campaign = await loadObject(join(root, "campaign.json")); + if ( + campaign.schema !== "paperclip.runner-protocol-eval.campaign/v1" || + !SAFE_CAMPAIGN.test(String(campaign.campaignId ?? "")) + ) { + throw new Error("Public report campaign metadata is invalid"); + } + return { files, campaign }; +} + +export async function createProtocolEvalBundleManifest(reportRoot, campaignId) { + if (!SAFE_CAMPAIGN.test(campaignId)) + throw new Error("Unsafe protocol eval campaign ID"); + const { files, campaign } = + await validatePublicProtocolEvalReport(reportRoot); + if (campaign.campaignId !== campaignId) + throw new Error("Report campaign ID does not match publication target"); + const entries = await Promise.all( + files.map(async (file) => { + const absolute = resolve(reportRoot, ...file.split("/")); + const [content, metadata] = await Promise.all([ + readFile(absolute), + stat(absolute), + ]); + return { + path: file, + sha256: createHash("sha256").update(content).digest("hex"), + bytes: metadata.size, + }; + }), + ); + return { + schema: "paperclip.runner-protocol-eval.bundle/v1", + campaignId, + bundleDigest: createHash("sha256") + .update(JSON.stringify(entries)) + .digest("hex"), + files: entries, + }; +} + +export function emptyProtocolEvalHistory() { + return { + schema: "paperclip.runner-protocol-eval.history/v1", + updatedAt: new Date(0).toISOString(), + latestCampaignId: null, + latestGreenCampaignId: null, + campaigns: [], + }; +} + +export function protocolEvalHistoryRecord(campaign, publicRoot) { + return { + campaignId: campaign.campaignId, + generatedAt: campaign.generatedAt, + publicUrl: `${publicRoot}/campaigns/${encodeURIComponent(campaign.campaignId)}/`, + complete: campaign.complete === true, + allPassed: campaign.allPassed === true, + totals: campaign.totals, + rosters: campaign.rosters, + source: campaign.source, + }; +} + +export function mergeProtocolEvalHistory(history, record) { + if (history.schema !== "paperclip.runner-protocol-eval.history/v1") { + throw new Error("Unsupported protocol eval history schema"); + } + const existing = history.campaigns.find( + (item) => item.campaignId === record.campaignId, + ); + if (existing && JSON.stringify(existing) !== JSON.stringify(record)) { + throw new Error( + `Immutable campaign history changed for ${record.campaignId}`, + ); + } + const campaigns = existing + ? history.campaigns + : [...history.campaigns, record]; + campaigns.sort((left, right) => + right.generatedAt.localeCompare(left.generatedAt), + ); + const latest = campaigns[0] ?? null; + const latestGreen = + campaigns.find((campaign) => campaign.complete && campaign.allPassed) ?? + null; + const retained = campaigns.slice(0, MAX_HISTORY_CAMPAIGNS); + if ( + latestGreen && + !retained.some( + (campaign) => campaign.campaignId === latestGreen.campaignId, + ) + ) { + retained[retained.length - 1] = latestGreen; + } + return { + schema: history.schema, + updatedAt: new Date().toISOString(), + latestCampaignId: latest?.campaignId ?? null, + latestGreenCampaignId: latestGreen?.campaignId ?? null, + campaigns: retained, + }; +} + +export function buildProtocolEvalPointers(history) { + const byId = new Map( + history.campaigns.map((campaign) => [campaign.campaignId, campaign]), + ); + const project = (id) => { + const campaign = id ? byId.get(id) : null; + return campaign + ? { + campaignId: campaign.campaignId, + generatedAt: campaign.generatedAt, + publicUrl: campaign.publicUrl, + paperclipSha: campaign.source?.paperclip?.sha ?? null, + evalsSha: campaign.source?.evals?.sha ?? null, + } + : null; + }; + return { + latest: { + schema: "paperclip.runner-protocol-eval.pointer/v1", + updatedAt: history.updatedAt, + campaign: project(history.latestCampaignId), + }, + latestGreen: { + schema: "paperclip.runner-protocol-eval.pointer/v1", + updatedAt: history.updatedAt, + campaign: project(history.latestGreenCampaignId), + }, + }; +} + +function date(value) { + return new Intl.DateTimeFormat("en-US", { + dateStyle: "medium", + timeStyle: "short", + timeZone: "UTC", + }).format(new Date(value)); +} + +export function renderProtocolEvalHistoryIndex(history) { + const rows = history.campaigns.length + ? history.campaigns + .map((campaign) => { + const status = + campaign.complete && campaign.allPassed ? "passed" : "failed"; + const rosters = campaign.rosters + .map( + (roster) => + `${html(roster.model)} · ${roster.passed}/${roster.selected}`, + ) + .join("
"); + return `${html(campaign.campaignId)}${html(date(campaign.generatedAt))} UTC${status}${html(campaign.totals.passed)}/${html(campaign.totals.selected)}${html(campaign.totals.behaviorFailures)} behavior · ${html(campaign.totals.infrastructureFailures)} infrastructure${rosters}${html(campaign.source?.paperclip?.sha?.slice(0, 8) ?? "unknown")}evals ${html(campaign.source?.evals?.sha?.slice(0, 8) ?? "unknown")}Open Evalbook →`; + }) + .join("") + : 'No campaigns have been published yet.'; + const latest = history.campaigns.find( + (campaign) => campaign.campaignId === history.latestCampaignId, + ); + const latestGreen = history.campaigns.find( + (campaign) => campaign.campaignId === history.latestGreenCampaignId, + ); + return ` +Runner protocol eval campaigns · Paperclip + +
Paperclip quality engineering

Runner protocol eval campaigns

Versioned direct live-runner Evalbook reports. Full provider transcripts, session identifiers, state, and raw tool evidence remain in access-controlled workflow artifacts.

+ +
${rows}
CampaignStatusCellsModels / rostersSource
Updated ${html(date(history.updatedAt))} UTC · Immutable campaign bundles · Canonical Evalbook layout with public-safe evidence projections
`; +} + +function awsObject(bucket, key) { + return `s3://${bucket}/${key}`; +} + +async function objectExists(bucket, key) { + try { + await execFileAsync("aws", [ + "s3api", + "head-object", + "--bucket", + bucket, + "--key", + key, + ]); + return true; + } catch (error) { + const detail = String(error?.stderr ?? error?.message ?? error); + if (/\b(?:404|Not Found|NoSuchKey)\b/iu.test(detail)) return false; + throw new Error( + `Unable to inspect protocol eval history object: ${detail.slice(0, 400)}`, + ); + } +} + +async function downloadJson(bucket, key, destination) { + if (!(await objectExists(bucket, key))) return null; + await execFileAsync("aws", [ + "s3", + "cp", + awsObject(bucket, key), + destination, + "--only-show-errors", + ]); + return loadObject(destination); +} + +async function uploadFile(bucket, key, file, cacheControl) { + const contentType = + extname(file) === ".html" ? "text/html; charset=utf-8" : "application/json"; + await execFileAsync("aws", [ + "s3", + "cp", + file, + awsObject(bucket, key), + "--only-show-errors", + "--content-type", + contentType, + "--cache-control", + cacheControl, + ]); +} + +async function uploadImmutableReport(bucket, prefix, reportRoot) { + const cacheControl = "public,max-age=31536000,immutable"; + await execFileAsync("aws", [ + "s3", + "cp", + reportRoot, + awsObject(bucket, prefix), + "--recursive", + "--only-show-errors", + "--cache-control", + cacheControl, + "--exclude", + "*.json", + ]); + await uploadFile( + bucket, + `${prefix}/campaign.json`, + resolve(reportRoot, "campaign.json"), + cacheControl, + ); +} + +export async function publishProtocolEvalHistory({ reportRoot, destination }) { + const validatedDestination = + validateProtocolEvalHistoryDestination(destination); + const { campaign } = await validatePublicProtocolEvalReport(reportRoot); + const manifest = await createProtocolEvalBundleManifest( + reportRoot, + campaign.campaignId, + ); + const temporary = await mkdtemp( + join(tmpdir(), "runner-protocol-eval-history-"), + ); + const historyKey = `${validatedDestination.prefix}/history.json`; + const history = mergeProtocolEvalHistory( + (await downloadJson( + validatedDestination.bucket, + historyKey, + join(temporary, "history.json"), + )) ?? emptyProtocolEvalHistory(), + protocolEvalHistoryRecord( + campaign, + `${validatedDestination.publicBaseUrl}/${validatedDestination.prefix}`, + ), + ); + const campaignPrefix = `${validatedDestination.prefix}/campaigns/${campaign.campaignId}`; + const manifestKey = `${campaignPrefix}/bundle-manifest.json`; + const existing = await downloadJson( + validatedDestination.bucket, + manifestKey, + join(temporary, "existing-manifest.json"), + ); + if (existing && existing.bundleDigest !== manifest.bundleDigest) { + throw new Error( + `Immutable campaign ${campaign.campaignId} already exists with a different digest`, + ); + } + if (!existing) { + await uploadImmutableReport( + validatedDestination.bucket, + campaignPrefix, + reportRoot, + ); + const manifestFile = join(temporary, "bundle-manifest.json"); + await writeFile(manifestFile, json(manifest)); + await uploadFile( + validatedDestination.bucket, + manifestKey, + manifestFile, + "public,max-age=31536000,immutable", + ); + } + const pointers = buildProtocolEvalPointers(history); + const mutable = { + "history.json": history, + "latest.json": pointers.latest, + "latest-green.json": pointers.latestGreen, + }; + for (const [name, value] of Object.entries(mutable)) { + const file = join(temporary, name); + await writeFile(file, json(value)); + await uploadFile( + validatedDestination.bucket, + `${validatedDestination.prefix}/${name}`, + file, + "no-cache", + ); + } + const index = join(temporary, "index.html"); + await writeFile(index, renderProtocolEvalHistoryIndex(history)); + await uploadFile( + validatedDestination.bucket, + `${validatedDestination.prefix}/index.html`, + index, + "no-cache", + ); + return { + campaignId: campaign.campaignId, + bundleDigest: manifest.bundleDigest, + historySize: history.campaigns.length, + }; +} + +async function main() { + const result = await publishProtocolEvalHistory({ + reportRoot: resolve( + process.env.PAPERCLIP_RUNNER_PROTOCOL_EVAL_PUBLIC_REPORT_DIR ?? + "runner-protocol-eval-public-report", + ), + destination: { + bucket: process.env.RUNNER_PROTOCOL_EVAL_HISTORY_S3_BUCKET ?? "", + prefix: + process.env.RUNNER_PROTOCOL_EVAL_HISTORY_PREFIX ?? + "runner-protocol-evals", + publicBaseUrl: + process.env.RUNNER_PROTOCOL_EVAL_HISTORY_PUBLIC_BASE_URL ?? "", + }, + }); + console.log( + `Published immutable protocol eval campaign ${result.campaignId} (${result.bundleDigest}) and ${result.historySize} history record(s)`, + ); +} + +if ( + process.argv[1] && + resolve(process.argv[1]) === resolve(import.meta.filename) +) { + await main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.test.mjs b/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.test.mjs new file mode 100644 index 0000000000..835144fdc5 --- /dev/null +++ b/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.test.mjs @@ -0,0 +1,221 @@ +import assert from "node:assert/strict"; +import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { + buildProtocolEvalPointers, + createProtocolEvalBundleManifest, + emptyProtocolEvalHistory, + isPublicProtocolEvalPath, + mergeProtocolEvalHistory, + protocolEvalHistoryRecord, + renderProtocolEvalHistoryIndex, + validateProtocolEvalHistoryDestination, + validatePublicProtocolEvalReport, +} from "./publish-runner-protocol-eval-history.mjs"; + +const roots = []; +test.afterEach(async () => { + await Promise.all( + roots.splice(0).map((root) => rm(root, { recursive: true, force: true })), + ); +}); + +function campaign(overrides = {}) { + return { + schema: "paperclip.runner-protocol-eval.campaign/v1", + campaignId: "gha-42-1", + generatedAt: "2026-09-05T00:00:00.000Z", + source: { + paperclip: { sha: "a".repeat(40), ref: "refs/heads/master" }, + evals: { repository: "paperclipai/paperclip-evals", sha: "b".repeat(40) }, + }, + complete: true, + allPassed: true, + totals: { + selected: 35, + passed: 35, + behaviorFailures: 0, + infrastructureFailures: 0, + }, + rosters: [ + { + rosterId: "protocol-live-mini", + model: "gpt-5.4-mini", + provider: "codex", + driver: "codex_app_server", + selected: 35, + passed: 35, + }, + ], + results: [], + ...overrides, + }; +} + +async function reportFixture() { + const root = await mkdtemp(join(tmpdir(), "runner-protocol-public-")); + roots.push(root); + await Promise.all([ + mkdir(join(root, "tests"), { recursive: true }), + mkdir(join(root, "attempts"), { recursive: true }), + ]); + await Promise.all([ + writeFile(join(root, "campaign.json"), JSON.stringify(campaign())), + writeFile( + join(root, "index.html"), + 'testattempt', + ), + writeFile( + join(root, "tests/get-context.html"), + 'overview', + ), + writeFile( + join(root, "attempts/attempt-01.html"), + 'overview', + ), + ]); + return root; +} + +test("accepts only credential-free HTTPS destinations and the dedicated prefix", () => { + assert.deepEqual( + validateProtocolEvalHistoryDestination({ + bucket: "paperclip-public-reports", + prefix: "/runner-protocol-evals/", + publicBaseUrl: "https://reports.paperclip.ing/", + }), + { + bucket: "paperclip-public-reports", + prefix: "runner-protocol-evals", + publicBaseUrl: "https://reports.paperclip.ing", + }, + ); + assert.throws( + () => + validateProtocolEvalHistoryDestination({ + bucket: "bad", + prefix: "../evals", + publicBaseUrl: "http://example.test", + }), + /prefix|HTTPS/, + ); +}); + +test("keeps publication to the canonical static Evalbook surface", () => { + for (const file of [ + "index.html", + "latest.html", + "inventory.html", + "tests/get-context.html", + "attempts/attempt-01.html", + "campaign.json", + ]) { + assert.equal(isPublicProtocolEvalPath(file), true, file); + } + for (const file of [ + "attempts/attempt-01/index.html", + "runs/attempt/artifact.json", + "provider-trace.log", + "../secret", + "assets/app.js", + ]) { + assert.equal(isPublicProtocolEvalPath(file), false, file); + } +}); + +test("validates inert linked reports and creates deterministic manifests", async () => { + const root = await reportFixture(); + const result = await validatePublicProtocolEvalReport(root); + assert.equal(result.files.length, 4); + const first = await createProtocolEvalBundleManifest(root, "gha-42-1"); + const second = await createProtocolEvalBundleManifest(root, "gha-42-1"); + assert.deepEqual(second, first); + assert.match(first.bundleDigest, /^[0-9a-f]{64}$/); +}); + +test("rejects scripts, remote resources, raw sessions, secret-shaped values, and broken links", async () => { + for (const bad of [ + "", + 'remote', + '
{"providerSessionId":"session"}
', + `
${"sk-" + "a".repeat(24)}
`, + 'broken', + ]) { + const root = await reportFixture(); + await writeFile(join(root, "index.html"), bad); + await assert.rejects(validatePublicProtocolEvalReport(root)); + } +}); + +test("retains immutable history and independent latest-green pointers", () => { + const green = protocolEvalHistoryRecord( + campaign(), + "https://reports.example/runner-protocol-evals", + ); + const red = protocolEvalHistoryRecord( + campaign({ + campaignId: "gha-43-1", + generatedAt: "2026-09-05T01:00:00.000Z", + allPassed: false, + totals: { + selected: 35, + passed: 34, + behaviorFailures: 1, + infrastructureFailures: 0, + }, + rosters: [{ ...campaign().rosters[0], passed: 34 }], + }), + "https://reports.example/runner-protocol-evals", + ); + let history = mergeProtocolEvalHistory(emptyProtocolEvalHistory(), green); + history = mergeProtocolEvalHistory(history, red); + assert.equal(history.latestCampaignId, "gha-43-1"); + assert.equal(history.latestGreenCampaignId, "gha-42-1"); + const pointers = buildProtocolEvalPointers(history); + assert.equal(pointers.latest.campaign.campaignId, "gha-43-1"); + assert.equal(pointers.latestGreen.campaign.campaignId, "gha-42-1"); + const index = renderProtocolEvalHistoryIndex(history); + assert.match(index, /Runner protocol eval campaigns/); + assert.match(index, /Open Evalbook/); + assert.match(index, /34\/35/); + assert.doesNotMatch(index, /private-session-id|sk-[A-Za-z0-9_-]{20,}/); + assert.throws( + () => mergeProtocolEvalHistory(history, { ...green, allPassed: false }), + /Immutable campaign history changed/, + ); +}); + +test("retains the latest green pointer outside the 200 newest campaigns", () => { + const green = protocolEvalHistoryRecord( + campaign(), + "https://reports.example/runner-protocol-evals", + ); + let history = mergeProtocolEvalHistory(emptyProtocolEvalHistory(), green); + for (let index = 0; index < 201; index += 1) { + history = mergeProtocolEvalHistory( + history, + protocolEvalHistoryRecord( + campaign({ + campaignId: `gha-${index + 43}-1`, + generatedAt: new Date( + Date.UTC(2026, 8, 5, 1, 0, index), + ).toISOString(), + allPassed: false, + }), + "https://reports.example/runner-protocol-evals", + ), + ); + } + + assert.equal(history.campaigns.length, 200); + assert.equal(history.latestCampaignId, "gha-243-1"); + assert.equal(history.latestGreenCampaignId, "gha-42-1"); + assert.equal(history.campaigns.at(-1).campaignId, "gha-42-1"); + assert.equal( + buildProtocolEvalPointers(history).latestGreen.campaign.campaignId, + "gha-42-1", + ); +}); diff --git a/packages/paperclip-runner/scripts/render-runner-workflow-evalbook.mjs b/packages/paperclip-runner/scripts/render-runner-workflow-evalbook.mjs new file mode 100644 index 0000000000..9ebafc19a4 --- /dev/null +++ b/packages/paperclip-runner/scripts/render-runner-workflow-evalbook.mjs @@ -0,0 +1,352 @@ +import { createHash } from "node:crypto"; +import { spawn } from "node:child_process"; +import { access, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; + +const EVAL_PROGRAM_RELATIVE_PATH = + "evals/paperclip-runner/tools/eval_program.py"; + +function json(value) { + return `${JSON.stringify(value, null, 2)}\n`; +} + +function sha256(value) { + return createHash("sha256").update(value).digest("hex"); +} + +function safeSegment(value) { + const segment = String(value) + .trim() + .replaceAll(/[^0-9A-Za-z._-]+/g, "-") + .replaceAll(/^-+|-+$/g, ""); + if (!segment) throw new Error("Evalbook attempt identity is empty"); + return segment; +} + +function candidateDescriptor(report, candidateId) { + const descriptor = + report.bundle.providerVersions?.[candidateId] ?? candidateId; + const separator = descriptor.indexOf(":"); + return separator < 0 + ? { driver: "unknown driver", model: descriptor } + : { + driver: descriptor.slice(0, separator), + model: descriptor.slice(separator + 1), + }; +} + +function scoreChecks(result) { + const dimensionChecks = Object.values(result.scorecard.dimensions).map( + (dimension) => ({ + id: dimension.dimension, + kind: "workflow_dimension", + passed: dimension.passed === true, + detail: + dimension.score === null + ? dimension.reasons.join("; ") || "not scored" + : `score ${dimension.score}${ + dimension.reasons.length === 0 + ? "" + : `; ${dimension.reasons.join("; ")}` + }`, + evidenceRefs: [], + }), + ); + const observationChecks = [ + ["lifecycle", result.observation.lifecycle?.checks ?? []], + ["continuation", result.observation.continuation?.checks ?? []], + ["presentation", result.observation.presentation?.checks ?? []], + ].flatMap(([group, checks]) => + checks.map((check) => ({ + id: `${group}.${check.id}`, + kind: `${group}_check`, + passed: check.passed === true, + detail: check.reason ?? (check.passed ? "passed" : "failed"), + evidenceRefs: [], + })), + ); + return [...dimensionChecks, ...observationChecks]; +} + +function disposition(result) { + if (result.scorecard.overall.passed === true) return "passed"; + if ( + result.observation.classification === "infrastructure_failure" || + result.observation.classification === "skipped" + ) { + return "infrastructure_failure"; + } + return "behavior_failure"; +} + +function infrastructureErrors(result) { + if (disposition(result) !== "infrastructure_failure") return []; + return [ + result.observation.failure?.message ?? + `workflow execution was ${result.observation.classification}`, + ]; +} + +export function runnerWorkflowEvalbookAttempt({ + report, + result, + caseDefinition, +}) { + const { driver, model } = candidateDescriptor(report, result.candidateId); + const identity = { + generatedAt: report.generatedAt, + bundleId: report.bundle.id, + caseId: result.scenarioId, + candidateId: result.candidateId, + }; + const timestamp = safeSegment(report.generatedAt); + const attemptId = `${timestamp}-${safeSegment(result.scenarioId)}-${safeSegment( + result.candidateId, + )}-${sha256(JSON.stringify(identity)).slice(0, 10)}`; + const checks = scoreChecks(result); + const costUsd = result.observation.metrics.costUsd; + const infrastructure = + disposition(result) === "infrastructure_failure" + ? result.observation.failure + : undefined; + const artifact = { + schema: "paperclip-runner/workflow-eval-artifact/v1", + attemptId, + createdAt: report.generatedAt, + requestedModel: model, + provider: result.observation.provider, + driver, + providerVersion: + report.bundle.runnerBuild ?? report.bundle.runnerVersion ?? "unknown", + providerSessionId: result.observation.base.trace.sessionId, + retainedSession: false, + retainedSessionStatus: "not retained by safe workflow eval projection", + usage: { + agentTurns: result.observation.metrics.attempts, + inputTokens: result.observation.metrics.totalTokens, + cachedInputTokens: 0, + outputTokens: 0, + reasoningTokens: 0, + ...(costUsd === undefined + ? {} + : { estimatedCostNanodollars: Math.round(costUsd * 1_000_000_000) }), + }, + turn: { + status: + result.observation.classification === "completed" + ? "completed" + : "failed", + }, + snapshot: { + createdAt: report.generatedAt, + providerModel: { + id: model, + provider: result.observation.provider, + }, + transcript: [], + evidence: [], + }, + devtools: { revisions: [] }, + ...(infrastructure === undefined + ? {} + : { + infrastructureFailure: { + class: infrastructure.code, + category: infrastructure.category, + retryable: infrastructure.retryable, + }, + }), + workflow: { + bundle: report.bundle, + observation: result.observation, + scorecard: result.scorecard, + }, + }; + const score = { + schema: "paperclip-runner/eval-score/v1", + attemptId, + caseId: result.scenarioId, + checks, + disposition: disposition(result), + infrastructureErrors: infrastructureErrors(result), + passed: result.scorecard.overall.passed === true, + digest: `sha256:${sha256(JSON.stringify({ identity, checks }))}`, + }; + const evalCase = { + schema: "paperclip-runner/workflow-eval-case/v1", + id: result.scenarioId, + title: caseDefinition?.title ?? result.scenarioId, + description: caseDefinition + ? `Runner workflow case. Tags: ${caseDefinition.tags.join(", ")}.` + : "Runner workflow case.", + prompt: + "Redacted by the safe Runner workflow eval projection; inspect the authored workflow case for orchestration steps.", + fixture: "runner-workflow-live-harness", + authority: { + controlPlaneOwned: result.observation.base.controlPlaneOwned, + }, + checks: Object.entries(caseDefinition?.assertions ?? {}).map( + ([id, expected]) => ({ + id, + kind: "workflow_assertion", + expected, + }), + ), + ...(caseDefinition === undefined + ? {} + : { workflowDefinition: caseDefinition }), + }; + const config = { + schema: "paperclip-runner/workflow-eval-config/v1", + id: result.candidateId, + model, + provider: result.observation.provider, + driver, + runnerVersion: report.bundle.runnerVersion, + runnerBuild: report.bundle.runnerBuild, + promptPolicyId: report.bundle.promptPolicyId, + }; + return { attemptId, artifact, score, case: evalCase, config }; +} + +async function writeImmutable(path, value) { + const content = json(value); + try { + const existing = await readFile(path, "utf8"); + if (existing !== content) { + throw new Error(`Immutable Evalbook record changed: ${path}`); + } + } catch (error) { + if (error?.code !== "ENOENT") throw error; + await writeFile(path, content, { flag: "wx", mode: 0o600 }); + } +} + +export async function writeRunnerWorkflowEvalbookAttempts({ + report, + runsRoot, + caseForId, +}) { + await mkdir(runsRoot, { recursive: true }); + const attempts = []; + for (const result of report.results) { + const attempt = runnerWorkflowEvalbookAttempt({ + report, + result, + caseDefinition: caseForId?.(result.scenarioId), + }); + const directory = resolve(runsRoot, attempt.attemptId); + await mkdir(directory, { recursive: true, mode: 0o700 }); + await Promise.all([ + writeImmutable(resolve(directory, "artifact.json"), attempt.artifact), + writeImmutable(resolve(directory, "score.json"), attempt.score), + writeImmutable(resolve(directory, "case.json"), attempt.case), + writeImmutable(resolve(directory, "config.json"), attempt.config), + ]); + attempts.push(attempt.attemptId); + } + return attempts; +} + +async function existingPath(paths) { + for (const path of paths.filter(Boolean)) { + try { + await access(path); + return resolve(path); + } catch { + // Continue through the explicit and conventional checkout locations. + } + } + return null; +} + +export async function resolveCanonicalEvalProgram(packageRoot, environment) { + const fromRoot = environment.PAPERCLIP_EVALS_ROOT + ? resolve(environment.PAPERCLIP_EVALS_ROOT, EVAL_PROGRAM_RELATIVE_PATH) + : null; + const program = await existingPath([ + environment.PAPERCLIP_EVALBOOK_PROGRAM, + fromRoot, + resolve(packageRoot, "../../.paperclip-evals", EVAL_PROGRAM_RELATIVE_PATH), + resolve( + packageRoot, + "../../../paperclip-evals", + EVAL_PROGRAM_RELATIVE_PATH, + ), + resolve( + packageRoot, + "../../../../paperclip-evals", + EVAL_PROGRAM_RELATIVE_PATH, + ), + ]); + if (program !== null) return program; + throw new Error( + `Canonical Evalbook generator not found. Set PAPERCLIP_EVALBOOK_PROGRAM to ${EVAL_PROGRAM_RELATIVE_PATH} in a paperclip-evals checkout.`, + ); +} + +async function run(command, args) { + await new Promise((accept, reject) => { + const child = spawn(command, args, { stdio: "inherit" }); + child.once("error", reject); + child.once("exit", (code, signal) => { + if (code === 0) accept(); + else + reject( + new Error( + `${command} exited ${code ?? `for signal ${signal ?? "unknown"}`}`, + ), + ); + }); + }); +} + +export async function renderRunnerWorkflowWithCanonicalEvalbook({ + packageRoot, + outputDirectory, + report, + caseForId, + environment = process.env, +}) { + const program = await resolveCanonicalEvalProgram(packageRoot, environment); + const runsRoot = resolve(outputDirectory, "evalbook-runs"); + await rm(runsRoot, { recursive: true, force: true }); + const attempts = await writeRunnerWorkflowEvalbookAttempts({ + report, + runsRoot, + caseForId, + }); + await Promise.all([ + rm(resolve(outputDirectory, "attempts"), { recursive: true, force: true }), + rm(resolve(outputDirectory, "tests"), { recursive: true, force: true }), + rm(resolve(outputDirectory, "index.html"), { force: true }), + rm(resolve(outputDirectory, "latest.html"), { force: true }), + rm(resolve(outputDirectory, "live-report.html"), { force: true }), + rm(resolve(outputDirectory, "deterministic-report.html"), { force: true }), + ]); + await run(environment.PYTHON ?? "python3", [ + program, + "report", + "--runs-root", + runsRoot, + "--output", + outputDirectory, + ]); + const programBytes = await readFile(program); + const manifest = { + schema: "paperclip.runner.workflow-evalbook-render.v1", + generatedAt: report.generatedAt, + generator: { + path: program, + sha256: `sha256:${sha256(programBytes)}`, + }, + sourceReport: "live-report.json", + attempts, + index: resolve(outputDirectory, "index.html"), + }; + await writeFile( + resolve(outputDirectory, "evalbook-manifest.json"), + json(manifest), + ); + return manifest; +} diff --git a/packages/paperclip-runner/scripts/render-runner-workflow-evalbook.test.mjs b/packages/paperclip-runner/scripts/render-runner-workflow-evalbook.test.mjs new file mode 100644 index 0000000000..54fe993633 --- /dev/null +++ b/packages/paperclip-runner/scripts/render-runner-workflow-evalbook.test.mjs @@ -0,0 +1,134 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { resolve } from "node:path"; +import test from "node:test"; + +import { + runnerWorkflowEvalbookAttempt, + writeRunnerWorkflowEvalbookAttempts, +} from "./render-runner-workflow-evalbook.mjs"; + +function fixture(overrides = {}) { + const report = { + generatedAt: "2026-09-05T16:30:00.000Z", + bundle: { + id: "runner-live-v2-test", + runnerVersion: "1.2.3", + runnerBuild: "abc123", + promptPolicyId: "runner-live-workflow-v1", + providerVersions: { + "codex-luna": "codex_app_server:gpt-5.6-luna", + }, + }, + results: [], + }; + const result = { + scenarioId: "verification-policy", + candidateId: "codex-luna", + observation: { + classification: "completed", + provider: "codex", + base: { + controlPlaneOwned: false, + trace: { sessionId: "eval-session-1" }, + }, + lifecycle: { + checks: [{ id: "terminal-authority", passed: true }], + }, + continuation: { checks: [] }, + presentation: { checks: [] }, + metrics: { + attempts: 1, + totalTokens: 1234, + costUsd: 0.0025, + }, + ...overrides.observation, + }, + scorecard: { + dimensions: { + semantic_outcome: { + dimension: "semantic_outcome", + score: 1, + passed: true, + reasons: [], + }, + }, + overall: { score: 1, passed: true }, + ...overrides.scorecard, + }, + }; + report.results.push(result); + return { report, result }; +} + +test("maps a workflow result to the canonical immutable-attempt inputs", () => { + const { report, result } = fixture(); + const attempt = runnerWorkflowEvalbookAttempt({ + report, + result, + caseDefinition: { + title: "Verify before completion", + tags: ["verification"], + }, + }); + + assert.match(attempt.attemptId, /verification-policy-codex-luna/); + assert.equal(attempt.artifact.requestedModel, "gpt-5.6-luna"); + assert.equal(attempt.artifact.driver, "codex_app_server"); + assert.equal(attempt.artifact.usage.estimatedCostNanodollars, 2_500_000); + assert.equal(attempt.score.disposition, "passed"); + assert.equal(attempt.config.id, "codex-luna"); + assert.equal(attempt.case.title, "Verify before completion"); + assert.match(attempt.case.prompt, /Redacted/); +}); + +test("preserves unscored infrastructure semantics for the grid", () => { + const { report, result } = fixture({ + observation: { + classification: "infrastructure_failure", + failure: { + code: "provider_timeout", + category: "provider", + retryable: true, + message: "provider timed out", + }, + }, + scorecard: { + overall: { score: null, passed: null }, + }, + }); + const attempt = runnerWorkflowEvalbookAttempt({ report, result }); + + assert.equal(attempt.score.disposition, "infrastructure_failure"); + assert.deepEqual(attempt.score.infrastructureErrors, ["provider timed out"]); + assert.deepEqual(attempt.artifact.infrastructureFailure, { + class: "provider_timeout", + category: "provider", + retryable: true, + }); +}); + +test("writes idempotent attempt records without raw provider content", async () => { + const root = await mkdtemp(resolve(tmpdir(), "runner-workflow-evalbook-")); + try { + const { report } = fixture(); + const options = { + report, + runsRoot: root, + caseForId: () => ({ title: "Verify", tags: ["verification"] }), + }; + const first = await writeRunnerWorkflowEvalbookAttempts(options); + const second = await writeRunnerWorkflowEvalbookAttempts(options); + assert.deepEqual(second, first); + + const artifact = JSON.parse( + await readFile(resolve(root, first[0], "artifact.json"), "utf8"), + ); + assert.deepEqual(artifact.snapshot.transcript, []); + assert.deepEqual(artifact.snapshot.evidence, []); + assert.equal(artifact.workflow.observation.provider, "codex"); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/packages/paperclip-runner/scripts/run-runner-live-eval-schedule.mjs b/packages/paperclip-runner/scripts/run-runner-live-eval-schedule.mjs index 19217335e3..e78643209f 100644 --- a/packages/paperclip-runner/scripts/run-runner-live-eval-schedule.mjs +++ b/packages/paperclip-runner/scripts/run-runner-live-eval-schedule.mjs @@ -9,6 +9,8 @@ import { } from "node:fs/promises"; import { resolve } from "node:path"; +import { renderRunnerWorkflowWithCanonicalEvalbook } from "./render-runner-workflow-evalbook.mjs"; + const packageRoot = resolve(import.meta.dirname, ".."); const evals = await import(resolve(packageRoot, "dist/eval/index.js")); const packageManifest = JSON.parse( @@ -21,8 +23,7 @@ if (mode !== "nightly" && mode !== "chaos") throw new Error(`unsupported workflow eval schedule mode: ${mode}`); const now = process.env.PAPERCLIP_EVAL_GENERATED_AT ?? new Date().toISOString(); const rotationDay = Number( - process.env.PAPERCLIP_EVAL_ROTATION_DAY ?? - evals.runnerLiveRotationWeek(now), + process.env.PAPERCLIP_EVAL_ROTATION_DAY ?? evals.runnerLiveRotationWeek(now), ); const seed = process.env.PAPERCLIP_EVAL_SCHEDULE_SEED ?? "runner-live-seven-week-v1"; @@ -35,6 +36,51 @@ const historyDirectory = resolve( ); await mkdir(outputDirectory, { recursive: true }); +function selectorValues(flag, environmentName) { + const values = []; + for (let index = 0; index < process.argv.length; index += 1) { + if (process.argv[index] !== flag) continue; + const value = process.argv[index + 1]; + if (!value || value.startsWith("--")) { + throw new Error(`${flag} requires a comma-separated value`); + } + values.push(value); + } + if (process.env[environmentName]) values.push(process.env[environmentName]); + return [ + ...new Set( + values.flatMap((value) => + value + .split(",") + .map((entry) => entry.trim()) + .filter(Boolean), + ), + ), + ]; +} + +function selectionLimit() { + const index = process.argv.indexOf("--limit"); + const raw = + index < 0 ? process.env.PAPERCLIP_EVAL_LIMIT : process.argv[index + 1]; + if (raw === undefined || raw === "") return undefined; + const value = Number(raw); + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error("--limit must be a positive integer"); + } + return value; +} + +const selection = { + candidateIds: selectorValues("--candidate", "PAPERCLIP_EVAL_CANDIDATE"), + caseIds: selectorValues("--case", "PAPERCLIP_EVAL_CASE"), + limit: selectionLimit(), +}; +const selectionActive = + selection.candidateIds.length > 0 || + selection.caseIds.length > 0 || + selection.limit !== undefined; + function safeBundleId(schedule) { const runnerBuild = process.env.PAPERCLIP_EVAL_RUNNER_BUILD ?? packageManifest.version; @@ -51,6 +97,13 @@ function safeBundleId(schedule) { reasoningEffort, }), ), + ...(selectionActive + ? { + selectedExecutions: schedule.entries.map( + (entry) => entry.executionId, + ), + } + : {}), }); return `runner-live-v2-${createHash("sha256").update(identity).digest("hex").slice(0, 16)}`; } @@ -110,7 +163,7 @@ async function retainHistory(report) { } if (mode === "nightly") { - const schedule = evals.buildRunnerLiveEvalSchedule({ + const fullSchedule = evals.buildRunnerLiveEvalSchedule({ seed, rotationDay, generatedAt: now, @@ -120,9 +173,12 @@ if (mode === "nightly") { throw new Error( `live schedule coverage failed: ${JSON.stringify(coverage)}`, ); + const schedule = selectionActive + ? evals.selectRunnerLiveEvalSchedule(fullSchedule, selection) + : fullSchedule; await writeFile( resolve(outputDirectory, "nightly-schedule.json"), - `${JSON.stringify({ schedule, coverage }, null, 2)}\n`, + `${JSON.stringify({ schedule, coverage, selection: selectionActive ? selection : null }, null, 2)}\n`, ); if (!execute) { process.stdout.write( @@ -230,9 +286,15 @@ if (mode === "nightly") { `${evals.renderRunnerWorkflowGitHubSummary(report, alerts)}\n## Previous compatible bundle\n\n${comparison === null ? "No compatible prior bundle is available." : `Pass-rate delta: ${comparison.passRateDelta}; overall delta: ${comparison.overallDelta}.`}\n`, ), ]); + await renderRunnerWorkflowWithCanonicalEvalbook({ + packageRoot, + outputDirectory, + report, + caseForId: evals.runnerWorkflowCase, + }); await retainHistory(report); process.stdout.write( - `Runner live evals complete: ${report.aggregate.passed}/${report.aggregate.scoreable} scoreable passed; ${report.aggregate.infrastructureFailures} infrastructure, ${report.aggregate.skipped} skipped, ${alerts.length} alert(s).\n`, + `Runner live evals complete: ${report.aggregate.passed}/${report.aggregate.scoreable} scoreable passed; ${report.aggregate.infrastructureFailures} infrastructure, ${report.aggregate.skipped} skipped, ${alerts.length} alert(s). Open ${resolve(outputDirectory, "index.html")}.\n`, ); } else { const payload = { diff --git a/packages/paperclip-runner/scripts/run-runner-workflow-evals.mjs b/packages/paperclip-runner/scripts/run-runner-workflow-evals.mjs index 80273bcd87..e52c720ad3 100644 --- a/packages/paperclip-runner/scripts/run-runner-workflow-evals.mjs +++ b/packages/paperclip-runner/scripts/run-runner-workflow-evals.mjs @@ -3,30 +3,62 @@ import { resolve } from "node:path"; const packageRoot = resolve(import.meta.dirname, ".."); const evals = await import(resolve(packageRoot, "dist/eval/index.js")); -const packageManifest = JSON.parse(await readFile(resolve(packageRoot, "package.json"), "utf8")); -const traceability = JSON.parse(await readFile(resolve(packageRoot, "spec/evals/stress-workflow-traceability.json"), "utf8")); -const traceabilitySummary = evals.validateStressTraceabilityManifest(traceability); -const results = await evals.runDeterministicRunnerWorkflowMatrix({ bundleId: "runner-workflows-deterministic-v1" }); +const packageManifest = JSON.parse( + await readFile(resolve(packageRoot, "package.json"), "utf8"), +); +const traceability = JSON.parse( + await readFile( + resolve(packageRoot, "spec/evals/stress-workflow-traceability.json"), + "utf8", + ), +); +const traceabilitySummary = + evals.validateStressTraceabilityManifest(traceability); +const results = await evals.runDeterministicRunnerWorkflowMatrix({ + bundleId: "runner-workflows-deterministic-v1", +}); const report = evals.buildRunnerWorkflowEvalReport({ source: "deterministic", bundle: { id: "runner-workflows-deterministic-v1", runnerVersion: packageManifest.version, promptPolicyId: "stress-sanitized-v1", - providerVersions: { codex: "fixture-v1", opencode: "fixture-v1", acpx: "fixture-v1" }, + providerVersions: { + codex: "fixture-v1", + opencode: "fixture-v1", + acpx: "fixture-v1", + }, }, results, traceability: traceabilitySummary, }); -if (report.aggregate.passed !== report.aggregate.scoreable) { - throw new Error(`Runner workflow evals passed ${report.aggregate.passed}/${report.aggregate.scoreable}`); -} -const outputDirectory = resolve(packageRoot, ".paperclip-local/evals/workflows"); +const outputDirectory = resolve( + packageRoot, + ".paperclip-local/evals/workflows", +); await mkdir(outputDirectory, { recursive: true }); await Promise.all([ - writeFile(resolve(outputDirectory, "deterministic-report.json"), `${JSON.stringify(report, null, 2)}\n`), - writeFile(resolve(outputDirectory, "deterministic-report.md"), evals.renderRunnerWorkflowMarkdown(report)), - writeFile(resolve(outputDirectory, "deterministic-report.junit.xml"), evals.renderRunnerWorkflowJUnit(report)), - writeFile(resolve(outputDirectory, "github-summary.md"), evals.renderRunnerWorkflowGitHubSummary(report)), + writeFile( + resolve(outputDirectory, "deterministic-report.json"), + `${JSON.stringify(report, null, 2)}\n`, + ), + writeFile( + resolve(outputDirectory, "deterministic-report.md"), + evals.renderRunnerWorkflowMarkdown(report), + ), + writeFile( + resolve(outputDirectory, "deterministic-report.junit.xml"), + evals.renderRunnerWorkflowJUnit(report), + ), + writeFile( + resolve(outputDirectory, "github-summary.md"), + evals.renderRunnerWorkflowGitHubSummary(report), + ), ]); -process.stdout.write(`Runner workflow evals passed: ${report.aggregate.passed}/${report.aggregate.scoreable}; coverage ${report.coverage.canonicalOperations} operations, ${report.coverage.capabilityCases} capability cases, ${report.coverage.workflows} workflows.\n`); +const outcome = + report.aggregate.passed === report.aggregate.scoreable + ? "all scoreable executions passed" + : `${report.aggregate.passed}/${report.aggregate.scoreable} scoreable executions passed; deterministic fixtures exercised fail-closed scoring`; +process.stdout.write( + `Runner workflow eval report ready: ${outcome}; coverage ${report.coverage.canonicalOperations} operations, ${report.coverage.capabilityCases} capability cases, ${report.coverage.workflows} workflows.\n`, +); diff --git a/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs new file mode 100644 index 0000000000..7314b63748 --- /dev/null +++ b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs @@ -0,0 +1,698 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { + appendFile, + cp, + lstat, + mkdir, + readFile, + readdir, + rm, + writeFile, +} from "node:fs/promises"; +import { basename, dirname, join, relative, resolve, sep } from "node:path"; + +const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$/; +const ATTEMPT_FILES = new Set([ + "request.json", + "artifact.json", + "score.json", + "compiled-fixture.json", + "case.json", + "config.json", +]); + +function json(value) { + return `${JSON.stringify(value, null, 2)}\n`; +} + +async function loadObject(path) { + const value = JSON.parse(await readFile(path, "utf8")); + if (value === null || Array.isArray(value) || typeof value !== "object") { + throw new Error(`Expected a JSON object: ${path}`); + } + return value; +} + +function safeId(value, label = "identifier") { + const result = String(value ?? ""); + if (!SAFE_ID.test(result)) throw new Error(`Unsafe ${label}: ${result}`); + return result; +} + +function inside(root, candidate, label) { + const rel = relative(resolve(root), resolve(candidate)); + if (!rel || rel === ".." || rel.startsWith(`..${sep}`)) { + throw new Error(`${label} escapes its declared root`); + } + return resolve(candidate); +} + +export function credentialForConfig(config) { + if (config.provider === "opencode") return "OPENROUTER_API_KEY"; + if (config.provider === "claude_managed") return "ANTHROPIC_API_KEY"; + if (config.provider === "aws_agentcore") return "AWS_AGENTCORE_OIDC"; + if (config.provider === "codex" || config.provider === undefined) { + return "OPENAI_API_KEY"; + } + if (config.provider === "acpx") { + if (config.acpxAgent === "pi") return "OPENROUTER_API_KEY"; + if (config.acpxAgent === "claude") return "ANTHROPIC_API_KEY"; + if (config.acpxAgent === "codex") return "OPENAI_API_KEY"; + } + throw new Error( + `No credential policy for ${config.provider ?? "codex"}/${config.acpxAgent ?? "default"}`, + ); +} + +function parseRosterSelection(value) { + if (!value?.trim() || value.trim() === "all") return null; + const selected = value + .split(",") + .map((entry) => entry.trim()) + .filter(Boolean); + if (selected.length === 0 || new Set(selected).size !== selected.length) { + throw new Error("Roster selection must contain unique comma-separated IDs"); + } + return new Set(selected); +} + +export async function buildProtocolEvalCatalog({ + evalsRoot, + rosterSelection = "all", + campaignId, + source = {}, + maxParallel = 100, +}) { + safeId(campaignId, "campaign ID"); + if ( + !Number.isSafeInteger(maxParallel) || + maxParallel < 2 || + maxParallel > 100 + ) { + throw new Error("maxParallel must be an integer from 2 through 100"); + } + const programRoot = resolve(evalsRoot, "evals/paperclip-runner"); + const rosterRoot = resolve(programRoot, "rosters"); + const selected = parseRosterSelection(rosterSelection); + const rosterFiles = (await readdir(rosterRoot, { withFileTypes: true })) + .filter( + (entry) => + entry.isFile() && + entry.name.startsWith("live-") && + entry.name.endsWith(".json"), + ) + .map((entry) => entry.name) + .sort(); + const rosters = []; + for (const rosterFile of rosterFiles) { + const rosterPath = resolve(rosterRoot, rosterFile); + const roster = await loadObject(rosterPath); + const rosterId = safeId(roster.id, "roster ID"); + if (selected && !selected.has(rosterId) && !selected.has(rosterFile)) { + continue; + } + if (roster.schema !== "paperclip-runner/live-roster/v1") { + throw new Error(`Unsupported live roster schema in ${rosterFile}`); + } + if (!Array.isArray(roster.cases) || roster.cases.length === 0) { + throw new Error(`Live roster ${rosterId} has no cases`); + } + const configPath = inside( + programRoot, + resolve(rosterRoot, String(roster.config ?? "")), + `Config for ${rosterId}`, + ); + const config = await loadObject(configPath); + const credentialName = credentialForConfig(config); + const cases = roster.cases.map((caseId) => safeId(caseId, "case ID")); + if (new Set(cases).size !== cases.length) { + throw new Error(`Live roster ${rosterId} repeats a case`); + } + rosters.push({ + rosterId, + rosterFile, + configFile: relative(programRoot, configPath).split(sep).join("/"), + model: String(roster.model ?? config.model ?? "unknown"), + provider: String(config.provider ?? "codex"), + driver: String(config.driver ?? "codex_app_server"), + credentialName, + cases, + }); + } + if (selected) { + const found = new Set( + rosters.flatMap((roster) => [roster.rosterId, roster.rosterFile]), + ); + const missing = [...selected].filter((entry) => !found.has(entry)); + if (missing.length > 0) { + throw new Error(`Unknown live roster selection: ${missing.join(", ")}`); + } + } + if (rosters.length === 0) throw new Error("No live rosters were selected"); + + const cells = rosters.flatMap((roster) => + roster.cases.map((caseId) => ({ + cellId: safeId(`${roster.rosterId}--${caseId}`, "cell ID"), + rosterId: roster.rosterId, + rosterFile: roster.rosterFile, + caseId, + model: roster.model, + provider: roster.provider, + driver: roster.driver, + credentialName: roster.credentialName, + })), + ); + const shards = [[], []]; + cells.forEach((cell, index) => shards[index % shards.length].push(cell)); + if (shards.some((shard) => shard.length > 256)) { + throw new Error( + "Protocol eval catalog exceeds the two-shard GitHub matrix limit", + ); + } + return { + schema: "paperclip.runner-protocol-eval.catalog/v1", + campaignId, + source, + rosters, + cells, + matrices: shards.map((include) => ({ include })), + maxParallel, + maxParallelPerShard: Math.floor(maxParallel / 2), + }; +} + +async function writeGithubOutput(entries) { + const output = process.env.GITHUB_OUTPUT; + if (!output) return; + await appendFile( + output, + Object.entries(entries) + .map(([key, value]) => `${key}=${value}\n`) + .join(""), + ); +} + +async function copyAttempt(source, destination) { + const metadata = await lstat(source); + if (!metadata.isDirectory() || metadata.isSymbolicLink()) { + throw new Error(`Attempt source is not a real directory: ${source}`); + } + await mkdir(destination, { recursive: false, mode: 0o700 }); + for (const entry of await readdir(source, { withFileTypes: true })) { + if ( + !entry.isFile() || + entry.isSymbolicLink() || + !ATTEMPT_FILES.has(entry.name) + ) { + throw new Error(`Unexpected attempt path ${join(source, entry.name)}`); + } + await cp(join(source, entry.name), join(destination, entry.name), { + errorOnExist: true, + }); + } + for (const required of [ + "artifact.json", + "score.json", + "case.json", + "config.json", + ]) { + await lstat(join(destination, required)); + } +} + +async function findFiles(root, name) { + const metadata = await lstat(root).catch(() => null); + if (!metadata) return []; + if (!metadata.isDirectory() || metadata.isSymbolicLink()) { + throw new Error(`Download root must be a real directory: ${root}`); + } + const found = []; + async function visit(directory) { + for (const entry of await readdir(directory, { withFileTypes: true })) { + const absolute = join(directory, entry.name); + if (entry.isSymbolicLink()) + throw new Error(`Refusing downloaded symlink: ${absolute}`); + if (entry.isDirectory()) await visit(absolute); + else if (entry.isFile() && entry.name === name) found.push(absolute); + } + } + await visit(root); + return found.sort(); +} + +function safeUsage(usage) { + const fields = [ + "agentTurns", + "providerRequests", + "inputTokens", + "outputTokens", + "cachedInputTokens", + "reasoningTokens", + "providerReportedCostNanodollars", + "estimatedCostNanodollars", + ]; + return Object.fromEntries( + fields + .filter((field) => Number.isFinite(usage?.[field]) && usage[field] >= 0) + .map((field) => [field, usage[field]]), + ); +} + +async function syntheticAttempt({ evalsRoot, runsOut, cell, campaignId }) { + const programRoot = resolve(evalsRoot, "evals/paperclip-runner"); + const roster = await loadObject( + resolve(programRoot, "rosters", cell.rosterFile), + ); + const casePath = resolve(programRoot, "cases", `${cell.caseId}.json`); + const configPath = inside( + programRoot, + resolve(programRoot, "rosters", String(roster.config)), + `Config for ${cell.rosterId}`, + ); + const [evalCase, config] = await Promise.all([ + loadObject(casePath), + loadObject(configPath), + ]); + const attemptId = safeId( + `${cell.cellId}-${campaignId}-missing`, + "synthetic attempt ID", + ); + const directory = resolve(runsOut, attemptId); + await mkdir(directory, { recursive: false, mode: 0o700 }); + const createdAt = new Date().toISOString(); + const artifact = { + schema: "paperclip-runner/eval-session-artifact/v1", + attemptId, + createdAt, + requestedModel: config.model, + provider: config.provider ?? "codex", + driver: config.driver ?? "codex_app_server", + providerVersion: config.opencodeVersion ?? null, + usage: {}, + turn: { status: "failed" }, + snapshot: { + createdAt, + providerModel: { + id: config.model, + provider: config.modelProvider ?? config.provider ?? "unknown", + }, + transcript: [], + evidence: [], + }, + devtools: { revisions: [] }, + infrastructureFailure: { + class: "ci_cell_artifact_missing", + category: "campaign_orchestration", + retryable: false, + }, + }; + const score = { + schema: "paperclip-runner/eval-score/v1", + attemptId, + caseId: cell.caseId, + disposition: "infrastructure_failure", + passed: false, + infrastructureErrors: [ + "The matrix cell did not retain a complete attempt artifact.", + ], + checks: [], + }; + score.digest = `sha256:${createHash("sha256").update(JSON.stringify(score)).digest("hex")}`; + await Promise.all([ + writeFile(join(directory, "artifact.json"), json(artifact), { + mode: 0o600, + }), + writeFile(join(directory, "score.json"), json(score), { mode: 0o600 }), + writeFile(join(directory, "case.json"), json(evalCase), { mode: 0o600 }), + writeFile(join(directory, "config.json"), json(config), { mode: 0o600 }), + ]); + return attemptId; +} + +export async function aggregateProtocolEvalCampaign({ + catalogPath, + downloadsRoot, + evalsRoot, + runsOut, + campaignOut, + source, +}) { + const catalog = await loadObject(catalogPath); + if (catalog.schema !== "paperclip.runner-protocol-eval.catalog/v1") { + throw new Error("Unsupported protocol eval catalog"); + } + await rm(runsOut, { recursive: true, force: true }); + await mkdir(runsOut, { recursive: true }); + const retainedByCell = new Map(); + const expectedByCell = new Map( + catalog.cells.map((cell) => [cell.cellId, cell]), + ); + const copiedAttemptIds = new Set(); + for (const statusPath of await findFiles(downloadsRoot, "cell.json")) { + const status = await loadObject(statusPath); + const cellId = safeId(status.cellId, "recorded cell ID"); + if (retainedByCell.has(cellId)) + throw new Error(`Duplicate cell artifact ${cellId}`); + const expected = expectedByCell.get(cellId); + if (!expected) + throw new Error(`Downloaded artifact names an unexpected cell ${cellId}`); + if ( + status.caseId !== expected.caseId || + status.rosterFile !== expected.rosterFile + ) { + throw new Error(`Downloaded cell metadata drifted for ${cellId}`); + } + const attemptRoot = resolve(dirname(statusPath), "runs"); + const attemptIds = []; + const attemptMetadata = await lstat(attemptRoot).catch(() => null); + if (attemptMetadata?.isDirectory() && !attemptMetadata.isSymbolicLink()) { + for (const entry of ( + await readdir(attemptRoot, { withFileTypes: true }) + ).sort((a, b) => a.name.localeCompare(b.name))) { + if (!entry.isDirectory() || entry.isSymbolicLink()) { + throw new Error( + `Unexpected cell run path ${join(attemptRoot, entry.name)}`, + ); + } + const attemptId = safeId(entry.name, "attempt ID"); + if (copiedAttemptIds.has(attemptId)) + throw new Error(`Duplicate attempt artifact ${attemptId}`); + const [score, artifact, config] = await Promise.all([ + loadObject(join(attemptRoot, entry.name, "score.json")), + loadObject(join(attemptRoot, entry.name, "artifact.json")), + loadObject(join(attemptRoot, entry.name, "config.json")), + ]); + if ( + score.attemptId !== attemptId || + artifact.attemptId !== attemptId || + score.caseId !== expected.caseId || + config.model !== expected.model || + (config.provider ?? "codex") !== expected.provider || + (config.driver ?? "codex_app_server") !== expected.driver + ) { + throw new Error(`Downloaded attempt identity drifted for ${cellId}`); + } + await copyAttempt( + join(attemptRoot, entry.name), + join(runsOut, attemptId), + ); + copiedAttemptIds.add(attemptId); + attemptIds.push(attemptId); + } + } + retainedByCell.set(cellId, { status, attemptIds }); + } + + const results = []; + for (const cell of catalog.cells) { + const retained = retainedByCell.get(cell.cellId); + const attemptIds = retained?.attemptIds?.length + ? retained.attemptIds + : [ + await syntheticAttempt({ + evalsRoot, + runsOut, + cell, + campaignId: catalog.campaignId, + }), + ]; + const finalAttemptId = attemptIds.at(-1); + const [score, artifact] = await Promise.all([ + loadObject(join(runsOut, finalAttemptId, "score.json")), + loadObject(join(runsOut, finalAttemptId, "artifact.json")), + ]); + if (score.caseId !== cell.caseId || score.attemptId !== finalAttemptId) { + throw new Error(`Final attempt identity drifted for ${cell.cellId}`); + } + results.push({ + cellId: cell.cellId, + rosterId: cell.rosterId, + caseId: cell.caseId, + model: cell.model, + provider: cell.provider, + driver: cell.driver, + attemptIds, + finalAttemptId, + disposition: score.disposition, + passed: score.passed === true, + usage: safeUsage(artifact.usage), + cellExitCode: Number.isSafeInteger(retained?.status?.exitCode) + ? retained.status.exitCode + : null, + }); + } + const totals = { + selected: results.length, + passed: results.filter((result) => result.passed).length, + behaviorFailures: results.filter( + (result) => result.disposition === "behavior_failure", + ).length, + infrastructureFailures: results.filter( + (result) => result.disposition === "infrastructure_failure", + ).length, + }; + const generatedAt = new Date().toISOString(); + const campaign = { + schema: "paperclip.runner-protocol-eval.campaign/v1", + campaignId: catalog.campaignId, + generatedAt, + source: { + paperclip: source.paperclip, + evals: source.evals, + workflowRunUrl: source.workflowRunUrl, + }, + complete: results.length === catalog.cells.length, + allPassed: totals.passed === totals.selected, + totals, + rosters: catalog.rosters.map((roster) => ({ + rosterId: roster.rosterId, + model: roster.model, + provider: roster.provider, + driver: roster.driver, + selected: roster.cases.length, + passed: results.filter( + (result) => result.rosterId === roster.rosterId && result.passed, + ).length, + })), + results, + }; + await mkdir(dirname(campaignOut), { recursive: true }); + await writeFile(campaignOut, json(campaign), { mode: 0o600 }); + return campaign; +} + +function publicArtifact(artifact) { + const model = artifact.snapshot?.providerModel ?? {}; + const infrastructure = artifact.infrastructureFailure; + const providerVersion = + artifact.provider === "claude_managed" || + artifact.provider === "aws_agentcore" + ? "remote profile redacted" + : (artifact.providerVersion ?? null); + return { + schema: artifact.schema, + attemptId: artifact.attemptId, + createdAt: artifact.createdAt, + requestedModel: artifact.requestedModel, + provider: artifact.provider, + driver: artifact.driver, + providerVersion, + retainedSession: false, + retainedSessionStatus: "redacted from the public report", + usage: safeUsage(artifact.usage), + turn: { status: artifact.turn?.status ?? "failed" }, + snapshot: { + createdAt: artifact.snapshot?.createdAt ?? artifact.createdAt, + providerModel: { + id: model.id ?? artifact.requestedModel, + provider: model.provider ?? artifact.provider, + }, + transcript: [], + evidence: [], + }, + devtools: { revisions: [] }, + ...(infrastructure && typeof infrastructure === "object" + ? { + infrastructureFailure: { + class: infrastructure.class, + category: infrastructure.category, + retryable: infrastructure.retryable === true, + }, + } + : {}), + }; +} + +function publicScore(score) { + return { + schema: score.schema, + attemptId: score.attemptId, + caseId: score.caseId, + disposition: score.disposition, + passed: score.passed === true, + infrastructureErrors: + score.disposition === "infrastructure_failure" + ? [ + "Infrastructure failure; diagnostic details remain in the access-controlled artifact.", + ] + : [], + checks: Array.isArray(score.checks) + ? score.checks.map((check) => ({ + id: check.id, + kind: check.kind, + passed: check.passed === true, + detail: check.passed === true ? "passed" : "failed", + evidenceRefs: [], + })) + : [], + digest: score.digest, + }; +} + +function publicConfig(config) { + return Object.fromEntries( + [ + "schema", + "id", + "model", + "provider", + "driver", + "opencodeVersion", + "acpxAgent", + "modelProvider", + ] + .filter((field) => config[field] !== undefined) + .map((field) => [field, config[field]]), + ); +} + +export async function sanitizeProtocolEvalRuns({ runsRoot, publicRunsRoot }) { + await rm(publicRunsRoot, { recursive: true, force: true }); + await mkdir(publicRunsRoot, { recursive: true }); + const attemptIds = []; + for (const entry of (await readdir(runsRoot, { withFileTypes: true })).sort( + (a, b) => a.name.localeCompare(b.name), + )) { + if (!entry.isDirectory() || entry.isSymbolicLink()) { + throw new Error( + `Unexpected merged run path ${join(runsRoot, entry.name)}`, + ); + } + const attemptId = safeId(entry.name, "attempt ID"); + const source = join(runsRoot, attemptId); + const [artifact, score, evalCase, config] = await Promise.all([ + loadObject(join(source, "artifact.json")), + loadObject(join(source, "score.json")), + loadObject(join(source, "case.json")), + loadObject(join(source, "config.json")), + ]); + if (artifact.attemptId !== attemptId || score.attemptId !== attemptId) { + throw new Error(`Attempt identity drifted in ${attemptId}`); + } + const destination = join(publicRunsRoot, attemptId); + await mkdir(destination, { mode: 0o700 }); + await Promise.all([ + writeFile( + join(destination, "artifact.json"), + json(publicArtifact(artifact)), + { mode: 0o600 }, + ), + writeFile(join(destination, "score.json"), json(publicScore(score)), { + mode: 0o600, + }), + writeFile(join(destination, "case.json"), json(evalCase), { + mode: 0o600, + }), + writeFile(join(destination, "config.json"), json(publicConfig(config)), { + mode: 0o600, + }), + ]); + attemptIds.push(attemptId); + } + return attemptIds; +} + +function argument(args, name, fallback) { + const index = args.indexOf(name); + return index < 0 ? fallback : args[index + 1]; +} + +async function main() { + const [command, ...args] = process.argv.slice(2); + if (command === "catalog") { + const output = resolve( + argument(args, "--output", "runner-protocol-eval-catalog.json"), + ); + const catalog = await buildProtocolEvalCatalog({ + evalsRoot: resolve(argument(args, "--evals-root", ".paperclip-evals")), + rosterSelection: argument(args, "--rosters", "all"), + campaignId: argument(args, "--campaign-id", `local-${Date.now()}`), + maxParallel: Number(argument(args, "--max-parallel", "100")), + source: { + paperclipSha: process.env.PAPERCLIP_PROTOCOL_EVAL_SOURCE_SHA ?? null, + evalsSha: process.env.PAPERCLIP_PROTOCOL_EVALS_SHA ?? null, + }, + }); + await mkdir(dirname(output), { recursive: true }); + await writeFile(output, json(catalog), { mode: 0o600 }); + await writeGithubOutput({ + matrix_0: JSON.stringify(catalog.matrices[0]), + matrix_1: JSON.stringify(catalog.matrices[1]), + matrix_1_present: String(catalog.matrices[1].include.length > 0), + max_parallel_per_shard: String(catalog.maxParallelPerShard), + selected: String(catalog.cells.length), + }); + console.log( + json({ + output, + selected: catalog.cells.length, + rosters: catalog.rosters.length, + }).trim(), + ); + return; + } + if (command === "aggregate") { + const campaign = await aggregateProtocolEvalCampaign({ + catalogPath: resolve(argument(args, "--catalog")), + downloadsRoot: resolve(argument(args, "--downloads")), + evalsRoot: resolve(argument(args, "--evals-root")), + runsOut: resolve(argument(args, "--runs-out")), + campaignOut: resolve(argument(args, "--campaign-out")), + source: { + paperclip: { + sha: process.env.PAPERCLIP_PROTOCOL_EVAL_SOURCE_SHA, + ref: process.env.PAPERCLIP_PROTOCOL_EVAL_SOURCE_REF, + }, + evals: { + repository: "paperclipai/paperclip-evals", + sha: process.env.PAPERCLIP_PROTOCOL_EVALS_SHA, + }, + workflowRunUrl: process.env.PAPERCLIP_PROTOCOL_EVAL_WORKFLOW_URL, + }, + }); + console.log(json(campaign.totals).trim()); + return; + } + if (command === "sanitize") { + const attemptIds = await sanitizeProtocolEvalRuns({ + runsRoot: resolve(argument(args, "--runs-root")), + publicRunsRoot: resolve(argument(args, "--output")), + }); + console.log(json({ attempts: attemptIds.length }).trim()); + return; + } + throw new Error( + "Usage: runner-protocol-eval-campaign.mjs ...", + ); +} + +if ( + process.argv[1] && + resolve(process.argv[1]) === resolve(import.meta.filename) +) { + await main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.test.mjs b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.test.mjs new file mode 100644 index 0000000000..474f89971b --- /dev/null +++ b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.test.mjs @@ -0,0 +1,349 @@ +import assert from "node:assert/strict"; +import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { + aggregateProtocolEvalCampaign, + buildProtocolEvalCatalog, + credentialForConfig, + sanitizeProtocolEvalRuns, +} from "./runner-protocol-eval-campaign.mjs"; + +const roots = []; +test.afterEach(async () => { + await Promise.all( + roots.splice(0).map((root) => rm(root, { recursive: true, force: true })), + ); +}); + +async function fixture() { + const root = await mkdtemp(join(tmpdir(), "runner-protocol-campaign-")); + roots.push(root); + const program = join(root, "evals/paperclip-runner"); + await Promise.all([ + mkdir(join(program, "rosters"), { recursive: true }), + mkdir(join(program, "configs"), { recursive: true }), + mkdir(join(program, "cases"), { recursive: true }), + ]); + const config = { + schema: "paperclip-runner/eval-config/v1", + id: "live-opencode-model", + provider: "opencode", + driver: "opencode_server", + model: "openrouter/example/model", + opencodeVersion: "1.18.17", + }; + const evalCase = { + schema: "paperclip-runner/eval-case/v1", + id: "get-task-context", + title: "Get context", + description: "Synthetic public fixture", + prompt: "Inspect the synthetic task.", + fixture: "../fixtures/company.json", + authority: { actorId: "agent-1", taskId: "task-1" }, + checks: [{ id: "context", kind: "semantic_operation" }], + }; + const roster = { + schema: "paperclip-runner/live-roster/v1", + id: "protocol-live-opencode-model", + model: config.model, + config: "../configs/live-opencode-model.json", + cases: [evalCase.id], + }; + await Promise.all([ + writeFile( + join(program, "configs/live-opencode-model.json"), + JSON.stringify(config), + ), + writeFile( + join(program, "cases/get-task-context.json"), + JSON.stringify(evalCase), + ), + writeFile( + join(program, "rosters/live-opencode-model.json"), + JSON.stringify(roster), + ), + ]); + return { root, program, config, evalCase, roster }; +} + +test("maps every qualified driver to one explicit credential boundary", () => { + assert.equal(credentialForConfig({ provider: "codex" }), "OPENAI_API_KEY"); + assert.equal( + credentialForConfig({ provider: "opencode" }), + "OPENROUTER_API_KEY", + ); + assert.equal( + credentialForConfig({ provider: "acpx", acpxAgent: "claude" }), + "ANTHROPIC_API_KEY", + ); + assert.equal( + credentialForConfig({ provider: "aws_agentcore" }), + "AWS_AGENTCORE_OIDC", + ); + assert.throws( + () => credentialForConfig({ provider: "unknown" }), + /credential policy/, + ); +}); + +test("catalogs roster plus case cells and emits bounded balanced shards", async () => { + const { root } = await fixture(); + const catalog = await buildProtocolEvalCatalog({ + evalsRoot: root, + campaignId: "gha-42-1", + maxParallel: 80, + }); + assert.equal(catalog.cells.length, 1); + assert.equal(catalog.cells[0].credentialName, "OPENROUTER_API_KEY"); + assert.equal(catalog.maxParallelPerShard, 40); + assert.equal(catalog.matrices[0].include.length, 1); + assert.equal(catalog.matrices[1].include.length, 0); + await assert.rejects( + buildProtocolEvalCatalog({ + evalsRoot: root, + campaignId: "gha-42-1", + rosterSelection: "missing-roster", + }), + /Unknown live roster/, + ); + await assert.rejects( + buildProtocolEvalCatalog({ + evalsRoot: root, + campaignId: "gha-42-1", + maxParallel: 1, + }), + /from 2 through 100/, + ); +}); + +test("aggregates retained attempts and synthesizes missing cells as infrastructure", async () => { + const { root, config, evalCase } = await fixture(); + const catalog = await buildProtocolEvalCatalog({ + evalsRoot: root, + campaignId: "gha-42-1", + }); + const catalogPath = join(root, "catalog.json"); + await writeFile(catalogPath, JSON.stringify(catalog)); + const campaign = await aggregateProtocolEvalCampaign({ + catalogPath, + downloadsRoot: join(root, "missing-downloads"), + evalsRoot: root, + runsOut: join(root, "merged-runs"), + campaignOut: join(root, "campaign.json"), + source: { + paperclip: { sha: "a".repeat(40), ref: "refs/heads/master" }, + evals: { repository: "paperclipai/paperclip-evals", sha: "b".repeat(40) }, + workflowRunUrl: "https://example.test/actions/runs/42", + }, + }); + assert.deepEqual(campaign.totals, { + selected: 1, + passed: 0, + behaviorFailures: 0, + infrastructureFailures: 1, + }); + assert.equal(campaign.results[0].disposition, "infrastructure_failure"); + assert.equal(campaign.complete, true); + assert.equal(campaign.allPassed, false); + const attempt = campaign.results[0].finalAttemptId; + assert.deepEqual( + JSON.parse(await readFile(join(root, "merged-runs", attempt, "case.json"))), + evalCase, + ); + assert.deepEqual( + JSON.parse( + await readFile(join(root, "merged-runs", attempt, "config.json")), + ), + config, + ); +}); + +test("rejects downloaded cells that were not declared by the immutable catalog", async () => { + const { root } = await fixture(); + const catalog = await buildProtocolEvalCatalog({ + evalsRoot: root, + campaignId: "gha-42-1", + }); + const catalogPath = join(root, "catalog.json"); + const downloads = join(root, "downloads/unexpected"); + await mkdir(downloads, { recursive: true }); + await Promise.all([ + writeFile(catalogPath, JSON.stringify(catalog)), + writeFile( + join(downloads, "cell.json"), + JSON.stringify({ + cellId: "protocol-live-opencode-model--undeclared", + rosterFile: "live-opencode-model.json", + caseId: "undeclared", + exitCode: 1, + }), + ), + ]); + await assert.rejects( + aggregateProtocolEvalCampaign({ + catalogPath, + downloadsRoot: join(root, "downloads"), + evalsRoot: root, + runsOut: join(root, "merged-runs"), + campaignOut: join(root, "campaign.json"), + source: {}, + }), + /unexpected cell/, + ); +}); + +test("public run projection removes provider sessions, traces, transcripts, evidence, and state", async () => { + const { root, config, evalCase } = await fixture(); + const attemptId = "get-task-context-opencode-gha-42-1-attempt-01"; + const source = join(root, "raw-runs", attemptId); + await mkdir(source, { recursive: true }); + await Promise.all([ + writeFile( + join(source, "artifact.json"), + JSON.stringify({ + schema: "paperclip-runner/eval-session-artifact/v1", + attemptId, + createdAt: "2026-09-05T00:00:00.000Z", + requestedModel: config.model, + provider: config.provider, + driver: config.driver, + providerVersion: config.opencodeVersion, + providerSessionId: "private-session-id", + trace: { secret: "private" }, + usage: { + agentTurns: 1, + inputTokens: 100, + estimatedCostNanodollars: 5000, + }, + turn: { status: "completed", turnId: "private-turn" }, + snapshot: { + createdAt: "2026-09-05T00:00:00.000Z", + providerModel: { id: config.model, provider: "openrouter" }, + transcript: [{ role: "assistant", text: "private transcript" }], + evidence: [{ kind: "tool_call", data: { token: "private" } }], + }, + issueThread: { messages: ["private"] }, + devtools: { revisions: [{ state: { private: true } }] }, + }), + ), + writeFile( + join(source, "score.json"), + JSON.stringify({ + schema: "paperclip-runner/eval-score/v1", + attemptId, + caseId: evalCase.id, + disposition: "passed", + passed: true, + infrastructureErrors: [], + checks: [ + { + id: "context", + kind: "semantic_operation", + passed: true, + detail: "private result", + evidenceRefs: ["event-1"], + }, + ], + digest: "sha256:test", + }), + ), + writeFile(join(source, "case.json"), JSON.stringify(evalCase)), + writeFile( + join(source, "config.json"), + JSON.stringify({ ...config, managedProfile: { profileId: "private" } }), + ), + ]); + await sanitizeProtocolEvalRuns({ + runsRoot: join(root, "raw-runs"), + publicRunsRoot: join(root, "public-runs"), + }); + const serialized = await readFile( + join(root, "public-runs", attemptId, "artifact.json"), + "utf8", + ); + assert.doesNotMatch( + serialized, + /private-session|private transcript|private-turn|issueThread|trace/, + ); + const artifact = JSON.parse(serialized); + assert.deepEqual(artifact.snapshot.transcript, []); + assert.deepEqual(artifact.snapshot.evidence, []); + assert.deepEqual(artifact.devtools.revisions, []); + const score = JSON.parse( + await readFile(join(root, "public-runs", attemptId, "score.json"), "utf8"), + ); + assert.deepEqual(score.checks[0], { + id: "context", + kind: "semantic_operation", + passed: true, + detail: "passed", + evidenceRefs: [], + }); + const publicConfig = await readFile( + join(root, "public-runs", attemptId, "config.json"), + "utf8", + ); + assert.doesNotMatch(publicConfig, /managedProfile|private/); +}); + +test("public run projection redacts remote profile version identities", async () => { + const { root, config, evalCase } = await fixture(); + const attemptId = "get-task-context-claude-managed-gha-42-1-attempt-01"; + const source = join(root, "raw-runs", attemptId); + await mkdir(source, { recursive: true }); + await Promise.all([ + writeFile( + join(source, "artifact.json"), + JSON.stringify({ + schema: "paperclip-runner/eval-session-artifact/v1", + attemptId, + createdAt: "2026-09-05T00:00:00.000Z", + requestedModel: "claude-sonnet-5", + provider: "claude_managed", + driver: "claude_managed_agents_api", + providerVersion: "private-agent-version-id", + usage: {}, + turn: { status: "completed" }, + snapshot: { + createdAt: "2026-09-05T00:00:00.000Z", + providerModel: { id: "claude-sonnet-5", provider: "anthropic" }, + }, + }), + ), + writeFile( + join(source, "score.json"), + JSON.stringify({ + schema: "paperclip-runner/eval-score/v1", + attemptId, + caseId: evalCase.id, + disposition: "passed", + passed: true, + checks: [], + }), + ), + writeFile(join(source, "case.json"), JSON.stringify(evalCase)), + writeFile( + join(source, "config.json"), + JSON.stringify({ + ...config, + id: "live-claude-managed", + model: "claude-sonnet-5", + provider: "claude_managed", + driver: "claude_managed_agents_api", + }), + ), + ]); + await sanitizeProtocolEvalRuns({ + runsRoot: join(root, "raw-runs"), + publicRunsRoot: join(root, "public-runs"), + }); + const publicArtifact = await readFile( + join(root, "public-runs", attemptId, "artifact.json"), + "utf8", + ); + assert.match(publicArtifact, /remote profile redacted/); + assert.doesNotMatch(publicArtifact, /private-agent-version-id/); +}); diff --git a/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs b/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs new file mode 100644 index 0000000000..5e9f990cb8 --- /dev/null +++ b/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs @@ -0,0 +1,109 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import test from "node:test"; + +const repositoryRoot = resolve(import.meta.dirname, "../../.."); +const workflowPath = resolve( + repositoryRoot, + ".github/workflows/runner-protocol-live-evals.yml", +); + +test("direct live eval workflow keeps paid execution behind stable actor authorization", async () => { + const workflow = await readFile(workflowPath, "utf8"); + assert.match(workflow, /^\s{2}authorize:/mu); + assert.match(workflow, /RUNNER_E2E_ALLOWED_ACTOR_IDS/u); + assert.match(workflow, /github\.actor_id/u); + assert.match(workflow, /github\.triggering_actor/u); + assert.match(workflow, /refs\/heads\/\$DEFAULT_BRANCH/u); + assert.match(workflow, /Reauthorize paid execution before provider access/u); + assert.match( + workflow, + /Reauthorize paid execution before provider access[\s\S]*actions\/checkout@[0-9a-f]{40}[\s\S]*Run one immutable direct protocol cell/u, + ); + assert.doesNotMatch( + workflow, + /^\s{2}(?:pull_request|pull_request_target|push|workflow_call|workflow_run):/mu, + ); + const actions = [ + ...workflow.matchAll(/^\s*(?:-\s*)?uses:\s*([^\s#]+)/gmu), + ].map((match) => match[1]); + assert.ok(actions.length > 0); + for (const action of actions) assert.match(action, /^[^@]+@[0-9a-f]{40}$/u); +}); + +test("resolves both repositories immutably and bounds total matrix concurrency", async () => { + const workflow = await readFile(workflowPath, "utf8"); + const authorize = workflow.slice( + workflow.indexOf(" authorize:"), + workflow.indexOf(" catalog:"), + ); + assert.match(authorize, /repos\/\$REPOSITORY\/branches\/\$encoded_branch/u); + assert.match(authorize, /\^\[0-9a-f\]\{40\}\$/u); + assert.match( + authorize, + /repos\/paperclipai\/paperclip-evals\/commits\/\$EVALS_SHA/u, + ); + assert.match(authorize, /COMMITPERCLIP_KEY/u); + assert.match( + authorize, + /GH_TOKEN: \$\{\{ steps\.evals_token\.outputs\.value \}\}/u, + ); + assert.match(authorize, /test "\$resolved" = "\$EVALS_SHA"/u); + const catalog = workflow.slice( + workflow.indexOf(" catalog:"), + workflow.indexOf(" build_runner:"), + ); + assert.match( + catalog, + /ref: \$\{\{ needs\.authorize\.outputs\.evals_sha \}\}/u, + ); + assert.match(catalog, /RUNNER_E2E_MAX_PARALLEL/u); + assert.match(catalog, /max_parallel_per_shard/u); + const privateCheckouts = [ + ...workflow.matchAll( + /repository: paperclipai\/paperclip-evals[\s\S]*?persist-credentials: false/gmu, + ), + ]; + assert.equal(privateCheckouts.length, 3); + for (const checkout of privateCheckouts) { + assert.match( + checkout[0], + /token: \$\{\{ steps\.evals_token\.outputs\.value \}\}/u, + ); + } + assert.match(workflow, /matrix_0/u); + assert.match(workflow, /matrix_1/u); +}); + +test("publishes only the separately sanitized Evalbook through trusted OIDC code", async () => { + const workflow = await readFile(workflowPath, "utf8"); + const report = workflow.slice( + workflow.indexOf(" report:"), + workflow.indexOf(" publish_history:"), + ); + assert.match( + report, + /Render the access-controlled canonical Evalbook report/u, + ); + assert.match( + report, + /--viewer-root runner-protocol-build\/extracted\/dist-issue-thread/u, + ); + assert.match(report, /runner-protocol-eval-campaign\.mjs sanitize/u); + assert.match( + report, + /Upload access-controlled canonical Evalbook and raw attempts/u, + ); + assert.match(report, /Upload publisher-only sanitized Evalbook/u); + + const publisher = workflow.slice(workflow.indexOf(" publish_history:")); + assert.match(publisher, /ref: \$\{\{ github\.sha \}\}/u); + assert.match(publisher, /id-token: write/u); + assert.match(publisher, /runner-protocol-eval-public-/u); + assert.match(publisher, /publish-runner-protocol-eval-history\.mjs/u); + assert.match(publisher, /runner-protocol-evals/u); + assert.doesNotMatch(publisher, /(?:OPENAI|ANTHROPIC|OPENROUTER)_API_KEY/u); + assert.doesNotMatch(publisher, /paperclipai\/paperclip-evals/u); + assert.doesNotMatch(publisher, /downloaded-runner-protocol-evals/u); +}); diff --git a/packages/paperclip-runner/scripts/stage-runner-binary.mjs b/packages/paperclip-runner/scripts/stage-runner-binary.mjs index 14fcee5a92..d75f4bbb6d 100644 --- a/packages/paperclip-runner/scripts/stage-runner-binary.mjs +++ b/packages/paperclip-runner/scripts/stage-runner-binary.mjs @@ -1,7 +1,11 @@ +import { execFile } from "node:child_process"; import { chmod, copyFile, mkdir } from "node:fs/promises"; import path from "node:path"; +import { promisify } from "node:util"; import { fileURLToPath } from "node:url"; +const execFileAsync = promisify(execFile); + const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const executable = process.platform === "win32" ? "paperclip-runnerd.exe" : "paperclip-runnerd"; const source = path.join(packageRoot, "runner", "target", "release", executable); @@ -11,3 +15,10 @@ const destination = path.join(destinationDirectory, executable); await mkdir(destinationDirectory, { recursive: true }); await copyFile(source, destination); if (process.platform !== "win32") await chmod(destination, 0o755); +// Rust's linker emits an ad-hoc Mach-O signature. Copying that executable to +// its package location preserves the bytes but can leave the kernel rejecting +// the new inode with SIGKILL. Re-sign the staged inode so local packaged-runner +// evals execute the same artifact that was just built. +if (process.platform === "darwin") { + await execFileAsync("codesign", ["--force", "--sign", "-", destination]); +} diff --git a/packages/paperclip-runner/src/cli/eval-session-contract.test.ts b/packages/paperclip-runner/src/cli/eval-session-contract.test.ts index 947c22cb99..e2765f0307 100644 --- a/packages/paperclip-runner/src/cli/eval-session-contract.test.ts +++ b/packages/paperclip-runner/src/cli/eval-session-contract.test.ts @@ -5,6 +5,10 @@ import { evalSessionUsage, parseEvalSessionRequest, } from "./eval-session-contract.js"; +import { + boundedEvalSessionUsage, + evalSessionProviderVersion, +} from "./eval-session.js"; function request(overrides: Record = {}): unknown { return { @@ -65,6 +69,43 @@ describe("eval-session request contract", () => { }))).toMatchObject({ provider: "acpx", acpxAgent: "claude" }); }); + it("accepts null optional fields from the original Evalbook v1 producer", () => { + const parsed = parseEvalSessionRequest(request({ + acpxAgent: null, + agentCoreProfile: null, + opencodeVersion: null, + })); + expect(parsed).toMatchObject({ + provider: "codex", + driver: "codex_app_server", + }); + expect(parsed).not.toHaveProperty("acpxAgent"); + expect(parsed).not.toHaveProperty("agentCoreProfile"); + expect(parsed).not.toHaveProperty("opencodeVersion"); + }); + + it("attributes managed providers to their immutable deployed revisions", () => { + expect(evalSessionProviderVersion(parseEvalSessionRequest(request({ + provider: "aws_agentcore", + driver: "aws_agentcore_harness_api", + model: "global.anthropic.claude-sonnet-4-6", + agentCoreProfile: agentCoreProfile(), + })))).toBe("aws-agentcore-harness-context-v2"); + expect(evalSessionProviderVersion(parseEvalSessionRequest(request({ + provider: "claude_managed", + driver: "claude_managed_agents_api", + model: "claude-sonnet-5", + managedProfile: { + profileId: "managed-qualified", + anthropicAgentId: "agent-test", + agentVersion: "17", + environmentId: "environment-test", + betaVersion: "managed-agents-2026-04-01", + maxSessionListCostUsd: 1, + }, + })))).toBe("17"); + }); + it("rejects Pi and accepts both qualified remote provider profiles", () => { expect(() => parseEvalSessionRequest(request({ provider: "acpx", @@ -143,6 +184,41 @@ describe("eval-session request contract", () => { }); describe("eval-session usage", () => { + it("retains durable failed turns even when their reported usage exceeds completed-turn limits", () => { + const parsed = parseEvalSessionRequest(request()); + const snapshot = { + usageLedger: [{ + receiptId: "receipt-failed", + attemptId: "attempt-1", + providerResponseId: "response-failed", + turnId: "turn-failed", + observedAt: "2026-09-05T00:00:00.000Z", + providerCalls: 2, + providerRequests: 2, + inputTokens: 1_000, + outputTokens: 100, + cachedInputTokens: 0, + reasoningTokens: 0, + costNanodollars: 200_000_000, + }], + } as unknown as CapabilityLiveSessionSnapshot; + const failedTurn = { + turnId: "turn-failed", + status: "failed" as const, + assistantText: "", + snapshot, + }; + + expect(boundedEvalSessionUsage(parsed, failedTurn)).toMatchObject({ + agentTurns: 2, + providerReportedCostNanodollars: 200_000_000, + }); + expect(() => boundedEvalSessionUsage(parsed, { + ...failedTurn, + status: "completed", + })).toThrow("agent turn limit exceeded"); + }); + it("deduplicates receipts and applies the versioned model price", () => { const receipt = { receiptId: "receipt-1", diff --git a/packages/paperclip-runner/src/cli/eval-session-contract.ts b/packages/paperclip-runner/src/cli/eval-session-contract.ts index 1caf1e04f0..9c48f987b9 100644 --- a/packages/paperclip-runner/src/cli/eval-session-contract.ts +++ b/packages/paperclip-runner/src/cli/eval-session-contract.ts @@ -233,7 +233,10 @@ export function parseEvalSessionRequest(value: unknown): EvalSessionRequest { if (input.driver !== undefined && input.driver !== driver) { throw new Error("eval-session provider/driver mismatch"); } - const acpxAgent = input.acpxAgent; + // The original Evalbook v1 producer serialized absent provider-specific + // options as JSON null. Preserve compatibility with those immutable request + // artifacts while continuing to reject non-null values for the wrong lane. + const acpxAgent = input.acpxAgent === null ? undefined : input.acpxAgent; if (acpxAgent === "pi") throw new Error("The Pi ACPX profile is not available"); if ( acpxAgent !== undefined && @@ -245,16 +248,22 @@ export function parseEvalSessionRequest(value: unknown): EvalSessionRequest { if (provider !== "acpx" && acpxAgent !== undefined) { throw new Error("eval-session acpxAgent requires provider acpx"); } + const managedProfileInput = input.managedProfile === null + ? undefined + : input.managedProfile; + const agentCoreProfileInput = input.agentCoreProfile === null + ? undefined + : input.agentCoreProfile; const managedProfile = provider === "claude_managed" - ? parseManagedProfile(input.managedProfile) + ? parseManagedProfile(managedProfileInput) : undefined; const agentCoreProfile = provider === "aws_agentcore" - ? parseAgentCoreProfile(input.agentCoreProfile) + ? parseAgentCoreProfile(agentCoreProfileInput) : undefined; - if (provider !== "claude_managed" && input.managedProfile !== undefined) { + if (provider !== "claude_managed" && managedProfileInput !== undefined) { throw new Error("eval-session managedProfile requires provider claude_managed"); } - if (provider !== "aws_agentcore" && input.agentCoreProfile !== undefined) { + if (provider !== "aws_agentcore" && agentCoreProfileInput !== undefined) { throw new Error("eval-session agentCoreProfile requires provider aws_agentcore"); } if (input.nativeResume !== undefined) { diff --git a/packages/paperclip-runner/src/cli/eval-session.ts b/packages/paperclip-runner/src/cli/eval-session.ts index f98e5ab1a8..609ec73090 100644 --- a/packages/paperclip-runner/src/cli/eval-session.ts +++ b/packages/paperclip-runner/src/cli/eval-session.ts @@ -53,7 +53,9 @@ async function sha256(path: string): Promise { return createHash("sha256").update(await readFile(path)).digest("hex"); } -function providerVersion(request: EvalSessionRequest): string | null { +export function evalSessionProviderVersion( + request: EvalSessionRequest, +): string | null { if (request.provider === "opencode") { const version = request.opencodeVersion ?? "1.18.17"; if (version !== "1.18.17") { @@ -71,7 +73,7 @@ function providerVersion(request: EvalSessionRequest): string | null { return request.managedProfile!.agentVersion; } if (request.provider === "aws_agentcore") { - return request.agentCoreProfile!.harnessVersion; + return request.agentCoreProfile!.qualificationRevision; } return null; } @@ -127,6 +129,32 @@ function usageIfAvailable( } } +export function boundedEvalSessionUsage( + request: EvalSessionRequest, + turn: CapabilityLiveTurnResult, +): EvalSessionUsage | null { + if (turn.status !== "completed") { + return usageIfAvailable(request, turn.snapshot); + } + const usage = evalSessionUsage(request.model, turn.snapshot); + if (usage.agentTurns > request.limits.maxAgentTurns) { + throw new Error("agent turn limit exceeded"); + } + if ( + usage.estimatedCostNanodollars > + request.limits.maxEstimatedCostNanodollars + ) { + throw new Error("estimated cost limit exceeded"); + } + if ( + usage.providerReportedCostNanodollars > + request.limits.maxEstimatedCostNanodollars + ) { + throw new Error("provider-reported cost limit exceeded"); + } + return usage; +} + async function closeSession( session: CapabilityLiveSession | null, reason: string, @@ -160,11 +188,16 @@ export async function runEvalSessionCli( const requestedProvider = request.provider ?? "codex"; const requestedDriver = request.driver ?? expectedEvalSessionDriver(requestedProvider); - const requestedProviderVersion = providerVersion(request); + const requestedProviderVersion = evalSessionProviderVersion(request); const service = options.serviceFactory?.(runnerdPath) ?? new CapabilityLiveSessionService({ transportOptions: { runnerBinary: runnerdPath, + // The transport performs the provider-specific allowlisting. Supplying + // the source environment here is still required: without it the + // isolated Codex home has no credential source and runnerd receives no + // executable PATH from the Evalbook CLI process. + environment: process.env, onDiagnostic: (message) => { process.stderr.write(`[eval-session runnerd] ${message}\n`); }, @@ -196,27 +229,11 @@ export async function runEvalSessionCli( } as unknown as CreateCapabilityLiveSessionInput; session = await service.create(createInput); turn = await session.sendMessage(request.prompt); - if (turn.status !== "completed") { - await session.completeAttempt("failed", `provider_turn_${turn.status}`); - throw new Error(`provider turn ended with status ${turn.status}`); - } - const usage = evalSessionUsage(request.model, turn.snapshot); - if (usage.agentTurns > request.limits.maxAgentTurns) { - throw new Error("agent turn limit exceeded"); - } - if ( - usage.estimatedCostNanodollars > - request.limits.maxEstimatedCostNanodollars - ) { - throw new Error("estimated cost limit exceeded"); - } - if ( - usage.providerReportedCostNanodollars > - request.limits.maxEstimatedCostNanodollars - ) { - throw new Error("provider-reported cost limit exceeded"); - } - await session.completeAttempt("succeeded"); + const usage = boundedEvalSessionUsage(request, turn); + await session.completeAttempt( + turn.status === "completed" ? "succeeded" : "failed", + turn.status === "completed" ? null : `provider_turn_${turn.status}`, + ); await closeSession(session, "eval session complete"); snapshot = session.snapshot(); @@ -259,7 +276,7 @@ export async function runEvalSessionCli( mode: "live", replaySource: "live", }), - usage, + ...(usage === null ? {} : { usage }), timing: { startedAt, finishedAt: new Date().toISOString(), diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts index b183da10a4..6d9a7e3e7e 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts @@ -48,7 +48,10 @@ const coreStateSchema = "paperclip.runner.durable.control-plane-state.v1"; const maxFrameBytes = 1024 * 1024; const maxCommandBytes = maxFrameBytes - 4 * 1024; const maxCommands = 500; -const maxCommittedEventWindow = 64; +// A provider can emit several 100-event runner batches before the transport's +// polling turn regains the event loop. Match the transport's explicit deferred +// event bound so a valid burst is not compacted before it can be observed. +const maxCommittedEventWindow = 4_096; const maxStateBytes = 192 * 1024 * 1024; const authChallengeTtlMs = 5_000; const stableIdPattern = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,239}$/; diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts index 2869830df4..2c4aca9107 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts @@ -1379,6 +1379,9 @@ describe("OpenCodeServerDriver", () => { "*": permissionMode, external_directory: "deny", }); + expect(config.provider.openrouter.models).toHaveProperty( + "deepseek/deepseek-v4-flash-0731", + ); expect(config.permission.paperclip_finish).toBeUndefined(); expect(config.permission["paperclip_*"]).toBe("allow"); await session.close({ reason: "permission mode test complete" }); @@ -1771,8 +1774,11 @@ describe("OpenCodeServerDriver", () => { workingDirectory: workspace, }); expect(session.ids().providerSessionId).toBe("ses_fake_1"); - expect(spawns).toHaveLength(2); - expect(commandLifecycle).toEqual(["before", "after", "before", "after"]); + expect(spawns.length).toBeGreaterThanOrEqual(2); + expect(spawns.length).toBeLessThanOrEqual(3); + expect(commandLifecycle).toEqual( + Array.from({ length: spawns.length }, () => ["before", "after"]).flat(), + ); await session.close({ reason: "test" }); }); diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts index ba0d68dc7e..a764d7b605 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts @@ -1866,6 +1866,8 @@ async function startRuntime(input: { ]); const instructionRoot = input.options.runtimeContext?.instructions.bundle.rootPath; + const [modelProvider, ...modelIdParts] = input.options.model.split("/"); + const providerModelId = modelIdParts.join("/"); const config = { $schema: "https://opencode.ai/config.json", model: input.options.model, @@ -1875,6 +1877,16 @@ async function startRuntime(input: { // system prompt; siblings remain available through the read-only root. instructions: [], plugin: [], + // OpenCode's bundled models.dev snapshot can lag behind OpenRouter's live + // catalog. Bind the already-qualified exact model slug into the built-in + // provider instead of silently falling back or rejecting a newer model. + provider: { + [modelProvider!]: { + models: { + [providerModelId]: { name: providerModelId }, + }, + }, + }, tools: { question: true, }, diff --git a/packages/paperclip-runner/src/eval/live-workflow-executor.test.ts b/packages/paperclip-runner/src/eval/live-workflow-executor.test.ts index 1f62eb9232..f791426794 100644 --- a/packages/paperclip-runner/src/eval/live-workflow-executor.test.ts +++ b/packages/paperclip-runner/src/eval/live-workflow-executor.test.ts @@ -86,7 +86,13 @@ vi.mock("../live/live-session.js", () => ({ })); import { runnerWorkflowCase } from "./workflow-catalog.js"; -import { executeLiveRunnerWorkflow } from "./live-workflow-executor.js"; +import { CapabilityMockControlPlaneAdapter } from "../mock-core/capability-mock-control-plane-adapter.js"; +import { + advanceDelegationReturnMockState, + executeLiveRunnerWorkflow, + scorableLiveWorkflowCalls, + unexpectedLiveWorkflowCalls, +} from "./live-workflow-executor.js"; import { RUNNER_LIVE_CANDIDATE_SLOTS, type RunnerLiveScheduleEntry, @@ -124,6 +130,117 @@ describe("live workflow executor infrastructure failures", () => { vi.unstubAllEnvs(); }); + it("allows read-only context support without permitting extra effects", () => { + const calls = [ + "get_task_context", + "get_task_history", + "finish_task", + "create_task", + ]; + expect(unexpectedLiveWorkflowCalls(calls, ["finish_task"])).toEqual([ + "create_task", + ]); + expect(scorableLiveWorkflowCalls(calls, ["finish_task"])).toEqual([ + "finish_task", + "create_task", + ]); + }); + + it("keeps required read-only calls in trajectory scoring", () => { + expect( + scorableLiveWorkflowCalls( + ["get_task_context", "get_task_history"], + ["get_task_context"], + ), + ).toEqual(["get_task_context"]); + }); + + it("advances a delegated child through the authoritative return wake", async () => { + const grants = ["delegation:tasks:create", "dependencies:write"]; + const adapter = new CapabilityMockControlPlaneAdapter({ + actors: [ + { + id: "actor-1", + companyId: "company-1", + name: "Workflow eval actor", + role: "engineer", + status: "active", + budgetId: "budget-actor-1", + capabilityGrants: grants, + }, + ], + }); + await adapter.start(); + await adapter.openFixtureRun({ + identity: { + runId: "delegation-parent", + sessionId: "delegation-session", + companyId: "company-1", + issueId: "task-1", + agentId: "actor-1", + }, + backendKind: "runner", + capabilities: grants, + }); + const created = await adapter.applyCommand({ + runId: "delegation-parent", + idempotencyKey: "create-child", + command: { + kind: "create_task", + title: "Independent verification", + description: "Verify the external boundary.", + assigneeActorId: "actor-1", + }, + }); + const childTaskId = created.entityRefs + .find((entityRef) => entityRef.startsWith("task:"))! + .slice("task:".length); + await adapter.applyCommand({ + runId: "delegation-parent", + idempotencyKey: "set-child-dependency", + command: { + kind: "set_dependencies", + blockedByTaskIds: [childTaskId], + }, + }); + + const advanced = await advanceDelegationReturnMockState({ + mockState: adapter.serialize(), + parentRunId: "delegation-parent", + parentSessionId: "delegation-session", + parentTaskId: "task-1", + capabilities: grants, + }); + expect(advanced).not.toBeNull(); + const restored = CapabilityMockControlPlaneAdapter.restore( + advanced!.mockState, + ); + expect(restored.snapshot().tasks).toEqual( + expect.arrayContaining([ + expect.objectContaining({ id: childTaskId, status: "done" }), + expect.objectContaining({ + id: "task-1", + status: "in_progress", + checkoutRunId: advanced!.returnRunId, + }), + ]), + ); + expect(restored.snapshot().blockers).toEqual([]); + expect(restored.snapshot().wakes).toContainEqual( + expect.objectContaining({ + taskId: "task-1", + reason: "blockers_resolved", + }), + ); + await expect( + restored.applyCommand({ + runId: advanced!.returnRunId, + idempotencyKey: "finish-parent", + command: { kind: "finish_task", summary: "Return completed." }, + }), + ).resolves.toMatchObject({ disposition: "applied" }); + }); + it("classifies shutdown failures as retryable infrastructure errors and redacts them", async () => { const leakedSecret = "sk-shutdown-secret-value"; liveSessionMocks.shutdown.mockRejectedValueOnce( diff --git a/packages/paperclip-runner/src/eval/live-workflow-executor.ts b/packages/paperclip-runner/src/eval/live-workflow-executor.ts index 6f3b54fa95..7d81d5ef4f 100644 --- a/packages/paperclip-runner/src/eval/live-workflow-executor.ts +++ b/packages/paperclip-runner/src/eval/live-workflow-executor.ts @@ -4,16 +4,24 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import type { CapabilityJsonValue } from "../mock-core/capability-control-plane-types.js"; +import { CapabilityMockControlPlaneAdapter } from "../mock-core/capability-mock-control-plane-adapter.js"; +import type { + PrpStructuredRunResult, + PrpTerminalState, +} from "../protocol/replay-contract.js"; import { capabilityFixtureRunCapabilities } from "../scenarios/fixture-run-capabilities.js"; import { CapabilityLiveSessionService, InMemoryCapabilityLiveSessionStore, type CapabilityLiveSession, + type CapabilityLiveSessionServiceOptions, type CapabilityLiveSessionSnapshot, + type CapabilityLiveSessionStore, type CapabilityLiveTurnEvent, type CapabilityLiveTurnResult, } from "../live/live-session.js"; import type { EvalObservation } from "./eval-scoring.js"; +import { isCapabilitySemanticReadOperation } from "../semantic-tools/policy.js"; import { RUNNER_WORKFLOW_OBSERVATION_SCHEMA, type RunnerWorkflowCheck, @@ -74,9 +82,261 @@ function continuationPrompt(evalCase: RunnerWorkflowEvalCase): string { if (evalCase.id === "steering-causality") { return "Steering update: use bullet points only. Apply only this formatting change and call finish_task exactly once without repeating prior work."; } + if (evalCase.id === "delegation-return") { + return "The plan is accepted. Create exactly one justified child, set the source task dependency to that child, then stop without calling finish_task. Wait for the authoritative child-completion result."; + } return "Continue from the authoritative typed interaction result. Do not redo completed work."; } +function workflowResult( + disposition: "done" | "blocked", + summary: string, + idempotencyKey: string, +): PrpStructuredRunResult { + return { + schema: "paperclip.run_result.v1", + reportedWorkDisposition: disposition, + summary, + completionClaim: { + contractRevision: "runner-workflow-eval-v1", + objectiveSatisfied: disposition === "done", + criteria: [], + remainingWork: + disposition === "done" + ? [] + : [ + { + description: "Wait for the delegated child to complete.", + blocksCompletion: true, + }, + ], + }, + evidence: [], + verification: [], + attentionRequests: [], + artifacts: [], + ...(disposition === "done" + ? {} + : { + blocker: { + reasonCode: "delegated_child_in_progress", + owner: { kind: "agent" as const, name: "Delegated child" }, + unblockAction: "Complete the delegated child task.", + scope: "current_track" as const, + }, + continuation: { + kind: "delegated_issue" as const, + summary: + "Resume the source task after the delegated child completes.", + idempotencyKey, + }, + }), + }; +} + +async function completeWorkflowRun( + adapter: CapabilityMockControlPlaneAdapter, + input: { + runId: string; + sessionId: string; + source: string; + disposition: "done" | "blocked"; + summary: string; + }, +): Promise { + const terminal: PrpTerminalState = { + schema: "paperclip.prp.terminal.v1", + turnTerminalState: "completed", + runTerminalState: "succeeded", + reportedWorkDisposition: input.disposition, + }; + const current = adapter.snapshot().runs.find((run) => run.id === input.runId); + if (current === undefined) { + throw new Error(`workflow eval run ${input.runId} is missing`); + } + if (current.result !== null) return; + if ( + !current.events.some( + (event) => "eventType" in event && event.eventType === "run.terminal", + ) + ) { + await adapter.appendEvent({ + schema: "paperclip.prp.event.v1", + sourceEventId: `${input.source}:terminal:1`, + sourceSeq: 1, + sourceInstanceId: input.source, + sourceKind: "runner", + runId: input.runId, + normalizedSessionId: input.sessionId, + turnId: `${input.source}-turn`, + eventType: "run.terminal", + schemaVersion: 1, + priority: 0, + emittedAt: new Date().toISOString(), + payload: terminal, + }); + } + await adapter.completeRun({ + result: workflowResult( + input.disposition, + input.summary, + `${input.source}-continuation`, + ), + terminal, + }); +} + +export interface AdvancedDelegationReturnState { + mockState: string; + childTaskId: string; + returnRunId: string; +} + +/** + * Advance the controlled mock authority through the child-return boundary. + * This is orchestration evidence, not a model-authored semantic call: the + * evaluated provider still has to author the child/dependency and final result. + */ +export async function advanceDelegationReturnMockState(input: { + mockState: string; + parentRunId: string; + parentSessionId: string; + parentTaskId: string; + capabilities: readonly string[]; +}): Promise { + const adapter = CapabilityMockControlPlaneAdapter.restore(input.mockState); + if (adapter.snapshot().lifecycle !== "running") await adapter.start(); + const initial = adapter.snapshot(); + const children = initial.tasks.filter( + (task) => task.parentId === input.parentTaskId, + ); + if (children.length !== 1) return null; + const child = children[0]!; + if ( + !initial.blockers.some( + (blocker) => + blocker.taskId === input.parentTaskId && + blocker.blockedByTaskId === child.id, + ) + ) { + return null; + } + + await completeWorkflowRun(adapter, { + runId: input.parentRunId, + sessionId: input.parentSessionId, + source: `${input.parentRunId}-delegation-wait`, + disposition: "blocked", + summary: "Source task is waiting on its delegated child.", + }); + + const childRunId = `${input.parentRunId}-child`; + const childSessionId = `${input.parentSessionId}-child`; + const actorId = child.assigneeActorId ?? initial.actors[0]!.id; + await adapter.openFixtureRun({ + identity: { + runId: childRunId, + sessionId: childSessionId, + companyId: child.companyId, + issueId: child.id, + agentId: actorId, + }, + backendKind: "runner", + sourceInstanceId: "runner-workflow-eval-child", + capabilities: [...input.capabilities], + }); + await adapter.applyCommand({ + runId: childRunId, + idempotencyKey: `${childRunId}-finish`, + command: { + kind: "finish_task", + taskId: child.id, + summary: "Delegated verification completed.", + }, + }); + await completeWorkflowRun(adapter, { + runId: childRunId, + sessionId: childSessionId, + source: `${childRunId}-completion`, + disposition: "done", + summary: "Delegated verification completed.", + }); + + const returnRunId = `${input.parentRunId}-return`; + await adapter.openFixtureRun({ + identity: { + runId: returnRunId, + sessionId: input.parentSessionId, + companyId: initial.company.id, + issueId: input.parentTaskId, + agentId: initial.actors[0]!.id, + }, + backendKind: "runner", + sourceInstanceId: "runner-workflow-eval-return", + capabilities: [...input.capabilities], + wake: { + reason: "blockers_resolved", + payload: { completedTaskId: child.id }, + }, + }); + return { mockState: adapter.serialize(), childTaskId: child.id, returnRunId }; +} + +async function restoreAfterDelegatedChild(input: { + session: CapabilityLiveSession; + store: CapabilityLiveSessionStore; + transportOptions: CapabilityLiveSessionServiceOptions["transportOptions"]; +}): Promise<{ + service: CapabilityLiveSessionService; + session: CapabilityLiveSession; +} | null> { + const before = input.session.snapshot(); + const advanced = await advanceDelegationReturnMockState({ + mockState: before.mockState, + parentRunId: before.authority.runId, + parentSessionId: before.sessionId, + parentTaskId: before.authority.taskId, + capabilities: before.authority.capabilities, + }); + if (advanced === null) return null; + await input.session.suspend("workflow eval delegated child completed"); + const checkpoint = await input.store.load(before.sessionId); + if (checkpoint === null) { + throw new Error("workflow eval delegation checkpoint is missing"); + } + const { + providerRunBinding: _providerRunBinding, + ...checkpointWithoutBinding + } = checkpoint; + const at = new Date().toISOString(); + const updated: CapabilityLiveSessionSnapshot = { + ...checkpointWithoutBinding, + revision: checkpoint.revision + 1, + updatedAt: at, + authority: { + ...checkpoint.authority, + runId: advanced.returnRunId, + }, + mockState: advanced.mockState, + stateHistory: [ + ...(checkpoint.stateHistory ?? []), + { + revision: JSON.parse(advanced.mockState).revision as number, + at, + turnId: null, + operationId: "eval.delegated_child_completed", + state: advanced.mockState, + }, + ], + }; + await input.store.save(updated); + const service = new CapabilityLiveSessionService({ + store: input.store, + transportOptions: input.transportOptions, + }); + return { service, session: await service.restore(before.sessionId) }; +} + function acpxAgent( candidate: RunnerLiveEvalCandidate, ): "claude" | "codex" | undefined { @@ -142,6 +402,33 @@ function observedCalls(snapshot: CapabilityLiveSessionSnapshot): string[] { ); } +export function unexpectedLiveWorkflowCalls( + calls: readonly string[], + expectedCalls: readonly string[], +): string[] { + const expected = new Set(expectedCalls); + return scorableLiveWorkflowCalls(calls, expectedCalls).filter( + (operationId) => !expected.has(operationId), + ); +} + +/** + * Keeps required reads and every stateful operation in trajectory scoring while + * ignoring optional read-only context gathering. The full call list remains in + * the workflow evidence and metrics for auditability. + */ +export function scorableLiveWorkflowCalls( + calls: readonly string[], + expectedCalls: readonly string[], +): string[] { + const expected = new Set(expectedCalls); + return calls.filter( + (operationId) => + expected.has(operationId) || + !isCapabilitySemanticReadOperation(operationId), + ); +} + function duplicateEffectSignals( snapshot: CapabilityLiveSessionSnapshot, ): string[] { @@ -591,6 +878,24 @@ export async function executeLiveRunnerWorkflow(input: { ), ); await settleInteractions(input.evalCase, session, turns, withinBudget); + if (input.evalCase.id === "delegation-return" && withinBudget()) { + const restored = await restoreAfterDelegatedChild({ + session, + store, + transportOptions, + }); + if (restored !== null) { + service = restored.service; + session = restored.session; + subscribeToSession(session); + turns.push( + await session.sendMessage( + "The delegated child completed successfully and the source task is unblocked. Use the authoritative child result and call finish_task exactly once without repeating the delegated work.", + { allowMissingUsage: input.allowMissingUsage }, + ), + ); + } + } if (input.evalCase.id === "steering-causality" && withinBudget()) { turns.push( await session.sendMessage(continuationPrompt(input.evalCase), { @@ -660,9 +965,7 @@ export async function executeLiveRunnerWorkflow(input: { const missingCalls = expectedCalls.filter( (operationId) => !calls.includes(operationId), ); - const extraCalls = calls.filter( - (operationId) => !expectedCalls.includes(operationId), - ); + const extraCalls = unexpectedLiveWorkflowCalls(calls, expectedCalls); const duplicateSignals = snapshot === undefined ? [] : duplicateEffectSignals(snapshot); const pendingInteractions = session?.pendingInteractions().length ?? 0; @@ -712,7 +1015,7 @@ export async function executeLiveRunnerWorkflow(input: { provenance: { source: "live_model", behavior: input.evalCase.id }, controlPlaneOwned: expectedCalls.length === 0, expectedCalls, - observedCalls: calls, + observedCalls: scorableLiveWorkflowCalls(calls, expectedCalls), forbiddenCalls: [], finalState: { expected: expectedCalls.length === 0 ? "unchanged" : "mutated", diff --git a/packages/paperclip-runner/src/eval/live-workflow-matrix.ts b/packages/paperclip-runner/src/eval/live-workflow-matrix.ts index c491bbe622..6ece8d18a4 100644 --- a/packages/paperclip-runner/src/eval/live-workflow-matrix.ts +++ b/packages/paperclip-runner/src/eval/live-workflow-matrix.ts @@ -247,6 +247,65 @@ export interface RunnerLiveEvalSchedule { expectedExecutions: number; } +export interface RunnerLiveEvalSelection { + candidateIds?: readonly string[]; + caseIds?: readonly string[]; + limit?: number; +} + +/** Selects a stable paid subset without weakening validation of the full schedule. */ +export function selectRunnerLiveEvalSchedule( + schedule: RunnerLiveEvalSchedule, + selection: RunnerLiveEvalSelection, +): RunnerLiveEvalSchedule { + const candidateIds = new Set(selection.candidateIds ?? []); + const caseIds = new Set(selection.caseIds ?? []); + for (const id of candidateIds) { + if (!schedule.candidates.some((candidate) => candidate.id === id)) { + throw new Error(`unknown Runner live eval candidate: ${id}`); + } + } + const scheduledCaseIds = new Set( + schedule.entries.map((entry) => entry.caseId), + ); + for (const id of caseIds) { + if (!scheduledCaseIds.has(id as RunnerWorkflowEvalCase["id"])) { + throw new Error(`unknown Runner live eval case: ${id}`); + } + } + if ( + selection.limit !== undefined && + (!Number.isSafeInteger(selection.limit) || selection.limit <= 0) + ) { + throw new Error( + "Runner live eval selection limit must be a positive integer", + ); + } + let entries = schedule.entries.filter( + (entry) => + (candidateIds.size === 0 || candidateIds.has(entry.candidateId)) && + (caseIds.size === 0 || caseIds.has(entry.caseId)), + ); + if (selection.limit !== undefined) + entries = entries.slice(0, selection.limit); + if (entries.length === 0) { + throw new Error( + "Runner live eval selection matched no scheduled executions", + ); + } + const selectedCandidateIds = new Set( + entries.map((entry) => entry.candidateId), + ); + return { + ...schedule, + candidates: schedule.candidates.filter((candidate) => + selectedCandidateIds.has(candidate.id), + ), + entries, + expectedExecutions: entries.length, + }; +} + export function assertRunnerLiveCandidateManifest(): void { const slots = RUNNER_LIVE_CANDIDATE_SLOTS.map((slot) => slot.id); const candidates = RUNNER_LIVE_CANDIDATE_SLOTS.flatMap( diff --git a/packages/paperclip-runner/src/eval/workflow-evals.test.ts b/packages/paperclip-runner/src/eval/workflow-evals.test.ts index aea24c13d3..b4e280dccb 100644 --- a/packages/paperclip-runner/src/eval/workflow-evals.test.ts +++ b/packages/paperclip-runner/src/eval/workflow-evals.test.ts @@ -24,6 +24,7 @@ import { resolvedNightlyCandidates, runnerLiveRotationWeek, runnerLiveScheduleCoverage, + selectRunnerLiveEvalSchedule, RunnerWorkflowInfrastructureError, } from "./live-workflow-matrix.js"; import { unavailableLiveRunnerWorkflowObservation } from "./live-workflow-executor.js"; @@ -306,6 +307,35 @@ describe("balanced live Runner workflow matrix", () => { ); }); + it("selects a stable bounded paid subset and rejects invalid selectors", () => { + const schedule = buildRunnerLiveEvalSchedule({ + seed: "subset-v1", + rotationDay: 0, + generatedAt: "2026-08-24T00:00:00.000Z", + }); + const selected = selectRunnerLiveEvalSchedule(schedule, { + candidateIds: ["codex-luna"], + limit: 2, + }); + expect(selected.expectedExecutions).toBe(2); + expect(selected.entries).toHaveLength(2); + expect( + selected.entries.every((entry) => entry.candidateId === "codex-luna"), + ).toBe(true); + expect(selected.candidates.map((candidate) => candidate.id)).toEqual([ + "codex-luna", + ]); + expect(schedule.expectedExecutions).toBe(40); + expect(() => + selectRunnerLiveEvalSchedule(schedule, { + candidateIds: ["unknown-candidate"], + }), + ).toThrow("unknown Runner live eval candidate"); + expect(() => selectRunnerLiveEvalSchedule(schedule, { limit: 0 })).toThrow( + "selection limit must be a positive integer", + ); + }); + it("retries only one retryable infrastructure failure", async () => { const schedule = buildRunnerLiveEvalSchedule({ seed: "retry-v1", @@ -409,7 +439,7 @@ describe("workflow reports and stress traceability", () => { } }); - it("renders safe JSON-derived Markdown, JUnit, and GitHub summaries", async () => { + it("renders safe JSON-derived Evalbook, Markdown, JUnit, and GitHub summaries", async () => { const results = await runDeterministicRunnerWorkflowMatrix(); const report = buildRunnerWorkflowEvalReport({ source: "deterministic", diff --git a/packages/paperclip-runner/src/evals/model-pricing.test.ts b/packages/paperclip-runner/src/evals/model-pricing.test.ts index 54c3a27c5d..b97495eb2e 100644 --- a/packages/paperclip-runner/src/evals/model-pricing.test.ts +++ b/packages/paperclip-runner/src/evals/model-pricing.test.ts @@ -29,4 +29,21 @@ describe("model pricing", () => { }), ).toThrow("model pricing unavailable"); }); + + it.each([ + ["openrouter/anthropic/claude-sonnet-5", 2, 0.2, 10], + ["openrouter/qwen/qwen3.8-max-0902", 2, 0.25, 6], + ["openrouter/google/gemini-3.8-flash", 0.75, 0.075, 3.75], + ["openrouter/z-ai/glm-5.3", 1.4, 0.14, 4.4], + ["openrouter/deepseek/deepseek-v4-flash-0731", 0.065, 0.016, 0.18], + ["openrouter/openai/gpt-6-astra", 10, 1, 50], + ])("pins the OpenRouter breadth price for %s", (model, input, cachedInput, output) => { + expect( + estimateModelCostNanodollars(model, { + inputTokens: 1_000, + cachedInputTokens: 400, + outputTokens: 100, + }).ratesUsdPerMillionTokens, + ).toEqual({ input, cachedInput, output }); + }); }); diff --git a/packages/paperclip-runner/src/evals/model-pricing.ts b/packages/paperclip-runner/src/evals/model-pricing.ts index 99cde385ed..8c374002a6 100644 --- a/packages/paperclip-runner/src/evals/model-pricing.ts +++ b/packages/paperclip-runner/src/evals/model-pricing.ts @@ -1,4 +1,4 @@ -export const MODEL_PRICING_VERSION = "provider-list-prices-2026-08-21" as const; +export const MODEL_PRICING_VERSION = "provider-list-prices-2026-09-05" as const; interface TokenRatesUsdPerMillion { input: number; @@ -18,6 +18,13 @@ const RATES: Readonly> = Object.freeze({ "claude-sonnet-5": { input: 3, cachedInput: 0.3, output: 15 }, // Amazon Bedrock global cross-region list price for Claude Sonnet 4.6. "global.anthropic.claude-sonnet-4-6": { input: 3, cachedInput: 0.3, output: 15 }, + // OpenRouter list-price snapshot used by the qualified OpenCode breadth lane. + "openrouter/anthropic/claude-sonnet-5": { input: 2, cachedInput: 0.2, output: 10 }, + "openrouter/qwen/qwen3.8-max-0902": { input: 2, cachedInput: 0.25, output: 6 }, + "openrouter/google/gemini-3.8-flash": { input: 0.75, cachedInput: 0.075, output: 3.75 }, + "openrouter/z-ai/glm-5.3": { input: 1.4, cachedInput: 0.14, output: 4.4 }, + "openrouter/deepseek/deepseek-v4-flash-0731": { input: 0.065, cachedInput: 0.016, output: 0.18 }, + "openrouter/openai/gpt-6-astra": { input: 10, cachedInput: 1, output: 50 }, }); export interface EstimatedModelCost { diff --git a/packages/paperclip-runner/src/live/live-session.test.ts b/packages/paperclip-runner/src/live/live-session.test.ts index 9ed9cbaf04..b569868318 100644 --- a/packages/paperclip-runner/src/live/live-session.test.ts +++ b/packages/paperclip-runner/src/live/live-session.test.ts @@ -87,6 +87,10 @@ class FakeCapabilityCodexTransport implements CodexAppServerTransport { if (method === "initialize") return { user: { sessionId: this.state.providerSessionId } }; if (method === "thread/start") { expect(Array.isArray(params.dynamicTools)).toBe(true); + expect(params.completionContract).toEqual({ + revision: "paperclip-capability-live-v1", + criterionIds: ["objective"], + }); return { model: "gpt-eval-test", modelProvider: "openai", @@ -633,10 +637,12 @@ describe("Capability live runnerd and Codex session", () => { it("persists the selected provider and passes it to the live runner transport", async () => { const state = providerState(); const providers: Array = []; + const permissionModes: Array = []; const baseFactory = fakeTransportFactory(state); const service = new CapabilityLiveSessionService({ transportFactory: (options) => { providers.push(options.provider); + permissionModes.push(options.opencodePermissionMode); return baseFactory(options); }, }); @@ -646,6 +652,7 @@ describe("Capability live runnerd and Codex session", () => { }); expect(providers).toEqual(["opencode"]); + expect(permissionModes).toEqual(["deny"]); expect(session.snapshot().config).toMatchObject({ provider: "opencode", driver: "opencode_server", @@ -655,6 +662,45 @@ describe("Capability live runnerd and Codex session", () => { await service.shutdown(session.id); }); + it("attributes Claude Managed sessions to the pinned immutable Agent version", async () => { + const state = providerState(); + const managedProfiles: Array | undefined> = []; + const baseFactory = fakeTransportFactory(state); + const service = new CapabilityLiveSessionService({ + transportFactory: (options) => { + managedProfiles.push(options.managedProfile); + return baseFactory(options); + }, + }); + const session = await service.create({ + provider: "claude_managed", + requestedModel: "claude-sonnet-5", + managedProfile: { + profileId: "managed-profile-1", + anthropicAgentId: "agent-1", + agentVersion: "17", + environmentId: "environment-1", + betaVersion: "managed-agents-2026-04-01", + maxSessionListCostUsd: 0.5, + }, + }); + + expect(managedProfiles).toEqual([ + expect.objectContaining({ + agentVersion: "17", + betaVersion: "managed-agents-2026-04-01", + model: "claude-sonnet-5", + }), + ]); + expect(session.snapshot().config).toMatchObject({ + provider: "claude_managed", + driver: "claude_managed_agents_api", + providerVersion: "17", + requestedModel: "claude-sonnet-5", + }); + await service.shutdown(session.id); + }); + it("opens lazy sessions with core and discovery gateways instead of optional schemas", async () => { const state = providerState(); const claim = "discovery:agents:read"; @@ -673,6 +719,74 @@ describe("Capability live runnerd and Codex session", () => { await service.shutdown(session.id); }); + it("keeps discovered invocations correlated to the provider gateway call", async () => { + const state = providerState(); + const claim = "discovery:agents:read"; + const service = new CapabilityLiveSessionService({ + transportFactory: fakeTransportFactory(state), + }); + const session = await service.create({ + runId: "run-live-lazy-invoke", + sessionId: "session-live-lazy-invoke", + toolExposure: "lazy", + capabilities: [claim], + explicitClaims: [claim], + scenario: { id: "lazy-invoke", claims: [claim] }, + }); + let gatewayResult: Record | null = null; + state.onTurnStart = async () => { + const transport = state.transports[0]!; + const turnId = [...state.turns.keys()].at(-1)!; + const discovery = await transport.invokeServerRequest({ + id: "request-discover", + method: "item/tool/call", + params: { + threadId: state.threadId, + turnId, + callId: "call-discover", + tool: "discover_capabilities", + arguments: { + query: "list company agents", + namespace: "discovery", + limit: 10, + }, + }, + }); + expect(discovery.success).toBe(true); + const invoked = await transport.invokeServerRequest({ + id: "request-invoke", + method: "item/tool/call", + params: { + threadId: state.threadId, + turnId, + callId: "call-invoke", + tool: "invoke_discovered_capability", + arguments: { operationId: "list_agents", input: {} }, + }, + }); + gatewayResult = JSON.parse( + String( + (invoked.contentItems as Array>)[0]?.text, + ), + ) as Record; + }; + + await session.sendMessage("Exercise the lazy discovery gateway."); + + expect(gatewayResult).toMatchObject({ + callId: "call-invoke", + operationId: "invoke_discovered_capability", + ok: true, + }); + expect( + session + .snapshot() + .evidence.filter((entry) => entry.kind === "tool_call") + .map((entry) => entry.data.operationId), + ).toContain("list_agents"); + await service.shutdown(session.id); + }); + it("returns typed mock results to the same multi-turn Codex thread without Paperclip network calls", async () => { const state = providerState(); const fetchSpy = vi.spyOn(globalThis, "fetch"); diff --git a/packages/paperclip-runner/src/live/live-session.ts b/packages/paperclip-runner/src/live/live-session.ts index dfd5cacefe..4f7b8c0f6b 100644 --- a/packages/paperclip-runner/src/live/live-session.ts +++ b/packages/paperclip-runner/src/live/live-session.ts @@ -54,6 +54,10 @@ import { } from "./workspace-file-reference.js"; const LIVE_SESSION_SCHEMA = "paperclip.capability.live-session.v1" as const; +const LIVE_COMPLETION_CONTRACT = Object.freeze({ + revision: "paperclip-capability-live-v1", + criterionIds: ["objective"], +}); const LIVE_BASE_INSTRUCTIONS = [ "You are operating one mock Paperclip issue through typed semantic tools.", "Use only the tools exposed in this thread; never call a Paperclip REST API.", @@ -922,7 +926,7 @@ export class CapabilityLiveSessionService { providerVersion: input.provider === "opencode" ? "1.18.17" : input.provider === "claude_managed" - ? input.managedProfile!.betaVersion + ? input.managedProfile!.agentVersion : input.provider === "aws_agentcore" ? input.agentCoreProfile!.qualificationRevision : input.provider === "acpx" ? acpxProfile!.acpxVersion : null, @@ -2269,6 +2273,15 @@ export class CapabilityLiveSession { const transportBundle = this.#transportFactory({ ...this.#transportOptions, provider, + ...(provider === "opencode" + ? { + // Capability-live sessions expose only governed semantic tools and + // use approvalPolicy=never. Keep OpenCode's ambient shell/file + // tools fail-closed instead of brokering broader permissions. + opencodePermissionMode: + this.#transportOptions.opencodePermissionMode ?? "deny", + } + : {}), ...(provider === "acpx" && this.#config.acpxAgent ? { acpxAgent: this.#config.acpxAgent, } : {}), @@ -2395,6 +2408,7 @@ export class CapabilityLiveSession { runtimeWorkspaceRoots: [this.#config.workingDirectory], approvalPolicy: "never", baseInstructions: LIVE_BASE_INSTRUCTIONS, + completionContract: LIVE_COMPLETION_CONTRACT, dynamicTools: [ ...tools.map(dynamicToolSpec), ...((this.#config.toolExposure ?? "eager") === "lazy" ? discoveryToolSpecs() : []), @@ -2554,7 +2568,15 @@ export class CapabilityLiveSession { this.#recordTerminalFact(this.#durableReplayTurnId(turnId), "completed"); } await this.#persist(); - return this.#codexToolResponse(result); + // The durable provider bridge correlates the outer semantic result with + // the exact dynamic tool Codex called. Keep the dispatched operation in + // evidence, but preserve the discovery gateway identity on the response + // envelope returned to runnerd. + const providerResult = + operationId === INVOKE_DISCOVERED_TOOL + ? { ...result, operationId, callId } + : result; + return this.#codexToolResponse(providerResult); } #isDurableTerminalReplay(turnId: string, input: unknown): boolean { @@ -2579,7 +2601,7 @@ export class CapabilityLiveSession { return `${interruptedTurnId}:durable-duplicate-replay`; } - #codexToolResponse(result: CapabilitySemanticToolResult): Record { + #codexToolResponse(result: { readonly ok: boolean }): Record { return { success: result.ok, contentItems: [{ diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index b8ffc2d367..08815660d1 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -59,6 +59,11 @@ import { withCodexCollaborationRuntimeInstructions, } from "./runnerd-codex-transport.js"; +it("launches runnerd with its production durable outbox limits", () => { + expect(runnerdLaunchProfileInternals.maxOutboxBytes).toBe(16 * 1024 * 1024); + expect(runnerdLaunchProfileInternals.p0ReserveBytes).toBe(1024 * 1024); +}); + it("replays the durable run attachment outcome and latest provider identity", () => { expect( runnerdRecoveryInternals.recoveredRunAttachment({ @@ -1421,6 +1426,75 @@ it("runs the lab provider boundary through authenticated durable PRP", async () }); }, 30_000); +it("continues rehydrating events after the committed-event window slides", async () => { + const stateDirectory = await mkdtemp( + join(tmpdir(), "runnerd-sliding-event-window-"), + ); + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory, "--split-event-burst"), + stateDirectory, + lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, + }); + bundle.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [ + { + type: "inputText", + text: JSON.stringify({ ok: true, result: { task: { id: "task-1" } } }), + }, + ], + })); + try { + await bundle.transport.request("initialize", {}); + await bundle.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [ + { + name: "get_task_context", + description: "Read the active task.", + inputSchema: { + type: "object", + properties: {}, + additionalProperties: false, + }, + }, + ], + }); + await bundle.transport.request("turn/start", { + input: [{ type: "text", text: "Emit a split event burst." }], + }); + const notifications = bundle.transport + .notifications() + [Symbol.asyncIterator](); + const methods: string[] = []; + const deadline = Date.now() + 20_000; + while (Date.now() < deadline) { + const next = await Promise.race([ + notifications.next(), + new Promise((_, reject) => + setTimeout( + () => + reject(new Error("sliding event window notification timeout")), + 10_000, + ), + ), + ]); + if (!next.value) break; + methods.push(next.value.method); + if (next.value.method === "turn/completed") break; + } + expect( + methods.filter((method) => method === "item/agentMessage/delta"), + ).toHaveLength(144); + expect(methods).toContain("turn/completed"); + } finally { + await bundle.transport.close(); + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 30_000); + it("binds an immediately failed durable turn before exposing its terminal", async () => { const stateDirectory = await mkdtemp( join(tmpdir(), "runnerd-fast-terminal-"), diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index fe384d7336..9698fb766a 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -76,6 +76,8 @@ const MAX_NOTIFICATION_COUNT = 2_048; const MAX_NOTIFICATION_BYTES = 4 * 1024 * 1024; const RUNNER_CLIENT_VERSION = "0.3.0"; const RUNNER_BOOTSTRAP_TICKET_TTL_MS = 60_000; +const RUNNERD_MAX_OUTBOX_BYTES = 16 * 1024 * 1024; +const RUNNERD_P0_RESERVE_BYTES = 1024 * 1024; function readLocalProcessStartedAt(pid: number): string | null { if (!Number.isInteger(pid) || pid <= 0) return null; @@ -1941,7 +1943,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { #handle: RunnerProcessHandle | null = null; #adoptedRunnerMonitor: NodeJS.Timeout | null = null; #pump: NodeJS.Timeout | null = null; - #eventIndex = 0; + #eventSourceSeq = 0; + #deferredTurnStartEvents: DurableRecoveryCommittedEvent[] = []; #threadId = ""; #sessionId: string | null = null; #providerIdentity: Record | null = null; @@ -2293,7 +2296,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { const previousRelease = this.#controlPlaneRelease; core.rotateRunIdentity(desired); - this.#eventIndex = 0; + this.#eventSourceSeq = 0; + this.#deferredTurnStartEvents = []; this.#durableTurnId = desired.turnId; this.#controlPlaneRelease = registration?.release ?? null; let previousReleased = false; @@ -3092,8 +3096,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.options.runnerStateDirectory ?? resolve(this.#root, "runner"), identity, ticket: core.issueBootstrapTicket(RUNNER_BOOTSTRAP_TICKET_TTL_MS), - maxOutboxBytes: 256 * 1024, - p0ReserveBytes: 64 * 1024, + maxOutboxBytes: RUNNERD_MAX_OUTBOX_BYTES, + p0ReserveBytes: RUNNERD_P0_RESERVE_BYTES, maxRuntimeMs: 60 * 60 * 1_000, reconnectGraceMs: this.options.runnerReconnectGraceMs, lifecyclePolicy: this.options.lifecyclePolicy, @@ -3432,10 +3436,11 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { // If attachment completed, replay only its latest identity event into the // transport's in-memory evidence; session events are consumed internally // and are not duplicated onto the provider notification stream. - this.#eventIndex = + this.#eventSourceSeq = runAttachment !== null && runAttachment.providerIdentityEventIndex >= 0 - ? runAttachment.providerIdentityEventIndex - : committedEvents.length; + ? committedEvents[runAttachment.providerIdentityEventIndex]!.sourceSeq - + 1 + : core.store.state.ackedSourceSeq; const adoptedProviderIdentityIndex = latestProviderIdentityEventIndex(committedEvents); if ( @@ -3494,8 +3499,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.options.runnerStateDirectory ?? resolve(this.#root, "runner"), identity, ticket: core.issueBootstrapTicket(RUNNER_BOOTSTRAP_TICKET_TTL_MS), - maxOutboxBytes: 256 * 1024, - p0ReserveBytes: 64 * 1024, + maxOutboxBytes: RUNNERD_MAX_OUTBOX_BYTES, + p0ReserveBytes: RUNNERD_P0_RESERVE_BYTES, maxRuntimeMs: 60 * 60 * 1_000, reconnectGraceMs: this.options.runnerReconnectGraceMs, lifecyclePolicy: this.options.lifecyclePolicy, @@ -3781,8 +3786,21 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { #pumpEvents(): void { this.#flushPendingTraceRehydrations(); const events = this.#core?.store.state.committedEvents ?? []; - while (this.#eventIndex < events.length) { - const event = events[this.#eventIndex]!; + for (;;) { + const deferredEvent = + !this.#turnStartResponsePending || this.#expectedProviderTurnId !== null + ? this.#deferredTurnStartEvents[0] + : undefined; + const event = + deferredEvent ?? + events.find((candidate) => candidate.sourceSeq > this.#eventSourceSeq); + if (event === undefined) return; + const fromDeferredQueue = deferredEvent !== undefined; + if (!fromDeferredQueue && event.sourceSeq !== this.#eventSourceSeq + 1) { + throw new Error( + `PRP provider event window advanced past source sequence ${this.#eventSourceSeq + 1}`, + ); + } const eventPayload = record(event.envelope.payload).payload; const turnStartWhileCommandResultPending = this.#turnStartResponsePending && @@ -3793,9 +3811,25 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { (notification) => notification.method === "turn/started", ))); // The durable command result is the only correlation authority for a - // provider-assigned turn id. Leave an early start at the cursor until - // that exact expected identity is installed. - if (turnStartWhileCommandResultPending) return; + // provider-assigned turn id. Copy an early start and its following + // events out of the control plane's sliding window until that exact + // expected identity is installed. + if ( + !fromDeferredQueue && + (turnStartWhileCommandResultPending || + (this.#turnStartResponsePending && + this.#expectedProviderTurnId === null && + this.#deferredTurnStartEvents.length > 0)) + ) { + if (this.#deferredTurnStartEvents.length >= 4_096) { + throw new Error( + "turn/start produced too many events before its durable command result", + ); + } + this.#eventSourceSeq = event.sourceSeq; + this.#deferredTurnStartEvents.push(structuredClone(event)); + continue; + } const terminalWhileTurnStartPending = this.#turnStartResponsePending && ([ @@ -3808,8 +3842,18 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { unwrapRunnerdProviderNotifications(eventPayload).some( (notification) => notification.method === "turn/completed", ))); - if (terminalWhileTurnStartPending) return; - this.#eventIndex += 1; + if (terminalWhileTurnStartPending) { + if (!fromDeferredQueue) { + this.#eventSourceSeq = event.sourceSeq; + this.#deferredTurnStartEvents.push(structuredClone(event)); + } + return; + } + // The control plane retains a sliding committed-event window. Track its + // durable protocol cursor rather than an array index: once that array is + // full, new events replace its prefix without increasing its length. + if (fromDeferredQueue) this.#deferredTurnStartEvents.shift(); + else this.#eventSourceSeq = event.sourceSeq; if ( event.eventType === "harness.ready" || event.eventType === "session.started" || @@ -4535,6 +4579,8 @@ export const runnerdLaunchProfileInternals = Object.freeze({ acpxProviderPackageAuthority, acpxRunnerLaunchProfile, resolveBuildOwnedCliArtifact, + maxOutboxBytes: RUNNERD_MAX_OUTBOX_BYTES, + p0ReserveBytes: RUNNERD_P0_RESERVE_BYTES, }); export const runnerdRecoveryInternals = Object.freeze({ diff --git a/packages/paperclip-runner/src/semantic-tools/discovery.ts b/packages/paperclip-runner/src/semantic-tools/discovery.ts index 0ff30e281f..c6adf41dcd 100644 --- a/packages/paperclip-runner/src/semantic-tools/discovery.ts +++ b/packages/paperclip-runner/src/semantic-tools/discovery.ts @@ -17,6 +17,8 @@ export interface CapabilityDiscoveryResult { readonly truncated: boolean; } +const MAX_DISCOVERY_RESULTS = 10; + const NAMESPACE: Readonly> = Object.freeze({ get_task_context: "active_task", get_task_history: "active_task", list_documents: "documents", read_document: "documents", list_document_revisions: "documents", @@ -47,7 +49,7 @@ export const CAPABILITY_DISCOVERY_GATEWAY_DEFINITIONS = Object.freeze([{ type: "object", properties: { query: { type: "string", minLength: 1, maxLength: 500 }, namespace: { type: "string" }, - limit: { type: "integer", minimum: 1, maximum: 8 }, + limit: { type: "integer", minimum: 1, maximum: MAX_DISCOVERY_RESULTS }, }, required: ["query"], additionalProperties: false, }, outputSchema: { @@ -90,7 +92,10 @@ export function discoverCapabilityDefinitions( ): CapabilityDiscoveryResult { const normalized = query.trim().toLowerCase(); if (normalized.length === 0) throw new Error("discovery_query_empty"); - const limit = Math.max(1, Math.min(options.limit ?? 5, 8)); + const limit = Math.max( + 1, + Math.min(options.limit ?? 5, MAX_DISCOVERY_RESULTS), + ); const tokens = normalized.split(/[^a-z0-9]+/).filter((token) => token.length > 1); const permitted = CAPABILITY_SEMANTIC_TOOL_CATALOG .filter((descriptor) => descriptor.exposure === "optional") diff --git a/packages/paperclip-runner/src/semantic-tools/policy.ts b/packages/paperclip-runner/src/semantic-tools/policy.ts index 0b077eea8d..6bcff09bec 100644 --- a/packages/paperclip-runner/src/semantic-tools/policy.ts +++ b/packages/paperclip-runner/src/semantic-tools/policy.ts @@ -8,7 +8,7 @@ import type { CapabilitySemanticToolDescriptor, } from "./types.js"; -const READ_OPERATIONS = new Set([ +export const CAPABILITY_SEMANTIC_READ_OPERATION_IDS = [ "get_task_context", "get_task_history", "list_documents", @@ -21,7 +21,15 @@ const READ_OPERATIONS = new Set([ "get_approval", "get_approval_context", "get_workspace_runtime", -]); +] as const satisfies readonly CapabilitySemanticOperationId[]; + +const READ_OPERATIONS = new Set( + CAPABILITY_SEMANTIC_READ_OPERATION_IDS, +); + +export function isCapabilitySemanticReadOperation(operationId: string): boolean { + return READ_OPERATIONS.has(operationId as CapabilitySemanticOperationId); +} export const DEFAULT_CAPABILITY_SCENARIO_POLICY: CapabilitySemanticScenarioPolicy = { id: "capability-default", diff --git a/packages/paperclip-runner/src/semantic-tools/semantic-tools.test.ts b/packages/paperclip-runner/src/semantic-tools/semantic-tools.test.ts index 1d0fa1cb8c..5d7ca42dc1 100644 --- a/packages/paperclip-runner/src/semantic-tools/semantic-tools.test.ts +++ b/packages/paperclip-runner/src/semantic-tools/semantic-tools.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest"; import type { CapabilityFixtureSeed } from "../mock-core/capability-control-plane-types.js"; import { CapabilityMockControlPlaneAdapter } from "../mock-core/capability-mock-control-plane-adapter.js"; import { CAPABILITY_SEMANTIC_TOOL_CATALOG } from "./catalog.js"; +import { CAPABILITY_DISCOVERY_GATEWAY_DEFINITIONS } from "./discovery.js"; import { CapabilitySemanticDispatcher } from "./dispatcher.js"; import { createCapabilityProviderNeutralBinding } from "./provider-neutral.js"; @@ -42,6 +43,13 @@ async function running( } describe("Capability semantic catalog and authorization", () => { + it("accepts the conventional ten-result capability discovery limit", () => { + expect( + CAPABILITY_DISCOVERY_GATEWAY_DEFINITIONS[0].inputSchema.properties.limit + .maximum, + ).toBe(10); + }); + it("publishes a stable narrow catalog without credentials or control-plane-owned tools", () => { const names = CAPABILITY_SEMANTIC_TOOL_CATALOG.map((tool) => tool.operationId); expect(new Set(names).size).toBe(names.length); diff --git a/packages/paperclip-runner/vite.issue-thread.config.ts b/packages/paperclip-runner/vite.issue-thread.config.ts index 17b33a483d..fe235b3172 100644 --- a/packages/paperclip-runner/vite.issue-thread.config.ts +++ b/packages/paperclip-runner/vite.issue-thread.config.ts @@ -14,7 +14,6 @@ export default defineConfig({ plugins: [react(), capabilityIssueThreadServerPlugin()], server: { host: "127.0.0.1" }, preview: { host: "127.0.0.1" }, - optimizeDeps: { esbuildOptions: { target: "esnext" } }, build: { outDir: resolve(packageRoot, "dist-issue-thread"), emptyOutDir: true, diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs index d98ccbfcdd..aaf3179215 100644 --- a/scripts/__tests__/release-verify-workflow.test.mjs +++ b/scripts/__tests__/release-verify-workflow.test.mjs @@ -248,6 +248,7 @@ test("Runner eval workflows pin actions and gate paid live execution", () => { readWorkflow("runner-chaos-evals.yml"), readWorkflow("runner-full-stack-e2e.yml"), readWorkflow("e2e.yml"), + readWorkflow("runner-protocol-live-evals.yml"), ]; for (const workflow of actionPinWorkflows) { @@ -286,6 +287,7 @@ test("Runner eval workflows pin actions and gate paid live execution", () => { "e2e.yml", "runner-full-stack-e2e.yml", "runner-live-evals.yml", + "runner-protocol-live-evals.yml", ]; const paidWorkflowNameSet = new Set(paidWorkflowNames); const providerSecretReference = @@ -322,7 +324,7 @@ test("Runner eval workflows pin actions and gate paid live execution", () => { assert.match(block, /\n environment:\n name: runner-e2e-paid\n/); assert.match( block, - /\n steps:\n(?:\s*\n)* - name: Reauthorize[^\n]*\n/, + /\n steps:(?: &[A-Za-z0-9_-]+)?\n(?:\s*\n)* - name: Reauthorize[^\n]*\n/, `${name} must reauthorize as the first provider-job step`, ); const reauthorize = block.indexOf(" - name: Reauthorize"); @@ -357,7 +359,11 @@ test("Runner eval workflows pin actions and gate paid live execution", () => { ); assert.doesNotMatch(historyPublisher, /^\s+cache: pnpm$/m); - for (const name of ["runner-full-stack-e2e.yml", "runner-live-evals.yml"]) { + for (const name of [ + "runner-full-stack-e2e.yml", + "runner-live-evals.yml", + "runner-protocol-live-evals.yml", + ]) { const workflow = readWorkflow(name); const crons = [...workflow.matchAll(/cron:\s*"([^"]+)"/g)].map( (match) => match[1], diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts index b875add0d3..03a68f6af8 100644 --- a/tests/runner-e2e/workflow-security.test.ts +++ b/tests/runner-e2e/workflow-security.test.ts @@ -121,15 +121,18 @@ describe("public repository paid workflow security", () => { it("gates every provider-secret job with stable actor IDs", async () => { const workflows = await Promise.all( - ["runner-full-stack-e2e.yml", "runner-live-evals.yml", "e2e.yml"].map( - async (name) => ({ - name, - contents: await readFile( - path.join(repositoryRoot, ".github/workflows", name), - "utf8", - ), - }), - ), + [ + "runner-full-stack-e2e.yml", + "runner-live-evals.yml", + "runner-protocol-live-evals.yml", + "e2e.yml", + ].map(async (name) => ({ + name, + contents: await readFile( + path.join(repositoryRoot, ".github/workflows", name), + "utf8", + ), + })), ); for (const { name, contents } of workflows) { @@ -482,6 +485,7 @@ describe("public repository paid workflow security", () => { "e2e.yml", "runner-full-stack-e2e.yml", "runner-live-evals.yml", + "runner-protocol-live-evals.yml", ]); const names = (await readdir(workflowDirectory)).filter((name) => /\.ya?ml$/.test(name), @@ -508,7 +512,11 @@ describe("public repository paid workflow security", () => { it("runs paid scheduled campaigns only on Sundays", async () => { const workflows = await Promise.all( - ["runner-full-stack-e2e.yml", "runner-live-evals.yml"].map((name) => + [ + "runner-full-stack-e2e.yml", + "runner-live-evals.yml", + "runner-protocol-live-evals.yml", + ].map((name) => readFile(path.join(repositoryRoot, ".github/workflows", name), "utf8"), ), );