From 165ca56a22adb60e5fda56045442d9c8498116a8 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sun, 6 Sep 2026 08:25:46 -0500 Subject: [PATCH] fix(runner): scope live eval tokens to eval repo (#12911) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - The merged direct live eval workflow must read the private `paperclip-evals` repository at an exact commit. > - Its first hosted dispatch failed before provider execution because the GitHub App token was minted from the `paperclip` repository installation. > - GitHub returned 404 while resolving the private eval commit, proving that token did not have the required repository scope. > - Minting each short-lived token from the exact private eval repository installation supplies only the cross-repository read boundary the workflow needs. > - A workflow regression now verifies every eval-token block keeps that exact scope. ## Linked Issues or Issue Description The first default-branch run of Runner Direct Live Protocol Evals failed in its immutable eval-commit verification step with HTTP 404. No provider jobs ran and no provider spend occurred. **What existing behavior does this improve?** It allows the protected direct live eval workflow to verify and check out the private `paperclipai/paperclip-evals` repository. **Current behavior** All four eval-token blocks set `GH_REPO` to `paperclipai/paperclip`, selecting a token installation that cannot read the private eval repository. **Proposed behavior** Set `GH_REPO` to the exact `paperclipai/paperclip-evals` repository in authorization, catalog, matrix, and report jobs. **Reason and benefit** The app mints a short-lived token from the correct repository installation while the main repository continues to use its ordinary read-only workflow token. **Breaking changes** None. ## What Changed - Scoped all four private-eval installation tokens to `paperclipai/paperclip-evals`. - Added a regression requiring that exact scope in every token block. ## Verification - `pnpm --filter @paperclipai/paperclip-runner test:runner-protocol-eval-publish` — 15 passed. - `node --test .github/scripts/tests/get-bot-token.test.mjs` — 3 passed. - `actionlint .github/workflows/runner-protocol-live-evals.yml` — passed. - `git diff --check` — passed. ## Risks - The workflow reads a private repository. The token is still short-lived, repository-specific, masked immediately, and used only by the protected default-branch workflow. - This changes no provider execution, Runner behavior, report content, S3 publishing, or browser E2E behavior. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used OpenAI Codex on GPT-5. The exact deployment ID and context-window size are not exposed. The model used reasoning, repository inspection, code editing, GitHub Actions diagnostics, and test execution. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change and contains no internal instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation where applicable - [x] I have considered and documented risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --- .github/workflows/runner-protocol-live-evals.yml | 8 ++++---- ...runner-protocol-eval-workflow-security.test.mjs | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/.github/workflows/runner-protocol-live-evals.yml b/.github/workflows/runner-protocol-live-evals.yml index 9f6352b033..060a7309bd 100644 --- a/.github/workflows/runner-protocol-live-evals.yml +++ b/.github/workflows/runner-protocol-live-evals.yml @@ -105,7 +105,7 @@ jobs: id: evals_token env: COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }} - GH_REPO: ${{ github.repository }} + GH_REPO: paperclipai/paperclip-evals run: | set -euo pipefail token="$(node .github/scripts/get-bot-token.mjs)" @@ -177,7 +177,7 @@ jobs: id: evals_token env: COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }} - GH_REPO: ${{ github.repository }} + GH_REPO: paperclipai/paperclip-evals run: | set -euo pipefail token="$(node .github/scripts/get-bot-token.mjs)" @@ -330,7 +330,7 @@ jobs: id: evals_token env: COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }} - GH_REPO: ${{ github.repository }} + GH_REPO: paperclipai/paperclip-evals run: | set -euo pipefail token="$(node .github/scripts/get-bot-token.mjs)" @@ -500,7 +500,7 @@ jobs: id: evals_token env: COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }} - GH_REPO: ${{ github.repository }} + GH_REPO: paperclipai/paperclip-evals run: | set -euo pipefail token="$(node .github/scripts/get-bot-token.mjs)" diff --git a/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs b/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs index 5e9f990cb8..d90a69e58e 100644 --- a/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs +++ b/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs @@ -45,6 +45,7 @@ test("resolves both repositories immutably and bounds total matrix concurrency", /repos\/paperclipai\/paperclip-evals\/commits\/\$EVALS_SHA/u, ); assert.match(authorize, /COMMITPERCLIP_KEY/u); + assert.match(authorize, /GH_REPO: paperclipai\/paperclip-evals/u); assert.match( authorize, /GH_TOKEN: \$\{\{ steps\.evals_token\.outputs\.value \}\}/u, @@ -66,6 +67,19 @@ test("resolves both repositories immutably and bounds total matrix concurrency", ), ]; assert.equal(privateCheckouts.length, 3); + const privateTokenSteps = [ + ...workflow.matchAll( + /^ - name: Generate private eval-repository token\n(?(?:^ {8,}.*\n?)*)/gmu, + ), + ]; + assert.equal(privateTokenSteps.length, 4); + for (const tokenStep of privateTokenSteps) { + assert.match( + tokenStep.groups.body, + /^ {10}GH_REPO: paperclipai\/paperclip-evals$/mu, + "every private-eval token must be minted from the eval repository installation", + ); + } for (const checkout of privateCheckouts) { assert.match( checkout[0],