diff --git a/.github/workflows/runner-protocol-live-evals.yml b/.github/workflows/runner-protocol-live-evals.yml index 9f6352b033..060a7309bd 100644 --- a/.github/workflows/runner-protocol-live-evals.yml +++ b/.github/workflows/runner-protocol-live-evals.yml @@ -105,7 +105,7 @@ jobs: id: evals_token env: COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }} - GH_REPO: ${{ github.repository }} + GH_REPO: paperclipai/paperclip-evals run: | set -euo pipefail token="$(node .github/scripts/get-bot-token.mjs)" @@ -177,7 +177,7 @@ jobs: id: evals_token env: COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }} - GH_REPO: ${{ github.repository }} + GH_REPO: paperclipai/paperclip-evals run: | set -euo pipefail token="$(node .github/scripts/get-bot-token.mjs)" @@ -330,7 +330,7 @@ jobs: id: evals_token env: COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }} - GH_REPO: ${{ github.repository }} + GH_REPO: paperclipai/paperclip-evals run: | set -euo pipefail token="$(node .github/scripts/get-bot-token.mjs)" @@ -500,7 +500,7 @@ jobs: id: evals_token env: COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }} - GH_REPO: ${{ github.repository }} + GH_REPO: paperclipai/paperclip-evals run: | set -euo pipefail token="$(node .github/scripts/get-bot-token.mjs)" diff --git a/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs b/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs index 5e9f990cb8..d90a69e58e 100644 --- a/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs +++ b/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs @@ -45,6 +45,7 @@ test("resolves both repositories immutably and bounds total matrix concurrency", /repos\/paperclipai\/paperclip-evals\/commits\/\$EVALS_SHA/u, ); assert.match(authorize, /COMMITPERCLIP_KEY/u); + assert.match(authorize, /GH_REPO: paperclipai\/paperclip-evals/u); assert.match( authorize, /GH_TOKEN: \$\{\{ steps\.evals_token\.outputs\.value \}\}/u, @@ -66,6 +67,19 @@ test("resolves both repositories immutably and bounds total matrix concurrency", ), ]; assert.equal(privateCheckouts.length, 3); + const privateTokenSteps = [ + ...workflow.matchAll( + /^ - name: Generate private eval-repository token\n(?(?:^ {8,}.*\n?)*)/gmu, + ), + ]; + assert.equal(privateTokenSteps.length, 4); + for (const tokenStep of privateTokenSteps) { + assert.match( + tokenStep.groups.body, + /^ {10}GH_REPO: paperclipai\/paperclip-evals$/mu, + "every private-eval token must be minted from the eval repository installation", + ); + } for (const checkout of privateCheckouts) { assert.match( checkout[0],