From 2a05b5ed3457ea33efd6895520447d1d97fe98d8 Mon Sep 17 00:00:00 2001 From: Nicky Leach Date: Thu, 10 Sep 2026 01:18:06 -0700 Subject: [PATCH] ci: split runner verification from build (#13142) ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Paperclip uses GitHub Actions to verify changes before release. > - The Paperclip Runner has a separate verification boundary. > - The build job currently runs this verification before the workspace build. > - This pull request moves runner verification into its own parallel job. > - The benefit is clearer CI results and less wait time for independent work. ## Linked Issues or Issue Description **What existing behavior does this improve?** The trusted PR and release verification workflows run Paperclip Runner verification inside the Build job. **Subsystem affected** Cross-cutting (GitHub Actions CI workflows). **Current behavior** The Build job runs `pnpm --filter @paperclipai/paperclip-runner check:all` before it builds the workspace. A runner verification failure appears as a Build failure. The workspace build cannot run in parallel with runner verification. **Proposed behavior** Each workflow has a `Verify Paperclip Runner` job with the same checkout, dependency install, and command. The Build job only builds its required outputs. Both jobs run after the same gate and policy jobs. **Reason and benefit** The runner command is an independent verification boundary. A dedicated job gives it a clear status and allows it to run in parallel with Build. **Breaking changes** None. The same runner verification command still runs in both workflows. ## What Changed - Added a dedicated `Verify Paperclip Runner` job to the trusted PR workflow. - Added a dedicated `Verify Paperclip Runner` job to the release verification workflow. - Kept the Build jobs independent and retained their existing build commands. - Updated the trusted-workflow policy test for the additional dependency-install job. ## Verification - Ran `git diff --check`. - Ran `node --test ./scripts/__tests__/e2e-shard.test.mjs`. - Ran `pnpm exec prettier --check .github/workflows/pr-trusted.yml .github/workflows/release-verify.yml`. - Confirmed both jobs retain their prior runner, dependency, and policy prerequisites. ## Risks Low risk. The runner verification job repeats the existing setup. It adds one parallel GitHub Actions runner to each affected workflow. ## Model Used OpenAI Codex, GPT-5.6, 128k context window, reasoning and tool-use capabilities. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- .github/workflows/pr-trusted.yml | 52 +++++++++++++++++-- .github/workflows/release-verify.yml | 31 ++++++++++- scripts/__tests__/e2e-shard.test.mjs | 10 +++- .../release-verify-workflow.test.mjs | 4 ++ 4 files changed, 90 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pr-trusted.yml b/.github/workflows/pr-trusted.yml index 40168b4565..0dd8d149a9 100644 --- a/.github/workflows/pr-trusted.yml +++ b/.github/workflows/pr-trusted.yml @@ -553,7 +553,7 @@ jobs: # Preserve the legacy required-check name while the underlying work runs in parallel. name: verify if: ${{ always() }} - needs: [gate, policy, typecheck_release_registry, general_tests, build, docker_context_integrity] + needs: [gate, policy, typecheck_release_registry, general_tests, verify_paperclip_runner, build, docker_context_integrity] runs-on: ${{ needs.gate.outputs.runner }} timeout-minutes: 5 @@ -564,6 +564,7 @@ jobs: POLICY_RESULT: ${{ needs.policy.result }} TYPECHECK_RELEASE_REGISTRY_RESULT: ${{ needs.typecheck_release_registry.result }} GENERAL_TESTS_RESULT: ${{ needs.general_tests.result }} + RUNNER_VERIFICATION_RESULT: ${{ needs.verify_paperclip_runner.result }} BUILD_RESULT: ${{ needs.build.result }} DOCKER_CONTEXT_INTEGRITY_RESULT: ${{ needs.docker_context_integrity.result }} run: | @@ -572,12 +573,14 @@ jobs: true) test "$TYPECHECK_RELEASE_REGISTRY_RESULT" = "success" test "$GENERAL_TESTS_RESULT" = "success" + test "$RUNNER_VERIFICATION_RESULT" = "success" test "$BUILD_RESULT" = "success" test "$DOCKER_CONTEXT_INTEGRITY_RESULT" = "success" ;; false) test "$TYPECHECK_RELEASE_REGISTRY_RESULT" = "skipped" test "$GENERAL_TESTS_RESULT" = "skipped" + test "$RUNNER_VERIFICATION_RESULT" = "skipped" test "$BUILD_RESULT" = "skipped" test "$DOCKER_CONTEXT_INTEGRITY_RESULT" = "skipped" ;; @@ -587,8 +590,8 @@ jobs: ;; esac - build: - name: Build + verify_paperclip_runner: + name: Verify Paperclip Runner needs: [gate, policy] if: ${{ needs.gate.outputs.full_ci == 'true' }} runs-on: ${{ needs.gate.outputs.runner }} @@ -633,6 +636,49 @@ jobs: - name: Verify Paperclip Runner run: pnpm --filter @paperclipai/paperclip-runner check:all + build: + name: Build + needs: [gate, policy] + if: ${{ needs.gate.outputs.full_ci == 'true' }} + runs-on: ${{ needs.gate.outputs.runner }} + timeout-minutes: 20 + + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Setup Node.js for pnpm bootstrap + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + + - name: Setup pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 + env: + NPM_CONFIG_AUDIT: "false" + NPM_CONFIG_FUND: "false" + NPM_CONFIG_UPDATE_NOTIFIER: "false" + with: + version: 9.15.4 + + - name: Restore regenerated PR lockfile (if policy uploaded one) + if: needs.policy.outputs.lockfile_regenerated == '1' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: pr-lockfile + path: . + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + - name: Build Runner Evalbook viewer run: pnpm --filter @paperclipai/paperclip-runner build:issue-thread diff --git a/.github/workflows/release-verify.yml b/.github/workflows/release-verify.yml index a8bc1abbf7..7193f38c84 100644 --- a/.github/workflows/release-verify.yml +++ b/.github/workflows/release-verify.yml @@ -191,8 +191,8 @@ jobs: - name: Run deterministic Runner workflow scorer tests run: pnpm test:runner-workflow-evals - build: - name: Build + verify_paperclip_runner: + name: Verify Paperclip Runner runs-on: ubuntu-latest timeout-minutes: 20 permissions: @@ -221,5 +221,32 @@ jobs: - name: Verify Paperclip Runner run: pnpm --filter @paperclipai/paperclip-runner check:all + build: + name: Build + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ inputs.ref }} + persist-credentials: false + + - name: Setup pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 + with: + version: 9.15.4 + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + + - name: Install dependencies + run: pnpm install --no-frozen-lockfile + - name: Build run: pnpm build diff --git a/scripts/__tests__/e2e-shard.test.mjs b/scripts/__tests__/e2e-shard.test.mjs index 1fed746a27..00893bc8bc 100644 --- a/scripts/__tests__/e2e-shard.test.mjs +++ b/scripts/__tests__/e2e-shard.test.mjs @@ -222,6 +222,7 @@ test("the trusted PR workflow limits full CI to merge-relevant stack layers", () for (const jobId of [ "typecheck_release_registry", "general_tests", + "verify_paperclip_runner", "build", "verify_serialized_server", "canary_dry_run", @@ -243,11 +244,16 @@ test("the trusted PR workflow limits full CI to merge-relevant stack layers", () const verify = jobs.get("verify"); assert.match( verify, - /^ {4}needs: \[gate, policy, typecheck_release_registry, general_tests, build, docker_context_integrity\]$/m, + /^ {4}needs: \[gate, policy, typecheck_release_registry, general_tests, verify_paperclip_runner, build, docker_context_integrity\]$/m, ); assert.match(verify, /POLICY_RESULT: \$\{\{ needs\.policy\.result \}\}/); assert.match(verify, /test "\$TYPECHECK_RELEASE_REGISTRY_RESULT" = "skipped"/); assert.match(verify, /test "\$GENERAL_TESTS_RESULT" = "skipped"/); + // Runner verification must participate in the legacy aggregate required + // check, or a runner regression could be merged while `verify` succeeds. + assert.match(verify, /RUNNER_VERIFICATION_RESULT: \$\{\{ needs\.verify_paperclip_runner\.result \}\}/); + assert.match(verify, /test "\$RUNNER_VERIFICATION_RESULT" = "success"/); + assert.match(verify, /test "\$RUNNER_VERIFICATION_RESULT" = "skipped"/); assert.match(verify, /test "\$BUILD_RESULT" = "skipped"/); // Both halves of the docker-context lane's gating: the result must be // wired into the aggregate's env AND asserted successful on full CI — @@ -327,7 +333,7 @@ test("the trusted PR workflow regenerates stale stacked lockfiles", () => { const restoreSteps = workflow.match( /- name: Restore regenerated PR lockfile \(if policy uploaded one\)\n if: needs\.policy\.outputs\.lockfile_regenerated == '1'/g, ) ?? []; - assert.equal(restoreSteps.length, 6, "every downstream install job must restore a required regenerated artifact"); + assert.equal(restoreSteps.length, 7, "every downstream install job must restore a required regenerated artifact"); assert.doesNotMatch( workflow, /- name: Restore regenerated PR lockfile \(if policy uploaded one\)[\s\S]{0,220}continue-on-error:/, diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs index aaf3179215..8b18a89039 100644 --- a/scripts/__tests__/release-verify-workflow.test.mjs +++ b/scripts/__tests__/release-verify-workflow.test.mjs @@ -202,6 +202,10 @@ test("release verify workflow covers the same split test surface as stable PR ve ); assert.match(verifyWorkflow, /pnpm test:runner-workflow-evals/); + const buildJob = verifyWorkflow.match(/ build:\n[\s\S]*?(?=\n [A-Za-z0-9_-]+:|$)/)?.[0] ?? ""; + assert.match(buildJob, /persist-credentials: false/); + assert.doesNotMatch(buildJob, /cache: pnpm/); + for (const group of [ "general-server", "general-workspaces-a",