From 974949a39bf87644e1579adb545ab485a083316c Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Fri, 11 Sep 2026 07:22:03 -0700 Subject: [PATCH] ci: spread cloud server verification across ten runners (#13227) ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Cloud waits for source verification before deploying a new image. > - The slowest server verification job spends about ten minutes running tests. > - Each job uses one test worker to preserve test isolation. > - This pull request distributes those suites across ten standard hosted runners. > - The benefit is a shorter verification path with the same test coverage. ## Linked Issues or Issue Description **Current behavior** In [readiness run 34572340764](https://github.com/paperclipai/paperclip/actions/runs/34572340764), the slowest server job ran for 638 seconds. Test execution used 594 seconds. This held readiness behind the image job. **Proposed behavior** Use ten general server jobs in the reusable release verification workflow. Keep the three chat jobs and every existing prerequisite. The complete partition test verifies that no server suite is omitted or duplicated. **Reason and benefit** Reduce merge-to-deployable time on the existing runner type. The next longest prerequisite was Runner verification at 526 seconds, so the initial expected total gain is about two minutes rather than a halving of readiness time. Measure actual queue and execution time before claiming a result. Related: #13198 introduced the separate chat lane. #12577 refreshes duration estimates; this change leaves that manifest alone. ## What Changed - Increase the general server matrix from five jobs to ten. - Verify the ten-way partition covers the complete server suite when combined with the chat lane. - Document runner demand and the unchanged local and PR grouping. ## Verification - `node --test scripts/__tests__/release-verify-workflow.test.mjs scripts/__tests__/run-vitest-stable-shard.test.mjs`: 29 passed. - `actionlint .github/workflows/release-verify.yml`: passed. - Full local `pnpm -r typecheck` and `pnpm build`: passed. - All latest-head GitHub CI checks passed, including the complete Linux test partition, build, typecheck, and browser gates. Greptile: 5/5 with zero open findings. - [Ten-shard timing probe](https://github.com/paperclipai/paperclip/actions/runs/34606772388): all 16 jobs passed; slowest server job 6m 23s versus 10m 38s in the earlier five-shard sample. This compares the server lane, not total readiness, and is not a controlled same-source A/B. - The full local `pnpm test:run` is also running. It has reproduced previously observed macOS-only failures in unchanged skill-cache and native-session suites; the corresponding Linux CI suites passed. Final local results will be attached separately. No affected-workflow test failed. ## Risks Five additional concurrent jobs per release verification run increase runner demand and repeated setup work. Queueing can offset the gain. Test workers, timeouts, permissions, and readiness requirements stay unchanged. Revert the matrix and its partition test to restore the previous split. ## Model Used OpenAI GPT-6 / Codex, with reasoning, tool use, and code execution. The exact serving model identifier and context-window size are not exposed by this session. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run the affected workflow tests locally and they pass; full-suite macOS limitations are disclosed above - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- .github/workflows/release-verify.yml | 42 ++++++++++++++----- doc/cloud-build-readiness.md | 6 +++ .../release-verify-workflow.test.mjs | 2 +- .../run-vitest-stable-shard.test.mjs | 4 +- 4 files changed, 40 insertions(+), 14 deletions(-) diff --git a/.github/workflows/release-verify.yml b/.github/workflows/release-verify.yml index 79adee9229..cf6dba8319 100644 --- a/.github/workflows/release-verify.yml +++ b/.github/workflows/release-verify.yml @@ -59,28 +59,48 @@ jobs: matrix: include: # Split the long chat file by collected test locations, and balance - # the remaining server files across five runners. Normal PR/local + # the remaining server files across ten runners. Normal PR/local # invocations retain their complete general-server group. - group: general-server-without-chat - group_label: server (1/5) + group_label: server (1/10) shard_index: 0 - shard_count: 5 + shard_count: 10 - group: general-server-without-chat - group_label: server (2/5) + group_label: server (2/10) shard_index: 1 - shard_count: 5 + shard_count: 10 - group: general-server-without-chat - group_label: server (3/5) + group_label: server (3/10) shard_index: 2 - shard_count: 5 + shard_count: 10 - group: general-server-without-chat - group_label: server (4/5) + group_label: server (4/10) shard_index: 3 - shard_count: 5 + shard_count: 10 - group: general-server-without-chat - group_label: server (5/5) + group_label: server (5/10) shard_index: 4 - shard_count: 5 + shard_count: 10 + - group: general-server-without-chat + group_label: server (6/10) + shard_index: 5 + shard_count: 10 + - group: general-server-without-chat + group_label: server (7/10) + shard_index: 6 + shard_count: 10 + - group: general-server-without-chat + group_label: server (8/10) + shard_index: 7 + shard_count: 10 + - group: general-server-without-chat + group_label: server (9/10) + shard_index: 8 + shard_count: 10 + - group: general-server-without-chat + group_label: server (10/10) + shard_index: 9 + shard_count: 10 - group: general-chat group_label: chat (1/3) shard_index: 0 diff --git a/doc/cloud-build-readiness.md b/doc/cloud-build-readiness.md index e5c45c1586..1e33170162 100644 --- a/doc/cloud-build-readiness.md +++ b/doc/cloud-build-readiness.md @@ -32,6 +32,12 @@ runner capacity to avoid waiting behind an older release. No verification gate i removed from npm publication. Watch organization-wide runner queues when measuring the result. +Release verification spreads the general server suites across ten standard hosted +runners, with the long chat suite split separately across three jobs. Each server +job still runs one test worker. The partition covers every suite exactly once; +normal PR and local test groups keep their existing shape. More jobs increase +concurrent runner demand, so compare queue time as well as test duration. + The artifact wait runs for up to 30 minutes and reports what is missing. Only an HTTP 404 means publication is pending; authorization errors, upstream outages, and identity mismatches fail the job. A failed, cancelled, or skipped prerequisite diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs index 9def9e3800..ff27c430c5 100644 --- a/scripts/__tests__/release-verify-workflow.test.mjs +++ b/scripts/__tests__/release-verify-workflow.test.mjs @@ -230,7 +230,7 @@ test("release verify workflow covers the same split test surface as stable PR ve for (const group of ["general-server-without-chat", "general-chat", "general-workspaces-a", "general-workspaces-b"]) { assert.match(verifyWorkflow, new RegExp(`group: ${group}`)); } - for (const [group, count] of [["general-server-without-chat", 5], ["general-chat", 3]]) { + for (const [group, count] of [["general-server-without-chat", 10], ["general-chat", 3]]) { const rows = [...verifyWorkflow.matchAll(new RegExp(`group: ${group}\\n\\s+group_label: [^\\n]+\\n\\s+shard_index: (\\d+)\\n\\s+shard_count: (\\d+)`, "g"))]; assert.deepEqual(rows.map((row) => [Number(row[1]), Number(row[2])]), Array.from({ length: count }, (_, index) => [index, count])); diff --git a/scripts/__tests__/run-vitest-stable-shard.test.mjs b/scripts/__tests__/run-vitest-stable-shard.test.mjs index 441657d173..3d5a329eb8 100644 --- a/scripts/__tests__/run-vitest-stable-shard.test.mjs +++ b/scripts/__tests__/run-vitest-stable-shard.test.mjs @@ -270,9 +270,9 @@ test("the real shard partition is duration-balanced", () => { test("release server shards plus the dedicated chat file cover the original server group exactly", () => { const full = dryRunJson(["--mode", "general", "--group", "general-server", "--shard-index", "0", "--shard-count", "1"]); - const shards = Array.from({ length: 5 }, (_, index) => dryRunJson([ + const shards = Array.from({ length: 10 }, (_, index) => dryRunJson([ "--mode", "general", "--group", "general-server-without-chat", - "--shard-index", String(index), "--shard-count", "5", + "--shard-index", String(index), "--shard-count", "10", ])); const files = shards.flatMap((shard) => shard.selectedGeneralServerSuites); const chat = "server/src/__tests__/chat-channels.integration.test.ts";