diff --git a/.github/workflows/release-verify.yml b/.github/workflows/release-verify.yml index 670aac3896..79adee9229 100644 --- a/.github/workflows/release-verify.yml +++ b/.github/workflows/release-verify.yml @@ -59,24 +59,28 @@ jobs: matrix: include: # Split the long chat file by collected test locations, and balance - # the remaining server files across four runners. Normal PR/local + # the remaining server files across five runners. Normal PR/local # invocations retain their complete general-server group. - group: general-server-without-chat - group_label: server (1/4) + group_label: server (1/5) shard_index: 0 - shard_count: 4 + shard_count: 5 - group: general-server-without-chat - group_label: server (2/4) + group_label: server (2/5) shard_index: 1 - shard_count: 4 + shard_count: 5 - group: general-server-without-chat - group_label: server (3/4) + group_label: server (3/5) shard_index: 2 - shard_count: 4 + shard_count: 5 - group: general-server-without-chat - group_label: server (4/4) + group_label: server (4/5) shard_index: 3 - shard_count: 4 + shard_count: 5 + - group: general-server-without-chat + group_label: server (5/5) + shard_index: 4 + shard_count: 5 - group: general-chat group_label: chat (1/3) shard_index: 0 diff --git a/doc/RELEASE-AUTOMATION-SETUP.md b/doc/RELEASE-AUTOMATION-SETUP.md index 3eaf5ceac2..43b6fee7bd 100644 --- a/doc/RELEASE-AUTOMATION-SETUP.md +++ b/doc/RELEASE-AUTOMATION-SETUP.md @@ -363,7 +363,7 @@ See [GitHub cache access restrictions](https://docs.github.com/en/actions/refere ## Chat integration test shards Release verification runs the large chat integration file on three independent -runners. Four other server shards cover every remaining general server file. +runners. Five other server shards cover every remaining general server file. The ordinary local test command and trusted PR workflow keep their complete `general-server` group. Each chat case shuts down its services, pauses its own still-active endpoints, and retires its active/waiting conversations after @@ -385,5 +385,9 @@ pnpm test:run:general -- --group general-chat --shard-index 0 --shard-count 3 Use indexes 0, 1, and 2 to run the complete chat suite. The CLI validates that each shard has work and that collection includes usable source locations. A Vitest collection or filtering change fails verification instead of dropping -tests. Splitting adds two release-verification jobs and repeats collection and +tests. Splitting adds three release-verification jobs and repeats collection and fixture setup; it does not make a single test faster. + +The file-duration manifest also records the native Codex Runner integration +suite's measured import and execution cost, so the existing file balancer +accounts for it in both ordinary PR and release verification. diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs index a1e9207c04..3ff0d35b27 100644 --- a/scripts/__tests__/release-verify-workflow.test.mjs +++ b/scripts/__tests__/release-verify-workflow.test.mjs @@ -209,7 +209,7 @@ test("release verify workflow covers the same split test surface as stable PR ve for (const group of ["general-server-without-chat", "general-chat", "general-workspaces-a", "general-workspaces-b"]) { assert.match(verifyWorkflow, new RegExp(`group: ${group}`)); } - for (const [group, count] of [["general-server-without-chat", 4], ["general-chat", 3]]) { + for (const [group, count] of [["general-server-without-chat", 5], ["general-chat", 3]]) { const rows = [...verifyWorkflow.matchAll(new RegExp(`group: ${group}\\n\\s+group_label: [^\\n]+\\n\\s+shard_index: (\\d+)\\n\\s+shard_count: (\\d+)`, "g"))]; assert.deepEqual(rows.map((row) => [Number(row[1]), Number(row[2])]), Array.from({ length: count }, (_, index) => [index, count])); diff --git a/scripts/__tests__/run-vitest-stable-shard.test.mjs b/scripts/__tests__/run-vitest-stable-shard.test.mjs index 51bc54420a..441657d173 100644 --- a/scripts/__tests__/run-vitest-stable-shard.test.mjs +++ b/scripts/__tests__/run-vitest-stable-shard.test.mjs @@ -270,9 +270,9 @@ test("the real shard partition is duration-balanced", () => { test("release server shards plus the dedicated chat file cover the original server group exactly", () => { const full = dryRunJson(["--mode", "general", "--group", "general-server", "--shard-index", "0", "--shard-count", "1"]); - const shards = Array.from({ length: 4 }, (_, index) => dryRunJson([ + const shards = Array.from({ length: 5 }, (_, index) => dryRunJson([ "--mode", "general", "--group", "general-server-without-chat", - "--shard-index", String(index), "--shard-count", "4", + "--shard-index", String(index), "--shard-count", "5", ])); const files = shards.flatMap((shard) => shard.selectedGeneralServerSuites); const chat = "server/src/__tests__/chat-channels.integration.test.ts"; diff --git a/scripts/general-server-shard-durations.json b/scripts/general-server-shard-durations.json index d4f71285ac..115a12669d 100644 --- a/scripts/general-server-shard-durations.json +++ b/scripts/general-server-shard-durations.json @@ -1,7 +1,9 @@ { "$comment": "Per-suite wall-clock durations (ms) for the general-server vitest lane, used by scripts/general-server-shard.mjs to balance suites across the PR shard matrix. Sampled from a real PR run of .github/workflows/pr.yml (actions run 32708351172, 2026-08-24) by diffing consecutive per-suite completion timestamps in the 'Run grouped general test suites' logs \u2014 that captures each suite's true serial cost (import + collect + tests), not just the vitest-reported test time. Suites missing here get the median weight, so the manifest only needs occasional refreshes.", "$chatSample": "chat-channels.integration.test.ts: actions run 34405038082, job 102646337040, 2026-09-09. The first suite completed at 21:19:52.9026416Z after Vitest RUN at 21:09:07.5491992Z: 645354ms rounded up, including startup/import/collection; the 985 tests themselves took 629654ms. All 2972 tests in the shard passed, but the job exceeded its unchanged 20-minute bound during cleanup. Recording this missing heavy-suite weight lets the existing LPT partition reserve one of the existing five shards without changing test coverage, isolation, or deadlines.", + "$nativeRunnerSample": "native-codex-runner.integration.test.ts: actions run 34555686996, job 103127786254, 2026-09-11. Consecutive suite completions at 02:50:34.2830368Z and 02:55:07.9837588Z give 273701ms including import/collection (test body 270773ms). This previously unweighted suite made one four-way server shard take 14m38s; recording its cost lets the existing LPT partition balance it in both PR and release runs.", "durations": { + "server/src/services/native-runtime/native-codex-runner.integration.test.ts": 273701, "server/src/__tests__/access-service.test.ts": 4757, "server/src/__tests__/access-validators.test.ts": 645, "server/src/__tests__/activity-log-responsible-user.test.ts": 4407,