ci: balance remaining server suites with measured native Runner cost
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Devin Foley
parent
9066878f18
commit
fbbb64bfa2
|
|
@ -59,24 +59,28 @@ jobs:
|
|||
matrix:
|
||||
include:
|
||||
# Split the long chat file by collected test locations, and balance
|
||||
# the remaining server files across four runners. Normal PR/local
|
||||
# the remaining server files across five runners. Normal PR/local
|
||||
# invocations retain their complete general-server group.
|
||||
- group: general-server-without-chat
|
||||
group_label: server (1/4)
|
||||
group_label: server (1/5)
|
||||
shard_index: 0
|
||||
shard_count: 4
|
||||
shard_count: 5
|
||||
- group: general-server-without-chat
|
||||
group_label: server (2/4)
|
||||
group_label: server (2/5)
|
||||
shard_index: 1
|
||||
shard_count: 4
|
||||
shard_count: 5
|
||||
- group: general-server-without-chat
|
||||
group_label: server (3/4)
|
||||
group_label: server (3/5)
|
||||
shard_index: 2
|
||||
shard_count: 4
|
||||
shard_count: 5
|
||||
- group: general-server-without-chat
|
||||
group_label: server (4/4)
|
||||
group_label: server (4/5)
|
||||
shard_index: 3
|
||||
shard_count: 4
|
||||
shard_count: 5
|
||||
- group: general-server-without-chat
|
||||
group_label: server (5/5)
|
||||
shard_index: 4
|
||||
shard_count: 5
|
||||
- group: general-chat
|
||||
group_label: chat (1/3)
|
||||
shard_index: 0
|
||||
|
|
|
|||
|
|
@ -363,7 +363,7 @@ See [GitHub cache access restrictions](https://docs.github.com/en/actions/refere
|
|||
## Chat integration test shards
|
||||
|
||||
Release verification runs the large chat integration file on three independent
|
||||
runners. Four other server shards cover every remaining general server file.
|
||||
runners. Five other server shards cover every remaining general server file.
|
||||
The ordinary local test command and trusted PR workflow keep their complete
|
||||
`general-server` group. Each chat case shuts down its services, pauses its own
|
||||
still-active endpoints, and retires its active/waiting conversations after
|
||||
|
|
@ -385,5 +385,9 @@ pnpm test:run:general -- --group general-chat --shard-index 0 --shard-count 3
|
|||
Use indexes 0, 1, and 2 to run the complete chat suite. The CLI validates that
|
||||
each shard has work and that collection includes usable source locations. A
|
||||
Vitest collection or filtering change fails verification instead of dropping
|
||||
tests. Splitting adds two release-verification jobs and repeats collection and
|
||||
tests. Splitting adds three release-verification jobs and repeats collection and
|
||||
fixture setup; it does not make a single test faster.
|
||||
|
||||
The file-duration manifest also records the native Codex Runner integration
|
||||
suite's measured import and execution cost, so the existing file balancer
|
||||
accounts for it in both ordinary PR and release verification.
|
||||
|
|
|
|||
|
|
@ -209,7 +209,7 @@ test("release verify workflow covers the same split test surface as stable PR ve
|
|||
for (const group of ["general-server-without-chat", "general-chat", "general-workspaces-a", "general-workspaces-b"]) {
|
||||
assert.match(verifyWorkflow, new RegExp(`group: ${group}`));
|
||||
}
|
||||
for (const [group, count] of [["general-server-without-chat", 4], ["general-chat", 3]]) {
|
||||
for (const [group, count] of [["general-server-without-chat", 5], ["general-chat", 3]]) {
|
||||
const rows = [...verifyWorkflow.matchAll(new RegExp(`group: ${group}\\n\\s+group_label: [^\\n]+\\n\\s+shard_index: (\\d+)\\n\\s+shard_count: (\\d+)`, "g"))];
|
||||
assert.deepEqual(rows.map((row) => [Number(row[1]), Number(row[2])]),
|
||||
Array.from({ length: count }, (_, index) => [index, count]));
|
||||
|
|
|
|||
|
|
@ -270,9 +270,9 @@ test("the real shard partition is duration-balanced", () => {
|
|||
|
||||
test("release server shards plus the dedicated chat file cover the original server group exactly", () => {
|
||||
const full = dryRunJson(["--mode", "general", "--group", "general-server", "--shard-index", "0", "--shard-count", "1"]);
|
||||
const shards = Array.from({ length: 4 }, (_, index) => dryRunJson([
|
||||
const shards = Array.from({ length: 5 }, (_, index) => dryRunJson([
|
||||
"--mode", "general", "--group", "general-server-without-chat",
|
||||
"--shard-index", String(index), "--shard-count", "4",
|
||||
"--shard-index", String(index), "--shard-count", "5",
|
||||
]));
|
||||
const files = shards.flatMap((shard) => shard.selectedGeneralServerSuites);
|
||||
const chat = "server/src/__tests__/chat-channels.integration.test.ts";
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
{
|
||||
"$comment": "Per-suite wall-clock durations (ms) for the general-server vitest lane, used by scripts/general-server-shard.mjs to balance suites across the PR shard matrix. Sampled from a real PR run of .github/workflows/pr.yml (actions run 32708351172, 2026-08-24) by diffing consecutive per-suite completion timestamps in the 'Run grouped general test suites' logs \u2014 that captures each suite's true serial cost (import + collect + tests), not just the vitest-reported test time. Suites missing here get the median weight, so the manifest only needs occasional refreshes.",
|
||||
"$chatSample": "chat-channels.integration.test.ts: actions run 34405038082, job 102646337040, 2026-09-09. The first suite completed at 21:19:52.9026416Z after Vitest RUN at 21:09:07.5491992Z: 645354ms rounded up, including startup/import/collection; the 985 tests themselves took 629654ms. All 2972 tests in the shard passed, but the job exceeded its unchanged 20-minute bound during cleanup. Recording this missing heavy-suite weight lets the existing LPT partition reserve one of the existing five shards without changing test coverage, isolation, or deadlines.",
|
||||
"$nativeRunnerSample": "native-codex-runner.integration.test.ts: actions run 34555686996, job 103127786254, 2026-09-11. Consecutive suite completions at 02:50:34.2830368Z and 02:55:07.9837588Z give 273701ms including import/collection (test body 270773ms). This previously unweighted suite made one four-way server shard take 14m38s; recording its cost lets the existing LPT partition balance it in both PR and release runs.",
|
||||
"durations": {
|
||||
"server/src/services/native-runtime/native-codex-runner.integration.test.ts": 273701,
|
||||
"server/src/__tests__/access-service.test.ts": 4757,
|
||||
"server/src/__tests__/access-validators.test.ts": 645,
|
||||
"server/src/__tests__/activity-log-responsible-user.test.ts": 4407,
|
||||
|
|
|
|||
Loading…
Reference in New Issue