From 710900bd5114b9396decd56005d14c06565bb1ae Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sat, 15 Aug 2026 15:43:33 -0700 Subject: [PATCH] test: full suite runs as N shard processes; scrub spec-sync child env MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two fixes from the wedge-hunt endgame: 1. Full-suite mode switches from one 'bun test --parallel' invocation to N concurrent shard PROCESSES, serial within each (the paid runner's proven model; N = min(6, cpus-2)). The single-invocation strategy hit three distinct Bun 1.3.13 worker pathologies in one day — a segfault whose crashed-worker retry wedged the run, a quarantined file's still-running file-level hooks stalling a worker, and spawn-heavy files hanging under load — and each one stalled the WHOLE invocation. Process shards isolate any wedge to its own shard. First full run under this model: no wedge, six epilogues, one real failure named. WORKER_HOSTILE stays as the paper trail; --parallel remains available per-shard for a future Bun. 2. That one real failure: spec-template-sync regenerates SKILL.md via a child that inherited the shard process's env — an earlier test's GSTACK_*/GBRAIN_* mutations changed generator output (failed in-suite, passed solo on an identical tree). The child now gets a scrubbed env: generator output must be a function of the templates, not of whichever test ran before. Co-Authored-By: Claude Fable 5 --- scripts/test-free-shards.ts | 57 ++++++++++++++++++--------------- test/spec-template-sync.test.ts | 11 +++++++ 2 files changed, 42 insertions(+), 26 deletions(-) diff --git a/scripts/test-free-shards.ts b/scripts/test-free-shards.ts index 4138aa9fc..8dadb86a6 100755 --- a/scripts/test-free-shards.ts +++ b/scripts/test-free-shards.ts @@ -22,7 +22,13 @@ * Execution strategy (decision ledger V3/D6 — evaluate the Bun built-in * first; probed 2026-08 on Bun 1.3.13): * - Full-suite runs (`bun test` via package.json, `bun run test:free`) use - * ONE child invocation with `--parallel`. Probes on real test files + * N CONCURRENT SHARD PROCESSES, serial within each (the paid runner's + * model). A single `--parallel` invocation was probed and initially + * adopted, then abandoned: three distinct Bun 1.3.13 worker pathologies + * (segfault + crash-retry wedge, skipped-file hooks stalling a worker, + * spawn-heavy files hanging under load) each stalled the whole + * invocation, while process shards isolate any wedge to its own shard. + * Original --parallel probe results, kept for the record: it * showed --parallel (a) prints the standard `Ran N tests across M files` * terminal summary, (b) exits non-zero when any file fails, (c) runs each * file in its own worker process (distinct pids, no shared globals), and @@ -59,7 +65,7 @@ * Exit codes: 0 pass, 1 fail, 124 wall-clock timeout. * * Usage: - * bun run scripts/test-free-shards.ts # full suite, one --parallel child + * bun run scripts/test-free-shards.ts # full suite, N concurrent shard processes * bun run scripts/test-free-shards.ts --list # show all * bun run scripts/test-free-shards.ts --windows-only --list # show curated * bun run scripts/test-free-shards.ts --windows-only # run curated @@ -203,11 +209,10 @@ export const DEFAULT_WALL_TIMEOUT_MS = 6 * 60_000; /** * Files that crash or wedge Bun's --parallel WORKERS but run fine in a plain - * serial process. Full-suite mode excludes them from the parallel invocation - * and runs them in their own serial child afterward (strict-classified like - * everything else — this is an execution-placement list, not a skip list). - * Each entry carries its reason; remove the entry when the underlying bug is - * fixed and a full parallel run stays green. + * serial process. Full-suite mode now uses shard PROCESSES (no workers), so + * this list is inert placement-wise — retained as the paper trail of why the + * one-invocation --parallel strategy was abandoned, and as the exclusion list + * should anyone re-attempt it on a newer Bun. */ export const WORKER_HOSTILE: Record = { 'browse/test/security-live-playwright.test.ts': @@ -980,27 +985,27 @@ async function main(): Promise { return exitCodeFor(outcome.status); } - // Full-suite mode: one bun invocation, files parallelized across per-file - // worker processes. See the header for the probe results that picked this - // over N spawned shard processes. Worker-hostile files are pulled out and - // run in their own serial child afterward. - const hostile = files.filter((f) => f in WORKER_HOSTILE); - const parallelFiles = files.filter((f) => !(f in WORKER_HOSTILE)); - const outcome = await runFreeShard(parallelFiles, 1, hostile.length > 0 ? 2 : 1, { - parallel: true, - wallTimeoutMs: options.wallTimeoutMs, - verbose: options.verbose, - }); - let worst = exitCodeFor(outcome.status); - if (hostile.length > 0) { - for (const f of hostile) console.log(`[test:free] serial (worker-hostile): ${f} — ${WORKER_HOSTILE[f]}`); - const serialOutcome = await runFreeShard(hostile, 2, 2, { + // Full-suite mode: N concurrent shard PROCESSES, serial within each — the + // paid runner's proven model. One `bun test --parallel` invocation was + // tried first (decision V3) and abandoned after three distinct + // worker-runtime pathologies in a single day on Bun 1.3.13: a segfault + // whose crashed-worker retry wedged the run (security-live-playwright), a + // gated file's still-running file-level hooks stalling a worker + // (compare-board), and spawn-heavy files hanging workers under load + // (session-runner-timeout). Plain child processes have none of these: + // proven spawn semantics, per-shard group-kill, per-shard logs, and a + // wedge only ever costs its own shard. WORKER_HOSTILE files are moot in + // process shards (no workers) and fold back into normal assignment. + const jobs = Math.max(1, Math.min(6, os.cpus().length - 2)); + const shards = assignFilesToShards(files, jobs); + console.log(`[test:free] full suite: ${files.length} files across ${jobs} shard processes`); + const outcomes = await Promise.all( + shards.map((shardFiles, index) => runFreeShard(shardFiles, index + 1, jobs, { wallTimeoutMs: options.wallTimeoutMs, verbose: options.verbose, - }); - worst = Math.max(worst, exitCodeFor(serialOutcome.status)); - } - return worst; + })), + ); + return Math.max(...outcomes.map((o) => exitCodeFor(o.status))); } if (import.meta.main) { diff --git a/test/spec-template-sync.test.ts b/test/spec-template-sync.test.ts index a498ca3b9..5d440db8e 100644 --- a/test/spec-template-sync.test.ts +++ b/test/spec-template-sync.test.ts @@ -20,6 +20,17 @@ describe('/spec template/generated sync', () => { cwd: ROOT, encoding: 'utf-8', timeout: 120_000, + // Scrubbed env: bun test runs a shard's files serially in ONE process, + // so an earlier test's env mutations (GSTACK_*/GBRAIN_* detection vars) + // leak into inherited process.env and change generator output — this + // test failed in-suite while passing solo on an identical tree. The + // generator's output must be a function of the templates, not of + // whichever test ran before this one. + env: { + PATH: process.env.PATH ?? '', + HOME: process.env.HOME ?? '', + TMPDIR: process.env.TMPDIR ?? '', + }, }); expect(res.status).toBe(0);