test: fix two parallelism-exposed flakes (probe re-run, live-tree census)

Both pass solo and on main but flaked under the parallel runner:

1. gstack-brain-context-load probed 'gbrain --version' PER QUERY with a
   500ms budget — a cold probe on a saturated box timed out (observed
   505ms), branding gbrain 'missing' for one query while siblings
   passed. The probe is now memoized (availability can't change
   mid-invocation) with a generous one-time 5s budget; query calls keep
   the tight timeout.

2. skill-size-budget's catalog estimate read the LIVE tree, so a
   concurrent worker's transient skill-shaped scratch dirs exactly
   doubled it (8356 vs 4177). The ratchet now counts git-TRACKED skills
   only — the catalog that ships, immune to sibling workers.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan 2026-08-15 09:13:18 -07:00
parent 1e0ff1e1e2
commit c9df02877c
No known key found for this signature in database
GPG Key ID: C1F69E85C74EFE1D
2 changed files with 35 additions and 7 deletions

View File

@ -190,16 +190,26 @@ function resolveSkillFile(args: CliArgs): string | null {
// ── Dispatchers ────────────────────────────────────────────────────────────
// Memoized: availability can't change mid-invocation, and the per-query
// re-probe was both wasteful (N probes per run) and load-flaky — a cold
// `gbrain --version` on a saturated box can exceed the 500ms budget, branding
// gbrain "missing" for one query while its siblings succeed (observed under
// the parallel free-suite runner: SKIP at dur=505ms with two OKs after it).
let _gbrainAvailable: boolean | null = null;
function gbrainAvailable(): boolean {
if (_gbrainAvailable !== null) return _gbrainAvailable;
try {
execFileSync("gbrain", ["--version"], {
stdio: "ignore",
timeout: MCP_TIMEOUT_MS,
// Generous first-probe budget: this runs ONCE, and a slow-to-start CLI
// is not a missing CLI. Query calls keep the tight MCP_TIMEOUT_MS.
timeout: 5_000,
});
return true;
_gbrainAvailable = true;
} catch {
return false;
_gbrainAvailable = false;
}
return _gbrainAvailable;
}
function dispatchVector(q: GbrainManifestQuery, args: CliArgs): QueryResult {

View File

@ -31,6 +31,7 @@
import { describe, test, expect } from 'bun:test';
import * as fs from 'fs';
import * as path from 'path';
import { execSync } from 'child_process';
import { captureBaseline, type ParityBaseline } from './helpers/capture-parity-baseline';
import { logBudgetOverride } from './helpers/budget-override';
import { CARVED_SKILLS } from './helpers/carve-guards';
@ -212,10 +213,27 @@ describe('SKILL.md size budget regression (gate, free)', () => {
test('catalog token estimate stays compressed (v1.45 target ≤ 7000)', () => {
const current = captureBaseline({ repoRoot: REPO_ROOT });
// Count only git-TRACKED skills. Under the parallel free-suite runner,
// concurrent test files can leave transient skill-shaped dirs in the live
// repo mid-run (observed: this estimate exactly DOUBLED, 8356 vs 4177,
// while a solo run passed). A repo-budget ratchet should measure the
// catalog that ships, not another worker's scratch state.
const tracked = new Set(
execSync('git ls-files -- "*/SKILL.md"', { cwd: REPO_ROOT, encoding: 'utf-8' })
.split('\n')
.filter(Boolean)
.filter((p) => p.split('/').length === 2)
.map((p) => p.split('/')[0]),
);
const catalogTokens = Math.round(
Object.values(current.skills)
.filter((s) => tracked.has(s.skill))
.reduce((sum, s) => sum + s.descriptionLen, 0) / 4,
);
const v145Target = 7000;
if (current.estTotalCatalogTokens <= v145Target) {
if (catalogTokens <= v145Target) {
// eslint-disable-next-line no-console
console.log(`[skill-size-budget] catalog OK: ~${current.estTotalCatalogTokens} tokens (target ≤${v145Target})`);
console.log(`[skill-size-budget] catalog OK: ~${catalogTokens} tokens (target ≤${v145Target}, ${tracked.size} tracked skills)`);
return;
}
const overrideReason = process.env.GSTACK_SIZE_BUDGET_OVERRIDE_REASON?.trim();
@ -223,12 +241,12 @@ describe('SKILL.md size budget regression (gate, free)', () => {
logBudgetOverride({
scope: 'skill-size-budget-catalog',
reason: overrideReason,
details: { target: v145Target, observed: current.estTotalCatalogTokens },
details: { target: v145Target, observed: catalogTokens },
});
return;
}
throw new Error(
`Catalog token estimate regressed past v1.45 target: ${current.estTotalCatalogTokens} tokens > ${v145Target}. ` +
`Catalog token estimate regressed past v1.45 target: ${catalogTokens} tokens > ${v145Target}. ` +
`T4 catalog trim should keep this under control. Override: set GSTACK_SIZE_BUDGET_OVERRIDE_REASON to allow.`,
);
});