test(catalog): aggregate discovery-surface budget with ratchet protocol

Every host loads every skill's frontmatter name + description at
discovery, every session. applyCatalogTrim in scripts/gen-skill-docs.ts
shapes each description and the 160KB per-file warn covers body size,
but nothing capped the aggregate frontmatter — the catalog could grow
one reasonable-looking description at a time. This test is that
enforcement layer.

Measures the catalog via test/helpers/skill-census.ts authoredSkills
(symlink-deduped, root router counted separately as the _gstack-command
alias line item): 53 skills + router = 4,420 bytes = 1,105
token-equivalents today, asserted <= 1,150 (~4% headroom). Per-skill
sub-cap of 260 bytes (largest today: design-consultation at 229), plus
a non-empty-description check.

Failure messages are self-service ratchets: they print the new total,
the delta, and the update protocol (bump the constant AND the
derivation comment in the same commit; trim instead of grow for
existing descriptions). Parser handles folded block scalars
(description: >-) for fork parity; import-free by design so it
survives generator refactors.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
(cherry picked from commit c106fb36f768181b80c257e5cff1cde4f435f9c0)
This commit is contained in:
Garry Tan 2026-08-12 10:56:45 -07:00
parent 8d1d15ff34
commit 2ae38785a1
No known key found for this signature in database
GPG Key ID: C1F69E85C74EFE1D
1 changed files with 122 additions and 0 deletions

122
test/catalog-budget.test.ts Normal file
View File

@ -0,0 +1,122 @@
import { describe, expect, test } from 'bun:test';
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
import { skillCensus } from './helpers/skill-census';
/**
* Aggregate discovery-surface budget: the sum of every skill's frontmatter
* `name` + `description` is what EVERY host loads at discovery, every session.
*
* This is the missing enforcement layer over the existing catalog-trim
* mechanism: `applyCatalogTrim` in scripts/gen-skill-docs.ts (~line 865)
* shapes each description, and the 160KB per-file warn (~line 1015) covers
* BODY size neither caps the aggregate frontmatter the catalog is made of.
*
* Import-free by design: parses skills' SKILL.md frontmatter directly. Do not
* import gen-skill-docs internals here this test must survive generator
* refactors.
*
* Budget derivation (re-derive it, do not trust the number):
* ref this commit
* method for each authored skill (test/helpers/skill-census.ts
* authoredSkills symlink-deduped, root router excluded) plus the
* root router's `_gstack-command` alias frontmatter as one separate
* line item, run parseFrontmatter() below and sum
* Buffer.byteLength(name) + Buffer.byteLength(description);
* token-equivalents = ceil(bytes / 4).
* result 53 authored skills = 4,371 bytes (1,093 token-equivalents);
* + root router alias 49 bytes = 4,420 bytes total
* = 1,105 token-equivalents (measured 2026-08-12)
* Ceiling is 1,150 token-equivalents (4,600 bytes), so headroom is 180 bytes
* (~4%). Dominant skill: design-consultation at 229 bytes name+description.
*/
const CATALOG_BUDGET_TOKEN_EQUIVALENTS = 1_150;
// Largest today: design-consultation at 229 bytes. A description that needs
// more than 260 bytes is a body paragraph, not a catalog entry.
const PER_SKILL_BYTE_CAP = 260;
const RATCHET_PROTOCOL =
'Adding a skill? Re-measure with: bun test test/catalog-budget.test.ts ' +
'(the failure prints the new total). Update CATALOG_BUDGET_TOKEN_EQUIVALENTS ' +
'AND the derivation comment (ref/date/value/which skill moved it) in the ' +
'SAME commit. Growing an existing description? Trim it instead — the ' +
'catalog is what every host loads at discovery, every session.';
const ROOT = join(import.meta.dir, '..');
function parseFrontmatter(body: string): { name: string; description: string } {
const name = body.match(/^name:\s*(.+)$/m)?.[1]?.trim() ?? '';
// Folded block scalar (description: >-) with two-space-indented continuation
// lines, falling back to a single-line description.
const folded = body.match(/^description:\s*>-?\r?\n((?: .*\r?\n)+)/m)?.[1];
const description = folded
? folded.split(/\r?\n/).map((line) => line.trim()).filter(Boolean).join(' ')
: body.match(/^description:\s*(?!>-?\s*$)(.+)$/m)?.[1]?.trim() ?? '';
return { name, description };
}
interface CatalogEntry {
skill: string;
name: string;
description: string;
bytes: number;
}
function catalogEntries(): CatalogEntry[] {
const entries: CatalogEntry[] = [];
for (const skill of skillCensus(ROOT).authoredSkills) {
const body = readFileSync(join(ROOT, skill, 'SKILL.md'), 'utf8');
const { name, description } = parseFrontmatter(body);
entries.push({
skill,
name,
description,
bytes: Buffer.byteLength(name) + Buffer.byteLength(description),
});
}
// The root SKILL.md is a router, registered by setup as the
// `_gstack-command` alias — not an authored skill, but its frontmatter
// still ships in the catalog, so it counts as one line item.
const router = parseFrontmatter(readFileSync(join(ROOT, 'SKILL.md'), 'utf8'));
if (router.name && router.description) {
entries.push({
skill: '(root router)',
name: router.name,
description: router.description,
bytes: Buffer.byteLength(router.name) + Buffer.byteLength(router.description),
});
}
return entries;
}
describe('catalog discovery-surface budget', () => {
test(`aggregate frontmatter stays within ${CATALOG_BUDGET_TOKEN_EQUIVALENTS} token-equivalents`, () => {
const entries = catalogEntries();
const totalBytes = entries.reduce((sum, e) => sum + e.bytes, 0);
const estimatedTokens = Math.ceil(totalBytes / 4);
const delta = estimatedTokens - CATALOG_BUDGET_TOKEN_EQUIVALENTS;
expect(
estimatedTokens,
`Catalog is ${estimatedTokens} token-equivalents (${totalBytes} bytes), ` +
`${delta} over the ${CATALOG_BUDGET_TOKEN_EQUIVALENTS} budget. ${RATCHET_PROTOCOL}`
).toBeLessThanOrEqual(CATALOG_BUDGET_TOKEN_EQUIVALENTS);
});
test(`every skill's name + description stays under ${PER_SKILL_BYTE_CAP} bytes`, () => {
for (const entry of catalogEntries()) {
expect(
entry.bytes,
`${entry.skill}: name + description is ${entry.bytes} bytes, ` +
`${entry.bytes - PER_SKILL_BYTE_CAP} over the ${PER_SKILL_BYTE_CAP}-byte ` +
`per-skill cap. ${RATCHET_PROTOCOL}`
).toBeLessThanOrEqual(PER_SKILL_BYTE_CAP);
}
});
test('every skill has a non-empty description', () => {
for (const entry of catalogEntries()) {
expect(entry.description, `${entry.skill}: empty or missing frontmatter description`).not.toBe('');
}
});
});