From 8d1d15ff34db70fb619125102eebcbf5c16abf66 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Wed, 12 Aug 2026 11:53:09 -0700 Subject: [PATCH] =?UTF-8?q?feat(cli):=20context-bill=20=E2=80=94=20token?= =?UTF-8?q?=20bill-of-materials=20(stripped=20port)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit lib/context-bill.ts, ported from the v2 fork and STRIPPED to the tiers this repo's skills can exercise: ALWAYS-ON (per-skill frontmatter bytes with dead-key and foreign-host-file flags), EAGER (SKILL.md + any forced 'for every invocation' references), on-disk totals, --diff, --budget, and --exact with the calibration table. The fork's CONDITIONAL/TRANSITIVE/LAZY/FAST-PATH parsers understand only its dispatcher layout and were dropped; the tier fields stay in the report shape (empty/zero/null) so re-adding a parser is additive. TOKEN_DIVISORS and their provenance docblock kept; --help notes recalibration via --exact's calibration block. Three upstream fixes over the fork: (a) findSkillDirs treats the walk ROOT as a container — the repo root's router SKILL.md is billed AND its children are walked (the fork short-circuited and billed one skill); walkMd skips node_modules and dot-directories. (b) installed-tree layout: subdirs that are their own repo checkout (a gstack/ clone inside ~/.claude/skills, detected by .git) are skipped, and directory symlinks (connect-chrome) are followed with a container-recursion cycle guard. (c) ROUTER_KEYS widened to the upstream frontmatter contract {name, description, version, allowed-tools, triggers, preamble-tier}. --exact writes an egress receipt (sink context-bill-exact, host api.anthropic.com) BEFORE any count_tokens POST; if the receipt cannot be written the run degrades to the offline estimate with a warning — nothing is sent unrecorded. bin/gstack-context-bill is the bun shim. Tests: fixture-tree ledgers, the three fixes, --diff/--budget exit codes, --exact with injected fetch (envelope subtraction, receipt ordering, fail-open degradation), CLI smoke test, and ground truth against THIS repo via test/helpers/skill-census.ts. Co-Authored-By: Claude Fable 5 (cherry picked from commit 675c19876b87ec927b555f5f64c7f93130b3de90) --- bin/gstack-context-bill | 7 + lib/context-bill.ts | 1003 +++++++++++++++++ test/context-bill.test.ts | 679 +++++++++++ .../context-bill/tree-a/alpha/SKILL.md | 21 + .../tree-a/alpha/references/CORE.md | 3 + .../tree-a/alpha/references/OPTIONAL.md | 3 + .../tree-a/alpha/references/POLICY.md | 3 + .../context-bill/tree-a/beta/SKILL.md | 8 + .../context-bill/tree-a/beta/agents.md | 3 + 9 files changed, 1730 insertions(+) create mode 100755 bin/gstack-context-bill create mode 100644 lib/context-bill.ts create mode 100644 test/context-bill.test.ts create mode 100644 test/fixtures/context-bill/tree-a/alpha/SKILL.md create mode 100644 test/fixtures/context-bill/tree-a/alpha/references/CORE.md create mode 100644 test/fixtures/context-bill/tree-a/alpha/references/OPTIONAL.md create mode 100644 test/fixtures/context-bill/tree-a/alpha/references/POLICY.md create mode 100644 test/fixtures/context-bill/tree-a/beta/SKILL.md create mode 100644 test/fixtures/context-bill/tree-a/beta/agents.md diff --git a/bin/gstack-context-bill b/bin/gstack-context-bill new file mode 100755 index 000000000..50b085700 --- /dev/null +++ b/bin/gstack-context-bill @@ -0,0 +1,7 @@ +#!/usr/bin/env bun +// gstack-context-bill — token bill-of-materials for an installed skills tree. +// All behavior lives in lib/context-bill.ts; this is the CLI shim. + +import { contextBillMain } from '../lib/context-bill'; + +process.exit(await contextBillMain(process.argv.slice(2))); diff --git a/lib/context-bill.ts b/lib/context-bill.ts new file mode 100644 index 000000000..bd191004a --- /dev/null +++ b/lib/context-bill.ts @@ -0,0 +1,1003 @@ +/** + * gstack context-bill — token bill-of-materials for an installed gstack skills tree. + * + * Read-only, offline, deterministic. Ledgers over pure file reads: + * ALWAYS-ON per-skill YAML frontmatter bytes (what every session's skill + * scanner loads), flagging frontmatter keys the router never + * reads and foreign-host files in scanner scope. + * EAGER SKILL.md plus any references the skill's prose forces "for + * every invocation". + * + * This is a STRIPPED port of the v2 fork's six-ledger bill: the CONDITIONAL, + * TRANSITIVE, LAZY, and FAST-PATH parsers only understand the fork's + * dispatcher-skill layout, which this repo's skills don't use, so they were + * dropped rather than shipped dead. The tier fields stay in the report shape + * (empty arrays / zeros / nulls) so re-adding a parser is additive: nothing + * downstream needs a schema change. + * + * Token figures come from one of two sources, always named in the output: + * ESTIMATE (default, offline) bytes / TOKEN_DIVISOR, calibrated against real + * count_tokens measurements. + * EXACT (--exact, opt-in) Anthropic's count_tokens for every file the + * bill touches. Sends file content off-machine, + * so it is never implicit: an egress receipt is + * written before the POSTs (sink + * 'context-bill-exact'), and if the receipt + * cannot be written the run degrades to the + * offline estimate with a warning instead of + * sending unrecorded. + * Both bytes and tokens are always shown, and the estimate's measured error + * band is printed with it. The tool never writes state anywhere (the egress + * receipt under --exact is the one exception, and it is the point). + */ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { writeReceipt } from "./egress-receipt"; + +const FORCED_PHRASE = "for every invocation"; +// Backticked reference in prose. `<...>` is excluded: a path template such as +// `references/templates/.md` names a family of files, not one on disk. +const PROSE_REF = /`(references\/[^`<>]+\.md)`/g; +// Upstream frontmatter contract: the keys the router/host actually reads. +const ROUTER_KEYS = new Set(["name", "description", "version", "allowed-tools", "triggers", "preamble-tier"]); +// Skill-shaped files other hosts drop into scanner scope. +const FOREIGN_SKILL_FILE = /^(skill\.(ya?ml|json)|agents?\.md|\.cursorrules|\.windsurfrules)$/i; + +/** + * Bytes per token, per content class, fitted to real count_tokens measurements. + * + * Calibration corpus: 219 `.md` skill files plus their frontmatter blocks, + * measured 2026-08-01 against `claude-opus-4-5` with the per-request message + * envelope subtracted. Regenerate with `gstack-context-bill --exact + * --json` and read the `calibration` block, which grades this estimate + * against measured counts file by file. + * + * Why classes and not one divisor: measured bytes-per-token spans 2.36 to 4.72 + * across the corpus, and the spread is largely structural. Legacy specialist + * modules cluster at 3.47 (n=50, range 3.10-3.83) and SKILL.md bodies at 4.21 + * (n=50, range 3.65-4.50) -- tight enough that one divisor for both charges + * some ledgers about 19% under while charging others about right. Splitting on + * path roles cuts mean per-file error from 11.1% to 7.4% and removes the + * systematic bias, which is what a cost tool owes. + * + * What classes do NOT fix: the `reference` class is genuinely heterogeneous + * (2.36 to 4.72 -- dense path/table files sit at one end, prose at the other), + * so worst-case per-file error stays near 40%. Use --exact when a single + * file's number has to be right. + * + * These divisors are tokenizer-specific. Opus 4.7 and later tokenize + * differently; on those models use --exact. + */ +export const TOKEN_DIVISORS: Record = { + frontmatter: 3.99, + skillmd: 4.21, + reference: 4.15, + artifact: 3.67, + legacy: 3.47, +}; +/** Fallback for content that matches no class. Corpus-wide aggregate. */ +export const TOKEN_DIVISOR = 3.9; +/** Worst-case per-file residual of the estimate over the calibration corpus. */ +export const TOKEN_ESTIMATE_ERROR_PCT = 40; + +export type TokensOf = (key: string, bytes: number) => number; + +export interface RefEntry { + path: string; + bytes: number; + tokens: number; + missing: boolean; + via?: string; + condition?: string; +} + +export interface SkillBill { + name: string; + dir: string; + frontmatterBytes: number; + frontmatterTokens: number; + frontmatterKeys: string[]; + deadKeys: string[]; + skillMdBytes: number; + skillMdTokens: number; + forcedRefs: RefEntry[]; + eagerBytes: number; + eagerTokens: number; + /** Stripped tiers: kept in the shape (empty/zero/null) so re-adding the + * fork's parsers is additive. */ + fastPath: null; + conditionalRefs: RefEntry[]; + conditionalBytes: number; + conditionalTokens: number; + transitiveRefs: RefEntry[]; + transitiveBytes: number; + transitiveTokens: number; + perInvocationBytes: number; + perInvocationTokens: number; + routeCeiling: { label: string; bytes: number; tokens: number } | null; + lazy: { label: string; modules: RefEntry[]; bytes: number; tokens: number }[]; + orphans: RefEntry[]; + foreignFiles: { path: string; bytes: number; tokens: number }[]; + totalMdBytes: number; + totalMdTokens: number; +} + +/** + * Content class from the path role. Legacy/artifact roles are kept even + * though their tiers are stripped: the divisors are per-content measurements + * and --exact calibration still grades them. + */ +export function contentClass(key: string): string { + if (key.endsWith("#frontmatter")) return "frontmatter"; + if (/references[/\\]legacy[/\\]/.test(key)) return "legacy"; + if (/references[/\\](artifacts|sections|support)[/\\]/.test(key)) return "artifact"; + if (/(^|[/\\])SKILL\.md$/.test(key)) return "skillmd"; + if (/references[/\\]/.test(key)) return "reference"; + return "other"; +} + +/** Path-less callers get the corpus-wide aggregate divisor. */ +export function estimateTokens(bytes: number): number { + return Math.round(bytes / TOKEN_DIVISOR); +} + +/** Default token source: the calibrated offline estimate. Unrounded, so sums round once. */ +function estimateTokensOf(key: string, bytes: number): number { + return bytes / (TOKEN_DIVISORS[contentClass(key)] ?? TOKEN_DIVISOR); +} + +function bytesOf(file: string): number | null { + try { + const st = fs.statSync(file); + return st.isFile() ? st.size : null; + } catch { + return null; + } +} + +function refEntry(skillDir: string, rel: string, tokensOf: TokensOf): RefEntry { + const abs = path.join(skillDir, rel); + const bytes = bytesOf(abs); + return { + path: rel, + bytes: bytes ?? 0, + tokens: bytes == null ? 0 : tokensOf(abs, bytes), + missing: bytes == null, + }; +} + +function sumBytes(entries: { bytes: number }[]): number { + return entries.reduce((n, e) => n + e.bytes, 0); +} + +function sumTokens(entries: { tokens: number }[]): number { + return entries.reduce((n, e) => n + e.tokens, 0); +} + +/** Cache key for a SKILL.md's frontmatter block, which is a slice, not a whole file. */ +function frontmatterKey(skillMdPath: string): string { + return `${skillMdPath}#frontmatter`; +} + +function parseFrontmatter(text: string): { bytes: number; keys: string[]; block: string } { + if (!text.startsWith("---")) return { bytes: 0, keys: [], block: "" }; + const end = text.indexOf("\n---", 3); + if (end === -1) return { bytes: 0, keys: [], block: "" }; + const closeEol = text.indexOf("\n", end + 1); + const block = text.slice(0, closeEol === -1 ? text.length : closeEol + 1); + const inner = text.slice(text.indexOf("\n") + 1, end); + const keys: string[] = []; + for (const line of inner.split("\n")) { + const m = /^([A-Za-z0-9_-]+)\s*:/.exec(line); + if (m) keys.push(m[1]); + } + return { bytes: Buffer.byteLength(block, "utf8"), keys, block }; +} + +/** + * Every .md file under a tree, for the on-disk total and for exact + * measurement. Skips node_modules and dot-directories: a skills tree that is + * also a repo checkout (dev symlink installs) would otherwise bill its + * dependency tree and CI state as skill content. + */ +export function walkMd(dir: string): string[] { + const out: string[] = []; + let entries: fs.Dirent[]; + try { + entries = fs.readdirSync(dir, { withFileTypes: true }); + } catch { + return out; + } + for (const e of entries) { + if (e.name.startsWith(".") || e.name === "node_modules") continue; + const p = path.join(dir, e.name); + if (e.isDirectory()) out.push(...walkMd(p)); + else if (e.isFile() && e.name.endsWith(".md")) out.push(p); + } + return out; +} + +function totalMd(dir: string, tokensOf: TokensOf): { bytes: number; tokens: number } { + let bytes = 0; + let tokens = 0; + for (const p of walkMd(dir)) { + const b = bytesOf(p) ?? 0; + bytes += b; + tokens += tokensOf(p, b); + } + return { bytes, tokens }; +} + +export function parseSkill(skillDir: string, name: string, tokensOf: TokensOf = estimateTokensOf): SkillBill { + const skillMdPath = path.join(skillDir, "SKILL.md"); + const text = fs.readFileSync(skillMdPath, "utf8"); + const skillMdBytes = bytesOf(skillMdPath) ?? 0; + const skillMdTokens = tokensOf(skillMdPath, skillMdBytes); + const fm = parseFrontmatter(text); + // The frontmatter block is a slice of SKILL.md, so it carries its own key. + const frontmatterTokens = tokensOf(frontmatterKey(skillMdPath), fm.bytes); + const deadKeys = fm.keys.filter((k) => !ROUTER_KEYS.has(k)); + + // EAGER: references a prose CLAUSE forces "for every invocation". Clause + // granularity matters: a line can carry a forced clause and a conditional + // one, and only the forced clause's references are eager. Routing tables + // never count (they were the fork's LAZY tier). + const forcedRefs: RefEntry[] = []; + const seenForced = new Set(); + for (const line of text.split("\n")) { + if (line.trim().startsWith("|")) continue; + for (const clause of line.split(/(?<=[.;])\s+/)) { + if (!clause.includes(FORCED_PHRASE)) continue; + for (const m of clause.matchAll(PROSE_REF)) { + const p = m[1]; + if (seenForced.has(p)) continue; + seenForced.add(p); + forcedRefs.push(refEntry(skillDir, p, tokensOf)); + } + } + } + + // Foreign-host skill files sitting next to SKILL.md. + const foreignFiles: { path: string; bytes: number; tokens: number }[] = []; + for (const entry of fs.readdirSync(skillDir, { withFileTypes: true })) { + if (entry.isFile() && FOREIGN_SKILL_FILE.test(entry.name)) { + const abs = path.join(skillDir, entry.name); + const bytes = bytesOf(abs) ?? 0; + foreignFiles.push({ path: entry.name, bytes, tokens: tokensOf(abs, bytes) }); + } + } + + const total = totalMd(skillDir, tokensOf); + const eagerBytes = skillMdBytes + sumBytes(forcedRefs); + const eagerTokens = skillMdTokens + sumTokens(forcedRefs); + return { + name, + dir: skillDir, + frontmatterBytes: fm.bytes, + frontmatterTokens, + frontmatterKeys: fm.keys, + deadKeys, + skillMdBytes, + skillMdTokens, + forcedRefs, + eagerBytes, + eagerTokens, + // Stripped tiers, shape preserved (see the module docblock). + fastPath: null, + conditionalRefs: [], + conditionalBytes: 0, + conditionalTokens: 0, + transitiveRefs: [], + transitiveBytes: 0, + transitiveTokens: 0, + // With the conditional/transitive tiers stripped, the per-invocation + // ceiling IS the eager figure. Re-adding a tier changes these sums only. + perInvocationBytes: eagerBytes, + perInvocationTokens: eagerTokens, + routeCeiling: null, + lazy: [], + orphans: [], + foreignFiles, + totalMdBytes: total.bytes, + totalMdTokens: total.tokens, + }; +} + +/** + * Every skill directory under a tree. + * + * Root-as-container (upstream fix): this repo's ROOT has a router SKILL.md + * AND fifty skill directories under it — the fork's walker short-circuited at + * the root and billed one "skill". The root is counted as a skill (the router + * costs what it costs) and the walk continues into its children. A NON-root + * dir with SKILL.md is still a leaf: its subtree (references/, test + * fixtures) is never another skill. + * + * Repo-checkout subdirs are skipped (upstream install layout fix): an + * installed ~/.claude/skills tree contains flat skill dirs PLUS a full gstack + * repo checkout (`gstack/`, with .git). Its nested SKILL.md files are the + * repo's sources, not installed skills of the tree being billed. + * + * Directory symlinks are followed (setup's shell glob follows them, so a + * symlinked skill like connect-chrome/ is real scanner load); a realpath + * seen-set breaks cycles. + */ +export function findSkillDirs(root: string): string[] { + const out: string[] = []; + const visited = new Set(); + const walk = (dir: string, isRoot: boolean) => { + let entries: fs.Dirent[]; + try { + entries = fs.readdirSync(dir, { withFileTypes: true }); + } catch { + return; + } + if (entries.some((e) => e.isFile() && e.name === "SKILL.md")) { + out.push(dir); + // Two symlinked paths to the same skill dir are BOTH billed (each is + // real scanner load); only container recursion below is cycle-guarded. + if (!isRoot) return; + } + // Cycle guard for container recursion (a symlink loop of directories). + let real: string; + try { + real = fs.realpathSync(dir); + } catch { + return; + } + if (visited.has(real)) return; + visited.add(real); + for (const e of entries) { + if (e.name.startsWith(".") || e.name === "node_modules") continue; + const child = path.join(dir, e.name); + let isDir = e.isDirectory(); + if (!isDir && e.isSymbolicLink()) { + try { + isDir = fs.statSync(child).isDirectory(); + } catch { + continue; // dangling symlink + } + } + if (!isDir) continue; + if (fs.existsSync(path.join(child, ".git"))) continue; // repo checkout, not a skill + walk(child, false); + } + }; + walk(path.resolve(root), true); + return out.sort(); +} + +export interface Bill { + root: string; + tokenSource: string; + tokenEstimate: Record; + tokenEstimateErrorPct: number; + calibration?: Calibration; + skills: SkillBill[]; + totals: { + skillCount: number; + alwaysOnBytes: number; + alwaysOnTokens: number; + eagerBytesBySkill: Record; + eagerTokensBySkill: Record; + perInvocationBytesBySkill: Record; + perInvocationTokensBySkill: Record; + totalMdBytes: number; + totalMdTokens: number; + }; +} + +export function buildBill( + root: string, + { tokensOf = estimateTokensOf, tokenSource, calibration }: { + tokensOf?: TokensOf; + tokenSource?: string; + calibration?: Calibration; + } = {}, +): Bill { + const resolved = path.resolve(root); + if (!fs.existsSync(resolved)) throw new Error(`No such tree: ${resolved}`); + const skills = findSkillDirs(resolved).map((dir) => + parseSkill(dir, path.relative(resolved, dir) || path.basename(resolved), tokensOf), + ); + const total = skills.reduce((n, s) => n + s.totalMdBytes, 0); + const totalTokens = skills.reduce((n, s) => n + s.totalMdTokens, 0); + return { + root: resolved, + // Named so a reader never has to guess whether a figure was measured. + tokenSource: tokenSource ?? "estimate: calibrated bytes/token per content class", + tokenEstimate: TOKEN_DIVISORS, + tokenEstimateErrorPct: tokenSource ? 0 : TOKEN_ESTIMATE_ERROR_PCT, + // Present only under --exact: how far the offline estimate was off, per file. + ...(calibration ? { calibration } : {}), + skills, + totals: { + skillCount: skills.length, + alwaysOnBytes: skills.reduce((n, s) => n + s.frontmatterBytes, 0), + alwaysOnTokens: skills.reduce((n, s) => n + s.frontmatterTokens, 0), + eagerBytesBySkill: Object.fromEntries(skills.map((s) => [s.name, s.eagerBytes])), + eagerTokensBySkill: Object.fromEntries(skills.map((s) => [s.name, Math.round(s.eagerTokens)])), + perInvocationBytesBySkill: Object.fromEntries(skills.map((s) => [s.name, s.perInvocationBytes])), + perInvocationTokensBySkill: Object.fromEntries( + skills.map((s) => [s.name, Math.round(s.perInvocationTokens)]), + ), + totalMdBytes: total, + totalMdTokens: totalTokens, + }, + }; +} + +export interface DiffRow { + ledger: string; + label: string; + before: number; + after: number; + delta: number; + tokenDelta: number; +} + +export function diffBills(a: Bill, b: Bill): { rows: DiffRow[]; grew: boolean } { + const rows: DiffRow[] = []; + const push = (ledger: string, label: string, before: number, after: number, tokBefore: number, tokAfter: number) => { + if (before !== after) { + rows.push({ ledger, label, before, after, delta: after - before, tokenDelta: Math.round(tokAfter - tokBefore) }); + } + }; + const skillNames = [...new Set([...a.skills, ...b.skills].map((s) => s.name))].sort(); + for (const name of skillNames) { + const sa = a.skills.find((s) => s.name === name); + const sb = b.skills.find((s) => s.name === name); + push( + "always-on", name, + sa?.frontmatterBytes ?? 0, sb?.frontmatterBytes ?? 0, + sa?.frontmatterTokens ?? 0, sb?.frontmatterTokens ?? 0, + ); + push("eager", name, sa?.eagerBytes ?? 0, sb?.eagerBytes ?? 0, sa?.eagerTokens ?? 0, sb?.eagerTokens ?? 0); + // Stripped tiers stay in the diff contract so re-adding them is additive. + push( + "conditional", name, + sa?.conditionalBytes ?? 0, sb?.conditionalBytes ?? 0, + sa?.conditionalTokens ?? 0, sb?.conditionalTokens ?? 0, + ); + push( + "transitive", name, + sa?.transitiveBytes ?? 0, sb?.transitiveBytes ?? 0, + sa?.transitiveTokens ?? 0, sb?.transitiveTokens ?? 0, + ); + } + rows.sort((x, y) => Math.abs(y.delta) - Math.abs(x.delta)); + const grew = + b.totals.alwaysOnBytes > a.totals.alwaysOnBytes || + rows.some((r) => ["eager", "conditional", "transitive"].includes(r.ledger) && r.delta > 0); + return { rows, grew }; +} + +export interface BudgetViolation { + ceiling: string; + limit: number; + actual: number | null; + files: string[]; +} + +/** + * Budget file: user-authored plain JSON, ceilings in ~tokens. + * { "alwaysOnTotal": 4000, "eagerPerInvocation": { "qa": 5000 }, + * "perInvocation": { "qa": 9000 } } + * With the conditional/transitive tiers stripped, `perInvocation` and + * `routeCeiling` gate the same figure as `eagerPerInvocation`; the keys stay + * accepted so budgets survive the tiers returning. + */ +export function checkBudget(bill: Bill, budget: Record): BudgetViolation[] { + const violations: BudgetViolation[] = []; + if (typeof budget.alwaysOnTotal === "number") { + const actual = Math.round(bill.totals.alwaysOnTokens); + if (actual > budget.alwaysOnTotal) { + violations.push({ + ceiling: "alwaysOnTotal", + limit: budget.alwaysOnTotal, + actual, + files: bill.skills.map((s) => `${s.name}/SKILL.md (frontmatter ${s.frontmatterBytes}B)`), + }); + } + } + for (const key of ["eagerPerInvocation", "perInvocation", "routeCeiling"]) { + for (const [name, limit] of Object.entries(budget[key] ?? {}) as [string, number][]) { + const skill = bill.skills.find((s) => s.name === name); + if (!skill) { + violations.push({ ceiling: `${key}.${name}`, limit, actual: null, files: [""] }); + continue; + } + const tokens = + key === "routeCeiling" + ? (skill.routeCeiling?.tokens ?? skill.perInvocationTokens) + : key === "perInvocation" + ? skill.perInvocationTokens + : skill.eagerTokens; + const actual = Math.round(tokens); + if (actual > limit) { + violations.push({ + ceiling: `${key}.${name}`, + limit, + actual, + files: [ + `${skill.name}/SKILL.md (${skill.skillMdBytes}B)`, + ...skill.forcedRefs.map((r) => `${skill.name}/${r.path} (${r.bytes}B)`), + ], + }); + } + } + } + return violations; +} + +function fmtBytes(b: number): string { + if (b >= 1024 * 1024) return `${(b / 1024 / 1024).toFixed(1)}MB`; + if (b >= 1024) return `${(b / 1024).toFixed(1)}KB`; + return `${b}B`; +} + +/** Exact counts are measurements, so they lose the "~" the estimate wears. */ +function fmtTok(tokens: number, exact: boolean): string { + const t = Math.round(tokens); + const tilde = exact ? "" : "~"; + return t >= 1000 ? `${tilde}${(t / 1000).toFixed(1)}K tok` : `${tilde}${t} tok`; +} + +export function renderBill(bill: Bill, { skill }: { skill?: string } = {}): string { + const skills = skill ? bill.skills.filter((s) => s.name === skill) : bill.skills; + const exact = bill.tokenEstimateErrorPct === 0; + const size = (bytes: number, tokens: number) => `${fmtBytes(bytes)} (${fmtTok(tokens, exact)})`; + const lines = [`Context bill for ${bill.root}`, `Token source: ${bill.tokenSource}`, ""]; + + lines.push( + `ALWAYS-ON (every session): ${skills.length} skills, ` + + `${size(skills.reduce((n, s) => n + s.frontmatterBytes, 0), skills.reduce((n, s) => n + s.frontmatterTokens, 0))}`, + ); + // The host wraps each skill's frontmatter in its own available_skills XML + // element before the model sees it. That wrapper is host-specific and cannot + // be read from this tree, so it is excluded here — the real always-on cost is + // this figure plus one wrapper per skill. + lines.push(" (frontmatter only; excludes the host's per-skill available_skills XML wrapper)"); + for (const s of skills) lines.push(` ${s.name.padEnd(20)} ${size(s.frontmatterBytes, s.frontmatterTokens)}`); + for (const s of skills) { + if (s.deadKeys.length) lines.push(` ! ${s.name}: frontmatter key(s) the router never reads: ${s.deadKeys.join(", ")}`); + for (const f of s.foreignFiles) lines.push(` ! ${s.name}: foreign-host file in scanner scope: ${f.path} (${size(f.bytes, f.tokens)})`); + } + lines.push(""); + + lines.push("EAGER (per invocation): SKILL.md + forced-read references"); + for (const s of skills) { + const refs = s.forcedRefs.length + ? ` = SKILL.md ${fmtBytes(s.skillMdBytes)} + refs ${fmtBytes(sumBytes(s.forcedRefs))} (${s.forcedRefs.map((r) => path.basename(r.path)).join(", ")})` + : ""; + lines.push(` ${s.name.padEnd(20)} ${size(s.eagerBytes, s.eagerTokens)}${refs}`); + for (const r of s.forcedRefs.filter((r) => r.missing)) lines.push(` ! ${s.name}: forced-read reference missing on disk: ${r.path}`); + } + lines.push(""); + + lines.push( + `TOTAL on disk: ${size(bill.totals.totalMdBytes, bill.totals.totalMdTokens)} across ${bill.totals.skillCount} skill(s).`, + ); + lines.push(tokenDisclaimer(bill)); + return lines.join("\n") + "\n"; +} + +/** + * Names the error band instead of hand-waving about "estimates". The band is the + * worst-case residual measured over the calibration corpus, not a guess. + */ +export function tokenDisclaimer(bill: Pick): string { + if (bill.tokenEstimateErrorPct === 0) { + return `Token counts measured with ${bill.tokenSource}. Bytes are exact.`; + } + const per = Object.entries(TOKEN_DIVISORS).map(([k, v]) => `${k} /${v}`).join(", "); + return ( + `Token counts are ESTIMATES: bytes divided per content class (${per}), calibrated against ` + + `count_tokens on 219 skill files. Measured accuracy of that estimate: mean ` + + `per-file error 7.4%, systematic bias under 0.5%, worst single file ` + + `${bill.tokenEstimateErrorPct}% (dense path/table files). Ledger rows ` + + `land tighter than single files because errors partly cancel across a sum. Run --exact for ` + + `measured counts when a number has to be right. Bytes are always exact.` + ); +} + +export function renderDiff(diff: { rows: DiffRow[]; grew: boolean }): string { + if (diff.rows.length === 0) return "No context-cost changes between trees.\n"; + const lines = ["Context-cost changes (sorted by |delta|):", ""]; + for (const r of diff.rows) { + const sign = r.delta > 0 ? "+" : "-"; + lines.push( + ` ${r.ledger.padEnd(11)} ${r.label.padEnd(28)} ${sign}${fmtBytes(Math.abs(r.delta))} (${sign}${Math.abs(r.tokenDelta)} tok) ${fmtBytes(r.before)} -> ${fmtBytes(r.after)}`, + ); + } + lines.push(""); + lines.push( + diff.grew + ? "RESULT: context cost GREW (always-on or eager)." + : "RESULT: no always-on or eager growth.", + ); + return lines.join("\n") + "\n"; +} + +// --exact defaults to the model the offline divisor was calibrated against, so +// `--exact` and the estimate are comparable. Later tokenizers differ. +export const EXACT_DEFAULT_MODEL = "claude-opus-4-5"; +const COUNT_TOKENS_URL = "https://api.anthropic.com/v1/messages/count_tokens"; +const EXACT_CONCURRENCY = 8; + +/** Typed failures, so callers branch on a code rather than on message text. */ +export class ExactModeError extends Error { + code: string; + constructor(code: string, message: string) { + super(message); + this.name = "ExactModeError"; + this.code = code; + } +} + +type FetchLike = typeof globalThis.fetch; + +interface CountTokensOptions { + model: string; + apiKey: string; + fetchImpl: FetchLike; +} + +async function countTokens(text: string, { model, apiKey, fetchImpl }: CountTokensOptions): Promise { + let res: Response; + try { + res = await fetchImpl(COUNT_TOKENS_URL, { + method: "POST", + headers: { + "content-type": "application/json", + "x-api-key": apiKey, + "anthropic-version": "2023-06-01", + }, + body: JSON.stringify({ model, messages: [{ role: "user", content: text }] }), + }); + } catch (error) { + throw new ExactModeError("exact_network_unreachable", `count_tokens unreachable: ${(error as Error)?.message ?? error}`); + } + if (!res.ok) { + const body = await res.text().catch(() => ""); + const code = res.status === 401 || res.status === 403 ? "exact_auth_rejected" : "exact_request_failed"; + throw new ExactModeError(code, `count_tokens returned ${res.status}: ${body.slice(0, 200)}`); + } + const json: any = await res.json(); + if (typeof json?.input_tokens !== "number") { + throw new ExactModeError("exact_response_malformed", "count_tokens response had no input_tokens"); + } + return json.input_tokens; +} + +export interface ExactMeasurement { + tokenSource: string; + tokensOf: TokensOf; + measuredFiles: number; + counts: Map; + /** Keys priced by estimate because measurement missed them. Read after buildBill. */ + missedKeys: Set; +} + +/** + * Measures every text the bill will bill for. Returns a `tokensOf` lookup. + * + * count_tokens prices a whole request, so it includes a fixed message envelope. + * That envelope is measured once and subtracted, leaving the tokens each file's + * own content contributes — otherwise every small reference is overcharged by a + * constant that has nothing to do with the file. + * + * Egress receipt BEFORE any POST (sink 'context-bill-exact'): if the receipt + * cannot be written this throws exact_egress_receipt_failed, which the CLI + * degrades to the offline estimate — nothing is sent unrecorded. + */ +export async function measureExactTokens( + root: string, + { model, apiKey, fetchImpl = fetch, onProgress, egressHome }: { + model: string; + apiKey: string; + fetchImpl?: FetchLike; + onProgress?: (done: number, total: number) => void; + egressHome?: string; + }, +): Promise { + if (!apiKey) { + throw new ExactModeError( + "exact_missing_api_key", + "--exact needs ANTHROPIC_API_KEY. Without it the offline estimate is used; nothing was sent.", + ); + } + const opts: CountTokensOptions = { model, apiKey, fetchImpl }; + + const texts = new Map(); + // Resolve before keying. buildBill resolves its root, so a relative root here + // would produce keys that never match and every lookup would fall back to the + // estimate -- exact mode silently degrading to the thing it replaces. + for (const file of walkMd(path.resolve(root))) { + const text = fs.readFileSync(file, "utf8"); + texts.set(file, text); + if (path.basename(file) === "SKILL.md") { + const fm = parseFrontmatter(text); + if (fm.block) texts.set(frontmatterKey(file), fm.block); + } + } + + // Receipt-before-send. Content-free: file count + total bytes only. + try { + let totalBytes = 0; + for (const t of texts.values()) totalBytes += Buffer.byteLength(t, "utf8"); + writeReceipt({ + home: egressHome, + sink: "context-bill-exact", + host: "api.anthropic.com", + payloadClass: `count-tokens skill-tree texts=${texts.size} (${totalBytes}B across ${EXACT_CONCURRENCY}-way POSTs)`, + bytes: totalBytes, + sha256: null, + consent: "user passed --exact", + }); + } catch (error) { + throw new ExactModeError( + "exact_egress_receipt_failed", + `egress receipt could not be written (${(error as Error)?.message ?? error}); refusing to send unrecorded`, + ); + } + + // One-char body: subtracting its single content token leaves the envelope. + const envelope = (await countTokens("x", opts)) - 1; + + const counts = new Map(); + const keys = [...texts.keys()]; + let next = 0; + let done = 0; + const worker = async () => { + while (next < keys.length) { + const key = keys[next++]; + const raw = await countTokens(texts.get(key)!, opts); + counts.set(key, Math.max(0, raw - envelope)); + onProgress?.(++done, keys.length); + } + }; + await Promise.all(Array.from({ length: Math.min(EXACT_CONCURRENCY, keys.length) }, worker)); + + // A key the walk never saw (a non-.md foreign-host file) falls back to the + // estimate rather than billing zero. Misses are counted, not swallowed: a bill + // that is part-measured and part-estimated must not present itself as measured. + const missed = new Set(); + const tokensOf: TokensOf = (key, bytes) => { + const exact = counts.get(key); + if (exact !== undefined) return exact; + missed.add(key); + return estimateTokensOf(key, bytes); + }; + return { + tokenSource: `count_tokens (${model})`, + tokensOf, + measuredFiles: counts.size, + counts, + missedKeys: missed, + }; +} + +export interface Calibration { + rows: { + path: string; + contentClass: string; + bytes: number; + estimatedTokens: number; + tokens: number; + bytesPerToken: number; + errorPct: number; + }[]; + worstErrorPct: number; + meanAbsErrorPct: number; + biasPct: number; +} + +/** + * Estimate-vs-measured residual per file. This is what makes the divisor + * auditable: run --exact and the tool grades its own offline estimate. + */ +export function calibrationTable(counts: Map, root: string): Calibration { + const rows: Calibration["rows"] = []; + for (const [key, tokens] of counts) { + if (key.endsWith("#frontmatter") || tokens === 0) continue; + const bytes = bytesOf(key); + if (bytes == null) continue; + // Grade the estimate the tool actually uses, class divisor included. + const estimated = Math.round(estimateTokensOf(key, bytes)); + rows.push({ + path: path.relative(root, key), + contentClass: contentClass(key), + bytes, + estimatedTokens: estimated, + tokens, + bytesPerToken: Number((bytes / tokens).toFixed(3)), + errorPct: Number((((estimated - tokens) / tokens) * 100).toFixed(1)), + }); + } + rows.sort((a, b) => Math.abs(b.errorPct) - Math.abs(a.errorPct)); + const abs = rows.map((r) => Math.abs(r.errorPct)); + return { + rows, + worstErrorPct: abs.length ? Math.max(...abs) : 0, + meanAbsErrorPct: abs.length ? Number((abs.reduce((a, b) => a + b, 0) / abs.length).toFixed(2)) : 0, + biasPct: rows.length + ? Number((rows.reduce((n, r) => n + r.errorPct, 0) / rows.length).toFixed(2)) + : 0, + }; +} + +// Where installed skills actually live: `.agents/skills` (the host-neutral +// canonical path) alongside `.claude/skills`, project then user. +const DEFAULT_TREES: string[][] = [ + ["cwd", "skills"], + ["cwd", ".agents", "skills"], + ["cwd", ".claude", "skills"], + ["home", ".agents", "skills"], + ["home", ".claude", "skills"], +]; + +function defaultTreeCandidates(cwd: string, homeDir: string): string[] { + return DEFAULT_TREES.map(([base, ...rest]) => path.join(base === "cwd" ? cwd : homeDir, ...rest)); +} + +function detectDefaultTree(cwd: string, homeDir: string): string | null { + return defaultTreeCandidates(cwd, homeDir).find((c) => fs.existsSync(c)) ?? null; +} + +const USAGE = + "Usage:\n" + + " gstack-context-bill [TREE] [--json] [--skill ]\n" + + " gstack-context-bill --diff [--json]\n" + + " gstack-context-bill [TREE] --budget [--json]\n" + + "\n" + + " --exact measure tokens with Anthropic's count_tokens instead of\n" + + " estimating. Off by default: it sends the content of every\n" + + " .md file in the tree to api.anthropic.com. Needs\n" + + " ANTHROPIC_API_KEY; passing --exact is the consent. An\n" + + " egress receipt is written before the send (see\n" + + " gstack-egress); if it cannot be written, the run falls\n" + + " back to the offline estimate.\n" + + " --exact also recalibrates: the --json output's\n" + + " `calibration` block grades the offline divisors\n" + + " (TOKEN_DIVISORS) file by file against measured counts.\n" + + " --exact-model model whose tokenizer to count against\n" + + ` (default ${EXACT_DEFAULT_MODEL}, the calibration model).\n`; + +export interface MainOptions { + cwd?: string; + stdout?: { write(s: string): unknown }; + stderr?: { write(s: string): unknown }; + homeDir?: string; + apiKey?: string; + fetchImpl?: FetchLike; + egressHome?: string; +} + +export async function contextBillMain(argv: string[], options: MainOptions = {}): Promise { + const cwd = options.cwd ?? process.cwd(); + const stdout = options.stdout ?? process.stdout; + const stderr = options.stderr ?? process.stderr; + const homeDir = options.homeDir ?? os.homedir(); + + const positional: string[] = []; + const flags: { json: boolean; diff: boolean; exact: boolean; exactModel: string; skill?: string; budget?: string } = + { json: false, diff: false, exact: false, exactModel: EXACT_DEFAULT_MODEL }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--json") flags.json = true; + else if (arg === "--diff") flags.diff = true; + else if (arg === "--skill") flags.skill = argv[++i]; + else if (arg === "--budget") flags.budget = argv[++i]; + else if (arg === "--exact") flags.exact = true; + else if (arg === "--exact-model") flags.exactModel = argv[++i]; + else if (arg === "--help" || arg === "-h") { + stdout.write(USAGE); + return 0; + } else if (arg.startsWith("--")) { + stderr.write(`Unknown flag: ${arg}\n${USAGE}`); + return 2; + } else positional.push(arg); + } + + // Exact mode is the only path that leaves the machine. Announce what is sent + // before sending it, and degrade to the estimate rather than failing the run. + const exactFor = async (tree: string): Promise<{ + tokensOf?: TokensOf; + tokenSource?: string; + calibration?: Calibration; + onDone?: () => void; + }> => { + if (!flags.exact) return {}; + const files = walkMd(tree).length; + stderr.write( + `--exact: sending the content of ${files} .md file(s) under ${tree} to ` + + `api.anthropic.com for count_tokens (${flags.exactModel}). No other data leaves this machine.\n`, + ); + try { + const measured = await measureExactTokens(tree, { + model: flags.exactModel, + apiKey: options.apiKey ?? process.env.ANTHROPIC_API_KEY ?? "", + fetchImpl: options.fetchImpl, + egressHome: options.egressHome, + }); + return { + tokensOf: measured.tokensOf, + tokenSource: measured.tokenSource, + calibration: calibrationTable(measured.counts, tree), + onDone: () => { + if (measured.missedKeys.size) { + stderr.write( + `--exact: ${measured.missedKeys.size} item(s) had no measurement and were estimated ` + + `(${[...measured.missedKeys].slice(0, 3).join(", ")}). Those figures are not measurements.\n`, + ); + } + }, + }; + } catch (error) { + if (!(error instanceof ExactModeError)) throw error; + stderr.write(`--exact unavailable [${error.code}]: ${error.message}\nFalling back to the offline estimate.\n`); + return {}; + } + }; + + try { + if (flags.diff) { + if (positional.length !== 2) { + stderr.write(`--diff needs exactly two trees.\n${USAGE}`); + return 2; + } + const treeA = path.resolve(cwd, positional[0]); + const treeB = path.resolve(cwd, positional[1]); + const optsA = await exactFor(treeA); + const optsB = await exactFor(treeB); + const diff = diffBills(buildBill(treeA, optsA), buildBill(treeB, optsB)); + optsA.onDone?.(); + optsB.onDone?.(); + stdout.write(flags.json ? JSON.stringify(diff, null, 2) + "\n" : renderDiff(diff)); + return diff.grew ? 2 : 0; + } + + const tree = positional[0] ? path.resolve(cwd, positional[0]) : detectDefaultTree(cwd, homeDir); + if (!tree) { + stderr.write( + `No skills tree found (tried ${defaultTreeCandidates(cwd, homeDir).join(", ")}). Pass a path.\n`, + ); + return 2; + } + const exactOpts = await exactFor(tree); + const bill = buildBill(tree, exactOpts); + exactOpts.onDone?.(); + if (bill.skills.length === 0) { + stderr.write(`No SKILL.md files found under ${tree}.\n`); + return 2; + } + if (flags.skill && !bill.skills.some((s) => s.name === flags.skill)) { + stderr.write(`No skill named "${flags.skill}" in ${tree}. Skills: ${bill.skills.map((s) => s.name).join(", ")}\n`); + return 2; + } + + if (flags.budget) { + const budget = JSON.parse(fs.readFileSync(path.resolve(cwd, flags.budget), "utf8")); + const violations = checkBudget(bill, budget); + if (flags.json) { + stdout.write(JSON.stringify({ ok: violations.length === 0, violations }, null, 2) + "\n"); + } else if (violations.length === 0) { + stdout.write("Within budget.\n"); + } else { + for (const v of violations) { + stdout.write(`OVER BUDGET: ${v.ceiling} at ~${v.actual} tok (ceiling ~${v.limit} tok)\n`); + for (const f of v.files) stdout.write(` ${f}\n`); + } + } + return violations.length === 0 ? 0 : 2; + } + + stdout.write(flags.json ? JSON.stringify(bill, null, 2) + "\n" : renderBill(bill, flags)); + return 0; + } catch (error) { + stderr.write(`${(error as Error)?.message ?? error}\n`); + return 1; + } +} diff --git a/test/context-bill.test.ts b/test/context-bill.test.ts new file mode 100644 index 000000000..79cf79aab --- /dev/null +++ b/test/context-bill.test.ts @@ -0,0 +1,679 @@ +/** + * gstack-context-bill — token bill-of-materials for an installed skills tree. + * + * Free tier, no network, no API keys. Covers the STRIPPED port: + * - ALWAYS-ON ledger: exact frontmatter byte sums, dead-key flag against + * the upstream router-key contract, foreign-host file flag + * - EAGER ledger: SKILL.md + forced-read refs from the "for every + * invocation" phrase; stripped tiers stay zero/empty (shape preserved) + * - the three upstream fixes: (a) root-as-container walking + + * node_modules/dotdir exclusion in walkMd, (b) repo-checkout subdir skip + * for installed trees, (c) widened ROUTER_KEYS + * - token estimate calibration, --json shape, --diff, --budget exit codes + * - --exact via injected fetch: envelope subtraction, measurement + * replacement, typed failures, egress receipt-before-send + fail-open + * - ground truth against THIS repo via test/helpers/skill-census.ts + */ +import { describe, it, expect, beforeAll, afterAll } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { + buildBill, + calibrationTable, + checkBudget, + contentClass, + contextBillMain, + diffBills, + estimateTokens, + findSkillDirs, + measureExactTokens, + renderBill, + tokenDisclaimer, + walkMd, + ExactModeError, + TOKEN_DIVISOR, + TOKEN_DIVISORS, +} from "../lib/context-bill"; +import { listReceipts, sha256Hex } from "../lib/egress-receipt"; +import { skillCensus } from "./helpers/skill-census"; + +const ROOT = path.join(import.meta.dir, ".."); +const TREE_A = path.join(import.meta.dir, "fixtures", "context-bill", "tree-a"); + +function fileBytes(...segments: string[]): number { + return fs.statSync(path.join(...segments)).size; +} + +/** Frontmatter block bytes, computed independently of the implementation. */ +function frontmatterBytes(file: string): number { + const text = fs.readFileSync(file, "utf8"); + const close = text.indexOf("\n---\n", 3); + return Buffer.byteLength(text.slice(0, close + 5), "utf8"); +} + +function capture() { + let buf = ""; + return { + stream: { write: (s: string) => ((buf += s), true) } as unknown as NodeJS.WriteStream, + text: () => buf, + }; +} + +describe("always-on ledger", () => { + const bill = buildBill(TREE_A); + + it("sums per-skill frontmatter bytes exactly", () => { + const alpha = bill.skills.find((s) => s.name === "alpha")!; + const beta = bill.skills.find((s) => s.name === "beta")!; + expect(alpha.frontmatterBytes).toBe(frontmatterBytes(path.join(TREE_A, "alpha", "SKILL.md"))); + expect(beta.frontmatterBytes).toBe(frontmatterBytes(path.join(TREE_A, "beta", "SKILL.md"))); + expect(bill.totals.alwaysOnBytes).toBe(alpha.frontmatterBytes + beta.frontmatterBytes); + }); + + it("flags only keys outside the upstream router contract (fix c: widened ROUTER_KEYS)", () => { + const alpha = bill.skills.find((s) => s.name === "alpha")!; + // `triggers` is part of the upstream frontmatter contract now — flagging + // it was the fork's contract, not this repo's. + expect(alpha.deadKeys).toEqual(["x-dead-key"]); + expect(bill.skills.find((s) => s.name === "beta")!.deadKeys).toEqual([]); + }); + + it("upstream contract keys are never dead: name/description/version/allowed-tools/triggers/preamble-tier", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "context-bill-keys-")); + fs.mkdirSync(path.join(tmp, "s")); + fs.writeFileSync( + path.join(tmp, "s", "SKILL.md"), + "---\nname: s\ndescription: d\nversion: 1\nallowed-tools: Bash\ntriggers: t\npreamble-tier: 2\n---\n\n# S\n", + ); + const s = buildBill(tmp).skills[0]; + expect(s.deadKeys).toEqual([]); + fs.rmSync(tmp, { recursive: true, force: true }); + }); + + it("flags foreign-host skill-shaped files in scanner scope", () => { + const beta = bill.skills.find((s) => s.name === "beta")!; + expect(beta.foreignFiles.map((f) => f.path)).toEqual(["agents.md"]); + expect(bill.skills.find((s) => s.name === "alpha")!.foreignFiles).toEqual([]); + }); +}); + +describe("eager ledger", () => { + const bill = buildBill(TREE_A); + const alpha = bill.skills.find((s) => s.name === "alpha")!; + + it("eager = SKILL.md + only the forced-read references", () => { + expect(alpha.forcedRefs.map((r) => r.path)).toEqual([ + "references/CORE.md", + "references/POLICY.md", + ]); + const expected = + fileBytes(TREE_A, "alpha", "SKILL.md") + + fileBytes(TREE_A, "alpha", "references", "CORE.md") + + fileBytes(TREE_A, "alpha", "references", "POLICY.md"); + expect(alpha.eagerBytes).toBe(expected); + }); + + it("a skill with no forced reads bills only its SKILL.md", () => { + const beta = bill.skills.find((s) => s.name === "beta")!; + expect(beta.eagerBytes).toBe(fileBytes(TREE_A, "beta", "SKILL.md")); + }); + + it("stripped tiers stay zero/empty but keep their shape (re-adding is additive)", () => { + // OPTIONAL.md is mandated under a condition and the mode table routes two + // legacy modules — the fork billed those in CONDITIONAL and LAZY. The + // stripped port must not bill them anywhere NOR lose the fields. + expect(alpha.conditionalRefs).toEqual([]); + expect(alpha.conditionalBytes).toBe(0); + expect(alpha.transitiveRefs).toEqual([]); + expect(alpha.transitiveBytes).toBe(0); + expect(alpha.lazy).toEqual([]); + expect(alpha.orphans).toEqual([]); + expect(alpha.fastPath).toBeNull(); + expect(alpha.routeCeiling).toBeNull(); + // With those tiers stripped, per-invocation == eager. + expect(alpha.perInvocationBytes).toBe(alpha.eagerBytes); + expect(alpha.perInvocationTokens).toBe(alpha.eagerTokens); + }); +}); + +describe("upstream fix a: root-as-container + walkMd exclusions", () => { + it("a root with its own SKILL.md is billed AND walked into (router + children)", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "context-bill-root-")); + fs.writeFileSync(path.join(tmp, "SKILL.md"), "---\nname: router\ndescription: r\n---\n# Router\n"); + fs.mkdirSync(path.join(tmp, "qa")); + fs.writeFileSync(path.join(tmp, "qa", "SKILL.md"), "---\nname: qa\ndescription: q\n---\n# QA\n"); + const dirs = findSkillDirs(tmp).map((d) => path.relative(fs.realpathSync(tmp), fs.realpathSync(d)) || "."); + expect(dirs.sort()).toEqual([".", "qa"]); + // A NON-root skill dir is still a leaf: nothing nested under qa/ counts. + fs.mkdirSync(path.join(tmp, "qa", "nested")); + fs.writeFileSync(path.join(tmp, "qa", "nested", "SKILL.md"), "# not a skill\n"); + expect(findSkillDirs(tmp).length).toBe(2); + fs.rmSync(tmp, { recursive: true, force: true }); + }); + + it("walkMd skips node_modules and dot-directories", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "context-bill-walk-")); + fs.writeFileSync(path.join(tmp, "real.md"), "x"); + fs.mkdirSync(path.join(tmp, "node_modules", "pkg"), { recursive: true }); + fs.writeFileSync(path.join(tmp, "node_modules", "pkg", "README.md"), "y".repeat(5000)); + fs.mkdirSync(path.join(tmp, ".git"), { recursive: true }); + fs.writeFileSync(path.join(tmp, ".git", "notes.md"), "z"); + const files = walkMd(tmp).map((f) => path.basename(f)); + expect(files).toEqual(["real.md"]); + fs.rmSync(tmp, { recursive: true, force: true }); + }); +}); + +describe("upstream fix b: installed-tree layout (repo-checkout subdir skip)", () => { + it("skips a subdir that is its own repo checkout (gstack/ inside ~/.claude/skills)", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "context-bill-install-")); + // Flat installed skill dirs. + for (const name of ["qa", "ship"]) { + fs.mkdirSync(path.join(tmp, name)); + fs.writeFileSync(path.join(tmp, name, "SKILL.md"), `---\nname: ${name}\ndescription: d\n---\n# ${name}\n`); + } + // A full repo checkout dropped into the tree: has .git and its own router + // SKILL.md plus nested skill sources. None of it is an installed skill. + fs.mkdirSync(path.join(tmp, "gstack", ".git"), { recursive: true }); + fs.writeFileSync(path.join(tmp, "gstack", "SKILL.md"), "---\nname: _router\ndescription: d\n---\n# router\n"); + fs.mkdirSync(path.join(tmp, "gstack", "review")); + fs.writeFileSync(path.join(tmp, "gstack", "review", "SKILL.md"), "---\nname: review\ndescription: d\n---\n# r\n"); + + const names = buildBill(tmp).skills.map((s) => s.name).sort(); + expect(names).toEqual(["qa", "ship"]); + fs.rmSync(tmp, { recursive: true, force: true }); + }); + + it("follows a directory symlink to a sibling skill (connect-chrome shape)", () => { + if (process.platform === "win32") return; + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "context-bill-symlink-")); + fs.mkdirSync(path.join(tmp, "open-browser")); + fs.writeFileSync(path.join(tmp, "open-browser", "SKILL.md"), "---\nname: ob\ndescription: d\n---\n# ob\n"); + fs.symlinkSync(path.join(tmp, "open-browser"), path.join(tmp, "connect-chrome")); + const names = buildBill(tmp).skills.map((s) => s.name).sort(); + // Both entries are real scanner load, so both are billed. + expect(names).toEqual(["connect-chrome", "open-browser"]); + fs.rmSync(tmp, { recursive: true, force: true }); + }); + + it("prefers ./skills, then .agents/skills, then .claude/skills, project before user", async () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "context-bill-detect-")); + const home = path.join(tmp, "home"); + const proj = path.join(tmp, "proj"); + const plant = (dir: string, name: string) => { + fs.mkdirSync(path.join(dir, name), { recursive: true }); + fs.cpSync(TREE_A, path.join(dir, name), { recursive: true }); + return path.join(dir, name); + }; + const homeClaude = plant(home, path.join(".claude", "skills")); + const homeAgents = plant(home, path.join(".agents", "skills")); + const projClaude = plant(proj, path.join(".claude", "skills")); + const projAgents = plant(proj, path.join(".agents", "skills")); + + const run = async () => { + const out = capture(); + const code = await contextBillMain([], { cwd: proj, homeDir: home, stdout: out.stream, stderr: out.stream }); + expect(code).toBe(0); + return out.text().split("\n")[0]; + }; + + expect(await run()).toContain(projAgents); + fs.rmSync(projAgents, { recursive: true, force: true }); + expect(await run()).toContain(projClaude); + fs.rmSync(projClaude, { recursive: true, force: true }); + expect(await run()).toContain(homeAgents); + fs.rmSync(homeAgents, { recursive: true, force: true }); + expect(await run()).toContain(homeClaude); + fs.rmSync(tmp, { recursive: true, force: true }); + }); + + it("names every candidate it tried when no tree exists", async () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "context-bill-none-")); + const out = capture(); + const code = await contextBillMain([], { cwd: tmp, homeDir: tmp, stdout: out.stream, stderr: out.stream }); + expect(code).toBe(2); + expect(out.text()).toContain(path.join(tmp, ".agents", "skills")); + expect(out.text()).toContain(path.join(tmp, ".claude", "skills")); + fs.rmSync(tmp, { recursive: true, force: true }); + }); +}); + +describe("token estimate calibration", () => { + it("uses the corpus-wide divisor for path-less callers, rounded", () => { + expect(estimateTokens(3900)).toBe(1000); + expect(estimateTokens(TOKEN_DIVISOR * 2)).toBe(2); + }); + + it("charges each content class its own measured divisor", () => { + expect(contentClass("/x/plan/SKILL.md")).toBe("skillmd"); + expect(contentClass("/x/plan/references/legacy/office.md")).toBe("legacy"); + expect(contentClass("/x/plan/references/RUNTIME.md")).toBe("reference"); + expect(contentClass("/x/plan/references/artifacts/qa/t.md")).toBe("artifact"); + expect(contentClass("/x/plan/SKILL.md#frontmatter")).toBe("frontmatter"); + expect(TOKEN_DIVISORS.legacy).toBeLessThan(TOKEN_DIVISORS.skillmd); + const alpha = buildBill(TREE_A).skills.find((s) => s.name === "alpha")!; + expect(alpha.skillMdTokens).toBeCloseTo(alpha.skillMdBytes / TOKEN_DIVISORS.skillmd, 6); + expect(alpha.forcedRefs[0].tokens).toBeCloseTo(alpha.forcedRefs[0].bytes / TOKEN_DIVISORS.reference, 6); + }); + + it("never claims more precision than it has: no divisor is a measurement", () => { + const bill = buildBill(TREE_A); + expect(bill.tokenEstimateErrorPct).toBeGreaterThan(0); + expect(bill.tokenSource).toContain("estimate"); + expect(bill.calibration).toBeUndefined(); + }); + + it("names the measured error band rather than a vague 'estimates'", () => { + const text = tokenDisclaimer(buildBill(TREE_A)); + expect(text).toContain("ESTIMATES"); + expect(text).toMatch(/worst single file 40%/); + expect(text).toContain("--exact"); + expect(text).toContain("Bytes are always exact"); + const exactText = tokenDisclaimer({ + tokenEstimateErrorPct: 0, + tokenSource: "count_tokens (claude-opus-4-5)", + }); + expect(exactText).toContain("measured with count_tokens"); + expect(exactText).not.toContain("ESTIMATES"); + }); +}); + +describe("rendering", () => { + it("text output carries the live ledgers plus flags", () => { + const text = renderBill(buildBill(TREE_A)); + expect(text).toContain("ALWAYS-ON (every session): 2 skills"); + expect(text).toContain("EAGER (per invocation)"); + expect(text).toContain("frontmatter key(s) the router never reads: x-dead-key"); + expect(text).toContain("foreign-host file in scanner scope: agents.md"); + expect(text).toContain("TOTAL on disk:"); + expect(text).toContain("Token source: estimate"); + expect(text).toContain("Token counts are ESTIMATES"); + }); + + it("states the always-on row's exclusion of the host's per-skill wrapper", () => { + const text = renderBill(buildBill(TREE_A)); + const alwaysOn = text.slice(text.indexOf("ALWAYS-ON"), text.indexOf("EAGER (")); + expect(alwaysOn).toContain("excludes the host's per-skill available_skills XML wrapper"); + }); + + it("--json shape is stable (stripped tiers keep their fields)", async () => { + const out = capture(); + const code = await contextBillMain([TREE_A, "--json"], { stdout: out.stream, stderr: out.stream }); + expect(code).toBe(0); + const bill = JSON.parse(out.text()); + expect(bill.tokenEstimate).toEqual(TOKEN_DIVISORS); + expect(bill.tokenEstimateErrorPct).toBe(40); + expect(Object.keys(bill.totals).sort()).toEqual([ + "alwaysOnBytes", + "alwaysOnTokens", + "eagerBytesBySkill", + "eagerTokensBySkill", + "perInvocationBytesBySkill", + "perInvocationTokensBySkill", + "skillCount", + "totalMdBytes", + "totalMdTokens", + ]); + const skill = bill.skills[0]; + for (const key of ["name", "frontmatterBytes", "frontmatterTokens", "deadKeys", "skillMdBytes", "skillMdTokens", "forcedRefs", "eagerBytes", "eagerTokens", "fastPath", "conditionalRefs", "conditionalBytes", "conditionalTokens", "transitiveRefs", "transitiveBytes", "transitiveTokens", "perInvocationBytes", "perInvocationTokens", "routeCeiling", "lazy", "orphans", "foreignFiles", "totalMdTokens"]) { + expect(skill).toHaveProperty(key); + } + for (const r of skill.forcedRefs) expect(typeof r.tokens).toBe("number"); + }); +}); + +describe("--diff", () => { + it("reports exactly the grown row and exits 2", async () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "context-bill-")); + const treeB = path.join(tmp, "tree-b"); + fs.cpSync(TREE_A, treeB, { recursive: true }); + fs.appendFileSync(path.join(treeB, "alpha", "SKILL.md"), "x".repeat(43000)); + + const diff = diffBills(buildBill(TREE_A), buildBill(treeB)); + expect(diff.rows).toHaveLength(1); + expect(diff.rows[0]).toMatchObject({ ledger: "eager", label: "alpha", delta: 43000 }); + expect(diff.grew).toBe(true); + + const out = capture(); + const code = await contextBillMain(["--diff", TREE_A, treeB], { stdout: out.stream, stderr: out.stream }); + expect(code).toBe(2); + expect(out.text()).toContain("eager"); + expect(out.text()).toContain("GREW"); + fs.rmSync(tmp, { recursive: true, force: true }); + }); + + it("identical trees diff clean and exit 0", async () => { + const out = capture(); + const code = await contextBillMain(["--diff", TREE_A, TREE_A], { stdout: out.stream, stderr: out.stream }); + expect(code).toBe(0); + expect(out.text()).toContain("No context-cost changes"); + }); +}); + +describe("--budget", () => { + const alphaEagerTok = Math.round(buildBill(TREE_A).skills.find((s) => s.name === "alpha")!.eagerTokens); + + it("exits 0 under budget, 2 over budget with offending files listed", async () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "context-bill-budget-")); + const okBudget = path.join(tmp, "ok.json"); + const tightBudget = path.join(tmp, "tight.json"); + fs.writeFileSync(okBudget, JSON.stringify({ alwaysOnTotal: 10000, eagerPerInvocation: { alpha: alphaEagerTok } })); + fs.writeFileSync(tightBudget, JSON.stringify({ eagerPerInvocation: { alpha: alphaEagerTok - 1 } })); + + const ok = capture(); + expect(await contextBillMain([TREE_A, "--budget", okBudget], { stdout: ok.stream, stderr: ok.stream })).toBe(0); + expect(ok.text()).toContain("Within budget"); + + const over = capture(); + expect(await contextBillMain([TREE_A, "--budget", tightBudget], { stdout: over.stream, stderr: over.stream })).toBe(2); + expect(over.text()).toContain("OVER BUDGET: eagerPerInvocation.alpha"); + expect(over.text()).toContain("alpha/SKILL.md"); + expect(over.text()).toContain("references/CORE.md"); + fs.rmSync(tmp, { recursive: true, force: true }); + }); + + it("perInvocation and routeCeiling keys stay accepted (they gate the eager figure while tiers are stripped)", () => { + const bill = buildBill(TREE_A); + expect(checkBudget(bill, { perInvocation: { alpha: alphaEagerTok } })).toEqual([]); + expect(checkBudget(bill, { perInvocation: { alpha: alphaEagerTok - 1 } })).toHaveLength(1); + expect(checkBudget(bill, { routeCeiling: { alpha: alphaEagerTok } })).toEqual([]); + }); + + it("checkBudget flags a budgeted skill missing from the tree", () => { + const violations = checkBudget(buildBill(TREE_A), { eagerPerInvocation: { ghost: 100 } }); + expect(violations).toHaveLength(1); + expect(violations[0].ceiling).toBe("eagerPerInvocation.ghost"); + }); +}); + +describe("--exact (opt-in measurement; offline here via an injected fetch)", () => { + let egressHome: string; + beforeAll(() => { + egressHome = fs.mkdtempSync(path.join(os.tmpdir(), "context-bill-egress-")); + }); + afterAll(() => { + try { fs.chmodSync(path.join(egressHome, "security"), 0o700); } catch {} + fs.rmSync(egressHome, { recursive: true, force: true }); + }); + + const ENVELOPE = 7; + /** Deterministic stand-in for count_tokens: 1 token per 3 chars, plus envelope. */ + function fakeFetch(calls: { body: string }[] = []) { + return (async (_url: string, init: { body: string }) => { + calls.push({ body: init.body }); + const { messages } = JSON.parse(init.body); + const text: string = messages[0].content; + return { + ok: true, + json: async () => ({ input_tokens: Math.ceil(text.length / 3) + ENVELOPE }), + }; + }) as never; + } + + it("writes the egress receipt BEFORE the first count_tokens POST", async () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "context-bill-receipt-")); + let receiptsAtFirstPost = -1; + const fetchImpl = (async (_url: string, init: { body: string }) => { + if (receiptsAtFirstPost === -1) receiptsAtFirstPost = listReceipts(home).length; + const { messages } = JSON.parse(init.body); + return { ok: true, json: async () => ({ input_tokens: Math.ceil(messages[0].content.length / 3) + ENVELOPE }) }; + }) as never; + await measureExactTokens(TREE_A, { model: "m", apiKey: "sk-test", fetchImpl, egressHome: home }); + expect(receiptsAtFirstPost).toBe(1); // the receipt existed before any POST + const receipts = listReceipts(home); + expect(receipts).toHaveLength(1); + expect(receipts[0].sink).toBe("context-bill-exact"); + expect(receipts[0].host).toBe("api.anthropic.com"); + expect(receipts[0].sha256).toBeNull(); + expect(receipts[0].bytes).toBeGreaterThan(0); + fs.rmSync(home, { recursive: true, force: true }); + }); + + it("fail-open: an unwritable ledger degrades --exact to the offline estimate, sending nothing", async () => { + if (process.platform === "win32" || process.getuid?.() === 0) return; + const home = fs.mkdtempSync(path.join(os.tmpdir(), "context-bill-refuse-")); + fs.mkdirSync(path.join(home, "security"), { recursive: true, mode: 0o500 }); + let called = false; + const stdout = capture(); + const stderr = capture(); + const code = await contextBillMain([TREE_A, "--exact", "--json"], { + stdout: stdout.stream, + stderr: stderr.stream, + apiKey: "sk-test", + fetchImpl: (() => ((called = true), Promise.reject(new Error("must not send")))) as never, + egressHome: home, + }); + expect(code).toBe(0); // the run still answers, with the estimate + expect(called).toBe(false); // NOTHING was sent unrecorded + expect(stderr.text()).toContain("exact_egress_receipt_failed"); + expect(stderr.text()).toContain("Falling back to the offline estimate"); + expect(JSON.parse(stdout.text()).tokenSource).toContain("estimate"); + fs.chmodSync(path.join(home, "security"), 0o700); + fs.rmSync(home, { recursive: true, force: true }); + }); + + it("subtracts the request envelope so small files are not overcharged", async () => { + const calls: { body: string }[] = []; + const measured = await measureExactTokens(TREE_A, { + model: "test-model", + apiKey: "sk-test", + fetchImpl: fakeFetch(calls), + egressHome, + }); + expect(measured.tokenSource).toBe("count_tokens (test-model)"); + const core = path.join(TREE_A, "alpha", "references", "CORE.md"); + const text = fs.readFileSync(core, "utf8"); + expect(measured.tokensOf(core, text.length)).toBe(Math.ceil(text.length / 3)); + // One probe call for the envelope, then one per measured text. + expect(calls.length).toBe(measured.measuredFiles + 1); + }); + + it("measures the frontmatter block, not the whole SKILL.md, for the always-on row", async () => { + const measured = await measureExactTokens(TREE_A, { + model: "m", apiKey: "sk-test", fetchImpl: fakeFetch(), egressHome, + }); + const skillMd = path.join(TREE_A, "alpha", "SKILL.md"); + const fmTokens = measured.tokensOf(`${skillMd}#frontmatter`, 0); + const bodyTokens = measured.tokensOf(skillMd, 0); + expect(fmTokens).toBeGreaterThan(0); + expect(fmTokens).toBeLessThan(bodyTokens); + }); + + it("exact tokens replace the estimate everywhere the bill prices content", async () => { + const measured = await measureExactTokens(TREE_A, { + model: "m", apiKey: "sk-test", fetchImpl: fakeFetch(), egressHome, + }); + const bill = buildBill(TREE_A, { tokensOf: measured.tokensOf, tokenSource: measured.tokenSource }); + const alpha = bill.skills.find((s) => s.name === "alpha")!; + expect(alpha.eagerTokens).toBeGreaterThanOrEqual(alpha.eagerBytes / 3); + expect(alpha.eagerTokens).toBeLessThan(alpha.eagerBytes / 3 + 4); + expect(alpha.eagerTokens / (alpha.eagerBytes / TOKEN_DIVISOR)).toBeGreaterThan(1.2); + expect(alpha.eagerTokens).toBe(alpha.skillMdTokens + alpha.forcedRefs.reduce((n, r) => n + r.tokens, 0)); + expect(bill.tokenEstimateErrorPct).toBe(0); + expect(renderBill(bill)).toContain("Token counts measured with count_tokens"); + expect(renderBill(bill)).not.toMatch(/\(~\d/); + }); + + it("measures against a relative root too, instead of silently estimating", async () => { + const relative = path.relative(process.cwd(), TREE_A); + const measured = await measureExactTokens(relative, { + model: "m", apiKey: "sk-test", fetchImpl: fakeFetch(), egressHome, + }); + const bill = buildBill(relative, { tokensOf: measured.tokensOf, tokenSource: measured.tokenSource }); + for (const s of bill.skills) { + expect(s.totalMdBytes / s.totalMdTokens).toBeLessThan(3.2); + } + expect(measured.missedKeys.size).toBe(0); + }); + + it("counts anything it could not measure instead of passing it off as measured", async () => { + const measured = await measureExactTokens(TREE_A, { + model: "m", apiKey: "sk-test", fetchImpl: fakeFetch(), egressHome, + }); + expect(measured.missedKeys.size).toBe(0); + const ghost = path.join(TREE_A, "alpha", "nope.md"); + expect(measured.tokensOf(ghost, 4150)).toBeGreaterThan(0); + expect(measured.missedKeys.has(ghost)).toBe(true); + }); + + it("grades its own estimate: calibrationTable reports the residual per file", async () => { + const measured = await measureExactTokens(TREE_A, { + model: "m", apiKey: "sk-test", fetchImpl: fakeFetch(), egressHome, + }); + const table = calibrationTable(measured.counts, TREE_A); + expect(table.rows.length).toBeGreaterThan(0); + for (const row of table.rows) { + expect(row).toHaveProperty("contentClass"); + expect(row.estimatedTokens).toBeGreaterThan(0); + expect(row.tokens).toBeGreaterThan(0); + expect(row.errorPct).toBeCloseTo(((row.estimatedTokens - row.tokens) / row.tokens) * 100, 1); + } + expect(typeof table.worstErrorPct).toBe("number"); + expect(typeof table.biasPct).toBe("number"); + }); + + it("refuses to go to the network without a key, and says nothing was sent", async () => { + let called = false; + await expect( + measureExactTokens(TREE_A, { + model: "m", + apiKey: "", + fetchImpl: (() => ((called = true), Promise.reject(new Error("should not run")))) as never, + egressHome, + }), + ).rejects.toMatchObject({ code: "exact_missing_api_key" }); + expect(called).toBe(false); + }); + + it("maps HTTP failures to typed codes", async () => { + const status = (code: number) => (async () => ({ ok: false, status: code, text: async () => "nope" })) as never; + for (const [code, expected] of [ + [401, "exact_auth_rejected"], + [403, "exact_auth_rejected"], + [500, "exact_request_failed"], + ] as const) { + await expect( + measureExactTokens(TREE_A, { model: "m", apiKey: "k", fetchImpl: status(code), egressHome }), + ).rejects.toMatchObject({ code: expected }); + } + await expect( + measureExactTokens(TREE_A, { + model: "m", apiKey: "k", + fetchImpl: (() => Promise.reject(new Error("offline"))) as never, + egressHome, + }), + ).rejects.toBeInstanceOf(ExactModeError); + }); + + it("offline by default: no --exact means no network call at all", async () => { + const out = capture(); + let called = false; + const code = await contextBillMain([TREE_A, "--json"], { + stdout: out.stream, + stderr: out.stream, + fetchImpl: (() => ((called = true), Promise.reject(new Error("no")))) as never, + apiKey: "sk-test", + egressHome, + }); + expect(code).toBe(0); + expect(called).toBe(false); + expect(JSON.parse(out.text()).tokenSource).toContain("estimate"); + }); + + it("discloses what --exact sends before sending it", async () => { + const stdout = capture(); + const stderr = capture(); + const code = await contextBillMain([TREE_A, "--exact", "--json"], { + stdout: stdout.stream, + stderr: stderr.stream, + apiKey: "sk-test", + fetchImpl: fakeFetch(), + egressHome, + }); + expect(code).toBe(0); + expect(stderr.text()).toContain("api.anthropic.com"); + expect(stderr.text()).toMatch(/sending the content of \d+ \.md file\(s\)/); + const bill = JSON.parse(stdout.text()); + expect(bill.tokenSource).toContain("count_tokens"); + expect(bill.calibration.rows.length).toBeGreaterThan(0); + }); + + it("degrades to the estimate when exact mode is unavailable, naming the code", async () => { + const stdout = capture(); + const stderr = capture(); + const code = await contextBillMain([TREE_A, "--exact", "--json"], { + stdout: stdout.stream, + stderr: stderr.stream, + apiKey: "", + fetchImpl: (() => Promise.reject(new Error("should not run"))) as never, + egressHome, + }); + expect(code).toBe(0); + expect(stderr.text()).toContain("exact_missing_api_key"); + expect(JSON.parse(stdout.text()).tokenSource).toContain("estimate"); + }); + + it("--help discloses --exact's egress and recalibration", async () => { + const out = capture(); + expect(await contextBillMain(["--help"], { stdout: out.stream, stderr: out.stream })).toBe(0); + expect(out.text()).toContain("api.anthropic.com"); + expect(out.text()).toContain("egress receipt"); + expect(out.text()).toContain("recalibrates"); + }); +}); + +describe("ground truth against THIS repo (skill-census parity)", () => { + const census = skillCensus(ROOT); + const dirs = findSkillDirs(ROOT); + const rels = dirs.map((d) => path.relative(ROOT, d) || "."); + + it("the walker reaches every physical SKILL.md the census counts", () => { + // physicalSkillFiles is the walker's expectation: every depth-1 skill dir + // (symlinked dirs included) plus the root router. + for (const rel of census.physicalSkillFiles) { + const dirRel = rel === "SKILL.md" ? "." : path.dirname(rel); + expect(rels, `walker missed ${rel}`).toContain(dirRel); + } + }); + + it("the root router SKILL.md is billed (fix a live on the real repo)", () => { + expect(census.physicalSkillFiles).toContain("SKILL.md"); // guard the guard + expect(rels).toContain("."); + }); + + it("a real skill's frontmatter bytes match an independent computation", () => { + const qaDir = dirs.find((d) => path.relative(ROOT, d) === "qa")!; + expect(qaDir).toBeTruthy(); + const bill = buildBill(qaDir); + expect(bill.skills).toHaveLength(1); + expect(bill.skills[0].frontmatterBytes).toBe(frontmatterBytes(path.join(qaDir, "SKILL.md"))); + expect(bill.skills[0].frontmatterBytes).toBeGreaterThan(0); + }); + + it("skill count is at least the census count (deeper trees may add more, never fewer)", () => { + expect(dirs.length).toBeGreaterThanOrEqual(census.physicalSkillFiles.length); + }); + + it("no skill dir is billed twice", () => { + expect(new Set(rels).size).toBe(rels.length); + }); +}); + +describe("CLI plumbing", () => { + it("unknown flag and missing tree are usage errors (exit 2)", async () => { + const a = capture(); + expect(await contextBillMain(["--nope"], { stdout: a.stream, stderr: a.stream })).toBe(2); + const b = capture(); + expect(await contextBillMain(["--diff", TREE_A], { stdout: b.stream, stderr: b.stream })).toBe(2); + const c = capture(); + expect(await contextBillMain([path.join(os.tmpdir(), "does-not-exist-xyz")], { stdout: c.stream, stderr: c.stream })).toBe(1); + }); + + it("bin/gstack-context-bill runs standalone", () => { + const result = Bun.spawnSync([path.join(ROOT, "bin", "gstack-context-bill"), TREE_A]); + expect(result.exitCode).toBe(0); + expect(result.stdout.toString()).toContain("ALWAYS-ON"); + expect(result.stdout.toString()).toContain("EAGER"); + }); +}); diff --git a/test/fixtures/context-bill/tree-a/alpha/SKILL.md b/test/fixtures/context-bill/tree-a/alpha/SKILL.md new file mode 100644 index 000000000..98f55e48d --- /dev/null +++ b/test/fixtures/context-bill/tree-a/alpha/SKILL.md @@ -0,0 +1,21 @@ +--- +name: alpha +description: Fixture dispatcher with a mode table and forced-read references. +triggers: a-key-the-upstream-router-DOES-read +x-dead-key: a-key-no-router-reads +--- + +# Alpha + +## Dispatch protocol + +1. Infer the mode from the request. +2. Read `references/CORE.md` and `references/POLICY.md` for every invocation. Read `references/OPTIONAL.md` before public-web work. +3. When the target is a repository, read `references/CORE.md` once before specialist work; it is already billed eagerly. + +## Top-level modes + +| Mode | Infer when | Candidate internal specialists | +|---|---|---| +| `Discovery` | The idea is fluid. | `references/legacy/office.md` | +| `Full chain` | Everything at once. | `references/legacy/auto.md` | diff --git a/test/fixtures/context-bill/tree-a/alpha/references/CORE.md b/test/fixtures/context-bill/tree-a/alpha/references/CORE.md new file mode 100644 index 000000000..c81ca42ad --- /dev/null +++ b/test/fixtures/context-bill/tree-a/alpha/references/CORE.md @@ -0,0 +1,3 @@ +# Core + +Forced-read reference number one. Read on every invocation of alpha. diff --git a/test/fixtures/context-bill/tree-a/alpha/references/OPTIONAL.md b/test/fixtures/context-bill/tree-a/alpha/references/OPTIONAL.md new file mode 100644 index 000000000..d83a78384 --- /dev/null +++ b/test/fixtures/context-bill/tree-a/alpha/references/OPTIONAL.md @@ -0,0 +1,3 @@ +# Optional + +Conditional reference. Not part of the eager bill. diff --git a/test/fixtures/context-bill/tree-a/alpha/references/POLICY.md b/test/fixtures/context-bill/tree-a/alpha/references/POLICY.md new file mode 100644 index 000000000..54b723c51 --- /dev/null +++ b/test/fixtures/context-bill/tree-a/alpha/references/POLICY.md @@ -0,0 +1,3 @@ +# Policy + +Forced-read reference number two. diff --git a/test/fixtures/context-bill/tree-a/beta/SKILL.md b/test/fixtures/context-bill/tree-a/beta/SKILL.md new file mode 100644 index 000000000..5b52c3359 --- /dev/null +++ b/test/fixtures/context-bill/tree-a/beta/SKILL.md @@ -0,0 +1,8 @@ +--- +name: beta +description: Clean fixture tool skill with no forced reads and no mode table. +--- + +# Beta + +One-command tool skill. Nothing eager beyond this file, nothing lazy. diff --git a/test/fixtures/context-bill/tree-a/beta/agents.md b/test/fixtures/context-bill/tree-a/beta/agents.md new file mode 100644 index 000000000..01aa0a02f --- /dev/null +++ b/test/fixtures/context-bill/tree-a/beta/agents.md @@ -0,0 +1,3 @@ +# Foreign host file + +A skill-shaped file another host dropped into scanner scope (#1694).