test(grok-build): dual-write parity + packaging behavior coverage

Add free tests for remaining code-review debt: runtimeRoot vs setup parity,
generateSpecSpawn fail-closed allowlist, GrokAdapter auth/HOME/prompt-file,
compat-audit fixture exit codes, and preflight-no-wipe setup smoke. Prefer
HOME for auth discovery so hermetic tests work under Bun; pipe stderr so
token-shaped CLI errors never inherit onto the parent console.
This commit is contained in:
Nehr 2026-07-13 02:04:30 +07:00
parent 4f6b2a58f8
commit 14eb95c32d
3 changed files with 432 additions and 1 deletions

View File

@ -0,0 +1,398 @@
/**
* Behavioral unit tests for Grok Build packaging surfaces (code-review #8):
* - generateSpecSpawn fail-closed allowlist + no default --always-approve
* - GrokAdapter available timeout / --prompt-file path / auth redaction
* - gstack-grok-compat-audit exit codes against fixture skills dirs
*
* Free (no API spend). Does not invoke real grok CLI for successful runs.
*/
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
import { spawnSync } from 'child_process';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { generateSpecSpawn, generateSpecExecuteFlag } from '../scripts/resolvers/spec-spawn';
import type { TemplateContext } from '../scripts/resolvers/types';
import { HOST_PATHS } from '../scripts/resolvers/types';
import { GrokAdapter } from './helpers/providers/grok';
const ROOT = path.resolve(import.meta.dir, '..');
const AUDIT_BIN = path.join(ROOT, 'bin', 'gstack-grok-compat-audit');
function makeCtx(host: 'grok-build' | 'claude'): TemplateContext {
return {
skillName: 'spec',
tmplPath: path.join(ROOT, 'spec', 'SKILL.md.tmpl'),
host,
paths: HOST_PATHS[host],
};
}
// ─── generateSpecSpawn ───────────────────────────────────────
describe('generateSpecSpawn (Grok fail-closed)', () => {
test('grok-build emits auth gate, ARCHIVE allowlist, and no default --always-approve', () => {
const out = generateSpecSpawn(makeCtx('grok-build'));
expect(out).toContain('command -v grok');
expect(out).toContain('ARCHIVE_PATH');
expect(out).toContain('SPAWN_PATH');
expect(out).toContain('GSTACK_STATE_ROOT');
expect(out).toContain('realpath');
expect(out).toContain('fail closed');
expect(out).toContain('grok --prompt-file');
// Default spawn must NOT enable elevated auto-approve
expect(out).not.toMatch(/grok --prompt-file[^\n]*--always-approve/);
expect(out).toMatch(/opt-in only|does \*\*not\*\* pass `--always-approve`|--always-approve` is opt-in/i);
});
test('grok-build --execute-claude path reuses ARCHIVE allowlist before cat|claude', () => {
const out = generateSpecSpawn(makeCtx('grok-build'));
expect(out).toContain('--execute-claude');
// Claude branch must re-check allowlist, not bare cat only
const claudeIdx = out.indexOf('--execute-claude');
const after = out.slice(claudeIdx);
expect(after).toContain('ARCHIVE_REAL');
expect(after).toContain('STATE_PROJECTS');
expect(after).toMatch(/cat "\$ARCHIVE_PATH" \| \(cd "\$SPAWN_PATH" && claude -p/);
});
test('claude host still uses classic stdin pipe (no Grok-only gates)', () => {
const out = generateSpecSpawn(makeCtx('claude'));
expect(out).toContain('claude -p');
expect(out).not.toContain('grok --prompt-file');
});
test('generateSpecExecuteFlag is host-aware', () => {
expect(generateSpecExecuteFlag(makeCtx('grok-build'))).toContain('grok --prompt-file');
expect(generateSpecExecuteFlag(makeCtx('claude'))).toContain('claude -p');
});
});
// ─── GrokAdapter ─────────────────────────────────────────────
describe('GrokAdapter behavioral unit', () => {
const adapter = new GrokAdapter();
let tmpHome: string;
let prevHome: string | undefined;
let prevXai: string | undefined;
let prevGrok: string | undefined;
beforeEach(() => {
tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-grok-adapter-'));
prevHome = process.env.HOME;
prevXai = process.env.XAI_API_KEY;
prevGrok = process.env.GROK_API_KEY;
process.env.HOME = tmpHome;
delete process.env.XAI_API_KEY;
delete process.env.GROK_API_KEY;
});
afterEach(() => {
if (prevHome === undefined) delete process.env.HOME;
else process.env.HOME = prevHome;
if (prevXai === undefined) delete process.env.XAI_API_KEY;
else process.env.XAI_API_KEY = prevXai;
if (prevGrok === undefined) delete process.env.GROK_API_KEY;
else process.env.GROK_API_KEY = prevGrok;
fs.rmSync(tmpHome, { recursive: true, force: true });
});
test('available() returns not-ok when CLI and auth both missing', async () => {
// Isolate PATH so grok cannot be found even if installed on the machine.
// Some environments still resolve `grok` via absolute exec caches — accept
// either binary-missing OR auth-missing as a valid not-ok outcome.
const prevPath = process.env.PATH;
process.env.PATH = '/nonexistent-bin-dir-for-grok-avail';
try {
const r = await adapter.available();
expect(r.ok).toBe(false);
expect(r.reason).toMatch(/not found|Install Grok|PATH|No Grok auth|XAI_API_KEY|GROK_API_KEY/i);
} finally {
process.env.PATH = prevPath;
}
});
test('available() fails when binary present but no auth file and no env key names', async () => {
// If grok is not on PATH in this environment, skip — unit still covers the
// auth-file/env-name branch when binary is present.
const which = spawnSync('sh', ['-c', 'command -v grok'], {
timeout: 2000,
encoding: 'utf-8',
stdio: ['ignore', 'pipe', 'ignore'],
});
if (which.status !== 0) {
// No binary: available should report CLI missing under empty HOME
const r = await adapter.available();
expect(r.ok).toBe(false);
return;
}
const r = await adapter.available();
expect(r.ok).toBe(false);
expect(r.reason).toMatch(/No Grok auth|XAI_API_KEY|GROK_API_KEY/i);
});
test('available() accepts auth.json presence under HOME (boolean only)', async () => {
const which = spawnSync('sh', ['-c', 'command -v grok'], {
timeout: 2000,
encoding: 'utf-8',
stdio: ['ignore', 'pipe', 'ignore'],
});
if (which.status !== 0) return; // cannot exercise auth-file path without binary
fs.mkdirSync(path.join(tmpHome, '.grok'), { recursive: true });
fs.writeFileSync(path.join(tmpHome, '.grok', 'auth.json'), '{}');
const r = await adapter.available();
expect(r.ok).toBe(true);
});
test('run() uses --prompt-file for multi-line / large prompts (ARG_MAX safety)', async () => {
// Force binary_missing path so we never hit a real CLI; still inspects argv construction
// via the ENOENT error path. Large prompt must not throw ARG_MAX.
const big = 'line\n'.repeat(500) + 'x'.repeat(3000);
const prevPath = process.env.PATH;
process.env.PATH = '/nonexistent-bin-dir-for-grok-test';
try {
const r = await adapter.run({
prompt: big,
workdir: tmpHome,
timeoutMs: 2000,
});
expect(r.error?.code).toBe('binary_missing');
expect(r.tokens).toEqual({ input: 0, output: 0 });
} finally {
process.env.PATH = prevPath;
}
});
test('run() redacts long token-shaped stderr on unknown errors', async () => {
// Drive the unknown error path with a fake binary that exits non-zero
const fakeBin = path.join(tmpHome, 'bin');
fs.mkdirSync(fakeBin, { recursive: true });
const grokSh = path.join(fakeBin, 'grok');
fs.writeFileSync(
grokSh,
`#!/bin/sh\necho "auth token SECRET_TOKEN_ABCDEFGHIJKLMNOPQRSTUVWXYZ123456" >&2\nexit 1\n`,
{ mode: 0o755 },
);
const prevPath = process.env.PATH;
process.env.PATH = `${fakeBin}:${prevPath ?? ''}`;
try {
const r = await adapter.run({
prompt: 'hi',
workdir: tmpHome,
timeoutMs: 3000,
});
// Auth keyword in stderr maps to auth code with redacted reason
expect(r.error).toBeDefined();
if (r.error?.code === 'auth') {
expect(r.error.reason).not.toContain('SECRET_TOKEN');
} else {
expect(r.error!.reason).not.toContain('SECRET_TOKEN_ABCDEFGHIJKLMNOPQRSTUVWXYZ123456');
expect(r.error!.reason).toMatch(/\[redacted\]|unknown|authentication/i);
}
} finally {
process.env.PATH = prevPath;
}
});
});
// ─── gstack-grok-compat-audit fixtures ───────────────────────
function writeMinimalRuntime(skillsDir: string, opts: { withReview?: boolean; withSpec?: boolean } = {}) {
const { withReview = true, withSpec = true } = opts;
const root = path.join(skillsDir, 'gstack');
for (const rel of [
'bin',
'browse/dist',
'browse/src',
'scripts',
'review/specialists',
]) {
fs.mkdirSync(path.join(root, rel), { recursive: true });
}
// When monorepo has design/dist + extension, audit requires them in the runtime root
if (fs.existsSync(path.join(ROOT, 'design', 'dist'))) {
fs.mkdirSync(path.join(root, 'design', 'dist'), { recursive: true });
}
if (fs.existsSync(path.join(ROOT, 'extension'))) {
fs.mkdirSync(path.join(root, 'extension'), { recursive: true });
}
// browse-client required by skillify audit
fs.mkdirSync(path.join(root, 'browse', 'src'), { recursive: true });
fs.writeFileSync(path.join(root, 'browse', 'src', 'browse-client.ts'), '// fixture\n');
if (withReview) {
fs.writeFileSync(path.join(root, 'review', 'checklist.md'), '# checklist\n');
fs.writeFileSync(path.join(root, 'review', 'TODOS-format.md'), '# todos\n');
}
// Audit compares fixture runtime against monorepo presence of design/extension.
// Mirror those when the monorepo has them so the fixture is "complete" for phase a.
if (fs.existsSync(path.join(ROOT, 'design', 'dist'))) {
fs.mkdirSync(path.join(root, 'design', 'dist'), { recursive: true });
}
if (fs.existsSync(path.join(ROOT, 'extension'))) {
fs.mkdirSync(path.join(root, 'extension'), { recursive: true });
fs.writeFileSync(path.join(root, 'extension', 'manifest.json'), '{}\n');
}
// connect-chrome → open-gstack-browser shape
const ogb = path.join(skillsDir, 'gstack-open-gstack-browser');
fs.mkdirSync(ogb, { recursive: true });
fs.writeFileSync(path.join(ogb, 'SKILL.md'), 'name: open-gstack-browser\n');
fs.symlinkSync(ogb, path.join(skillsDir, 'connect-chrome'));
if (withSpec) {
const spec = path.join(skillsDir, 'gstack-spec');
fs.mkdirSync(spec, { recursive: true });
fs.writeFileSync(
path.join(spec, 'SKILL.md'),
[
'---',
'name: spec',
'---',
'Spawn **Grok** headless with --prompt-file.',
'```bash',
'grok --prompt-file "$ARCHIVE_PATH" --cwd "$SPAWN_PATH"',
'```',
'$GSTACK_ROOT',
].join('\n'),
);
}
}
function runAudit(skillsDir: string, phase = 'a'): { status: number | null; stdout: string; stderr: string } {
const r = spawnSync('bun', ['run', AUDIT_BIN, '--skills-dir', skillsDir, '--phase', phase], {
cwd: ROOT,
encoding: 'utf-8',
timeout: 30000,
});
return {
status: r.status,
stdout: r.stdout?.toString() ?? '',
stderr: r.stderr?.toString() ?? '',
};
}
describe('gstack-grok-compat-audit fixtures', () => {
let skillsDir: string;
beforeEach(() => {
skillsDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-grok-audit-'));
});
afterEach(() => {
fs.rmSync(skillsDir, { recursive: true, force: true });
});
test('missing skills dir exits 1 INCOMPATIBLE', () => {
const missing = path.join(skillsDir, 'does-not-exist');
const r = runAudit(missing, 'a');
expect(r.status).toBe(1);
expect(r.stdout).toMatch(/INCOMPATIBLE|skills dir missing/i);
});
test('minimal complete runtime exits 0 COMPATIBLE for phase a', () => {
writeMinimalRuntime(skillsDir);
const r = runAudit(skillsDir, 'a');
expect(r.status).toBe(0);
expect(r.stdout).toContain('COMPATIBLE');
});
test('missing review/checklist.md fails phase a', () => {
writeMinimalRuntime(skillsDir, { withReview: false });
const r = runAudit(skillsDir, 'a');
expect(r.status).toBe(1);
expect(r.stdout).toMatch(/missing review\/checklist\.md|INCOMPATIBLE/);
});
test('missing gstack runtime root fails', () => {
fs.mkdirSync(skillsDir, { recursive: true });
// empty skills dir — no gstack/
const r = runAudit(skillsDir, 'a');
expect(r.status).toBe(1);
expect(r.stdout).toMatch(/missing runtime root|INCOMPATIBLE/);
});
test('phase ab fails when neither gstack-spec nor spec package installed', () => {
writeMinimalRuntime(skillsDir, { withSpec: false });
const r = runAudit(skillsDir, 'ab');
expect(r.status).toBe(1);
expect(r.stdout).toMatch(/neither gstack-spec nor spec|INCOMPATIBLE/);
});
});
// ─── setup create_grok_runtime_root preflight smoke ──────────
describe('create_grok_runtime_root preflight (setup smoke)', () => {
test('refuses wipe when required monorepo assets missing', () => {
const emptyRepo = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-grok-setup-empty-'));
const dest = path.join(emptyRepo, 'install-target', 'gstack');
// Seed a live install that must NOT be wiped when preflight fails
fs.mkdirSync(dest, { recursive: true });
fs.writeFileSync(path.join(dest, 'KEEPME'), 'live');
const setupPath = path.join(ROOT, 'setup');
// Source extracted functions from a temp file — do NOT eval via $(...) because
// create_grok_runtime_root uses $$ for the staging suffix and command
// substitution would expand it early.
const extractScript = path.join(emptyRepo, 'extract-and-run.sh');
fs.writeFileSync(
extractScript,
`#!/usr/bin/env bash
set +e
SETUP=${JSON.stringify(setupPath)}
GSTACK_DIR=${JSON.stringify(emptyRepo)}
DEST=${JSON.stringify(dest)}
_link_or_copy() { ln -sfn "$1" "$2" 2>/dev/null || cp -R "$1" "$2"; }
extract_fn() {
local name="$1"
local out="$2"
awk -v name="$name" '
$0 ~ "^" name "\\\\(\\\\)" {grab=1}
grab {
print
for (i=1;i<=length($0);i++) {
c=substr($0,i,1)
if (c=="{") depth++
if (c=="}") {
depth--
if (depth==0) { exit }
}
}
}
' "$SETUP" > "$out"
}
FN_DIR=$(mktemp -d)
extract_fn _grok_link_under_monorepo "$FN_DIR/link.sh"
extract_fn create_grok_runtime_root "$FN_DIR/create.sh"
# shellcheck source=/dev/null
. "$FN_DIR/link.sh"
# shellcheck source=/dev/null
. "$FN_DIR/create.sh"
create_grok_runtime_root "$GSTACK_DIR" "$DEST"
rc=$?
rm -rf "$FN_DIR"
if [ $rc -eq 0 ]; then
echo "UNEXPECTED_SUCCESS"
exit 2
fi
echo "PREFLIGHT_FAILED_AS_EXPECTED"
if [ -f "$DEST/KEEPME" ]; then
echo "LIVE_PRESERVED"
exit 0
fi
echo "LIVE_WIPED"
exit 3
`,
{ mode: 0o755 },
);
const r = spawnSync('bash', [extractScript], {
encoding: 'utf-8',
timeout: 15000,
});
fs.rmSync(emptyRepo, { recursive: true, force: true });
expect(r.stdout + r.stderr).toMatch(/preflight|required monorepo asset missing/i);
expect(r.stdout).toContain('PREFLIGHT_FAILED_AS_EXPECTED');
expect(r.stdout).toContain('LIVE_PRESERVED');
expect(r.status).toBe(0);
});
});

View File

@ -42,7 +42,10 @@ export class GrokAdapter implements ProviderAdapter {
};
}
const authPath = path.join(os.homedir(), '.grok', 'auth.json');
// Prefer HOME when set so hermetic tests / agent envs can isolate auth
// discovery. Bun's os.homedir() ignores process.env.HOME (unlike Node).
const home = process.env.HOME || os.homedir();
const authPath = path.join(home, '.grok', 'auth.json');
const hasAuthFile = fs.existsSync(authPath);
// Presence of env *names* only — never read/log values
const hasKey = !!(process.env.XAI_API_KEY || process.env.GROK_API_KEY);
@ -82,6 +85,8 @@ export class GrokAdapter implements ProviderAdapter {
timeout: opts.timeoutMs,
encoding: 'utf-8',
maxBuffer: 32 * 1024 * 1024,
// Pipe stderr so auth-shaped tokens never inherit onto the parent console
stdio: ['ignore', 'pipe', 'pipe'],
env: { ...process.env, GSTACK_HEADLESS: '1' },
});
return {

View File

@ -568,6 +568,34 @@ describe('host config correctness', () => {
expect(grokBuild.pathRewrites.some(r => r.from === 'MODEL_OVERLAY: claude' && r.to === 'MODEL_OVERLAY: none')).toBe(true);
expect(grokBuild.toolRewrites?.['AskUserQuestion']).toBe('ask_user_question');
});
// Dual-write parity (#5): hosts/grok-build.ts runtimeRoot must appear in setup's
// create_grok_runtime_root so packaging cannot skew silently.
test('grok-build runtimeRoot dual-write parity with create_grok_runtime_root', () => {
const setupPath = path.join(ROOT, 'setup');
const setup = fs.readFileSync(setupPath, 'utf-8');
const fnStart = setup.indexOf('create_grok_runtime_root()');
expect(fnStart).toBeGreaterThanOrEqual(0);
// Function ends at link_grok_skill_dirs (next sibling) — slice that range
const fnEnd = setup.indexOf('link_grok_skill_dirs()', fnStart);
expect(fnEnd).toBeGreaterThan(fnStart);
const fnBody = setup.slice(fnStart, fnEnd);
for (const link of grokBuild.runtimeRoot.globalSymlinks) {
// Each asset path must appear as a monorepo source under create_grok_runtime_root
expect(fnBody).toContain(link);
}
for (const f of grokBuild.runtimeRoot.globalFiles?.review ?? []) {
expect(fnBody).toContain(f);
}
// Preflight + atomic stage (review #6) stay wired
expect(fnBody).toContain('preflight');
expect(fnBody).toMatch(/\.next\.\$\$|staging/);
// Core review files fail-closed (review #7): required=1 on link
expect(fnBody).toMatch(
/for f in checklist\.md TODOS-format\.md; do[\s\S]*?_grok_link_under_monorepo[^\n]* 1 \|\| return 1/,
);
});
});
describe('resolveDistBinary (U2 double-home fix)', () => {