diff --git a/test/grok-packaging-behavior.test.ts b/test/grok-packaging-behavior.test.ts new file mode 100644 index 000000000..193640e05 --- /dev/null +++ b/test/grok-packaging-behavior.test.ts @@ -0,0 +1,398 @@ +/** + * Behavioral unit tests for Grok Build packaging surfaces (code-review #8): + * - generateSpecSpawn fail-closed allowlist + no default --always-approve + * - GrokAdapter available timeout / --prompt-file path / auth redaction + * - gstack-grok-compat-audit exit codes against fixture skills dirs + * + * Free (no API spend). Does not invoke real grok CLI for successful runs. + */ + +import { describe, test, expect, beforeEach, afterEach } from 'bun:test'; +import { spawnSync } from 'child_process'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import { generateSpecSpawn, generateSpecExecuteFlag } from '../scripts/resolvers/spec-spawn'; +import type { TemplateContext } from '../scripts/resolvers/types'; +import { HOST_PATHS } from '../scripts/resolvers/types'; +import { GrokAdapter } from './helpers/providers/grok'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const AUDIT_BIN = path.join(ROOT, 'bin', 'gstack-grok-compat-audit'); + +function makeCtx(host: 'grok-build' | 'claude'): TemplateContext { + return { + skillName: 'spec', + tmplPath: path.join(ROOT, 'spec', 'SKILL.md.tmpl'), + host, + paths: HOST_PATHS[host], + }; +} + +// ─── generateSpecSpawn ─────────────────────────────────────── + +describe('generateSpecSpawn (Grok fail-closed)', () => { + test('grok-build emits auth gate, ARCHIVE allowlist, and no default --always-approve', () => { + const out = generateSpecSpawn(makeCtx('grok-build')); + expect(out).toContain('command -v grok'); + expect(out).toContain('ARCHIVE_PATH'); + expect(out).toContain('SPAWN_PATH'); + expect(out).toContain('GSTACK_STATE_ROOT'); + expect(out).toContain('realpath'); + expect(out).toContain('fail closed'); + expect(out).toContain('grok --prompt-file'); + // Default spawn must NOT enable elevated auto-approve + expect(out).not.toMatch(/grok --prompt-file[^\n]*--always-approve/); + expect(out).toMatch(/opt-in only|does \*\*not\*\* pass `--always-approve`|--always-approve` is opt-in/i); + }); + + test('grok-build --execute-claude path reuses ARCHIVE allowlist before cat|claude', () => { + const out = generateSpecSpawn(makeCtx('grok-build')); + expect(out).toContain('--execute-claude'); + // Claude branch must re-check allowlist, not bare cat only + const claudeIdx = out.indexOf('--execute-claude'); + const after = out.slice(claudeIdx); + expect(after).toContain('ARCHIVE_REAL'); + expect(after).toContain('STATE_PROJECTS'); + expect(after).toMatch(/cat "\$ARCHIVE_PATH" \| \(cd "\$SPAWN_PATH" && claude -p/); + }); + + test('claude host still uses classic stdin pipe (no Grok-only gates)', () => { + const out = generateSpecSpawn(makeCtx('claude')); + expect(out).toContain('claude -p'); + expect(out).not.toContain('grok --prompt-file'); + }); + + test('generateSpecExecuteFlag is host-aware', () => { + expect(generateSpecExecuteFlag(makeCtx('grok-build'))).toContain('grok --prompt-file'); + expect(generateSpecExecuteFlag(makeCtx('claude'))).toContain('claude -p'); + }); +}); + +// ─── GrokAdapter ───────────────────────────────────────────── + +describe('GrokAdapter behavioral unit', () => { + const adapter = new GrokAdapter(); + let tmpHome: string; + let prevHome: string | undefined; + let prevXai: string | undefined; + let prevGrok: string | undefined; + + beforeEach(() => { + tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-grok-adapter-')); + prevHome = process.env.HOME; + prevXai = process.env.XAI_API_KEY; + prevGrok = process.env.GROK_API_KEY; + process.env.HOME = tmpHome; + delete process.env.XAI_API_KEY; + delete process.env.GROK_API_KEY; + }); + + afterEach(() => { + if (prevHome === undefined) delete process.env.HOME; + else process.env.HOME = prevHome; + if (prevXai === undefined) delete process.env.XAI_API_KEY; + else process.env.XAI_API_KEY = prevXai; + if (prevGrok === undefined) delete process.env.GROK_API_KEY; + else process.env.GROK_API_KEY = prevGrok; + fs.rmSync(tmpHome, { recursive: true, force: true }); + }); + + test('available() returns not-ok when CLI and auth both missing', async () => { + // Isolate PATH so grok cannot be found even if installed on the machine. + // Some environments still resolve `grok` via absolute exec caches — accept + // either binary-missing OR auth-missing as a valid not-ok outcome. + const prevPath = process.env.PATH; + process.env.PATH = '/nonexistent-bin-dir-for-grok-avail'; + try { + const r = await adapter.available(); + expect(r.ok).toBe(false); + expect(r.reason).toMatch(/not found|Install Grok|PATH|No Grok auth|XAI_API_KEY|GROK_API_KEY/i); + } finally { + process.env.PATH = prevPath; + } + }); + + test('available() fails when binary present but no auth file and no env key names', async () => { + // If grok is not on PATH in this environment, skip — unit still covers the + // auth-file/env-name branch when binary is present. + const which = spawnSync('sh', ['-c', 'command -v grok'], { + timeout: 2000, + encoding: 'utf-8', + stdio: ['ignore', 'pipe', 'ignore'], + }); + if (which.status !== 0) { + // No binary: available should report CLI missing under empty HOME + const r = await adapter.available(); + expect(r.ok).toBe(false); + return; + } + const r = await adapter.available(); + expect(r.ok).toBe(false); + expect(r.reason).toMatch(/No Grok auth|XAI_API_KEY|GROK_API_KEY/i); + }); + + test('available() accepts auth.json presence under HOME (boolean only)', async () => { + const which = spawnSync('sh', ['-c', 'command -v grok'], { + timeout: 2000, + encoding: 'utf-8', + stdio: ['ignore', 'pipe', 'ignore'], + }); + if (which.status !== 0) return; // cannot exercise auth-file path without binary + fs.mkdirSync(path.join(tmpHome, '.grok'), { recursive: true }); + fs.writeFileSync(path.join(tmpHome, '.grok', 'auth.json'), '{}'); + const r = await adapter.available(); + expect(r.ok).toBe(true); + }); + + test('run() uses --prompt-file for multi-line / large prompts (ARG_MAX safety)', async () => { + // Force binary_missing path so we never hit a real CLI; still inspects argv construction + // via the ENOENT error path. Large prompt must not throw ARG_MAX. + const big = 'line\n'.repeat(500) + 'x'.repeat(3000); + const prevPath = process.env.PATH; + process.env.PATH = '/nonexistent-bin-dir-for-grok-test'; + try { + const r = await adapter.run({ + prompt: big, + workdir: tmpHome, + timeoutMs: 2000, + }); + expect(r.error?.code).toBe('binary_missing'); + expect(r.tokens).toEqual({ input: 0, output: 0 }); + } finally { + process.env.PATH = prevPath; + } + }); + + test('run() redacts long token-shaped stderr on unknown errors', async () => { + // Drive the unknown error path with a fake binary that exits non-zero + const fakeBin = path.join(tmpHome, 'bin'); + fs.mkdirSync(fakeBin, { recursive: true }); + const grokSh = path.join(fakeBin, 'grok'); + fs.writeFileSync( + grokSh, + `#!/bin/sh\necho "auth token SECRET_TOKEN_ABCDEFGHIJKLMNOPQRSTUVWXYZ123456" >&2\nexit 1\n`, + { mode: 0o755 }, + ); + const prevPath = process.env.PATH; + process.env.PATH = `${fakeBin}:${prevPath ?? ''}`; + try { + const r = await adapter.run({ + prompt: 'hi', + workdir: tmpHome, + timeoutMs: 3000, + }); + // Auth keyword in stderr maps to auth code with redacted reason + expect(r.error).toBeDefined(); + if (r.error?.code === 'auth') { + expect(r.error.reason).not.toContain('SECRET_TOKEN'); + } else { + expect(r.error!.reason).not.toContain('SECRET_TOKEN_ABCDEFGHIJKLMNOPQRSTUVWXYZ123456'); + expect(r.error!.reason).toMatch(/\[redacted\]|unknown|authentication/i); + } + } finally { + process.env.PATH = prevPath; + } + }); +}); + +// ─── gstack-grok-compat-audit fixtures ─────────────────────── + +function writeMinimalRuntime(skillsDir: string, opts: { withReview?: boolean; withSpec?: boolean } = {}) { + const { withReview = true, withSpec = true } = opts; + const root = path.join(skillsDir, 'gstack'); + for (const rel of [ + 'bin', + 'browse/dist', + 'browse/src', + 'scripts', + 'review/specialists', + ]) { + fs.mkdirSync(path.join(root, rel), { recursive: true }); + } + // When monorepo has design/dist + extension, audit requires them in the runtime root + if (fs.existsSync(path.join(ROOT, 'design', 'dist'))) { + fs.mkdirSync(path.join(root, 'design', 'dist'), { recursive: true }); + } + if (fs.existsSync(path.join(ROOT, 'extension'))) { + fs.mkdirSync(path.join(root, 'extension'), { recursive: true }); + } + // browse-client required by skillify audit + fs.mkdirSync(path.join(root, 'browse', 'src'), { recursive: true }); + fs.writeFileSync(path.join(root, 'browse', 'src', 'browse-client.ts'), '// fixture\n'); + if (withReview) { + fs.writeFileSync(path.join(root, 'review', 'checklist.md'), '# checklist\n'); + fs.writeFileSync(path.join(root, 'review', 'TODOS-format.md'), '# todos\n'); + } + // Audit compares fixture runtime against monorepo presence of design/extension. + // Mirror those when the monorepo has them so the fixture is "complete" for phase a. + if (fs.existsSync(path.join(ROOT, 'design', 'dist'))) { + fs.mkdirSync(path.join(root, 'design', 'dist'), { recursive: true }); + } + if (fs.existsSync(path.join(ROOT, 'extension'))) { + fs.mkdirSync(path.join(root, 'extension'), { recursive: true }); + fs.writeFileSync(path.join(root, 'extension', 'manifest.json'), '{}\n'); + } + // connect-chrome → open-gstack-browser shape + const ogb = path.join(skillsDir, 'gstack-open-gstack-browser'); + fs.mkdirSync(ogb, { recursive: true }); + fs.writeFileSync(path.join(ogb, 'SKILL.md'), 'name: open-gstack-browser\n'); + fs.symlinkSync(ogb, path.join(skillsDir, 'connect-chrome')); + + if (withSpec) { + const spec = path.join(skillsDir, 'gstack-spec'); + fs.mkdirSync(spec, { recursive: true }); + fs.writeFileSync( + path.join(spec, 'SKILL.md'), + [ + '---', + 'name: spec', + '---', + 'Spawn **Grok** headless with --prompt-file.', + '```bash', + 'grok --prompt-file "$ARCHIVE_PATH" --cwd "$SPAWN_PATH"', + '```', + '$GSTACK_ROOT', + ].join('\n'), + ); + } +} + +function runAudit(skillsDir: string, phase = 'a'): { status: number | null; stdout: string; stderr: string } { + const r = spawnSync('bun', ['run', AUDIT_BIN, '--skills-dir', skillsDir, '--phase', phase], { + cwd: ROOT, + encoding: 'utf-8', + timeout: 30000, + }); + return { + status: r.status, + stdout: r.stdout?.toString() ?? '', + stderr: r.stderr?.toString() ?? '', + }; +} + +describe('gstack-grok-compat-audit fixtures', () => { + let skillsDir: string; + + beforeEach(() => { + skillsDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-grok-audit-')); + }); + + afterEach(() => { + fs.rmSync(skillsDir, { recursive: true, force: true }); + }); + + test('missing skills dir exits 1 INCOMPATIBLE', () => { + const missing = path.join(skillsDir, 'does-not-exist'); + const r = runAudit(missing, 'a'); + expect(r.status).toBe(1); + expect(r.stdout).toMatch(/INCOMPATIBLE|skills dir missing/i); + }); + + test('minimal complete runtime exits 0 COMPATIBLE for phase a', () => { + writeMinimalRuntime(skillsDir); + const r = runAudit(skillsDir, 'a'); + expect(r.status).toBe(0); + expect(r.stdout).toContain('COMPATIBLE'); + }); + + test('missing review/checklist.md fails phase a', () => { + writeMinimalRuntime(skillsDir, { withReview: false }); + const r = runAudit(skillsDir, 'a'); + expect(r.status).toBe(1); + expect(r.stdout).toMatch(/missing review\/checklist\.md|INCOMPATIBLE/); + }); + + test('missing gstack runtime root fails', () => { + fs.mkdirSync(skillsDir, { recursive: true }); + // empty skills dir — no gstack/ + const r = runAudit(skillsDir, 'a'); + expect(r.status).toBe(1); + expect(r.stdout).toMatch(/missing runtime root|INCOMPATIBLE/); + }); + + test('phase ab fails when neither gstack-spec nor spec package installed', () => { + writeMinimalRuntime(skillsDir, { withSpec: false }); + const r = runAudit(skillsDir, 'ab'); + expect(r.status).toBe(1); + expect(r.stdout).toMatch(/neither gstack-spec nor spec|INCOMPATIBLE/); + }); +}); + +// ─── setup create_grok_runtime_root preflight smoke ────────── + +describe('create_grok_runtime_root preflight (setup smoke)', () => { + test('refuses wipe when required monorepo assets missing', () => { + const emptyRepo = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-grok-setup-empty-')); + const dest = path.join(emptyRepo, 'install-target', 'gstack'); + // Seed a live install that must NOT be wiped when preflight fails + fs.mkdirSync(dest, { recursive: true }); + fs.writeFileSync(path.join(dest, 'KEEPME'), 'live'); + + const setupPath = path.join(ROOT, 'setup'); + // Source extracted functions from a temp file — do NOT eval via $(...) because + // create_grok_runtime_root uses $$ for the staging suffix and command + // substitution would expand it early. + const extractScript = path.join(emptyRepo, 'extract-and-run.sh'); + fs.writeFileSync( + extractScript, + `#!/usr/bin/env bash +set +e +SETUP=${JSON.stringify(setupPath)} +GSTACK_DIR=${JSON.stringify(emptyRepo)} +DEST=${JSON.stringify(dest)} +_link_or_copy() { ln -sfn "$1" "$2" 2>/dev/null || cp -R "$1" "$2"; } +extract_fn() { + local name="$1" + local out="$2" + awk -v name="$name" ' + $0 ~ "^" name "\\\\(\\\\)" {grab=1} + grab { + print + for (i=1;i<=length($0);i++) { + c=substr($0,i,1) + if (c=="{") depth++ + if (c=="}") { + depth-- + if (depth==0) { exit } + } + } + } + ' "$SETUP" > "$out" +} +FN_DIR=$(mktemp -d) +extract_fn _grok_link_under_monorepo "$FN_DIR/link.sh" +extract_fn create_grok_runtime_root "$FN_DIR/create.sh" +# shellcheck source=/dev/null +. "$FN_DIR/link.sh" +# shellcheck source=/dev/null +. "$FN_DIR/create.sh" +create_grok_runtime_root "$GSTACK_DIR" "$DEST" +rc=$? +rm -rf "$FN_DIR" +if [ $rc -eq 0 ]; then + echo "UNEXPECTED_SUCCESS" + exit 2 +fi +echo "PREFLIGHT_FAILED_AS_EXPECTED" +if [ -f "$DEST/KEEPME" ]; then + echo "LIVE_PRESERVED" + exit 0 +fi +echo "LIVE_WIPED" +exit 3 +`, + { mode: 0o755 }, + ); + const r = spawnSync('bash', [extractScript], { + encoding: 'utf-8', + timeout: 15000, + }); + fs.rmSync(emptyRepo, { recursive: true, force: true }); + + expect(r.stdout + r.stderr).toMatch(/preflight|required monorepo asset missing/i); + expect(r.stdout).toContain('PREFLIGHT_FAILED_AS_EXPECTED'); + expect(r.stdout).toContain('LIVE_PRESERVED'); + expect(r.status).toBe(0); + }); +}); diff --git a/test/helpers/providers/grok.ts b/test/helpers/providers/grok.ts index fd7d74c4c..bc1e8068b 100644 --- a/test/helpers/providers/grok.ts +++ b/test/helpers/providers/grok.ts @@ -42,7 +42,10 @@ export class GrokAdapter implements ProviderAdapter { }; } - const authPath = path.join(os.homedir(), '.grok', 'auth.json'); + // Prefer HOME when set so hermetic tests / agent envs can isolate auth + // discovery. Bun's os.homedir() ignores process.env.HOME (unlike Node). + const home = process.env.HOME || os.homedir(); + const authPath = path.join(home, '.grok', 'auth.json'); const hasAuthFile = fs.existsSync(authPath); // Presence of env *names* only — never read/log values const hasKey = !!(process.env.XAI_API_KEY || process.env.GROK_API_KEY); @@ -82,6 +85,8 @@ export class GrokAdapter implements ProviderAdapter { timeout: opts.timeoutMs, encoding: 'utf-8', maxBuffer: 32 * 1024 * 1024, + // Pipe stderr so auth-shaped tokens never inherit onto the parent console + stdio: ['ignore', 'pipe', 'pipe'], env: { ...process.env, GSTACK_HEADLESS: '1' }, }); return { diff --git a/test/host-config.test.ts b/test/host-config.test.ts index 4dddf3af1..4ccec763f 100644 --- a/test/host-config.test.ts +++ b/test/host-config.test.ts @@ -568,6 +568,34 @@ describe('host config correctness', () => { expect(grokBuild.pathRewrites.some(r => r.from === 'MODEL_OVERLAY: claude' && r.to === 'MODEL_OVERLAY: none')).toBe(true); expect(grokBuild.toolRewrites?.['AskUserQuestion']).toBe('ask_user_question'); }); + + // Dual-write parity (#5): hosts/grok-build.ts runtimeRoot must appear in setup's + // create_grok_runtime_root so packaging cannot skew silently. + test('grok-build runtimeRoot dual-write parity with create_grok_runtime_root', () => { + const setupPath = path.join(ROOT, 'setup'); + const setup = fs.readFileSync(setupPath, 'utf-8'); + const fnStart = setup.indexOf('create_grok_runtime_root()'); + expect(fnStart).toBeGreaterThanOrEqual(0); + // Function ends at link_grok_skill_dirs (next sibling) — slice that range + const fnEnd = setup.indexOf('link_grok_skill_dirs()', fnStart); + expect(fnEnd).toBeGreaterThan(fnStart); + const fnBody = setup.slice(fnStart, fnEnd); + + for (const link of grokBuild.runtimeRoot.globalSymlinks) { + // Each asset path must appear as a monorepo source under create_grok_runtime_root + expect(fnBody).toContain(link); + } + for (const f of grokBuild.runtimeRoot.globalFiles?.review ?? []) { + expect(fnBody).toContain(f); + } + // Preflight + atomic stage (review #6) stay wired + expect(fnBody).toContain('preflight'); + expect(fnBody).toMatch(/\.next\.\$\$|staging/); + // Core review files fail-closed (review #7): required=1 on link + expect(fnBody).toMatch( + /for f in checklist\.md TODOS-format\.md; do[\s\S]*?_grok_link_under_monorepo[^\n]* 1 \|\| return 1/, + ); + }); }); describe('resolveDistBinary (U2 double-home fix)', () => {