mirror of https://github.com/garrytan/gstack.git
399 lines
15 KiB
TypeScript
399 lines
15 KiB
TypeScript
/**
|
|
* Behavioral unit tests for Grok Build packaging surfaces (code-review #8):
|
|
* - generateSpecSpawn fail-closed allowlist + no default --always-approve
|
|
* - GrokAdapter available timeout / --prompt-file path / auth redaction
|
|
* - gstack-grok-compat-audit exit codes against fixture skills dirs
|
|
*
|
|
* Free (no API spend). Does not invoke real grok CLI for successful runs.
|
|
*/
|
|
|
|
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
|
import { spawnSync } from 'child_process';
|
|
import * as fs from 'fs';
|
|
import * as path from 'path';
|
|
import * as os from 'os';
|
|
import { generateSpecSpawn, generateSpecExecuteFlag } from '../scripts/resolvers/spec-spawn';
|
|
import type { TemplateContext } from '../scripts/resolvers/types';
|
|
import { HOST_PATHS } from '../scripts/resolvers/types';
|
|
import { GrokAdapter } from './helpers/providers/grok';
|
|
|
|
const ROOT = path.resolve(import.meta.dir, '..');
|
|
const AUDIT_BIN = path.join(ROOT, 'bin', 'gstack-grok-compat-audit');
|
|
|
|
function makeCtx(host: 'grok-build' | 'claude'): TemplateContext {
|
|
return {
|
|
skillName: 'spec',
|
|
tmplPath: path.join(ROOT, 'spec', 'SKILL.md.tmpl'),
|
|
host,
|
|
paths: HOST_PATHS[host],
|
|
};
|
|
}
|
|
|
|
// ─── generateSpecSpawn ───────────────────────────────────────
|
|
|
|
describe('generateSpecSpawn (Grok fail-closed)', () => {
|
|
test('grok-build emits auth gate, ARCHIVE allowlist, and no default --always-approve', () => {
|
|
const out = generateSpecSpawn(makeCtx('grok-build'));
|
|
expect(out).toContain('command -v grok');
|
|
expect(out).toContain('ARCHIVE_PATH');
|
|
expect(out).toContain('SPAWN_PATH');
|
|
expect(out).toContain('GSTACK_STATE_ROOT');
|
|
expect(out).toContain('realpath');
|
|
expect(out).toContain('fail closed');
|
|
expect(out).toContain('grok --prompt-file');
|
|
// Default spawn must NOT enable elevated auto-approve
|
|
expect(out).not.toMatch(/grok --prompt-file[^\n]*--always-approve/);
|
|
expect(out).toMatch(/opt-in only|does \*\*not\*\* pass `--always-approve`|--always-approve` is opt-in/i);
|
|
});
|
|
|
|
test('grok-build --execute-claude path reuses ARCHIVE allowlist before cat|claude', () => {
|
|
const out = generateSpecSpawn(makeCtx('grok-build'));
|
|
expect(out).toContain('--execute-claude');
|
|
// Claude branch must re-check allowlist, not bare cat only
|
|
const claudeIdx = out.indexOf('--execute-claude');
|
|
const after = out.slice(claudeIdx);
|
|
expect(after).toContain('ARCHIVE_REAL');
|
|
expect(after).toContain('STATE_PROJECTS');
|
|
expect(after).toMatch(/cat "\$ARCHIVE_PATH" \| \(cd "\$SPAWN_PATH" && claude -p/);
|
|
});
|
|
|
|
test('claude host still uses classic stdin pipe (no Grok-only gates)', () => {
|
|
const out = generateSpecSpawn(makeCtx('claude'));
|
|
expect(out).toContain('claude -p');
|
|
expect(out).not.toContain('grok --prompt-file');
|
|
});
|
|
|
|
test('generateSpecExecuteFlag is host-aware', () => {
|
|
expect(generateSpecExecuteFlag(makeCtx('grok-build'))).toContain('grok --prompt-file');
|
|
expect(generateSpecExecuteFlag(makeCtx('claude'))).toContain('claude -p');
|
|
});
|
|
});
|
|
|
|
// ─── GrokAdapter ─────────────────────────────────────────────
|
|
|
|
describe('GrokAdapter behavioral unit', () => {
|
|
const adapter = new GrokAdapter();
|
|
let tmpHome: string;
|
|
let prevHome: string | undefined;
|
|
let prevXai: string | undefined;
|
|
let prevGrok: string | undefined;
|
|
|
|
beforeEach(() => {
|
|
tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-grok-adapter-'));
|
|
prevHome = process.env.HOME;
|
|
prevXai = process.env.XAI_API_KEY;
|
|
prevGrok = process.env.GROK_API_KEY;
|
|
process.env.HOME = tmpHome;
|
|
delete process.env.XAI_API_KEY;
|
|
delete process.env.GROK_API_KEY;
|
|
});
|
|
|
|
afterEach(() => {
|
|
if (prevHome === undefined) delete process.env.HOME;
|
|
else process.env.HOME = prevHome;
|
|
if (prevXai === undefined) delete process.env.XAI_API_KEY;
|
|
else process.env.XAI_API_KEY = prevXai;
|
|
if (prevGrok === undefined) delete process.env.GROK_API_KEY;
|
|
else process.env.GROK_API_KEY = prevGrok;
|
|
fs.rmSync(tmpHome, { recursive: true, force: true });
|
|
});
|
|
|
|
test('available() returns not-ok when CLI and auth both missing', async () => {
|
|
// Isolate PATH so grok cannot be found even if installed on the machine.
|
|
// Some environments still resolve `grok` via absolute exec caches — accept
|
|
// either binary-missing OR auth-missing as a valid not-ok outcome.
|
|
const prevPath = process.env.PATH;
|
|
process.env.PATH = '/nonexistent-bin-dir-for-grok-avail';
|
|
try {
|
|
const r = await adapter.available();
|
|
expect(r.ok).toBe(false);
|
|
expect(r.reason).toMatch(/not found|Install Grok|PATH|No Grok auth|XAI_API_KEY|GROK_API_KEY/i);
|
|
} finally {
|
|
process.env.PATH = prevPath;
|
|
}
|
|
});
|
|
|
|
test('available() fails when binary present but no auth file and no env key names', async () => {
|
|
// If grok is not on PATH in this environment, skip — unit still covers the
|
|
// auth-file/env-name branch when binary is present.
|
|
const which = spawnSync('sh', ['-c', 'command -v grok'], {
|
|
timeout: 2000,
|
|
encoding: 'utf-8',
|
|
stdio: ['ignore', 'pipe', 'ignore'],
|
|
});
|
|
if (which.status !== 0) {
|
|
// No binary: available should report CLI missing under empty HOME
|
|
const r = await adapter.available();
|
|
expect(r.ok).toBe(false);
|
|
return;
|
|
}
|
|
const r = await adapter.available();
|
|
expect(r.ok).toBe(false);
|
|
expect(r.reason).toMatch(/No Grok auth|XAI_API_KEY|GROK_API_KEY/i);
|
|
});
|
|
|
|
test('available() accepts auth.json presence under HOME (boolean only)', async () => {
|
|
const which = spawnSync('sh', ['-c', 'command -v grok'], {
|
|
timeout: 2000,
|
|
encoding: 'utf-8',
|
|
stdio: ['ignore', 'pipe', 'ignore'],
|
|
});
|
|
if (which.status !== 0) return; // cannot exercise auth-file path without binary
|
|
fs.mkdirSync(path.join(tmpHome, '.grok'), { recursive: true });
|
|
fs.writeFileSync(path.join(tmpHome, '.grok', 'auth.json'), '{}');
|
|
const r = await adapter.available();
|
|
expect(r.ok).toBe(true);
|
|
});
|
|
|
|
test('run() uses --prompt-file for multi-line / large prompts (ARG_MAX safety)', async () => {
|
|
// Force binary_missing path so we never hit a real CLI; still inspects argv construction
|
|
// via the ENOENT error path. Large prompt must not throw ARG_MAX.
|
|
const big = 'line\n'.repeat(500) + 'x'.repeat(3000);
|
|
const prevPath = process.env.PATH;
|
|
process.env.PATH = '/nonexistent-bin-dir-for-grok-test';
|
|
try {
|
|
const r = await adapter.run({
|
|
prompt: big,
|
|
workdir: tmpHome,
|
|
timeoutMs: 2000,
|
|
});
|
|
expect(r.error?.code).toBe('binary_missing');
|
|
expect(r.tokens).toEqual({ input: 0, output: 0 });
|
|
} finally {
|
|
process.env.PATH = prevPath;
|
|
}
|
|
});
|
|
|
|
test('run() redacts long token-shaped stderr on unknown errors', async () => {
|
|
// Drive the unknown error path with a fake binary that exits non-zero
|
|
const fakeBin = path.join(tmpHome, 'bin');
|
|
fs.mkdirSync(fakeBin, { recursive: true });
|
|
const grokSh = path.join(fakeBin, 'grok');
|
|
fs.writeFileSync(
|
|
grokSh,
|
|
`#!/bin/sh\necho "auth token SECRET_TOKEN_ABCDEFGHIJKLMNOPQRSTUVWXYZ123456" >&2\nexit 1\n`,
|
|
{ mode: 0o755 },
|
|
);
|
|
const prevPath = process.env.PATH;
|
|
process.env.PATH = `${fakeBin}:${prevPath ?? ''}`;
|
|
try {
|
|
const r = await adapter.run({
|
|
prompt: 'hi',
|
|
workdir: tmpHome,
|
|
timeoutMs: 3000,
|
|
});
|
|
// Auth keyword in stderr maps to auth code with redacted reason
|
|
expect(r.error).toBeDefined();
|
|
if (r.error?.code === 'auth') {
|
|
expect(r.error.reason).not.toContain('SECRET_TOKEN');
|
|
} else {
|
|
expect(r.error!.reason).not.toContain('SECRET_TOKEN_ABCDEFGHIJKLMNOPQRSTUVWXYZ123456');
|
|
expect(r.error!.reason).toMatch(/\[redacted\]|unknown|authentication/i);
|
|
}
|
|
} finally {
|
|
process.env.PATH = prevPath;
|
|
}
|
|
});
|
|
});
|
|
|
|
// ─── gstack-grok-compat-audit fixtures ───────────────────────
|
|
|
|
function writeMinimalRuntime(skillsDir: string, opts: { withReview?: boolean; withSpec?: boolean } = {}) {
|
|
const { withReview = true, withSpec = true } = opts;
|
|
const root = path.join(skillsDir, 'gstack');
|
|
for (const rel of [
|
|
'bin',
|
|
'browse/dist',
|
|
'browse/src',
|
|
'scripts',
|
|
'review/specialists',
|
|
]) {
|
|
fs.mkdirSync(path.join(root, rel), { recursive: true });
|
|
}
|
|
// When monorepo has design/dist + extension, audit requires them in the runtime root
|
|
if (fs.existsSync(path.join(ROOT, 'design', 'dist'))) {
|
|
fs.mkdirSync(path.join(root, 'design', 'dist'), { recursive: true });
|
|
}
|
|
if (fs.existsSync(path.join(ROOT, 'extension'))) {
|
|
fs.mkdirSync(path.join(root, 'extension'), { recursive: true });
|
|
}
|
|
// browse-client required by skillify audit
|
|
fs.mkdirSync(path.join(root, 'browse', 'src'), { recursive: true });
|
|
fs.writeFileSync(path.join(root, 'browse', 'src', 'browse-client.ts'), '// fixture\n');
|
|
if (withReview) {
|
|
fs.writeFileSync(path.join(root, 'review', 'checklist.md'), '# checklist\n');
|
|
fs.writeFileSync(path.join(root, 'review', 'TODOS-format.md'), '# todos\n');
|
|
}
|
|
// Audit compares fixture runtime against monorepo presence of design/extension.
|
|
// Mirror those when the monorepo has them so the fixture is "complete" for phase a.
|
|
if (fs.existsSync(path.join(ROOT, 'design', 'dist'))) {
|
|
fs.mkdirSync(path.join(root, 'design', 'dist'), { recursive: true });
|
|
}
|
|
if (fs.existsSync(path.join(ROOT, 'extension'))) {
|
|
fs.mkdirSync(path.join(root, 'extension'), { recursive: true });
|
|
fs.writeFileSync(path.join(root, 'extension', 'manifest.json'), '{}\n');
|
|
}
|
|
// connect-chrome → open-gstack-browser shape
|
|
const ogb = path.join(skillsDir, 'gstack-open-gstack-browser');
|
|
fs.mkdirSync(ogb, { recursive: true });
|
|
fs.writeFileSync(path.join(ogb, 'SKILL.md'), 'name: open-gstack-browser\n');
|
|
fs.symlinkSync(ogb, path.join(skillsDir, 'connect-chrome'));
|
|
|
|
if (withSpec) {
|
|
const spec = path.join(skillsDir, 'gstack-spec');
|
|
fs.mkdirSync(spec, { recursive: true });
|
|
fs.writeFileSync(
|
|
path.join(spec, 'SKILL.md'),
|
|
[
|
|
'---',
|
|
'name: spec',
|
|
'---',
|
|
'Spawn **Grok** headless with --prompt-file.',
|
|
'```bash',
|
|
'grok --prompt-file "$ARCHIVE_PATH" --cwd "$SPAWN_PATH"',
|
|
'```',
|
|
'$GSTACK_ROOT',
|
|
].join('\n'),
|
|
);
|
|
}
|
|
}
|
|
|
|
function runAudit(skillsDir: string, phase = 'a'): { status: number | null; stdout: string; stderr: string } {
|
|
const r = spawnSync('bun', ['run', AUDIT_BIN, '--skills-dir', skillsDir, '--phase', phase], {
|
|
cwd: ROOT,
|
|
encoding: 'utf-8',
|
|
timeout: 30000,
|
|
});
|
|
return {
|
|
status: r.status,
|
|
stdout: r.stdout?.toString() ?? '',
|
|
stderr: r.stderr?.toString() ?? '',
|
|
};
|
|
}
|
|
|
|
describe('gstack-grok-compat-audit fixtures', () => {
|
|
let skillsDir: string;
|
|
|
|
beforeEach(() => {
|
|
skillsDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-grok-audit-'));
|
|
});
|
|
|
|
afterEach(() => {
|
|
fs.rmSync(skillsDir, { recursive: true, force: true });
|
|
});
|
|
|
|
test('missing skills dir exits 1 INCOMPATIBLE', () => {
|
|
const missing = path.join(skillsDir, 'does-not-exist');
|
|
const r = runAudit(missing, 'a');
|
|
expect(r.status).toBe(1);
|
|
expect(r.stdout).toMatch(/INCOMPATIBLE|skills dir missing/i);
|
|
});
|
|
|
|
test('minimal complete runtime exits 0 COMPATIBLE for phase a', () => {
|
|
writeMinimalRuntime(skillsDir);
|
|
const r = runAudit(skillsDir, 'a');
|
|
expect(r.status).toBe(0);
|
|
expect(r.stdout).toContain('COMPATIBLE');
|
|
});
|
|
|
|
test('missing review/checklist.md fails phase a', () => {
|
|
writeMinimalRuntime(skillsDir, { withReview: false });
|
|
const r = runAudit(skillsDir, 'a');
|
|
expect(r.status).toBe(1);
|
|
expect(r.stdout).toMatch(/missing review\/checklist\.md|INCOMPATIBLE/);
|
|
});
|
|
|
|
test('missing gstack runtime root fails', () => {
|
|
fs.mkdirSync(skillsDir, { recursive: true });
|
|
// empty skills dir — no gstack/
|
|
const r = runAudit(skillsDir, 'a');
|
|
expect(r.status).toBe(1);
|
|
expect(r.stdout).toMatch(/missing runtime root|INCOMPATIBLE/);
|
|
});
|
|
|
|
test('phase ab fails when neither gstack-spec nor spec package installed', () => {
|
|
writeMinimalRuntime(skillsDir, { withSpec: false });
|
|
const r = runAudit(skillsDir, 'ab');
|
|
expect(r.status).toBe(1);
|
|
expect(r.stdout).toMatch(/neither gstack-spec nor spec|INCOMPATIBLE/);
|
|
});
|
|
});
|
|
|
|
// ─── setup create_grok_runtime_root preflight smoke ──────────
|
|
|
|
describe('create_grok_runtime_root preflight (setup smoke)', () => {
|
|
test('refuses wipe when required monorepo assets missing', () => {
|
|
const emptyRepo = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-grok-setup-empty-'));
|
|
const dest = path.join(emptyRepo, 'install-target', 'gstack');
|
|
// Seed a live install that must NOT be wiped when preflight fails
|
|
fs.mkdirSync(dest, { recursive: true });
|
|
fs.writeFileSync(path.join(dest, 'KEEPME'), 'live');
|
|
|
|
const setupPath = path.join(ROOT, 'setup');
|
|
// Source extracted functions from a temp file — do NOT eval via $(...) because
|
|
// create_grok_runtime_root uses $$ for the staging suffix and command
|
|
// substitution would expand it early.
|
|
const extractScript = path.join(emptyRepo, 'extract-and-run.sh');
|
|
fs.writeFileSync(
|
|
extractScript,
|
|
`#!/usr/bin/env bash
|
|
set +e
|
|
SETUP=${JSON.stringify(setupPath)}
|
|
GSTACK_DIR=${JSON.stringify(emptyRepo)}
|
|
DEST=${JSON.stringify(dest)}
|
|
_link_or_copy() { ln -sfn "$1" "$2" 2>/dev/null || cp -R "$1" "$2"; }
|
|
extract_fn() {
|
|
local name="$1"
|
|
local out="$2"
|
|
awk -v name="$name" '
|
|
$0 ~ "^" name "\\\\(\\\\)" {grab=1}
|
|
grab {
|
|
print
|
|
for (i=1;i<=length($0);i++) {
|
|
c=substr($0,i,1)
|
|
if (c=="{") depth++
|
|
if (c=="}") {
|
|
depth--
|
|
if (depth==0) { exit }
|
|
}
|
|
}
|
|
}
|
|
' "$SETUP" > "$out"
|
|
}
|
|
FN_DIR=$(mktemp -d)
|
|
extract_fn _grok_link_under_monorepo "$FN_DIR/link.sh"
|
|
extract_fn create_grok_runtime_root "$FN_DIR/create.sh"
|
|
# shellcheck source=/dev/null
|
|
. "$FN_DIR/link.sh"
|
|
# shellcheck source=/dev/null
|
|
. "$FN_DIR/create.sh"
|
|
create_grok_runtime_root "$GSTACK_DIR" "$DEST"
|
|
rc=$?
|
|
rm -rf "$FN_DIR"
|
|
if [ $rc -eq 0 ]; then
|
|
echo "UNEXPECTED_SUCCESS"
|
|
exit 2
|
|
fi
|
|
echo "PREFLIGHT_FAILED_AS_EXPECTED"
|
|
if [ -f "$DEST/KEEPME" ]; then
|
|
echo "LIVE_PRESERVED"
|
|
exit 0
|
|
fi
|
|
echo "LIVE_WIPED"
|
|
exit 3
|
|
`,
|
|
{ mode: 0o755 },
|
|
);
|
|
const r = spawnSync('bash', [extractScript], {
|
|
encoding: 'utf-8',
|
|
timeout: 15000,
|
|
});
|
|
fs.rmSync(emptyRepo, { recursive: true, force: true });
|
|
|
|
expect(r.stdout + r.stderr).toMatch(/preflight|required monorepo asset missing/i);
|
|
expect(r.stdout).toContain('PREFLIGHT_FAILED_AS_EXPECTED');
|
|
expect(r.stdout).toContain('LIVE_PRESERVED');
|
|
expect(r.status).toBe(0);
|
|
});
|
|
});
|