diff --git a/test/grok-packaging-behavior.test.ts b/test/grok-packaging-behavior.test.ts index 193640e05..a39eb3982 100644 --- a/test/grok-packaging-behavior.test.ts +++ b/test/grok-packaging-behavior.test.ts @@ -15,7 +15,8 @@ import * as os from 'os'; import { generateSpecSpawn, generateSpecExecuteFlag } from '../scripts/resolvers/spec-spawn'; import type { TemplateContext } from '../scripts/resolvers/types'; import { HOST_PATHS } from '../scripts/resolvers/types'; -import { GrokAdapter } from './helpers/providers/grok'; +import { GrokAdapter, parseGrokOutput, isStructurallyValidGrokAuthFile } from './helpers/providers/grok'; +import { estimateCostUsd, PRICING } from './helpers/pricing'; const ROOT = path.resolve(import.meta.dir, '..'); const AUDIT_BIN = path.join(ROOT, 'bin', 'gstack-grok-compat-audit'); @@ -132,7 +133,7 @@ describe('GrokAdapter behavioral unit', () => { expect(r.reason).toMatch(/No Grok auth|XAI_API_KEY|GROK_API_KEY/i); }); - test('available() accepts auth.json presence under HOME (boolean only)', async () => { + test('available() accepts non-empty auth.json object under HOME', async () => { const which = spawnSync('sh', ['-c', 'command -v grok'], { timeout: 2000, encoding: 'utf-8', @@ -140,11 +141,94 @@ describe('GrokAdapter behavioral unit', () => { }); if (which.status !== 0) return; // cannot exercise auth-file path without binary fs.mkdirSync(path.join(tmpHome, '.grok'), { recursive: true }); - fs.writeFileSync(path.join(tmpHome, '.grok', 'auth.json'), '{}'); + // OAuth-shaped key (URL) without logging real secrets — structure only + fs.writeFileSync( + path.join(tmpHome, '.grok', 'auth.json'), + JSON.stringify({ 'https://auth.x.ai::fixture-user': { access: 'x' } }), + ); const r = await adapter.available(); expect(r.ok).toBe(true); }); + test('available() fails closed on empty object auth.json {}', async () => { + const which = spawnSync('sh', ['-c', 'command -v grok'], { + timeout: 2000, + encoding: 'utf-8', + stdio: ['ignore', 'pipe', 'ignore'], + }); + if (which.status !== 0) return; + fs.mkdirSync(path.join(tmpHome, '.grok'), { recursive: true }); + fs.writeFileSync(path.join(tmpHome, '.grok', 'auth.json'), '{}'); + const r = await adapter.available(); + expect(r.ok).toBe(false); + expect(r.reason).toMatch(/No Grok auth|auth\.json|XAI_API_KEY|GROK_API_KEY/i); + // Reason must never echo file contents / secrets + expect(r.reason).not.toMatch(/access_token|sk-|Bearer/i); + }); + + test('available() fails closed on zero-byte and invalid auth.json', async () => { + const which = spawnSync('sh', ['-c', 'command -v grok'], { + timeout: 2000, + encoding: 'utf-8', + stdio: ['ignore', 'pipe', 'ignore'], + }); + if (which.status !== 0) return; + fs.mkdirSync(path.join(tmpHome, '.grok'), { recursive: true }); + const authPath = path.join(tmpHome, '.grok', 'auth.json'); + + fs.writeFileSync(authPath, ''); + expect((await adapter.available()).ok).toBe(false); + + fs.writeFileSync(authPath, ' \n'); + expect((await adapter.available()).ok).toBe(false); + + fs.writeFileSync(authPath, 'not-json'); + expect((await adapter.available()).ok).toBe(false); + + fs.writeFileSync(authPath, 'null'); + expect((await adapter.available()).ok).toBe(false); + + fs.writeFileSync(authPath, '[]'); + expect((await adapter.available()).ok).toBe(false); + }); + + test('available() fails closed on whitespace-only env keys', async () => { + const which = spawnSync('sh', ['-c', 'command -v grok'], { + timeout: 2000, + encoding: 'utf-8', + stdio: ['ignore', 'pipe', 'ignore'], + }); + if (which.status !== 0) return; + process.env.XAI_API_KEY = ' '; + process.env.GROK_API_KEY = '\t'; + const r = await adapter.available(); + expect(r.ok).toBe(false); + expect(r.reason).not.toContain(' '); + }); + + test('available() accepts non-blank env key without auth file', async () => { + const which = spawnSync('sh', ['-c', 'command -v grok'], { + timeout: 2000, + encoding: 'utf-8', + stdio: ['ignore', 'pipe', 'ignore'], + }); + if (which.status !== 0) return; + process.env.XAI_API_KEY = 'test-not-a-real-key'; + const r = await adapter.available(); + expect(r.ok).toBe(true); + }); + + test('isStructurallyValidGrokAuthFile pure checks', () => { + const p = path.join(tmpHome, 'auth.json'); + fs.writeFileSync(p, ''); + expect(isStructurallyValidGrokAuthFile(p)).toBe(false); + fs.writeFileSync(p, '{}'); + expect(isStructurallyValidGrokAuthFile(p)).toBe(false); + fs.writeFileSync(p, '{"k":1}'); + expect(isStructurallyValidGrokAuthFile(p)).toBe(true); + expect(isStructurallyValidGrokAuthFile(path.join(tmpHome, 'missing.json'))).toBe(false); + }); + test('run() uses --prompt-file for multi-line / large prompts (ARG_MAX safety)', async () => { // Force binary_missing path so we never hit a real CLI; still inspects argv construction // via the ENOENT error path. Large prompt must not throw ARG_MAX. @@ -194,8 +278,126 @@ describe('GrokAdapter behavioral unit', () => { process.env.PATH = prevPath; } }); + + test('run() passes --output-format json and parses usage from fixture CLI', async () => { + const fakeBin = path.join(tmpHome, 'bin'); + fs.mkdirSync(fakeBin, { recursive: true }); + const grokSh = path.join(fakeBin, 'grok'); + // Echo argv so we can assert --output-format json; emit usage-bearing JSON on stdout + const payload = JSON.stringify({ + text: 'hello', + usage: { input_tokens: 1000, output_tokens: 500 }, + model: 'grok', + }); + fs.writeFileSync( + grokSh, + `#!/bin/sh +# record argv for assertions +printf '%s\\n' "$*" > "${tmpHome}/grok-argv.txt" +echo '${payload}' +`, + { mode: 0o755 }, + ); + const prevPath = process.env.PATH; + process.env.PATH = `${fakeBin}:${prevPath ?? ''}`; + try { + const r = await adapter.run({ + prompt: 'hi', + workdir: tmpHome, + timeoutMs: 3000, + }); + expect(r.error).toBeUndefined(); + expect(r.output).toBe('hello'); + expect(r.tokens).toEqual({ input: 1000, output: 500 }); + expect(r.modelUsed).toBe('grok'); + // Cost honesty: non-zero tokens × official rates → non-zero USD + const cost = adapter.estimateCost(r.tokens, r.modelUsed); + expect(cost).toBeGreaterThan(0); + const argv = fs.readFileSync(path.join(tmpHome, 'grok-argv.txt'), 'utf-8'); + expect(argv).toMatch(/--output-format\s+json|--output-format json/); + } finally { + process.env.PATH = prevPath; + } + }); }); +// ─── parseGrokOutput + pricing honesty ─────────────────────── + +describe('parseGrokOutput + Grok pricing', () => { + test('parses characterized headless json shape (no usage → zero tokens)', () => { + const raw = JSON.stringify({ + text: 'pong', + stopReason: 'EndTurn', + sessionId: '019f57be-82a4-7990-89ed-47a030fbaeb7', + requestId: 'fe579ce3-777b-42e5-aea5-f2e25c24398d', + thought: 'simple reply', + }); + const p = parseGrokOutput(raw); + expect(p.output).toBe('pong'); + expect(p.tokens).toEqual({ input: 0, output: 0 }); + }); + + test('parses usage when present (input_tokens / output_tokens)', () => { + const p = parseGrokOutput( + JSON.stringify({ + text: 'ok', + usage: { input_tokens: 1_000_000, output_tokens: 1_000_000 }, + }), + ); + expect(p.tokens).toEqual({ input: 1_000_000, output: 1_000_000 }); + // With U1 rates for `grok` ($1/$2 per MTok): 1 + 2 = $3 + expect(estimateCostUsd(p.tokens, 'grok')).toBe(3); + }); + + test('parses prompt_tokens / completion_tokens aliases', () => { + const p = parseGrokOutput( + JSON.stringify({ + text: 'x', + usage: { prompt_tokens: 100, completion_tokens: 50 }, + }), + ); + expect(p.tokens).toEqual({ input: 100, output: 50 }); + }); + + test('plain text without usage → zero tokens, no throw', () => { + const p = parseGrokOutput('just plain assistant text'); + expect(p.output).toBe('just plain assistant text'); + expect(p.tokens).toEqual({ input: 0, output: 0 }); + }); + + test('streaming-json NDJSON accumulates text; zero tokens without usage', () => { + const raw = [ + '{"type":"thought","data":"hmm"}', + '{"type":"text","data":"hel"}', + '{"type":"text","data":"lo"}', + '{"type":"end","stopReason":"EndTurn","sessionId":"x"}', + ].join('\n'); + const p = parseGrokOutput(raw); + expect(p.output).toBe('hello'); + expect(p.tokens).toEqual({ input: 0, output: 0 }); + }); + + test('official Grok rates are non-zero and match rate math', () => { + expect(PRICING['grok']?.input_per_mtok).toBe(1); + expect(PRICING['grok']?.output_per_mtok).toBe(2); + expect(PRICING['grok-build-0.1']?.input_per_mtok).toBe(1); + expect(PRICING['grok-4.5']?.input_per_mtok).toBe(2); + expect(PRICING['grok-4.5']?.output_per_mtok).toBe(6); + // 1M in + 1M out at $1/$2 → $3.00 + expect(estimateCostUsd({ input: 1_000_000, output: 1_000_000 }, 'grok')).toBe(3); + // 1M in + 1M out at $2/$6 → $8.00 + expect(estimateCostUsd({ input: 1_000_000, output: 1_000_000 }, 'grok-4.5')).toBe(8); + }); + + test('unknown model returns 0 and does not throw; peer rows unchanged', () => { + expect(estimateCostUsd({ input: 1_000_000, output: 1_000_000 }, 'not-a-real-model-xyz')).toBe(0); + expect(PRICING['claude-opus-4-7']?.input_per_mtok).toBe(15); + expect(PRICING['gpt-5.4']?.input_per_mtok).toBe(2.5); + expect(PRICING['gemini-2.5-pro']?.input_per_mtok).toBe(1.25); + }); +}); + + // ─── gstack-grok-compat-audit fixtures ─────────────────────── function writeMinimalRuntime(skillsDir: string, opts: { withReview?: boolean; withSpec?: boolean } = {}) { diff --git a/test/helpers/pricing.ts b/test/helpers/pricing.ts index 604703c17..cecaf2e80 100644 --- a/test/helpers/pricing.ts +++ b/test/helpers/pricing.ts @@ -6,9 +6,17 @@ * - Anthropic: https://www.anthropic.com/pricing#api * - OpenAI: https://openai.com/api/pricing/ * - Google AI: https://ai.google.dev/pricing + * - xAI: https://docs.x.ai/developers/pricing * * When a model isn't in the table, estimateCost returns 0 with a console warning. * Prefer adding a new row to the table over guessing. + * + * xAI rates (official docs, as of 2026-07): Code API grok-build-0.1 $1/$2; + * Chat API grok-4.5 $2/$6; grok-4.3 and grok-4.20-* $1.25/$2.50. + * Benchmark adapter default id `grok` maps to Code API grok-build-0.1 + * (Grok Build headless default). Alias `grok-4` maps to Chat mid-tier + * grok-4.3 rates. Run cost still needs real token counts from the adapter — + * rates alone cannot invent usage. */ export interface ModelPricing { @@ -33,9 +41,15 @@ export const PRICING: Record = { 'gemini-2.5-pro': { input_per_mtok: 1.25, output_per_mtok: 5.00, as_of: '2026-04' }, 'gemini-2.5-flash': { input_per_mtok: 0.30, output_per_mtok: 1.20, as_of: '2026-04' }, - // xAI Grok (placeholder rates — update when public API pricing is pinned) - 'grok': { input_per_mtok: 3.00, output_per_mtok: 15.00, as_of: '2026-07' }, - 'grok-4': { input_per_mtok: 3.00, output_per_mtok: 15.00, as_of: '2026-07' }, + // xAI Grok — https://docs.x.ai/developers/pricing (as of 2026-07) + 'grok-build-0.1': { input_per_mtok: 1.00, output_per_mtok: 2.00, as_of: '2026-07' }, + 'grok': { input_per_mtok: 1.00, output_per_mtok: 2.00, as_of: '2026-07' }, // alias → Code API + 'grok-4.5': { input_per_mtok: 2.00, output_per_mtok: 6.00, as_of: '2026-07' }, + 'grok-4.3': { input_per_mtok: 1.25, output_per_mtok: 2.50, as_of: '2026-07' }, + 'grok-4': { input_per_mtok: 1.25, output_per_mtok: 2.50, as_of: '2026-07' }, // alias → Chat mid-tier + 'grok-4.20-0309-reasoning': { input_per_mtok: 1.25, output_per_mtok: 2.50, as_of: '2026-07' }, + 'grok-4.20-0309-non-reasoning': { input_per_mtok: 1.25, output_per_mtok: 2.50, as_of: '2026-07' }, + 'grok-4.20-multi-agent-0309': { input_per_mtok: 1.25, output_per_mtok: 2.50, as_of: '2026-07' }, }; const WARNED = new Set(); diff --git a/test/helpers/providers/grok.ts b/test/helpers/providers/grok.ts index bc1e8068b..e4231a2a1 100644 --- a/test/helpers/providers/grok.ts +++ b/test/helpers/providers/grok.ts @@ -5,11 +5,202 @@ import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; +export type GrokParsedOutput = { + output: string; + tokens: { input: number; output: number; cached?: number }; + toolCalls: number; + modelUsed?: string; +}; + +/** + * Parse Grok headless stdout. + * + * Characterized 2026-07 against grok 0.2.93: + * --output-format json → + * { text, stopReason, sessionId, requestId, thought? } + * --output-format streaming-json → + * NDJSON {type:"thought"|"text"|"end", ...} — end has stopReason/sessionId + * Neither shape exposes usage today. When a future CLI adds usage, accept + * common field names (input_tokens/output_tokens, prompt_tokens/completion_tokens, + * nested usage object). Never invent token counts from prompt/output length. + */ +export function parseGrokOutput(raw: string): GrokParsedOutput { + const trimmed = raw.trim(); + if (!trimmed) { + return { output: '', tokens: { input: 0, output: 0 }, toolCalls: 0 }; + } + + // Single JSON object (default headless json format) + try { + const obj = JSON.parse(trimmed); + if (obj && typeof obj === 'object' && !Array.isArray(obj)) { + return parseGrokJsonObject(obj as Record); + } + } catch { + // fall through to NDJSON / plain text + } + + // streaming-json NDJSON lines + if (trimmed.includes('\n') || trimmed.startsWith('{')) { + const lines = trimmed.split('\n').map((l) => l.trim()).filter(Boolean); + let looksNdjson = 0; + let textParts: string[] = []; + let tokens = { input: 0, output: 0 } as { input: number; output: number; cached?: number }; + let toolCalls = 0; + let modelUsed: string | undefined; + let sawUsage = false; + + for (const line of lines) { + try { + const ev = JSON.parse(line) as Record; + looksNdjson++; + const t = ev.type; + if (t === 'text' && typeof ev.data === 'string') { + textParts.push(ev.data); + } else if (t === 'message' && typeof ev.data === 'string') { + textParts.push(ev.data); + } else if (typeof ev.text === 'string') { + textParts.push(ev.text); + } + const fromEv = extractUsage(ev); + if (fromEv) { + tokens = fromEv.tokens; + sawUsage = true; + if (fromEv.cached !== undefined) tokens = { ...tokens, cached: fromEv.cached }; + } + if (typeof ev.model === 'string') modelUsed = ev.model; + if (typeof ev.modelUsed === 'string') modelUsed = ev.modelUsed; + if (typeof ev.num_turns === 'number') toolCalls = ev.num_turns; + if (typeof ev.tool_calls === 'number') toolCalls = ev.tool_calls; + if (typeof ev.toolCallCount === 'number') toolCalls = ev.toolCallCount; + } catch { + // non-JSON line — ignore for NDJSON path + } + } + + if (looksNdjson > 0 && (textParts.length > 0 || sawUsage || lines.length === looksNdjson)) { + return { + output: textParts.join(''), + tokens: sawUsage ? tokens : { input: 0, output: 0 }, + toolCalls, + modelUsed, + }; + } + } + + // Plain text fallback — zero tokens (do not estimate from length) + return { output: raw, tokens: { input: 0, output: 0 }, toolCalls: 0 }; +} + +function parseGrokJsonObject(obj: Record): GrokParsedOutput { + // Preferred text fields for Grok Build; also accept Claude-like `result` + let output = ''; + if (typeof obj.text === 'string') output = obj.text; + else if (typeof obj.result === 'string') output = obj.result; + else if (typeof obj.message === 'string') output = obj.message; + else if (obj.result != null) output = String(obj.result); + + const usage = extractUsage(obj); + const tokens = usage + ? { input: usage.tokens.input, output: usage.tokens.output, ...(usage.cached !== undefined ? { cached: usage.cached } : {}) } + : { input: 0, output: 0 }; + + const toolCalls = + (typeof obj.num_turns === 'number' ? obj.num_turns : undefined) ?? + (typeof obj.tool_calls === 'number' ? obj.tool_calls : undefined) ?? + (typeof obj.toolCallCount === 'number' ? obj.toolCallCount : undefined) ?? + 0; + + const modelUsed = + (typeof obj.model === 'string' ? obj.model : undefined) ?? + (typeof obj.modelUsed === 'string' ? obj.modelUsed : undefined) ?? + (typeof obj.model_id === 'string' ? obj.model_id : undefined); + + return { output, tokens, toolCalls, modelUsed }; +} + +function numField(obj: Record, ...keys: string[]): number | undefined { + for (const k of keys) { + const v = obj[k]; + if (typeof v === 'number' && Number.isFinite(v) && v >= 0) return v; + } + return undefined; +} + +/** Extract token usage from a JSON object if present; otherwise undefined. */ +function extractUsage( + obj: Record, +): { tokens: { input: number; output: number }; cached?: number } | undefined { + const nested = + obj.usage && typeof obj.usage === 'object' && !Array.isArray(obj.usage) + ? (obj.usage as Record) + : null; + + const sources = nested ? [nested, obj] : [obj]; + for (const src of sources) { + const input = numField( + src, + 'input_tokens', + 'prompt_tokens', + 'inputTokens', + 'promptTokens', + 'input_token_count', + ); + const output = numField( + src, + 'output_tokens', + 'completion_tokens', + 'outputTokens', + 'completionTokens', + 'output_token_count', + ); + if (input !== undefined || output !== undefined) { + const cached = numField( + src, + 'cache_read_input_tokens', + 'cached_prompt_tokens', + 'cached_tokens', + 'cached', + ); + return { + tokens: { input: input ?? 0, output: output ?? 0 }, + ...(cached !== undefined ? { cached } : {}), + }; + } + } + return undefined; +} + +/** + * Structural auth check only — never log file or env values. + * Valid auth.json: non-empty JSON object with ≥1 key (OAuth shapes use URL keys). + * Reject: missing, empty/whitespace, invalid JSON, null, arrays, {}. + */ +export function isStructurallyValidGrokAuthFile(authPath: string): boolean { + try { + if (!fs.existsSync(authPath)) return false; + const raw = fs.readFileSync(authPath, 'utf-8'); + if (!raw.trim()) return false; + const parsed = JSON.parse(raw) as unknown; + if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) return false; + return Object.keys(parsed as object).length > 0; + } catch { + return false; + } +} + +function hasNonBlankEnvKey(): boolean { + const xai = process.env.XAI_API_KEY?.trim(); + const grok = process.env.GROK_API_KEY?.trim(); + return !!(xai || grok); +} + /** * Grok adapter — wraps the `grok` CLI via -p / --single / --prompt-file. * - * Auth readiness is boolean only: CLI present + (~/.grok/auth.json OR - * XAI_API_KEY / GROK_API_KEY env names present). Never log token values. + * Auth readiness: CLI present + (structurally valid ~/.grok/auth.json OR + * non-blank XAI_API_KEY / GROK_API_KEY after trim). Never log secret values. + * No network probe in available(). */ export class GrokAdapter implements ProviderAdapter { readonly name = 'grok'; @@ -46,13 +237,13 @@ export class GrokAdapter implements ProviderAdapter { // discovery. Bun's os.homedir() ignores process.env.HOME (unlike Node). const home = process.env.HOME || os.homedir(); const authPath = path.join(home, '.grok', 'auth.json'); - const hasAuthFile = fs.existsSync(authPath); - // Presence of env *names* only — never read/log values - const hasKey = !!(process.env.XAI_API_KEY || process.env.GROK_API_KEY); - if (!hasAuthFile && !hasKey) { + const hasValidAuthFile = isStructurallyValidGrokAuthFile(authPath); + const hasKey = hasNonBlankEnvKey(); + if (!hasValidAuthFile && !hasKey) { return { ok: false, - reason: 'No Grok auth found. Log in via `grok` interactive session, or export XAI_API_KEY / GROK_API_KEY.', + reason: + 'No Grok auth found. Log in via `grok` interactive session (non-empty auth.json), or export a non-blank XAI_API_KEY / GROK_API_KEY.', }; } return { ok: true }; @@ -76,6 +267,8 @@ export class GrokAdapter implements ProviderAdapter { } else { args = ['--single', opts.prompt]; } + // Request JSON so we can parse usage when the CLI exposes it (currently often omitted). + args.push('--output-format', 'json'); if (opts.model) args.push('--model', opts.model); if (opts.extraArgs) args.push(...opts.extraArgs); @@ -89,12 +282,14 @@ export class GrokAdapter implements ProviderAdapter { stdio: ['ignore', 'pipe', 'pipe'], env: { ...process.env, GSTACK_HEADLESS: '1' }, }); + const raw = typeof out === 'string' ? out : String(out); + const parsed = parseGrokOutput(raw); return { - output: typeof out === 'string' ? out : String(out), - tokens: { input: 0, output: 0 }, + output: parsed.output, + tokens: parsed.tokens, durationMs: Date.now() - start, - toolCalls: 0, - modelUsed: opts.model || 'grok', + toolCalls: parsed.toolCalls, + modelUsed: parsed.modelUsed || opts.model || 'grok', }; } catch (err: unknown) { const durationMs = Date.now() - start;