From b5ebda1dca0b118a403b35bf70b38870fb981655 Mon Sep 17 00:00:00 2001 From: Valentin Marchaud Date: Wed, 12 Aug 2026 02:38:23 +0200 Subject: [PATCH] fix(grok-local): report real token usage and cost instead of hardcoded zeros (#10433) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Cost/usage tracking is core to that: the dashboard shows per-agent spend so a team can see what their AI workforce is costing them > - The `grok_local` adapter (xAI's Grok Build CLI) is a newer adapter than `claude_local`/`codex_local`, and its usage/cost wiring was left incomplete > - Every `grok_local` run persists `usage.inputTokens/outputTokens/cachedInputTokens = 0` and `costUsd = null` in `heartbeat_runs`, unconditionally, even though the underlying `grok` CLI reports real, non-zero token counts and cost per turn in its own JSON stream > - This pull request wires the parser to actually read `usage`/`total_cost_usd` from the CLI's terminal `end` event, threads those values into the adapter's execution result, and marks them `usageBasis: "per_run"` so the heartbeat service doesn't incorrectly delta them against a prior run on a resumed session (matching how `claude_local`/`codex_local` already do this) > - The benefit is accurate cost/usage visibility for any self-hosted Paperclip instance running Grok Build agents, instead of a dashboard that always reads zero ## Linked Issues or Issue Description Fixes: #10432 ## What Changed - `packages/adapters/grok-local/src/server/parse.ts`: `parseGrokJsonl()` now reads `usage.input_tokens` / `usage.output_tokens` / `usage.cache_read_input_tokens` / `total_cost_usd` from the terminal `end` event and returns them on `ParsedGrokJsonl` (previously discarded entirely). - `packages/adapters/grok-local/src/server/execute.ts`: `toResult()` now populates `usage.inputTokens/outputTokens/cachedInputTokens` from the parsed values instead of hardcoded `0`, sets `usageBasis: "per_run"` (each `--single` invocation reports usage for just that process, not a running session total), and surfaces `costUsd` only when `billingType === "api"` (metered) — subscription/OAuth billing has no marginal dollar cost, so it stays `null` there, but token counts are populated for both billing types since usage visibility is useful regardless of billing model. - `packages/adapters/grok-local/src/server/parse.test.ts`: added a test asserting usage/cost extraction from a representative `end` event payload, and updated the existing exact-equality test for the new fields. - `packages/adapters/grok-local/src/server/execute.test.ts`: added a test covering both subscription billing (tokens populated, `costUsd: null`) and API-key billing (tokens populated, real `costUsd`), and asserting `usageBasis: "per_run"` in both cases. ## Verification - `pnpm vitest run packages/adapters/grok-local/src/server/parse.test.ts packages/adapters/grok-local/src/server/execute.test.ts` — 9/9 passed - `tsc --noEmit` on the `grok-local` package — clean - Verified against a real self-hosted Paperclip instance running `grok` CLI `0.2.112` with SuperGrok subscription (OAuth) auth: confirmed the raw CLI stream reports real `usage`/`total_cost_usd` (e.g. `"usage":{"input_tokens":21560,...},"total_cost_usd":0.0564448`) that was previously discarded before ever reaching `heartbeat_runs.usage_json`, which always showed all-zero tokens regardless of real usage. ## Risks - Low risk, additive change scoped entirely to the `grok_local` adapter's usage/cost reporting path — no change to control flow, session handling, or process execution. - `usageBasis: "per_run"` mirrors the existing, already-tested pattern in `claude_local`/`codex_local` execute paths, so the heartbeat service's per-run vs. session-cumulative delta logic is exercised the same way. - `costUsd` is intentionally left `null` for subscription/OAuth billing (no behavior change there beyond now-populated token counts) to avoid implying a dollar cost that doesn't exist for flat-rate billing. ## Model Used Claude Sonnet 5 (`claude-sonnet-5`), via Claude Code, no extended thinking. Root cause was found by comparing real `grok` CLI JSON stream output (captured directly from a live invocation) against the persisted `heartbeat_runs.usage_json` row for the same run on a self-hosted instance, then reading `parse.ts`/`execute.ts` source to confirm the hardcoded zero values. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above (none found) - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (`fix/grok-local-usage-cost-tracking`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [ ] I have updated relevant documentation to reflect my changes (no user-facing docs reference this internal usage-reporting behavior) - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green (pending at time of writing) - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups (addressed the one P1 raised — `usageBasis: "per_run"`) - [x] I will address all Greptile and reviewer comments before requesting merge --- .../grok-local/src/server/execute.test.ts | 64 +++++++++++++++++++ .../adapters/grok-local/src/server/execute.ts | 14 ++-- .../grok-local/src/server/parse.test.ts | 23 +++++++ .../adapters/grok-local/src/server/parse.ts | 22 ++++++- 4 files changed, 118 insertions(+), 5 deletions(-) diff --git a/packages/adapters/grok-local/src/server/execute.test.ts b/packages/adapters/grok-local/src/server/execute.test.ts index b520bf13e6..3d7f5d6c9b 100644 --- a/packages/adapters/grok-local/src/server/execute.test.ts +++ b/packages/adapters/grok-local/src/server/execute.test.ts @@ -138,6 +138,70 @@ describe("grok_local execute", () => { expect(logs.map((entry) => entry.chunk)).not.toEqual([]); }); + it("reports real per-run token usage, marks it as per_run, and only surfaces cost for API billing", async () => { + const root = await makeTempRoot(); + + runProcessMock.mockImplementation(async () => ({ + exitCode: 0, + signal: null, + timedOut: false, + stdout: [ + JSON.stringify({ type: "text", data: "done" }), + JSON.stringify({ + type: "end", + stopReason: "EndTurn", + sessionId: "sess-1", + requestId: "req-1", + usage: { input_tokens: 2384, output_tokens: 261, cache_read_input_tokens: 23040 }, + total_cost_usd: 0.013246, + }), + ].join("\n"), + stderr: "", + })); + + const baseCtx: AdapterExecutionContext = { + runId: "run-1", + agent: { + id: "agent-1", + companyId: "company-1", + name: "Grok Agent", + adapterType: "grok_local", + adapterConfig: {}, + }, + runtime: { sessionId: null, sessionParams: null, sessionDisplayId: null, taskKey: null }, + config: { cwd: root }, + context: {}, + authToken: "run-token", + onLog: async () => {}, + }; + + // Subscription billing (no XAI_API_KEY): token usage is populated, but + // there is no marginal dollar cost so costUsd stays null. + const subscriptionResult = await execute(baseCtx); + expect(subscriptionResult).toMatchObject({ + usage: { inputTokens: 2384, outputTokens: 261, cachedInputTokens: 23040 }, + usageBasis: "per_run", + billingType: "subscription", + costUsd: null, + }); + + // API-key billing: same token usage, plus the real dollar cost. + const previousApiKey = process.env.XAI_API_KEY; + process.env.XAI_API_KEY = "test-key"; + try { + const apiResult = await execute(baseCtx); + expect(apiResult).toMatchObject({ + usage: { inputTokens: 2384, outputTokens: 261, cachedInputTokens: 23040 }, + usageBasis: "per_run", + billingType: "api", + costUsd: 0.013246, + }); + } finally { + if (previousApiKey === undefined) delete process.env.XAI_API_KEY; + else process.env.XAI_API_KEY = previousApiKey; + } + }); + it("cleans up staged assets when setup fails before the Grok process starts", async () => { const root = await makeTempRoot(); const instructionsPath = path.join(root, "managed", "AGENTS.md"); diff --git a/packages/adapters/grok-local/src/server/execute.ts b/packages/adapters/grok-local/src/server/execute.ts index 45ae407bc7..e82ff1cec6 100644 --- a/packages/adapters/grok-local/src/server/execute.ts +++ b/packages/adapters/grok-local/src/server/execute.ts @@ -541,10 +541,14 @@ export async function execute(ctx: AdapterExecutionContext): Promise { errorMessage: null, stopReason: "EndTurn", requestId: "req-1", + inputTokens: 0, + outputTokens: 0, + cachedInputTokens: 0, + costUsd: null, }); }); + it("extracts token usage and cost from the end event", () => { + const parsed = parseGrokJsonl([ + JSON.stringify({ type: "text", data: "hi" }), + JSON.stringify({ + type: "end", + stopReason: "EndTurn", + sessionId: "sess-1", + requestId: "req-1", + usage: { input_tokens: 21560, output_tokens: 960, cache_read_input_tokens: 25216 }, + total_cost_usd: 0.0564448, + }), + ].join("\n")); + + expect(parsed.inputTokens).toBe(21560); + expect(parsed.outputTokens).toBe(960); + expect(parsed.cachedInputTokens).toBe(25216); + expect(parsed.costUsd).toBe(0.0564448); + }); + it("reads structured error payloads", () => { const parsed = parseGrokJsonl([ JSON.stringify({ type: "error", error: { message: "Authentication required" } }), diff --git a/packages/adapters/grok-local/src/server/parse.ts b/packages/adapters/grok-local/src/server/parse.ts index 7fc415a239..5a36de0509 100644 --- a/packages/adapters/grok-local/src/server/parse.ts +++ b/packages/adapters/grok-local/src/server/parse.ts @@ -1,4 +1,4 @@ -import { asString, parseJson, parseObject } from "@paperclipai/adapter-utils/server-utils"; +import { asNumber, asString, parseJson, parseObject } from "@paperclipai/adapter-utils/server-utils"; import { applyTurnBoundary, createTurnBoundaryState } from "../shared/turn-boundary.js"; export interface ParsedGrokJsonl { @@ -8,6 +8,10 @@ export interface ParsedGrokJsonl { errorMessage: string | null; stopReason: string | null; requestId: string | null; + inputTokens: number; + outputTokens: number; + cachedInputTokens: number; + costUsd: number | null; } function errorText(value: unknown): string { @@ -31,6 +35,10 @@ export function parseGrokJsonl(stdout: string): ParsedGrokJsonl { let stopReason: string | null = null; let requestId: string | null = null; let errorMessage: string | null = null; + let inputTokens = 0; + let outputTokens = 0; + let cachedInputTokens = 0; + let costUsd: number | null = null; const thoughtParts: string[] = []; const textParts: string[] = []; const thoughtBoundary = createTurnBoundaryState(); @@ -59,6 +67,14 @@ export function parseGrokJsonl(stdout: string): ParsedGrokJsonl { sessionId = asString(event.sessionId, "").trim() || sessionId; stopReason = asString(event.stopReason, "").trim() || stopReason; requestId = asString(event.requestId, "").trim() || requestId; + const usage = parseObject(event.usage); + inputTokens = asNumber(usage.input_tokens, inputTokens); + outputTokens = asNumber(usage.output_tokens, outputTokens); + cachedInputTokens = asNumber(usage.cache_read_input_tokens, cachedInputTokens); + const totalCostUsd = event.total_cost_usd; + if (typeof totalCostUsd === "number" && Number.isFinite(totalCostUsd)) { + costUsd = totalCostUsd; + } continue; } @@ -75,6 +91,10 @@ export function parseGrokJsonl(stdout: string): ParsedGrokJsonl { errorMessage, stopReason, requestId, + inputTokens, + outputTokens, + cachedInputTokens, + costUsd, }; }