From efcce9cc8e17e2b30f6f2edf30e00a77b0bf19bb Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Mon, 13 Jul 2026 20:44:38 -0500 Subject: [PATCH] fix(adapters): record unpriced CLI usage (#9505) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source control plane people use to manage AI-agent companies > - Budgets and spend telemetry are control-plane safety features, not just reporting > - Local Codex and Claude adapters can execute through either ACP or their native CLI engines > - The ACP lane records usage and reported cost, but CLI JSON output often reports tokens without a price > - The CLI lane was either losing per-run usage semantics or coercing missing cost to zero, making real usage indistinguishable from a genuinely free run > - This pull request preserves CLI usage as per-run totals and records token-bearing runs without a reported price as explicitly unpriced ledger events > - The benefit is accurate usage accounting and a visible pricing gap instead of silently misleading zero-cost telemetry ## Linked Issues or Issue Description Refs #9471 Refs #9230 **Bug description** A `codex_local` run using the CLI engine can emit a final `turn.completed` event with millions of input tokens and tens of thousands of output tokens while the agent's spend ledger remains indistinguishable from a true zero-usage, zero-cost run. Claude CLI output has the same missing-price edge case. **Expected behavior** Token-bearing CLI runs should persist their usage. If the adapter reports a price, the ledger should record it as reported; if the CLI reports usage but no price, the ledger should explicitly mark the event as unpriced rather than silently treating missing price data as a reported `$0` cost. **Reproduction shape** 1. Configure `codex_local` with `engine: cli`. 2. Run a task that produces a `turn.completed` usage payload. 3. Observe token usage in the run stream. 4. Before this change, missing price data is represented as ordinary zero-cost spend and the CLI usage basis is not consistently propagated. ## What Changed - Mark Codex and Claude native CLI usage totals as `per_run` and propagate that basis through success and failure results. - Stop coercing missing Claude CLI cost to `0`. - Add `cost_status` to cost events with `reported` and `unpriced` values, including an idempotent migration and shared validation/types. - Persist token-bearing runs without a reported price as `unpriced` ledger events while retaining zero cents until an authoritative price exists. - Add parser, execute-path, heartbeat-accounting, and cost-service regression coverage for both local CLI adapters. - Document the cost-status invariant and CLI accounting behavior. ## Verification - `pnpm exec vitest run packages/adapters/codex-local/src/server/parse.test.ts packages/adapters/claude-local/src/server/parse.test.ts server/src/__tests__/codex-local-execute.test.ts server/src/__tests__/claude-local-execute.test.ts server/src/__tests__/heartbeat-cost-accounting.test.ts server/src/__tests__/costs-service.test.ts` — 6 files / 102 tests passed. - `pnpm --filter @paperclipai/shared typecheck` - `pnpm --filter @paperclipai/db typecheck` — includes migration numbering and safety checks. - `pnpm --filter @paperclipai/adapter-codex-local typecheck` - `pnpm --filter @paperclipai/adapter-claude-local typecheck` - `pnpm --filter @paperclipai/server typecheck` ## Risks - Existing cost rows default to `reported`, preserving current interpretation; only new token-bearing events with absent cost are marked `unpriced`. - This change does not invent model pricing. Budget hard stops still cannot charge an unknown amount, but operators and evals can now distinguish missing pricing from a genuinely reported zero cost. - Consumers that enumerate cost-event fields should tolerate the additive `costStatus` field. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - OpenAI Codex, exact model `gpt-5.3-codex`, with repository tool use and code execution; default reasoning mode. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge Co-authored-by: Paperclip --- doc/SPEC-implementation.md | 1 + doc/spec/agent-runs.md | 2 +- .../claude-local/src/server/execute.ts | 2 +- .../claude-local/src/server/parse.test.ts | 2 +- .../adapters/claude-local/src/server/parse.ts | 2 +- .../codex-local/src/server/execute.ts | 2 + .../codex-local/src/server/parse.test.ts | 2 + .../adapters/codex-local/src/server/parse.ts | 1 + .../src/migrations/0147_cost_event_status.sql | 1 + packages/db/src/migrations/meta/_journal.json | 7 +++ packages/db/src/schema/cost_events.ts | 1 + packages/shared/src/constants.ts | 3 ++ packages/shared/src/index.ts | 1 + packages/shared/src/types/cost.ts | 3 +- packages/shared/src/validators/cost.ts | 3 +- .../__tests__/claude-local-execute.test.ts | 3 ++ .../src/__tests__/codex-local-execute.test.ts | 3 ++ server/src/__tests__/costs-service.test.ts | 43 +++++++++++++++++++ .../heartbeat-cost-accounting.test.ts | 22 ++++++++++ server/src/services/heartbeat.ts | 24 +++++++++++ 20 files changed, 122 insertions(+), 6 deletions(-) create mode 100644 packages/db/src/migrations/0147_cost_event_status.sql create mode 100644 server/src/__tests__/heartbeat-cost-accounting.test.ts diff --git a/doc/SPEC-implementation.md b/doc/SPEC-implementation.md index 295195409f..13793545ad 100644 --- a/doc/SPEC-implementation.md +++ b/doc/SPEC-implementation.md @@ -292,6 +292,7 @@ Invariants: - `billing_code` text null - `provider` text not null - `model` text not null +- `cost_status` text not null default `reported`; `unpriced` when usage exists but no price was reported - `input_tokens` int not null default 0 - `output_tokens` int not null default 0 - `cost_cents` int not null diff --git a/doc/spec/agent-runs.md b/doc/spec/agent-runs.md index 56e2eae134..ab98996c32 100644 --- a/doc/spec/agent-runs.md +++ b/doc/spec/agent-runs.md @@ -313,7 +313,7 @@ Codex emits JSONL events. Parse line-by-line and extract: - `cached_input_tokens` - `output_tokens` -Codex JSONL currently may not include cost; store token usage and leave cost null/unknown unless available. +Codex JSONL currently may not include cost; store token usage as per-run totals and mark the ledger row `unpriced` unless a cost is available. ## 7.3 Common local adapter process handling diff --git a/packages/adapters/claude-local/src/server/execute.ts b/packages/adapters/claude-local/src/server/execute.ts index ea4e2759eb..961e0c9eb2 100644 --- a/packages/adapters/claude-local/src/server/execute.ts +++ b/packages/adapters/claude-local/src/server/execute.ts @@ -1067,7 +1067,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise { outputTokens: 1_800, cachedInputTokens: 20, }); - expect(parsed.usageBasis).toBeNull(); + expect(parsed.usageBasis).toBe("per_run"); }); }); diff --git a/packages/adapters/claude-local/src/server/parse.ts b/packages/adapters/claude-local/src/server/parse.ts index 852a7f1f01..2707ed2c9a 100644 --- a/packages/adapters/claude-local/src/server/parse.ts +++ b/packages/adapters/claude-local/src/server/parse.ts @@ -111,7 +111,7 @@ export function parseClaudeStreamJson(stdout: string) { usage, // modelUsage covers exactly this CLI invocation, so mark it per-run to // keep the server from applying its session-cumulative delta heuristic. - usageBasis: modelUsageTotals ? ("per_run" as const) : null, + usageBasis: "per_run" as const, summary, resultJson: finalResult, }; diff --git a/packages/adapters/codex-local/src/server/execute.ts b/packages/adapters/codex-local/src/server/execute.ts index 976b63cf98..cd9b9f1806 100644 --- a/packages/adapters/codex-local/src/server/execute.ts +++ b/packages/adapters/codex-local/src/server/execute.ts @@ -976,6 +976,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise { cachedInputTokens: 2, outputTokens: 4, }, + usageBasis: "per_run", errorMessage: "resume failed", }); }); @@ -63,6 +64,7 @@ describe("parseCodexJsonl", () => { cachedInputTokens: 2, outputTokens: 4, }, + usageBasis: "per_run", errorMessage: null, }); }); diff --git a/packages/adapters/codex-local/src/server/parse.ts b/packages/adapters/codex-local/src/server/parse.ts index 535d872f28..0c4e0a7312 100644 --- a/packages/adapters/codex-local/src/server/parse.ts +++ b/packages/adapters/codex-local/src/server/parse.ts @@ -70,6 +70,7 @@ export function parseCodexJsonl(stdout: string) { sessionId, summary: finalMessage?.trim() ?? "", usage, + usageBasis: "per_run" as const, errorMessage, }; } diff --git a/packages/db/src/migrations/0147_cost_event_status.sql b/packages/db/src/migrations/0147_cost_event_status.sql new file mode 100644 index 0000000000..ed9879237a --- /dev/null +++ b/packages/db/src/migrations/0147_cost_event_status.sql @@ -0,0 +1 @@ +ALTER TABLE "cost_events" ADD COLUMN IF NOT EXISTS "cost_status" text DEFAULT 'reported' NOT NULL; diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index fd11d6dbb9..a28906ec31 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -1016,6 +1016,13 @@ "when": 1783822632557, "tag": "0146_routine_activity_gate", "breakpoints": true + }, + { + "idx": 147, + "version": "7", + "when": 1783953514660, + "tag": "0147_cost_event_status", + "breakpoints": true } ] } diff --git a/packages/db/src/schema/cost_events.ts b/packages/db/src/schema/cost_events.ts index f8a36847e6..7652290fbd 100644 --- a/packages/db/src/schema/cost_events.ts +++ b/packages/db/src/schema/cost_events.ts @@ -20,6 +20,7 @@ export const costEvents = pgTable( provider: text("provider").notNull(), biller: text("biller").notNull().default("unknown"), billingType: text("billing_type").notNull().default("unknown"), + costStatus: text("cost_status").notNull().default("reported"), model: text("model").notNull(), inputTokens: integer("input_tokens").notNull().default(0), cachedInputTokens: integer("cached_input_tokens").notNull().default(0), diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.ts index 801019a47a..b9db9c76b9 100644 --- a/packages/shared/src/constants.ts +++ b/packages/shared/src/constants.ts @@ -687,6 +687,9 @@ export const BILLING_TYPES = [ ] as const; export type BillingType = (typeof BILLING_TYPES)[number]; +export const COST_STATUSES = ["reported", "unpriced"] as const; +export type CostStatus = (typeof COST_STATUSES)[number]; + export const FINANCE_EVENT_KINDS = [ "inference_charge", "platform_fee", diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 9b90f1bb8a..59ba736c6e 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -373,6 +373,7 @@ export { type SecretScope, type StorageProvider, type BillingType, + type CostStatus, type FinanceEventKind, type FinanceDirection, type FinanceUnit, diff --git a/packages/shared/src/types/cost.ts b/packages/shared/src/types/cost.ts index 11d9a7652c..88e2af361d 100644 --- a/packages/shared/src/types/cost.ts +++ b/packages/shared/src/types/cost.ts @@ -1,4 +1,4 @@ -import type { BillingType } from "../constants.js"; +import type { BillingType, CostStatus } from "../constants.js"; export interface CostEvent { id: string; @@ -12,6 +12,7 @@ export interface CostEvent { provider: string; biller: string; billingType: BillingType; + costStatus: CostStatus; model: string; inputTokens: number; cachedInputTokens: number; diff --git a/packages/shared/src/validators/cost.ts b/packages/shared/src/validators/cost.ts index ed22d7ebc2..d38ec43681 100644 --- a/packages/shared/src/validators/cost.ts +++ b/packages/shared/src/validators/cost.ts @@ -1,5 +1,5 @@ import { z } from "zod"; -import { BILLING_TYPES } from "../constants.js"; +import { BILLING_TYPES, COST_STATUSES } from "../constants.js"; export const createCostEventSchema = z.object({ agentId: z.string().uuid(), @@ -11,6 +11,7 @@ export const createCostEventSchema = z.object({ provider: z.string().min(1), biller: z.string().min(1).optional(), billingType: z.enum(BILLING_TYPES).optional().default("unknown"), + costStatus: z.enum(COST_STATUSES).optional().default("reported"), model: z.string().min(1), inputTokens: z.number().int().nonnegative().optional().default(0), cachedInputTokens: z.number().int().nonnegative().optional().default(0), diff --git a/server/src/__tests__/claude-local-execute.test.ts b/server/src/__tests__/claude-local-execute.test.ts index 7fe01345a7..af8312a883 100644 --- a/server/src/__tests__/claude-local-execute.test.ts +++ b/server/src/__tests__/claude-local-execute.test.ts @@ -720,6 +720,9 @@ describe("claude execute", () => { expect(result.exitCode).toBe(0); expect(result.errorMessage).toBeNull(); + expect(result.usage).toEqual({ inputTokens: 1, cachedInputTokens: 0, outputTokens: 1 }); + expect(result.usageBasis).toBe("per_run"); + expect(result.costUsd).toBeNull(); expect(loggedCommand).toBe(commandPath); expect(loggedEnv.HOME).toBe(root); expect(loggedEnv.CLAUDE_CONFIG_DIR).toBe(claudeConfigDir); diff --git a/server/src/__tests__/codex-local-execute.test.ts b/server/src/__tests__/codex-local-execute.test.ts index cdadb2f45c..b7d719fce1 100644 --- a/server/src/__tests__/codex-local-execute.test.ts +++ b/server/src/__tests__/codex-local-execute.test.ts @@ -178,6 +178,9 @@ describe("codex execute", () => { expect(result.exitCode).toBe(0); expect(result.errorMessage).toBeNull(); + expect(result.usage).toEqual({ inputTokens: 1, cachedInputTokens: 0, outputTokens: 1 }); + expect(result.usageBasis).toBe("per_run"); + expect(result.costUsd).toBeNull(); const capture = JSON.parse(await fs.readFile(capturePath, "utf8")) as CapturePayload; expect(capture.codexHome).toBe(managedCodexHome); diff --git a/server/src/__tests__/costs-service.test.ts b/server/src/__tests__/costs-service.test.ts index 1effd44490..3ce190c093 100644 --- a/server/src/__tests__/costs-service.test.ts +++ b/server/src/__tests__/costs-service.test.ts @@ -3,6 +3,7 @@ import request from "supertest"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { afterAll, afterEach, beforeAll } from "vitest"; import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; import { createDb, companies, @@ -432,6 +433,48 @@ describeEmbeddedPostgres("cost and finance aggregate overflow handling", () => { await tempDb?.cleanup(); }); + it("persists unpriced token usage without inflating monthly spend", async () => { + const companyId = randomUUID(); + const agentId = randomUUID(); + + await db.insert(companies).values({ + id: companyId, + name: "Paperclip", + issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`, + requireBoardApprovalForNewAgents: false, + }); + await db.insert(agents).values({ + id: agentId, + companyId, + name: "CLI Agent", + role: "engineer", + status: "active", + adapterType: "codex_local", + adapterConfig: {}, + runtimeConfig: {}, + permissions: {}, + }); + + const event = await costs.createEvent(companyId, { + agentId, + provider: "openai", + biller: "chatgpt", + billingType: "subscription_included", + costStatus: "unpriced", + model: "gpt-5.6-terra", + inputTokens: 2_732_577, + cachedInputTokens: 2_632_998, + outputTokens: 32_644, + costCents: 0, + occurredAt: new Date("2026-07-13T14:22:54.000Z"), + }); + + expect(event.costStatus).toBe("unpriced"); + expect(event.inputTokens).toBe(2_732_577); + const [agent] = await db.select().from(agents).where(eq(agents.id, agentId)); + expect(agent?.spentMonthlyCents).toBe(0); + }); + it("aggregates cost event sums above int32 without raising Postgres integer overflow", async () => { const companyId = randomUUID(); const agentId = randomUUID(); diff --git a/server/src/__tests__/heartbeat-cost-accounting.test.ts b/server/src/__tests__/heartbeat-cost-accounting.test.ts new file mode 100644 index 0000000000..9f55385d69 --- /dev/null +++ b/server/src/__tests__/heartbeat-cost-accounting.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from "vitest"; +import { resolveLedgerCostStatus } from "../services/heartbeat.js"; + +describe("heartbeat cost accounting", () => { + it("marks token-bearing CLI usage without a reported cost as unpriced", () => { + expect(resolveLedgerCostStatus({ + costUsd: null, + inputTokens: 2_732_577, + cachedInputTokens: 2_632_998, + outputTokens: 32_644, + })).toBe("unpriced"); + }); + + it("marks reported CLI cost as priced", () => { + expect(resolveLedgerCostStatus({ + costUsd: 1.25, + inputTokens: 2_090, + cachedInputTokens: 300_000, + outputTokens: 77_000, + })).toBe("reported"); + }); +}); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 2c792bef19..a9418cd063 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -12,6 +12,7 @@ import { envBindingSchema, isEnvironmentDriverSupportedForAdapter, type BillingType, + type CostStatus, type EnvironmentLeaseStatus, type ExecutionWorkspace, type ExecutionWorkspaceConfig, @@ -2367,6 +2368,16 @@ function normalizeBilledCostCents(costUsd: number | null | undefined, billingTyp return Math.max(0, Math.round(costUsd * 100)); } +export function resolveLedgerCostStatus(input: { + costUsd: number | null | undefined; + inputTokens: number; + cachedInputTokens: number; + outputTokens: number; +}): CostStatus { + const hasTokenUsage = input.inputTokens > 0 || input.cachedInputTokens > 0 || input.outputTokens > 0; + return input.costUsd == null && hasTokenUsage ? "unpriced" : "reported"; +} + async function resolveLedgerScopeForRun( db: Db, companyId: string, @@ -10867,6 +10878,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const billingType = normalizeLedgerBillingType(result.billingType); const additionalCostCents = normalizeBilledCostCents(result.costUsd, billingType); const hasTokenUsage = inputTokens > 0 || outputTokens > 0 || cachedInputTokens > 0; + const costStatus = resolveLedgerCostStatus({ + costUsd: result.costUsd, + inputTokens, + cachedInputTokens, + outputTokens, + }); const provider = result.provider ?? "unknown"; const biller = resolveLedgerBiller(result); const ledgerScope = await resolveLedgerScopeForRun(db, agent.companyId, run); @@ -10897,6 +10914,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) provider, biller, billingType, + costStatus, model: result.model ?? "unknown", inputTokens, cachedInputTokens, @@ -12967,6 +12985,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) biller: resolveLedgerBiller(adapterResult), model: readNonEmptyString(adapterResult.model) ?? "unknown", ...(adapterResult.costUsd != null ? { costUsd: adapterResult.costUsd } : {}), + costStatus: resolveLedgerCostStatus({ + costUsd: adapterResult.costUsd, + inputTokens: normalizedUsage?.inputTokens ?? 0, + cachedInputTokens: normalizedUsage?.cachedInputTokens ?? 0, + outputTokens: normalizedUsage?.outputTokens ?? 0, + }), billingType: normalizeLedgerBillingType(adapterResult.billingType), } as Record) : null;