From ee851fc364087aa36951aaec78afe2be42ce27e3 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sat, 1 Aug 2026 07:15:52 -0700 Subject: [PATCH] feat(heartbeat): expose cache-adjusted run cost (#10349) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The heartbeat service is the control plane that runs agents through adapters and records each run's usage and cost in the finance/cost ledger > - Adapters report a provider cost (`costUsd`), but there is no way to represent the provider-billed cost *after* prompt-cache discounts, so cache-heavy runs are priced wrong and some paid runs end up exported with a zero/null cost > - A benchmark comparing Paperclip-orchestrated runs against direct harness invocation of the same tasks measured 1.5–3.1× higher apparent USD per pair, largely because cache-discounted billing was not represented in the exported cost data > - This pull request adds an optional `cacheAdjustedCostUsd` field to `AdapterExecutionResult` and a `resolveCacheAdjustedCostUsd` helper in the heartbeat service that prefers the explicit cache-adjusted figure and falls back to the reported `costUsd`, persisting it into the run's usage/ledger JSON > - The benefit is that paid runs are no longer exported as zero/null cost and cache-heavy runs can be priced correctly, so operators comparing orchestrated vs. direct costs see real numbers ## Linked Issues or Issue Description No single existing public issue covers this exactly; closely related cost-reporting issues: - Refs #8947 — hermes adapter never reports usage/cost to Paperclip, so budget limits never trigger - Refs #6716 — hermes_local cost/usage capture returns zero - Refs #3320 — expose per-run token counts in activity log and dashboard **Problem (feature-request form):** Adapters can only report a single `costUsd`. Providers with prompt caching bill less than the nominal token cost, and the heartbeat cost accounting has no field for the cache-adjusted billed amount. As a result, cache-heavy paid runs are either priced at the undiscounted figure or, when the adapter withholds the ambiguous number, exported as zero/null. **Proposed solution:** an explicit optional `cacheAdjustedCostUsd` on the adapter execution result, resolved server-side with a safe fallback to `costUsd`. **Alternatives considered:** recomputing cache discounts server-side from token counts (rejected: provider pricing tables drift and cache billing rules are provider-specific; the adapter is the source of truth). ## What Changed - `packages/adapter-utils/src/types.ts`: added optional `cacheAdjustedCostUsd?: number | null` to `AdapterExecutionResult`, with a doc comment on adapter expectations - `server/src/services/heartbeat.ts`: added exported `resolveCacheAdjustedCostUsd()` (explicit cache-adjusted value wins when a finite non-negative number; otherwise falls back to a finite non-negative `costUsd`; otherwise `null`), and consistently uses the resolved billed value for ledger cents, cost status, and run usage JSON - `server/src/__tests__/heartbeat-cost-accounting.test.ts`: added unit coverage for explicit precedence, fallback, invalid values, adjusted-only pricing, and discounted ledger billing ## Verification - `pnpm exec vitest run server/src/__tests__/heartbeat-cost-accounting.test.ts` — 1 file, 7 tests passed - `pnpm --filter @paperclipai/adapter-utils typecheck` — passed - `pnpm --filter @paperclipai/server typecheck` — passed - GitHub Actions on head `dc9c3830bf694b9afb3b27c5c8c36bff38e7fdcb` — all 26 checks clean/skipped; one unrelated E2E checkout-contention flake passed on its single failed-job rerun - Greptile review on the same head — 5/5 with zero unresolved threads ## Risks - Low risk: the field is optional and additive; when absent, behavior falls back to the existing `costUsd` path - Ledger/usage JSON gains a new optional `cacheAdjustedCostUsd` key — consumers that strictly validate keys would need to tolerate it (usage JSON is already open-shaped) - No migrations, no API-breaking changes > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - Implementation: Anthropic Claude Fable 5, model ID `claude-fable-5`, standard context window, agentic coding mode with shell/file tool use - PR preparation and verification: OpenAI Codex on GPT-5 (the runtime did not expose a more specific serving snapshot or context-window value), reasoning mode with shell and GitHub tool use ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes (doc comment on the new field; no user-facing docs affected) - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- packages/adapter-utils/src/types.ts | 6 +++ .../heartbeat-cost-accounting.test.ts | 47 ++++++++++++++++++- server/src/services/heartbeat.ts | 26 ++++++++-- 3 files changed, 74 insertions(+), 5 deletions(-) diff --git a/packages/adapter-utils/src/types.ts b/packages/adapter-utils/src/types.ts index d065722def..b6d7439a5d 100644 --- a/packages/adapter-utils/src/types.ts +++ b/packages/adapter-utils/src/types.ts @@ -101,6 +101,12 @@ export interface AdapterExecutionResult { model?: string | null; billingType?: AdapterBillingType | null; costUsd?: number | null; + /** + * Provider-billed cost after prompt-cache discounts. Adapters should set + * this when they expose it separately; otherwise the server treats a + * provider-reported `costUsd` as the cache-adjusted billed amount. + */ + cacheAdjustedCostUsd?: number | null; resultJson?: Record | null; runtimeServices?: AdapterRuntimeServiceReport[]; /** diff --git a/server/src/__tests__/heartbeat-cost-accounting.test.ts b/server/src/__tests__/heartbeat-cost-accounting.test.ts index 9f55385d69..182b8acbb0 100644 --- a/server/src/__tests__/heartbeat-cost-accounting.test.ts +++ b/server/src/__tests__/heartbeat-cost-accounting.test.ts @@ -1,5 +1,8 @@ import { describe, expect, it } from "vitest"; -import { resolveLedgerCostStatus } from "../services/heartbeat.js"; +import { + resolveCacheAdjustedCostUsd, + resolveLedgerCostStatus, +} from "../services/heartbeat.js"; describe("heartbeat cost accounting", () => { it("marks token-bearing CLI usage without a reported cost as unpriced", () => { @@ -19,4 +22,46 @@ describe("heartbeat cost accounting", () => { outputTokens: 77_000, })).toBe("reported"); }); + + it("uses an explicit cache-adjusted provider cost when available", () => { + expect(resolveCacheAdjustedCostUsd({ + costUsd: 1.25, + cacheAdjustedCostUsd: 0.92, + })).toBe(0.92); + }); + + it("attributes provider-reported billed cost as cache-adjusted by default", () => { + expect(resolveCacheAdjustedCostUsd({ + costUsd: 1.25, + cacheAdjustedCostUsd: null, + })).toBe(1.25); + }); + + it("does not attribute invalid or unavailable costs", () => { + expect(resolveCacheAdjustedCostUsd({ + costUsd: null, + cacheAdjustedCostUsd: Number.NaN, + })).toBeNull(); + }); + + it("prices a run that only reports a cache-adjusted cost", () => { + const billedCostUsd = resolveCacheAdjustedCostUsd({ + costUsd: null, + cacheAdjustedCostUsd: 0.42, + }); + expect(billedCostUsd).toBe(0.42); + expect(resolveLedgerCostStatus({ + costUsd: billedCostUsd, + inputTokens: 1_000, + cachedInputTokens: 900_000, + outputTokens: 5_000, + })).toBe("reported"); + }); + + it("bills the discounted amount when both nominal and cache-adjusted costs are reported", () => { + expect(resolveCacheAdjustedCostUsd({ + costUsd: 3.1, + cacheAdjustedCostUsd: 1.5, + })).toBe(1.5); + }); }); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index aed926c578..491d804a0e 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -3413,6 +3413,21 @@ export function resolveLedgerCostStatus(input: { return input.costUsd == null && hasTokenUsage ? "unpriced" : "reported"; } +export function resolveCacheAdjustedCostUsd(input: { + costUsd?: number | null; + cacheAdjustedCostUsd?: number | null; +}) { + const explicit = input.cacheAdjustedCostUsd; + if (typeof explicit === "number" && Number.isFinite(explicit) && explicit >= 0) { + return explicit; + } + const reported = input.costUsd; + if (typeof reported === "number" && Number.isFinite(reported) && reported >= 0) { + return reported; + } + return null; +} + export async function resolveLedgerScopeForRun( db: Db, companyId: string, @@ -12551,10 +12566,11 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const outputTokens = usage?.outputTokens ?? 0; const cachedInputTokens = usage?.cachedInputTokens ?? 0; const billingType = normalizeLedgerBillingType(result.billingType); - const additionalCostCents = normalizeBilledCostCents(result.costUsd, billingType); + const billedCostUsd = resolveCacheAdjustedCostUsd(result); + const additionalCostCents = normalizeBilledCostCents(billedCostUsd, billingType); const hasTokenUsage = inputTokens > 0 || outputTokens > 0 || cachedInputTokens > 0; const costStatus = resolveLedgerCostStatus({ - costUsd: result.costUsd, + costUsd: billedCostUsd, inputTokens, cachedInputTokens, outputTokens, @@ -14809,8 +14825,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) ? "timed_out" : "failed"; + const cacheAdjustedCostUsd = resolveCacheAdjustedCostUsd(adapterResult); const usageJson = - normalizedUsage || adapterResult.costUsd != null + normalizedUsage || adapterResult.costUsd != null || cacheAdjustedCostUsd != null ? ({ ...(normalizedUsage ?? {}), ...(rawUsage ? { @@ -14836,8 +14853,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) biller: resolveLedgerBiller(adapterResult), model: readNonEmptyString(adapterResult.model) ?? "unknown", ...(adapterResult.costUsd != null ? { costUsd: adapterResult.costUsd } : {}), + ...(cacheAdjustedCostUsd != null ? { cacheAdjustedCostUsd } : {}), costStatus: resolveLedgerCostStatus({ - costUsd: adapterResult.costUsd, + costUsd: cacheAdjustedCostUsd, inputTokens: normalizedUsage?.inputTokens ?? 0, cachedInputTokens: normalizedUsage?.cachedInputTokens ?? 0, outputTokens: normalizedUsage?.outputTokens ?? 0,