From 9b1fd42ac180cd6a40097bc237520ca78ca87f3d Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Thu, 13 Aug 2026 13:58:05 -0700 Subject: [PATCH] test(grok-local): isolate billing env in usage cost test (#11285) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Local adapters report run output, token use, and cost data. > - The Grok local adapter now reports real token use and cost data. > - Its new billing test must prove the no-key path and the API-key path. > - The no-key assertion used the caller environment without isolation. > - This made the test fail when `XAI_API_KEY` was already set. > - This pull request isolates that environment state in the test. > - The benefit is stable coverage for the cost gate from #10433. ## Linked Issues or Issue Description Refs #10433 **What happened?** The Grok local usage and cost test asserted subscription billing while it still used the ambient process environment. If `XAI_API_KEY` was set before the test ran, the adapter selected API billing instead. The subscription assertion could then fail on a developer machine or a CI runner with provider credentials. **Expected behavior** The test should prove the subscription path with no `XAI_API_KEY`. It should also prove the API billing path with a test key. **Steps to reproduce** 1. Start from `master` after #10433. 2. Set `XAI_API_KEY` in the shell environment. 3. Run `vitest` for `packages/adapters/grok-local/src/server/execute.test.ts`. 4. Observe that the subscription half can take the API billing branch without test isolation. **Paperclip version or commit** `master` after #10433. **Deployment mode** Built from source. ## What Changed - Isolated `XAI_API_KEY` with save, delete, set, and restore logic around both billing assertions. - Gave the subscription and API billing checks separate run ids and temp roots. ## Verification - `XAI_API_KEY=ambient-test-key corepack pnpm exec vitest run packages/adapters/grok-local/src/server/execute.test.ts` - `corepack pnpm --filter @paperclipai/adapter-grok-local typecheck` ## Risks Low risk. This changes test setup only. It does not change Grok local adapter runtime behavior. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used OpenAI GPT-5 Codex local coding agent. The agent used shell tools, GitHub CLI, and local test execution. The context window size was not exposed in this run. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge Co-authored-by: Claude --- .../grok-local/src/server/execute.test.ts | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/packages/adapters/grok-local/src/server/execute.test.ts b/packages/adapters/grok-local/src/server/execute.test.ts index 3d7f5d6c9b..2b03c7dcbd 100644 --- a/packages/adapters/grok-local/src/server/execute.test.ts +++ b/packages/adapters/grok-local/src/server/execute.test.ts @@ -139,8 +139,6 @@ describe("grok_local execute", () => { }); it("reports real per-run token usage, marks it as per_run, and only surfaces cost for API billing", async () => { - const root = await makeTempRoot(); - runProcessMock.mockImplementation(async () => ({ exitCode: 0, signal: null, @@ -159,8 +157,8 @@ describe("grok_local execute", () => { stderr: "", })); - const baseCtx: AdapterExecutionContext = { - runId: "run-1", + const makeCtx = async (runId: string): Promise => ({ + runId, agent: { id: "agent-1", companyId: "company-1", @@ -169,27 +167,30 @@ describe("grok_local execute", () => { adapterConfig: {}, }, runtime: { sessionId: null, sessionParams: null, sessionDisplayId: null, taskKey: null }, - config: { cwd: root }, + config: { cwd: await makeTempRoot() }, context: {}, authToken: "run-token", onLog: async () => {}, - }; - - // Subscription billing (no XAI_API_KEY): token usage is populated, but - // there is no marginal dollar cost so costUsd stays null. - const subscriptionResult = await execute(baseCtx); - expect(subscriptionResult).toMatchObject({ - usage: { inputTokens: 2384, outputTokens: 261, cachedInputTokens: 23040 }, - usageBasis: "per_run", - billingType: "subscription", - costUsd: null, }); - // API-key billing: same token usage, plus the real dollar cost. const previousApiKey = process.env.XAI_API_KEY; - process.env.XAI_API_KEY = "test-key"; try { - const apiResult = await execute(baseCtx); + // Subscription billing (no XAI_API_KEY): token usage is populated, but + // there is no marginal dollar cost so costUsd stays null. Clear the key + // explicitly so the ambient environment (dev machine or CI with provider + // secrets) cannot flip this branch to API billing. + delete process.env.XAI_API_KEY; + const subscriptionResult = await execute(await makeCtx("run-subscription")); + expect(subscriptionResult).toMatchObject({ + usage: { inputTokens: 2384, outputTokens: 261, cachedInputTokens: 23040 }, + usageBasis: "per_run", + billingType: "subscription", + costUsd: null, + }); + + // API-key billing: same token usage, plus the real dollar cost. + process.env.XAI_API_KEY = "test-key"; + const apiResult = await execute(await makeCtx("run-api")); expect(apiResult).toMatchObject({ usage: { inputTokens: 2384, outputTokens: 261, cachedInputTokens: 23040 }, usageBasis: "per_run",