From 24e0b9776bd386b4c8fbbb9b649e9487afc67e90 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 3 Sep 2026 17:57:56 -0500 Subject: [PATCH] fix(runner): isolate local smoke credentials per profile --- packages/paperclip-runner/package.json | 2 +- .../local-provider-smoke-environment.mjs | 15 ++++++ .../local-provider-smoke-environment.test.mjs | 34 +++++++++++++ .../scripts/run-local-provider-smoke.mjs | 49 ++++++++++++------- 4 files changed, 81 insertions(+), 19 deletions(-) create mode 100644 packages/paperclip-runner/scripts/local-provider-smoke-environment.mjs create mode 100644 packages/paperclip-runner/scripts/local-provider-smoke-environment.test.mjs diff --git a/packages/paperclip-runner/package.json b/packages/paperclip-runner/package.json index 6a7f8832be..4b9429086d 100644 --- a/packages/paperclip-runner/package.json +++ b/packages/paperclip-runner/package.json @@ -73,7 +73,7 @@ "typecheck:rust": "cargo fmt --manifest-path runner/Cargo.toml --all -- --check && cargo check --manifest-path runner/Cargo.toml --locked --workspace", "typecheck:browser": "tsc -p tsconfig.browser.json --noEmit", "test": "pnpm run test:typescript && pnpm run test:rust", - "test:typescript": "pnpm run ensure:eval-build-deps && pnpm run build:rust && node --test test/protocol-contract.test.mjs test/acpx-sidecar-contract.test.mjs test/acpx-codex-package-contract.test.mjs scripts/aws-agentcore-provisioning.test.mjs scripts/build-verified-provider-entrypoints.test.mjs scripts/materialize-opencode-binary.test.mjs && vitest run", + "test:typescript": "pnpm run ensure:eval-build-deps && pnpm run build:rust && node --test test/protocol-contract.test.mjs test/acpx-sidecar-contract.test.mjs test/acpx-codex-package-contract.test.mjs scripts/aws-agentcore-provisioning.test.mjs scripts/build-verified-provider-entrypoints.test.mjs scripts/local-provider-smoke-environment.test.mjs scripts/materialize-opencode-binary.test.mjs && vitest run", "test:rust": "cargo test --release --manifest-path runner/Cargo.toml --locked --workspace", "test:codex": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --test codex_provider", "test:durable": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core durable::", diff --git a/packages/paperclip-runner/scripts/local-provider-smoke-environment.mjs b/packages/paperclip-runner/scripts/local-provider-smoke-environment.mjs new file mode 100644 index 0000000000..a1200497d4 --- /dev/null +++ b/packages/paperclip-runner/scripts/local-provider-smoke-environment.mjs @@ -0,0 +1,15 @@ +export async function withIsolatedProfileCredentials(input) { + for (const name of input.providerCredentialNames) { + delete input.environment[name]; + } + for (const [name, value] of Object.entries(input.profileCredentials)) { + input.environment[name] = value; + } + try { + return await input.run(); + } finally { + for (const name of input.providerCredentialNames) { + delete input.environment[name]; + } + } +} diff --git a/packages/paperclip-runner/scripts/local-provider-smoke-environment.test.mjs b/packages/paperclip-runner/scripts/local-provider-smoke-environment.test.mjs new file mode 100644 index 0000000000..1a60070aa5 --- /dev/null +++ b/packages/paperclip-runner/scripts/local-provider-smoke-environment.test.mjs @@ -0,0 +1,34 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { withIsolatedProfileCredentials } from "./local-provider-smoke-environment.mjs"; + +test("a later smoke profile cannot observe another provider credential", async () => { + const environment = { + PATH: "/bin", + OPENAI_API_KEY: "credential-from-an-earlier-profile", + }; + const providerCredentialNames = ["OPENAI_API_KEY", "ANTHROPIC_API_KEY"]; + + await assert.rejects( + withIsolatedProfileCredentials({ + environment, + providerCredentialNames, + profileCredentials: { + ANTHROPIC_API_KEY: "credential-for-current-profile", + }, + run: async () => { + assert.equal(environment.OPENAI_API_KEY, undefined); + assert.equal( + environment.ANTHROPIC_API_KEY, + "credential-for-current-profile", + ); + assert.equal(environment.PATH, "/bin"); + throw new Error("provider failed"); + }, + }), + /provider failed/, + ); + + assert.deepEqual(environment, { PATH: "/bin" }); +}); diff --git a/packages/paperclip-runner/scripts/run-local-provider-smoke.mjs b/packages/paperclip-runner/scripts/run-local-provider-smoke.mjs index ad1428389f..cc856be840 100644 --- a/packages/paperclip-runner/scripts/run-local-provider-smoke.mjs +++ b/packages/paperclip-runner/scripts/run-local-provider-smoke.mjs @@ -4,6 +4,8 @@ import { access, mkdir, mkdtemp, readdir, rm, stat } from "node:fs/promises"; import { tmpdir } from "node:os"; import { basename, join, resolve } from "node:path"; +import { withIsolatedProfileCredentials } from "./local-provider-smoke-environment.mjs"; + const PROFILE_IDS = [ "runner-acpx-claude", "runner-acpx-codex", @@ -244,12 +246,14 @@ try { liveCandidate(profile, RUNNER_LIVE_CANDIDATE_SLOTS), ]), ); + const credentialsByProfile = new Map(); for (const [profile, candidate] of candidates) { if (!candidate) throw new Error(`Missing live candidate for ${profile}`); const missingCredentials = []; + const profileCredentials = {}; for (const name of candidate.qualification.requiredEnvironment) { const value = ambientEnvironment[name]?.trim(); - if (value) process.env[name] = value; + if (value) profileCredentials[name] = value; else missingCredentials.push(name); } if (missingCredentials.length > 0) { @@ -257,13 +261,16 @@ try { `${profile} requires ${missingCredentials.join(", ")} in the smoke process environment`, ); } + credentialsByProfile.set(profile, profileCredentials); } + const providerCredentialNames = new Set( + [...credentialsByProfile.values()].flatMap((credentials) => + Object.keys(credentials), + ), + ); const credentialValues = new Set( - [...candidates.values()].flatMap((candidate) => - candidate.qualification.requiredEnvironment.flatMap((name) => { - const value = ambientEnvironment[name]?.trim(); - return value ? [value] : []; - }), + [...credentialsByProfile.values()].flatMap((credentials) => + Object.values(credentials), ), ); const evalCase = runnerWorkflowCase("completion-robustness"); @@ -283,18 +290,24 @@ try { budget: candidate.budget, }; try { - const observation = await executeLiveRunnerWorkflow({ - entry, - candidate, - evalCase, - workingDirectory: workspace, - // This smoke proves the provider launch/message/semantic-terminal path. - // Usage conformance remains covered by the dedicated eval campaign. - allowMissingUsage: true, - expectedAssistantText: "PAPERCLIP_LOCAL_PROVIDER_SMOKE_OK", - promptOverride: - "Reply with exactly PAPERCLIP_LOCAL_PROVIDER_SMOKE_OK and no other text. Do not call tools.", - runnerBinary: runnerd, + const observation = await withIsolatedProfileCredentials({ + environment: process.env, + providerCredentialNames, + profileCredentials: credentialsByProfile.get(profile), + run: () => + executeLiveRunnerWorkflow({ + entry, + candidate, + evalCase, + workingDirectory: workspace, + // This smoke proves the provider launch/message/semantic-terminal path. + // Usage conformance remains covered by the dedicated eval campaign. + allowMissingUsage: true, + expectedAssistantText: "PAPERCLIP_LOCAL_PROVIDER_SMOKE_OK", + promptOverride: + "Reply with exactly PAPERCLIP_LOCAL_PROVIDER_SMOKE_OK and no other text. Do not call tools.", + runnerBinary: runnerd, + }), }); const failures = failedChecks(observation); const passed = failures.length === 0;