From bf51b6728568ea11566a5873156e62023e95326b Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 3 Sep 2026 11:59:32 -0500 Subject: [PATCH] test(runner): add isolated local provider smoke loop --- packages/paperclip-runner/package.json | 1 + .../scripts/run-local-provider-smoke.mjs | 331 ++++++++++++++++++ .../src/eval/live-workflow-executor.test.ts | 51 +++ .../src/eval/live-workflow-executor.ts | 38 +- .../paperclip-runner/src/live/live-session.ts | 11 +- tests/runner-e2e/playwright.config.ts | 14 +- 6 files changed, 439 insertions(+), 7 deletions(-) create mode 100644 packages/paperclip-runner/scripts/run-local-provider-smoke.mjs diff --git a/packages/paperclip-runner/package.json b/packages/paperclip-runner/package.json index 38478ea72d..0598e0636c 100644 --- a/packages/paperclip-runner/package.json +++ b/packages/paperclip-runner/package.json @@ -144,6 +144,7 @@ "demo:live-console": "pnpm run build:typescript && node scripts/live-console-demo-server.mjs", "smoke:capability:ui": "pnpm run build:typescript && cargo build --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --bin paperclip-runnerd && node scripts/capability-issue-thread-smoke.mjs", "smoke:capability:cleanroom": "pnpm run build:typescript && cargo build --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --bin paperclip-runnerd && node scripts/capability-clean-room-smoke.mjs", + "smoke:local-provider": "node scripts/run-local-provider-smoke.mjs", "console:live-console": "pnpm run build:typescript && vite --config vite.config.ts --host 127.0.0.1 --port 4180", "console:sdk": "pnpm run build:typescript && vite --config vite.sdk.config.ts --host 127.0.0.1 --port 4181", "browser:dev": "pnpm run build:typescript && pnpm run build:runner-binaries && vite --config vite.config.ts", diff --git a/packages/paperclip-runner/scripts/run-local-provider-smoke.mjs b/packages/paperclip-runner/scripts/run-local-provider-smoke.mjs new file mode 100644 index 0000000000..ad1428389f --- /dev/null +++ b/packages/paperclip-runner/scripts/run-local-provider-smoke.mjs @@ -0,0 +1,331 @@ +#!/usr/bin/env node + +import { access, mkdir, mkdtemp, readdir, rm, stat } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, join, resolve } from "node:path"; + +const PROFILE_IDS = [ + "runner-acpx-claude", + "runner-acpx-codex", + "runner-codex", + "runner-opencode", +]; + +function parseProfiles(args) { + const selected = []; + for (let index = 0; index < args.length; index += 1) { + if (args[index] !== "--profile" || !args[index + 1]) { + throw new Error( + `Usage: pnpm smoke:local-provider -- --profile <${PROFILE_IDS.join("|")}|all>`, + ); + } + selected.push(args[++index]); + } + if (selected.length === 0) return ["runner-acpx-claude"]; + const expanded = selected.flatMap((profile) => + profile === "all" ? PROFILE_IDS : [profile], + ); + for (const profile of expanded) { + if (!PROFILE_IDS.includes(profile)) { + throw new Error(`Unsupported local provider smoke profile: ${profile}`); + } + } + return [...new Set(expanded)]; +} + +function liveCandidate(id, candidateSlots) { + const candidates = candidateSlots.flatMap((slot) => slot.candidates); + if (id === "runner-acpx-claude") { + return candidates.find( + (candidate) => candidate.id === "acpx-claude-sonnet", + ); + } + if (id === "runner-acpx-codex") { + return candidates.find((candidate) => candidate.id === "acpx-codex-sol"); + } + if (id === "runner-codex") { + const source = candidates.find( + (candidate) => candidate.id === "codex-luna", + ); + return ( + source && { + ...source, + id: "runner-codex-sol", + model: "gpt-5.6-sol", + } + ); + } + const source = candidates.find( + (candidate) => candidate.id === "opencode-kimi", + ); + return ( + source && { + ...source, + id: "runner-opencode-deepseek", + model: "openrouter/deepseek/deepseek-v4-flash-0731", + } + ); +} + +function failedChecks(observation) { + const smokeChecks = new Set([ + "terminal-authority", + "first-visible-progress", + "substantive-response", + "expected-assistant-text", + "no-empty-comment", + "terminal-presentation", + ]); + return [...observation.lifecycle.checks, ...observation.presentation.checks] + .filter((check) => smokeChecks.has(check.id) && !check.passed) + .map((check) => check.id); +} + +function safeErrorMessage(error, credentialValues) { + let message = error instanceof Error ? error.message : String(error); + for (const credential of credentialValues) { + if (credential.length > 0) + message = message.split(credential).join("[REDACTED]"); + } + return message + .replace(/\bsk-[A-Za-z0-9_-]{8,}\b/g, "[REDACTED]") + .replace(/\bBearer\s+\S+/gi, "Bearer [REDACTED]") + .replace(/\bAKIA[0-9A-Z]{16}\b/g, "[REDACTED]") + .slice(0, 1_000); +} + +async function filesUnder(directory, include, skipDirectory = () => false) { + const files = []; + for (const entry of await readdir(directory, { withFileTypes: true })) { + const path = join(directory, entry.name); + if (entry.isDirectory()) { + if (!skipDirectory(path)) { + files.push(...(await filesUnder(path, include, skipDirectory))); + } + } else if (entry.isFile() && include(path)) { + files.push(path); + } + } + return files; +} + +async function assertArtifactsFresh(label, sources, artifacts) { + const sourceTimes = await Promise.all( + sources.map(async (source) => (await stat(source)).mtimeMs), + ); + const artifactTimes = await Promise.all( + artifacts.map(async (artifact) => (await stat(artifact)).mtimeMs), + ); + if (Math.max(...sourceTimes) > Math.min(...artifactTimes)) { + throw new Error( + `${label} smoke artifacts are older than their source. Rebuild the targeted artifacts before running the smoke.`, + ); + } +} + +const profiles = parseProfiles(process.argv.slice(2)); +const packageRoot = resolve(import.meta.dirname, ".."); +const runnerd = resolve( + packageRoot, + "runner", + "target", + "debug", + process.platform === "win32" ? "paperclip-runnerd.exe" : "paperclip-runnerd", +); +const requiredArtifacts = [ + runnerd, + resolve(packageRoot, "dist", "eval", "index.js"), + ...(profiles.some((profile) => profile.startsWith("runner-acpx-")) + ? [resolve(packageRoot, "dist", "cli", "acpx-runtime-sidecar.cjs")] + : []), + ...(profiles.includes("runner-opencode") + ? [resolve(packageRoot, "dist", "cli", "opencode-app-server-proxy.cjs")] + : []), +]; +await Promise.all(requiredArtifacts.map((artifact) => access(artifact))).catch( + () => { + throw new Error( + "Local provider smoke artifacts are missing. Run build:typescript and build:runner-binaries once.", + ); + }, +); + +const typescriptSources = await filesUnder( + resolve(packageRoot, "src"), + (path) => path.endsWith(".ts") && !path.endsWith(".test.ts"), + (path) => + /\/(?:browser|devtools|issue-thread|react|scenarios|standalone)$/u.test( + path, + ), +); +await assertArtifactsFresh( + "TypeScript", + [ + resolve(packageRoot, "package.json"), + resolve(packageRoot, "tsconfig.json"), + ...typescriptSources, + ], + requiredArtifacts.filter((artifact) => artifact !== runnerd), +); +const rustSources = await filesUnder( + resolve(packageRoot, "runner"), + (path) => + path.endsWith(".rs") || + path.endsWith("Cargo.toml") || + path.endsWith("Cargo.lock"), + (path) => path.endsWith("/target"), +); +await assertArtifactsFresh("runnerd", rustSources, [runnerd]); + +const smokeRoot = await mkdtemp( + join(tmpdir(), "paperclip-local-provider-smoke-"), +); +if (!basename(smokeRoot).startsWith("paperclip-local-provider-smoke-")) { + throw new Error("Refusing an unrecognized local provider smoke root"); +} +await Promise.all( + ["tmp", "home", "paperclip-home", "codex-home", "claude-home"].map( + (directory) => mkdir(join(smokeRoot, directory), { recursive: true }), + ), +); + +const ambientEnvironment = { ...process.env }; +for (const name of Object.keys(process.env)) delete process.env[name]; +for (const name of [ + "PATH", + "SystemRoot", + "ComSpec", + "PATHEXT", + "LANG", + "LC_ALL", + "LC_CTYPE", + "TZ", + "SSL_CERT_FILE", + "SSL_CERT_DIR", + "NODE_EXTRA_CA_CERTS", +]) { + if (ambientEnvironment[name] !== undefined) { + process.env[name] = ambientEnvironment[name]; + } +} +Object.assign(process.env, { + TMPDIR: join(smokeRoot, "tmp"), + HOME: join(smokeRoot, "home"), + PAPERCLIP_HOME: join(smokeRoot, "paperclip-home"), + CODEX_HOME: join(smokeRoot, "codex-home"), + CLAUDE_CONFIG_DIR: join(smokeRoot, "claude-home"), + NO_BROWSER: "1", + PAPERCLIP_OPEN_ON_LISTEN: "false", +}); + +let cleanupPromise; +const cleanup = () => { + cleanupPromise ??= rm(smokeRoot, { recursive: true, force: true }); + return cleanupPromise; +}; +const exitAfterCleanup = (status) => { + void cleanup().finally(() => process.exit(status)); +}; +process.once("SIGINT", () => exitAfterCleanup(130)); +process.once("SIGTERM", () => exitAfterCleanup(143)); + +let failed = false; +try { + // Import only after the disposable homes are authoritative so no provider + // module can snapshot the developer's normal Paperclip or provider state. + const { + RUNNER_LIVE_CANDIDATE_SLOTS, + executeLiveRunnerWorkflow, + runnerWorkflowCase, + } = await import("../dist/eval/index.js"); + const candidates = new Map( + profiles.map((profile) => [ + profile, + liveCandidate(profile, RUNNER_LIVE_CANDIDATE_SLOTS), + ]), + ); + for (const [profile, candidate] of candidates) { + if (!candidate) throw new Error(`Missing live candidate for ${profile}`); + const missingCredentials = []; + for (const name of candidate.qualification.requiredEnvironment) { + const value = ambientEnvironment[name]?.trim(); + if (value) process.env[name] = value; + else missingCredentials.push(name); + } + if (missingCredentials.length > 0) { + throw new Error( + `${profile} requires ${missingCredentials.join(", ")} in the smoke process environment`, + ); + } + } + const credentialValues = new Set( + [...candidates.values()].flatMap((candidate) => + candidate.qualification.requiredEnvironment.flatMap((name) => { + const value = ambientEnvironment[name]?.trim(); + return value ? [value] : []; + }), + ), + ); + const evalCase = runnerWorkflowCase("completion-robustness"); + for (const profile of profiles) { + const candidate = candidates.get(profile); + const workspace = join(smokeRoot, `workspace-${profile}`); + await mkdir(workspace, { recursive: true }); + const entry = { + // Avoid a three-segment dotted identity: durable redaction correctly + // treats that shape as a possible JWT and refuses to rewrite IDs. + executionId: `local-smoke-${profile}-${String(Date.now())}`, + caseId: evalCase.id, + candidateId: candidate.id, + slotId: candidate.slotId, + repetition: 1, + providerTrace: "raw", + budget: candidate.budget, + }; + try { + const observation = await executeLiveRunnerWorkflow({ + entry, + candidate, + evalCase, + workingDirectory: workspace, + // This smoke proves the provider launch/message/semantic-terminal path. + // Usage conformance remains covered by the dedicated eval campaign. + allowMissingUsage: true, + expectedAssistantText: "PAPERCLIP_LOCAL_PROVIDER_SMOKE_OK", + promptOverride: + "Reply with exactly PAPERCLIP_LOCAL_PROVIDER_SMOKE_OK and no other text. Do not call tools.", + runnerBinary: runnerd, + }); + const failures = failedChecks(observation); + const passed = failures.length === 0; + failed ||= !passed; + process.stdout.write( + `${JSON.stringify({ + profile, + status: passed ? "passed" : "failed", + classification: passed + ? "message_completed" + : observation.classification, + settlementMs: observation.metrics.settlementMs ?? null, + totalTokens: observation.metrics.totalTokens ?? null, + costUsd: observation.metrics.costUsd ?? null, + failedChecks: failures, + failureCode: observation.failure?.code ?? null, + })}\n`, + ); + } catch (error) { + failed = true; + process.stderr.write( + `${JSON.stringify({ + profile, + status: "failed", + infrastructureError: safeErrorMessage(error, credentialValues), + })}\n`, + ); + } + } +} finally { + await cleanup(); +} + +if (failed) process.exitCode = 1; diff --git a/packages/paperclip-runner/src/eval/live-workflow-executor.test.ts b/packages/paperclip-runner/src/eval/live-workflow-executor.test.ts index 7705595f44..1f62eb9232 100644 --- a/packages/paperclip-runner/src/eval/live-workflow-executor.test.ts +++ b/packages/paperclip-runner/src/eval/live-workflow-executor.test.ts @@ -219,6 +219,57 @@ describe("live workflow executor infrastructure failures", () => { } }); + it("keeps launch-only smoke exceptions explicit and rejects a wrong marker", async () => { + liveSessionMocks.snapshot.mockReturnValue({ + sessionId: "session-smoke-marker", + authority: {}, + mockState: JSON.stringify({ tasks: [] }), + transcript: [ + { + id: "assistant-smoke-marker", + role: "assistant", + text: "a different response", + }, + ], + evidence: [], + authorizationRecords: [], + attempts: [], + usageLedger: [], + stateHistory: [], + workspaceDiffs: [], + }); + const candidate = RUNNER_LIVE_CANDIDATE_SLOTS[0]!.candidates[0]!; + const entry: RunnerLiveScheduleEntry = { + executionId: "local-smoke-marker", + caseId: "final-response", + candidateId: candidate.id, + slotId: candidate.slotId, + repetition: 1, + providerTrace: "raw", + budget: candidate.budget, + }; + + const observation = await executeLiveRunnerWorkflow({ + entry, + candidate, + evalCase: runnerWorkflowCase(entry.caseId), + allowMissingUsage: true, + expectedAssistantText: "PAPERCLIP_LOCAL_PROVIDER_SMOKE_OK", + promptOverride: "Return the smoke marker.", + }); + + expect(liveSessionMocks.sendMessage).toHaveBeenCalledWith( + "Return the smoke marker.", + { allowMissingUsage: true }, + ); + expect(observation.presentation.checks).toContainEqual( + expect.objectContaining({ + id: "expected-assistant-text", + passed: false, + }), + ); + }); + it("fails the candidate budget and stops before a paid continuation", async () => { liveSessionMocks.snapshot.mockReturnValue({ sessionId: "session-budget-test", diff --git a/packages/paperclip-runner/src/eval/live-workflow-executor.ts b/packages/paperclip-runner/src/eval/live-workflow-executor.ts index 9448d613d5..6f3b54fa95 100644 --- a/packages/paperclip-runner/src/eval/live-workflow-executor.ts +++ b/packages/paperclip-runner/src/eval/live-workflow-executor.ts @@ -479,6 +479,10 @@ export async function executeLiveRunnerWorkflow(input: { candidate: RunnerLiveEvalCandidate; evalCase: RunnerWorkflowEvalCase; workingDirectory?: string; + allowMissingUsage?: boolean; + expectedAssistantText?: string; + promptOverride?: string; + runnerBinary?: string; }): Promise { const runtimeRoot = await mkdtemp( join(tmpdir(), "paperclip-runner-live-eval-"), @@ -487,6 +491,9 @@ export async function executeLiveRunnerWorkflow(input: { const store = new InMemoryCapabilityLiveSessionStore(); const transportOptions = { environment: candidateTransportEnvironment(input.candidate, tracePath), + ...(input.runnerBinary === undefined + ? {} + : { runnerBinary: input.runnerBinary }), }; let service = new CapabilityLiveSessionService({ store, transportOptions }); let session: CapabilityLiveSession | null = null; @@ -558,7 +565,9 @@ export async function executeLiveRunnerWorkflow(input: { capabilities: capabilityFixtureRunCapabilities(LIVE_GRANTS), explicitClaims: [...LIVE_GRANTS], runId: input.entry.executionId, - sessionId: `session-${input.entry.executionId}`, + // Durable session identity is validation-bearing and must not look like + // credential material (for example a `session-...` token). + sessionId: `eval-${input.entry.executionId}`, attemptId: `attempt-${input.entry.executionId}`, turnTimeoutMs: input.candidate.budget.maxLatencyMs, lifecyclePolicy: { mode: "warm", idleTimeoutMs: 300_000 }, @@ -573,11 +582,20 @@ export async function executeLiveRunnerWorkflow(input: { const settled = await Promise.allSettled([pending]); if (settled[0]?.status === "fulfilled") turns.push(settled[0].value); } else { - turns.push(await session.sendMessage(promptFor(input.evalCase))); + turns.push( + await session.sendMessage( + input.promptOverride ?? promptFor(input.evalCase), + { + allowMissingUsage: input.allowMissingUsage, + }, + ), + ); await settleInteractions(input.evalCase, session, turns, withinBudget); if (input.evalCase.id === "steering-causality" && withinBudget()) { turns.push( - await session.sendMessage(continuationPrompt(input.evalCase)), + await session.sendMessage(continuationPrompt(input.evalCase), { + allowMissingUsage: input.allowMissingUsage, + }), ); } if (input.evalCase.id === "restart-recovery" && withinBudget()) { @@ -590,7 +608,9 @@ export async function executeLiveRunnerWorkflow(input: { session = await service.restore(sessionId); subscribeToSession(session); turns.push( - await session.sendMessage(continuationPrompt(input.evalCase)), + await session.sendMessage(continuationPrompt(input.evalCase), { + allowMissingUsage: input.allowMissingUsage, + }), ); } } @@ -799,6 +819,16 @@ export async function executeLiveRunnerWorkflow(input: { assistantTexts.some((text) => text.trim().length >= 2), "provider emitted no user-facing response", ), + ...(input.expectedAssistantText === undefined + ? [] + : [ + check( + "expected-assistant-text", + assistantTexts.length === 1 && + assistantTexts[0]?.trim() === input.expectedAssistantText, + "provider response did not exactly match the smoke marker", + ), + ]), check( "no-empty-comment", assistantTexts.every((text) => text.trim().length > 0), diff --git a/packages/paperclip-runner/src/live/live-session.ts b/packages/paperclip-runner/src/live/live-session.ts index 9bb3c04829..dfd5cacefe 100644 --- a/packages/paperclip-runner/src/live/live-session.ts +++ b/packages/paperclip-runner/src/live/live-session.ts @@ -1448,7 +1448,11 @@ export class CapabilityLiveSession { return this.snapshot(); } - async sendMessage(message: string): Promise { + async sendMessage( + message: string, + /** Launch-only diagnostics may opt out; qualification campaigns must not. */ + options: { allowMissingUsage?: boolean } = {}, + ): Promise { const value = message.trim(); if (value.length === 0) throw new Error("Capability live messages cannot be empty"); if (this.#status === "suspended" || this.#transport === null) { @@ -1650,7 +1654,10 @@ export class CapabilityLiveSession { }, }); } - await this.#captureTurnUsage(result.turnId, result.status !== "completed"); + await this.#captureTurnUsage( + result.turnId, + result.status !== "completed" || options.allowMissingUsage === true, + ); await this.#persist(); await this.#afterTurnSettled(); return { ...result, snapshot: this.snapshot() }; diff --git a/tests/runner-e2e/playwright.config.ts b/tests/runner-e2e/playwright.config.ts index 0bbc899847..8bc57d0cb2 100644 --- a/tests/runner-e2e/playwright.config.ts +++ b/tests/runner-e2e/playwright.config.ts @@ -14,6 +14,13 @@ const privateDir = required("PAPERCLIP_RUNNER_E2E_PRIVATE_DIR"); const paperclipHome = required("PAPERCLIP_HOME"); const configPath = required("PAPERCLIP_CONFIG"); const baseURL = `http://127.0.0.1:${port}`; +const chromiumExecutable = + process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE?.trim(); +if (chromiumExecutable && !path.isAbsolute(chromiumExecutable)) { + throw new Error( + "PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE must be an absolute path", + ); +} required("PAPERCLIP_INSTANCE_ID"); required("PAPERCLIP_AGENT_JWT_SECRET"); required("PAPERCLIP_DECISION_SIGNING_SECRET"); @@ -38,12 +45,17 @@ export default defineConfig({ use: { baseURL, browserName: "chromium", + ...(chromiumExecutable + ? { launchOptions: { executablePath: chromiumExecutable } } + : {}), headless: true, actionTimeout: 30_000, navigationTimeout: 30_000, screenshot: "only-on-failure", trace: "retain-on-failure", - video: "retain-on-failure", + // A developer-supplied system Chromium keeps the local smoke loop + // installation-free; CI's managed browser retains failure video as usual. + video: chromiumExecutable ? "off" : "retain-on-failure", }, webServer: { // Do not put an env object here: Playwright serializes webServer config in