test(runner): add isolated local provider smoke loop

packages/paperclip-runner/package.json

@@ -144,6 +144,7 @@
     "demo:live-console": "pnpm run build:typescript && node scripts/live-console-demo-server.mjs",
     "smoke:capability:ui": "pnpm run build:typescript && cargo build --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --bin paperclip-runnerd && node scripts/capability-issue-thread-smoke.mjs",
     "smoke:capability:cleanroom": "pnpm run build:typescript && cargo build --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --bin paperclip-runnerd && node scripts/capability-clean-room-smoke.mjs",
+    "smoke:local-provider": "node scripts/run-local-provider-smoke.mjs",
     "console:live-console": "pnpm run build:typescript && vite --config vite.config.ts --host 127.0.0.1 --port 4180",
     "console:sdk": "pnpm run build:typescript && vite --config vite.sdk.config.ts --host 127.0.0.1 --port 4181",
     "browser:dev": "pnpm run build:typescript && pnpm run build:runner-binaries && vite --config vite.config.ts",

packages/paperclip-runner/scripts/run-local-provider-smoke.mjs

@@ -0,0 +1,331 @@
+#!/usr/bin/env node
+
+import { access, mkdir, mkdtemp, readdir, rm, stat } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { basename, join, resolve } from "node:path";
+
+const PROFILE_IDS = [
+  "runner-acpx-claude",
+  "runner-acpx-codex",
+  "runner-codex",
+  "runner-opencode",
+];
+
+function parseProfiles(args) {
+  const selected = [];
+  for (let index = 0; index < args.length; index += 1) {
+    if (args[index] !== "--profile" || !args[index + 1]) {
+      throw new Error(
+        `Usage: pnpm smoke:local-provider -- --profile <${PROFILE_IDS.join("|")}|all>`,
+      );
+    }
+    selected.push(args[++index]);
+  }
+  if (selected.length === 0) return ["runner-acpx-claude"];
+  const expanded = selected.flatMap((profile) =>
+    profile === "all" ? PROFILE_IDS : [profile],
+  );
+  for (const profile of expanded) {
+    if (!PROFILE_IDS.includes(profile)) {
+      throw new Error(`Unsupported local provider smoke profile: ${profile}`);
+    }
+  }
+  return [...new Set(expanded)];
+}
+
+function liveCandidate(id, candidateSlots) {
+  const candidates = candidateSlots.flatMap((slot) => slot.candidates);
+  if (id === "runner-acpx-claude") {
+    return candidates.find(
+      (candidate) => candidate.id === "acpx-claude-sonnet",
+    );
+  }
+  if (id === "runner-acpx-codex") {
+    return candidates.find((candidate) => candidate.id === "acpx-codex-sol");
+  }
+  if (id === "runner-codex") {
+    const source = candidates.find(
+      (candidate) => candidate.id === "codex-luna",
+    );
+    return (
+      source && {
+        ...source,
+        id: "runner-codex-sol",
+        model: "gpt-5.6-sol",
+      }
+    );
+  }
+  const source = candidates.find(
+    (candidate) => candidate.id === "opencode-kimi",
+  );
+  return (
+    source && {
+      ...source,
+      id: "runner-opencode-deepseek",
+      model: "openrouter/deepseek/deepseek-v4-flash-0731",
+    }
+  );
+}
+
+function failedChecks(observation) {
+  const smokeChecks = new Set([
+    "terminal-authority",
+    "first-visible-progress",
+    "substantive-response",
+    "expected-assistant-text",
+    "no-empty-comment",
+    "terminal-presentation",
+  ]);
+  return [...observation.lifecycle.checks, ...observation.presentation.checks]
+    .filter((check) => smokeChecks.has(check.id) && !check.passed)
+    .map((check) => check.id);
+}
+
+function safeErrorMessage(error, credentialValues) {
+  let message = error instanceof Error ? error.message : String(error);
+  for (const credential of credentialValues) {
+    if (credential.length > 0)
+      message = message.split(credential).join("[REDACTED]");
+  }
+  return message
+    .replace(/\bsk-[A-Za-z0-9_-]{8,}\b/g, "[REDACTED]")
+    .replace(/\bBearer\s+\S+/gi, "Bearer [REDACTED]")
+    .replace(/\bAKIA[0-9A-Z]{16}\b/g, "[REDACTED]")
+    .slice(0, 1_000);
+}
+
+async function filesUnder(directory, include, skipDirectory = () => false) {
+  const files = [];
+  for (const entry of await readdir(directory, { withFileTypes: true })) {
+    const path = join(directory, entry.name);
+    if (entry.isDirectory()) {
+      if (!skipDirectory(path)) {
+        files.push(...(await filesUnder(path, include, skipDirectory)));
+      }
+    } else if (entry.isFile() && include(path)) {
+      files.push(path);
+    }
+  }
+  return files;
+}
+
+async function assertArtifactsFresh(label, sources, artifacts) {
+  const sourceTimes = await Promise.all(
+    sources.map(async (source) => (await stat(source)).mtimeMs),
+  );
+  const artifactTimes = await Promise.all(
+    artifacts.map(async (artifact) => (await stat(artifact)).mtimeMs),
+  );
+  if (Math.max(...sourceTimes) > Math.min(...artifactTimes)) {
+    throw new Error(
+      `${label} smoke artifacts are older than their source. Rebuild the targeted artifacts before running the smoke.`,
+    );
+  }
+}
+
+const profiles = parseProfiles(process.argv.slice(2));
+const packageRoot = resolve(import.meta.dirname, "..");
+const runnerd = resolve(
+  packageRoot,
+  "runner",
+  "target",
+  "debug",
+  process.platform === "win32" ? "paperclip-runnerd.exe" : "paperclip-runnerd",
+);
+const requiredArtifacts = [
+  runnerd,
+  resolve(packageRoot, "dist", "eval", "index.js"),
+  ...(profiles.some((profile) => profile.startsWith("runner-acpx-"))
+    ? [resolve(packageRoot, "dist", "cli", "acpx-runtime-sidecar.cjs")]
+    : []),
+  ...(profiles.includes("runner-opencode")
+    ? [resolve(packageRoot, "dist", "cli", "opencode-app-server-proxy.cjs")]
+    : []),
+];
+await Promise.all(requiredArtifacts.map((artifact) => access(artifact))).catch(
+  () => {
+    throw new Error(
+      "Local provider smoke artifacts are missing. Run build:typescript and build:runner-binaries once.",
+    );
+  },
+);
+
+const typescriptSources = await filesUnder(
+  resolve(packageRoot, "src"),
+  (path) => path.endsWith(".ts") && !path.endsWith(".test.ts"),
+  (path) =>
+    /\/(?:browser|devtools|issue-thread|react|scenarios|standalone)$/u.test(
+      path,
+    ),
+);
+await assertArtifactsFresh(
+  "TypeScript",
+  [
+    resolve(packageRoot, "package.json"),
+    resolve(packageRoot, "tsconfig.json"),
+    ...typescriptSources,
+  ],
+  requiredArtifacts.filter((artifact) => artifact !== runnerd),
+);
+const rustSources = await filesUnder(
+  resolve(packageRoot, "runner"),
+  (path) =>
+    path.endsWith(".rs") ||
+    path.endsWith("Cargo.toml") ||
+    path.endsWith("Cargo.lock"),
+  (path) => path.endsWith("/target"),
+);
+await assertArtifactsFresh("runnerd", rustSources, [runnerd]);
+
+const smokeRoot = await mkdtemp(
+  join(tmpdir(), "paperclip-local-provider-smoke-"),
+);
+if (!basename(smokeRoot).startsWith("paperclip-local-provider-smoke-")) {
+  throw new Error("Refusing an unrecognized local provider smoke root");
+}
+await Promise.all(
+  ["tmp", "home", "paperclip-home", "codex-home", "claude-home"].map(
+    (directory) => mkdir(join(smokeRoot, directory), { recursive: true }),
+  ),
+);
+
+const ambientEnvironment = { ...process.env };
+for (const name of Object.keys(process.env)) delete process.env[name];
+for (const name of [
+  "PATH",
+  "SystemRoot",
+  "ComSpec",
+  "PATHEXT",
+  "LANG",
+  "LC_ALL",
+  "LC_CTYPE",
+  "TZ",
+  "SSL_CERT_FILE",
+  "SSL_CERT_DIR",
+  "NODE_EXTRA_CA_CERTS",
+]) {
+  if (ambientEnvironment[name] !== undefined) {
+    process.env[name] = ambientEnvironment[name];
+  }
+}
+Object.assign(process.env, {
+  TMPDIR: join(smokeRoot, "tmp"),
+  HOME: join(smokeRoot, "home"),
+  PAPERCLIP_HOME: join(smokeRoot, "paperclip-home"),
+  CODEX_HOME: join(smokeRoot, "codex-home"),
+  CLAUDE_CONFIG_DIR: join(smokeRoot, "claude-home"),
+  NO_BROWSER: "1",
+  PAPERCLIP_OPEN_ON_LISTEN: "false",
+});
+
+let cleanupPromise;
+const cleanup = () => {
+  cleanupPromise ??= rm(smokeRoot, { recursive: true, force: true });
+  return cleanupPromise;
+};
+const exitAfterCleanup = (status) => {
+  void cleanup().finally(() => process.exit(status));
+};
+process.once("SIGINT", () => exitAfterCleanup(130));
+process.once("SIGTERM", () => exitAfterCleanup(143));
+
+let failed = false;
+try {
+  // Import only after the disposable homes are authoritative so no provider
+  // module can snapshot the developer's normal Paperclip or provider state.
+  const {
+    RUNNER_LIVE_CANDIDATE_SLOTS,
+    executeLiveRunnerWorkflow,
+    runnerWorkflowCase,
+  } = await import("../dist/eval/index.js");
+  const candidates = new Map(
+    profiles.map((profile) => [
+      profile,
+      liveCandidate(profile, RUNNER_LIVE_CANDIDATE_SLOTS),
+    ]),
+  );
+  for (const [profile, candidate] of candidates) {
+    if (!candidate) throw new Error(`Missing live candidate for ${profile}`);
+    const missingCredentials = [];
+    for (const name of candidate.qualification.requiredEnvironment) {
+      const value = ambientEnvironment[name]?.trim();
+      if (value) process.env[name] = value;
+      else missingCredentials.push(name);
+    }
+    if (missingCredentials.length > 0) {
+      throw new Error(
+        `${profile} requires ${missingCredentials.join(", ")} in the smoke process environment`,
+      );
+    }
+  }
+  const credentialValues = new Set(
+    [...candidates.values()].flatMap((candidate) =>
+      candidate.qualification.requiredEnvironment.flatMap((name) => {
+        const value = ambientEnvironment[name]?.trim();
+        return value ? [value] : [];
+      }),
+    ),
+  );
+  const evalCase = runnerWorkflowCase("completion-robustness");
+  for (const profile of profiles) {
+    const candidate = candidates.get(profile);
+    const workspace = join(smokeRoot, `workspace-${profile}`);
+    await mkdir(workspace, { recursive: true });
+    const entry = {
+      // Avoid a three-segment dotted identity: durable redaction correctly
+      // treats that shape as a possible JWT and refuses to rewrite IDs.
+      executionId: `local-smoke-${profile}-${String(Date.now())}`,
+      caseId: evalCase.id,
+      candidateId: candidate.id,
+      slotId: candidate.slotId,
+      repetition: 1,
+      providerTrace: "raw",
+      budget: candidate.budget,
+    };
+    try {
+      const observation = await executeLiveRunnerWorkflow({
+        entry,
+        candidate,
+        evalCase,
+        workingDirectory: workspace,
+        // This smoke proves the provider launch/message/semantic-terminal path.
+        // Usage conformance remains covered by the dedicated eval campaign.
+        allowMissingUsage: true,
+        expectedAssistantText: "PAPERCLIP_LOCAL_PROVIDER_SMOKE_OK",
+        promptOverride:
+          "Reply with exactly PAPERCLIP_LOCAL_PROVIDER_SMOKE_OK and no other text. Do not call tools.",
+        runnerBinary: runnerd,
+      });
+      const failures = failedChecks(observation);
+      const passed = failures.length === 0;
+      failed ||= !passed;
+      process.stdout.write(
+        `${JSON.stringify({
+          profile,
+          status: passed ? "passed" : "failed",
+          classification: passed
+            ? "message_completed"
+            : observation.classification,
+          settlementMs: observation.metrics.settlementMs ?? null,
+          totalTokens: observation.metrics.totalTokens ?? null,
+          costUsd: observation.metrics.costUsd ?? null,
+          failedChecks: failures,
+          failureCode: observation.failure?.code ?? null,
+        })}\n`,
+      );
+    } catch (error) {
+      failed = true;
+      process.stderr.write(
+        `${JSON.stringify({
+          profile,
+          status: "failed",
+          infrastructureError: safeErrorMessage(error, credentialValues),
+        })}\n`,
+      );
+    }
+  }
+} finally {
+  await cleanup();
+}
+
+if (failed) process.exitCode = 1;

packages/paperclip-runner/src/eval/live-workflow-executor.test.ts

@@ -219,6 +219,57 @@ describe("live workflow executor infrastructure failures", () => {
     }
   });
 
+  it("keeps launch-only smoke exceptions explicit and rejects a wrong marker", async () => {
+    liveSessionMocks.snapshot.mockReturnValue({
+      sessionId: "session-smoke-marker",
+      authority: {},
+      mockState: JSON.stringify({ tasks: [] }),
+      transcript: [
+        {
+          id: "assistant-smoke-marker",
+          role: "assistant",
+          text: "a different response",
+        },
+      ],
+      evidence: [],
+      authorizationRecords: [],
+      attempts: [],
+      usageLedger: [],
+      stateHistory: [],
+      workspaceDiffs: [],
+    });
+    const candidate = RUNNER_LIVE_CANDIDATE_SLOTS[0]!.candidates[0]!;
+    const entry: RunnerLiveScheduleEntry = {
+      executionId: "local-smoke-marker",
+      caseId: "final-response",
+      candidateId: candidate.id,
+      slotId: candidate.slotId,
+      repetition: 1,
+      providerTrace: "raw",
+      budget: candidate.budget,
+    };
+
+    const observation = await executeLiveRunnerWorkflow({
+      entry,
+      candidate,
+      evalCase: runnerWorkflowCase(entry.caseId),
+      allowMissingUsage: true,
+      expectedAssistantText: "PAPERCLIP_LOCAL_PROVIDER_SMOKE_OK",
+      promptOverride: "Return the smoke marker.",
+    });
+
+    expect(liveSessionMocks.sendMessage).toHaveBeenCalledWith(
+      "Return the smoke marker.",
+      { allowMissingUsage: true },
+    );
+    expect(observation.presentation.checks).toContainEqual(
+      expect.objectContaining({
+        id: "expected-assistant-text",
+        passed: false,
+      }),
+    );
+  });
+
   it("fails the candidate budget and stops before a paid continuation", async () => {
     liveSessionMocks.snapshot.mockReturnValue({
       sessionId: "session-budget-test",

packages/paperclip-runner/src/eval/live-workflow-executor.ts

@@ -479,6 +479,10 @@ export async function executeLiveRunnerWorkflow(input: {
   candidate: RunnerLiveEvalCandidate;
   evalCase: RunnerWorkflowEvalCase;
   workingDirectory?: string;
+  allowMissingUsage?: boolean;
+  expectedAssistantText?: string;
+  promptOverride?: string;
+  runnerBinary?: string;
 }): Promise<RunnerWorkflowObservation> {
   const runtimeRoot = await mkdtemp(
     join(tmpdir(), "paperclip-runner-live-eval-"),
@@ -487,6 +491,9 @@ export async function executeLiveRunnerWorkflow(input: {
   const store = new InMemoryCapabilityLiveSessionStore();
   const transportOptions = {
     environment: candidateTransportEnvironment(input.candidate, tracePath),
+    ...(input.runnerBinary === undefined
+      ? {}
+      : { runnerBinary: input.runnerBinary }),
   };
   let service = new CapabilityLiveSessionService({ store, transportOptions });
   let session: CapabilityLiveSession | null = null;
@@ -558,7 +565,9 @@ export async function executeLiveRunnerWorkflow(input: {
       capabilities: capabilityFixtureRunCapabilities(LIVE_GRANTS),
       explicitClaims: [...LIVE_GRANTS],
       runId: input.entry.executionId,
-      sessionId: `session-${input.entry.executionId}`,
+      // Durable session identity is validation-bearing and must not look like
+      // credential material (for example a `session-...` token).
+      sessionId: `eval-${input.entry.executionId}`,
       attemptId: `attempt-${input.entry.executionId}`,
       turnTimeoutMs: input.candidate.budget.maxLatencyMs,
       lifecyclePolicy: { mode: "warm", idleTimeoutMs: 300_000 },
@@ -573,11 +582,20 @@ export async function executeLiveRunnerWorkflow(input: {
         const settled = await Promise.allSettled([pending]);
         if (settled[0]?.status === "fulfilled") turns.push(settled[0].value);
       } else {
-        turns.push(await session.sendMessage(promptFor(input.evalCase)));
+        turns.push(
+          await session.sendMessage(
+            input.promptOverride ?? promptFor(input.evalCase),
+            {
+              allowMissingUsage: input.allowMissingUsage,
+            },
+          ),
+        );
         await settleInteractions(input.evalCase, session, turns, withinBudget);
         if (input.evalCase.id === "steering-causality" && withinBudget()) {
           turns.push(
-            await session.sendMessage(continuationPrompt(input.evalCase)),
+            await session.sendMessage(continuationPrompt(input.evalCase), {
+              allowMissingUsage: input.allowMissingUsage,
+            }),
           );
         }
         if (input.evalCase.id === "restart-recovery" && withinBudget()) {
@@ -590,7 +608,9 @@ export async function executeLiveRunnerWorkflow(input: {
           session = await service.restore(sessionId);
           subscribeToSession(session);
           turns.push(
-            await session.sendMessage(continuationPrompt(input.evalCase)),
+            await session.sendMessage(continuationPrompt(input.evalCase), {
+              allowMissingUsage: input.allowMissingUsage,
+            }),
           );
         }
       }
@@ -799,6 +819,16 @@ export async function executeLiveRunnerWorkflow(input: {
         assistantTexts.some((text) => text.trim().length >= 2),
       "provider emitted no user-facing response",
     ),
+    ...(input.expectedAssistantText === undefined
+      ? []
+      : [
+          check(
+            "expected-assistant-text",
+            assistantTexts.length === 1 &&
+              assistantTexts[0]?.trim() === input.expectedAssistantText,
+            "provider response did not exactly match the smoke marker",
+          ),
+        ]),
     check(
       "no-empty-comment",
       assistantTexts.every((text) => text.trim().length > 0),

packages/paperclip-runner/src/live/live-session.ts

@@ -1448,7 +1448,11 @@ export class CapabilityLiveSession {
     return this.snapshot();
   }
 
-  async sendMessage(message: string): Promise<CapabilityLiveTurnResult> {
+  async sendMessage(
+    message: string,
+    /** Launch-only diagnostics may opt out; qualification campaigns must not. */
+    options: { allowMissingUsage?: boolean } = {},
+  ): Promise<CapabilityLiveTurnResult> {
     const value = message.trim();
     if (value.length === 0) throw new Error("Capability live messages cannot be empty");
     if (this.#status === "suspended" || this.#transport === null) {
@@ -1650,7 +1654,10 @@ export class CapabilityLiveSession {
         },
       });
     }
-    await this.#captureTurnUsage(result.turnId, result.status !== "completed");
+    await this.#captureTurnUsage(
+      result.turnId,
+      result.status !== "completed" || options.allowMissingUsage === true,
+    );
     await this.#persist();
     await this.#afterTurnSettled();
     return { ...result, snapshot: this.snapshot() };

tests/runner-e2e/playwright.config.ts

@@ -14,6 +14,13 @@ const privateDir = required("PAPERCLIP_RUNNER_E2E_PRIVATE_DIR");
 const paperclipHome = required("PAPERCLIP_HOME");
 const configPath = required("PAPERCLIP_CONFIG");
 const baseURL = `http://127.0.0.1:${port}`;
+const chromiumExecutable =
+  process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE?.trim();
+if (chromiumExecutable && !path.isAbsolute(chromiumExecutable)) {
+  throw new Error(
+    "PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE must be an absolute path",
+  );
+}
 required("PAPERCLIP_INSTANCE_ID");
 required("PAPERCLIP_AGENT_JWT_SECRET");
 required("PAPERCLIP_DECISION_SIGNING_SECRET");
@@ -38,12 +45,17 @@ export default defineConfig({
   use: {
     baseURL,
     browserName: "chromium",
+    ...(chromiumExecutable
+      ? { launchOptions: { executablePath: chromiumExecutable } }
+      : {}),
     headless: true,
     actionTimeout: 30_000,
     navigationTimeout: 30_000,
     screenshot: "only-on-failure",
     trace: "retain-on-failure",
-    video: "retain-on-failure",
+    // A developer-supplied system Chromium keeps the local smoke loop
+    // installation-free; CI's managed browser retains failure video as usual.
+    video: chromiumExecutable ? "off" : "retain-on-failure",
   },
   webServer: {
     // Do not put an env object here: Playwright serializes webServer config in