test(evals): execute generated chat reports before publication

Verify real viewer rendering, read-only controls, tool expansion, navigation, reload and responsive visibility for both report projections.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

.github/workflows/runner-protocol-live-evals.yml

@@ -614,6 +614,18 @@ jobs:
             --coverage-matrix .paperclip-evals/evals/paperclip-runner/coverage-matrix.json
           cp runner-protocol-merged/campaign.json runner-protocol-merged/public-report/campaign.json
 
+      - name: Set up report browser verification
+        uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
+        with:
+          version: 9.15.4
+
+      - name: Verify the actual chat viewer before publication
+        run: |
+          pnpm install --frozen-lockfile --ignore-scripts
+          pnpm --filter @paperclipai/paperclip-runner exec playwright install --with-deps chromium
+          node packages/paperclip-runner/scripts/verify-runner-evalbook-viewer.mjs --report-root runner-protocol-merged/public-report --screenshots runner-protocol-merged/viewer-proof
+          node packages/paperclip-runner/scripts/verify-runner-evalbook-viewer.mjs --report-root runner-protocol-merged/report
+
       - name: Enforce the static public allowlist
         id: public_report
         run: |

packages/paperclip-runner/docs/runner-protocol-live-evals.md

@@ -23,6 +23,16 @@ The CSP prohibits network calls, forms and external resources. Supply
 `PAPERCLIP_RUNNER_PROTOCOL_EVAL_VIEWER_DIR` to the publisher. The workflow sends
 a viewer-only artifact to that job; raw attempts and provider secrets stay out.
 
+Before uploading, the report job runs the actual built application in Chromium
+against representative passing, failing and missing-recording pages in both
+reports. It checks initial rendering, tool expansion, read-only controls,
+navigation, reload, a narrow viewport and the public no-API-request boundary.
+Run the same check locally with
+`node packages/paperclip-runner/scripts/verify-runner-evalbook-viewer.mjs --report-root /path/to/report`.
+Add `--screenshots /path/to/proof` for visual evidence. Public screenshots are
+retained in the aggregate Actions artifact, not added to the public report's
+closed file allowlist.
+
 ### Refresh a completed report without calling models
 
 Download the aggregate Actions artifact, then:

packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs

@@ -136,6 +136,14 @@ test("publishes only the separately sanitized Evalbook through trusted OIDC code
     /Upload access-controlled canonical Evalbook and raw attempts/u,
   );
   assert.match(report, /Upload publisher-only sanitized Evalbook/u);
+  assert.match(
+    report,
+    /verify-runner-evalbook-viewer\.mjs --report-root runner-protocol-merged\/public-report/u,
+  );
+  assert.match(
+    report,
+    /verify-runner-evalbook-viewer\.mjs --report-root runner-protocol-merged\/report/u,
+  );
   assert.match(
     report,
     /--viewer-root runner-protocol-build\/extracted\/dist-issue-thread\s*\\\n\s*--public-viewer/u,

packages/paperclip-runner/scripts/verify-runner-evalbook-viewer.mjs

@@ -0,0 +1,151 @@
+#!/usr/bin/env node
+// Execute the built viewer against actual generated pages, without a runner API.
+import assert from "node:assert/strict";
+import { createServer } from "node:http";
+import { readFile, readdir, mkdir } from "node:fs/promises";
+import { resolve, join, extname, sep } from "node:path";
+import { chromium } from "@playwright/test";
+
+const arg = (name) => process.argv[process.argv.indexOf(name) + 1];
+if (!process.argv.includes("--report-root"))
+  throw new Error("--report-root is required");
+const root = resolve(arg("--report-root"));
+const screenshots = process.argv.includes("--screenshots")
+  ? resolve(arg("--screenshots"))
+  : null;
+const samples = new Map();
+for (const entry of await readdir(join(root, "attempts"), {
+  withFileTypes: true,
+})) {
+  if (!entry.isDirectory()) continue;
+  const route = `attempts/${entry.name}/index.html`;
+  const html = await readFile(join(root, route), "utf8");
+  const encoded =
+    html.match(
+      /<script type="application\/json" id="paperclip-eval-report">([^<]*)<\/script>/u,
+    )?.[1] ??
+    html.match(/window\.__PAPERCLIP_EVAL_REPORT__=(.*?);<\/script>/su)?.[1];
+  assert.ok(encoded, `Attempt lacks canonical viewer payload: ${route}`);
+  const payload = JSON.parse(encoded);
+  const messages = payload.view.turns
+    .flatMap((turn) => turn.items)
+    .filter((item) => ["user_message", "agent_message"].includes(item.kind));
+  const kind = !messages.length
+    ? "missing-recording"
+    : payload.passed
+      ? "passed"
+      : "failed";
+  if (!samples.has(kind)) samples.set(kind, { route, payload });
+}
+assert.ok(samples.size, "No attempt pages to verify");
+const server = createServer(async (req, res) => {
+  try {
+    const url = new URL(req.url, "http://report.invalid");
+    const path = resolve(root, `.${decodeURIComponent(url.pathname)}`);
+    if (!path.startsWith(`${root}${sep}`)) {
+      res.writeHead(403).end();
+      return;
+    }
+    const mime = {
+      ".html": "text/html",
+      ".js": "text/javascript",
+      ".css": "text/css",
+      ".woff2": "font/woff2",
+    };
+    res.writeHead(200, {
+      "Content-Type": mime[extname(path)] ?? "application/octet-stream",
+    });
+    res.end(await readFile(path));
+  } catch {
+    res.writeHead(404).end();
+  }
+});
+await new Promise((done) => server.listen(0, "127.0.0.1", done));
+const origin = `http://127.0.0.1:${server.address().port}`;
+let browser;
+try {
+  browser = await chromium.launch({ headless: true });
+  if (screenshots) await mkdir(screenshots, { recursive: true });
+  for (const [kind, { route, payload }] of samples) {
+    const page = await browser.newPage({
+      viewport: { width: 1440, height: 1000 },
+    });
+    const errors = [];
+    const requests = [];
+    page.on("pageerror", (error) => errors.push(error.message));
+    page.on("request", (request) => requests.push(request.url()));
+    await page.goto(`${origin}/${route}`, { waitUntil: "networkidle" });
+    await page.locator(".pit-thread").waitFor({ state: "visible" });
+    assert.equal(
+      await page.locator("h1").textContent(),
+      payload.view.issue.title,
+    );
+    assert.equal(
+      await page
+        .getByRole("button", { name: "Reset scenario", exact: true })
+        .count(),
+      0,
+    );
+    assert.equal(
+      await page.getByText("Loading company state…", { exact: true }).count(),
+      0,
+    );
+    assert.ok(
+      await page.getByText(payload.run.model, { exact: false }).count(),
+      "Model missing from inspector",
+    );
+    const tool = page
+      .locator('[data-thread-item="tool_activity"] summary')
+      .first();
+    if (await tool.count()) {
+      await tool.click();
+      assert.ok(
+        await page.locator('[data-thread-item="tool_activity"][open]').count(),
+      );
+      if (payload.publication)
+        assert.ok(
+          await page
+            .getByText("Arguments withheld from public replay.", {
+              exact: false,
+            })
+            .count(),
+        );
+    }
+    if (payload.publication) {
+      assert.ok(
+        await page
+          .getByText(payload.publication.notice, { exact: true })
+          .count(),
+      );
+      assert.ok(
+        requests.every(
+          (url) => url.startsWith(origin) && !url.includes("/api/"),
+        ),
+        "Replay attempted a remote/API request",
+      );
+    }
+    assert.deepEqual(errors, [], `Viewer errors for ${kind}`);
+    if (screenshots)
+      await page.screenshot({
+        path: join(screenshots, `${kind}.png`),
+        fullPage: true,
+      });
+    await page
+      .getByRole("link", { name: "← All results", exact: true })
+      .click();
+    assert.equal(new URL(page.url()).pathname, "/index.html");
+    await page.goto(`${origin}/${route}`, { waitUntil: "networkidle" });
+    await page.reload({ waitUntil: "networkidle" });
+    await page.locator(".pit-thread").waitFor({ state: "visible" });
+    assert.deepEqual(errors, [], `Reload errors for ${kind}`);
+    await page.setViewportSize({ width: 390, height: 844 });
+    await page.locator(".pit-thread").waitFor({ state: "visible" });
+    await page.close();
+    console.log(
+      `Verified ${kind}: chat, read-only controls, navigation, reload and narrow viewport`,
+    );
+  }
+} finally {
+  await browser?.close();
+  await new Promise((done) => server.close(done));
+}