fix(evals): preserve qualification identity across report refreshes

Show a notice for fully withheld turns, keep render time separate from measurement time, pin only actual qualifications, and prove index/assets symlink rejection.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

packages/paperclip-runner/docs/runner-protocol-live-evals.md

@@ -59,6 +59,10 @@ adds a labeled refresh and retains the source campaign, original measurement
 timestamp, renderer digest, and `providerCalls: 0`. Scores and evaluated source
 revisions do not change. This is not a new model qualification run. Future live
 runs create chat reports automatically.
+Refreshes are ordered by their render time in the history list, but keep the
+original measurement timestamp and never replace the latest or latest-green
+qualification pointers. Those two real-run records remain retained even when
+many report revisions fill the history window.
 
 This is the provider-backed, one-turn protocol qualification layer in
 `paperclipai/paperclip-evals/evals/paperclip-runner`. It is intentionally

packages/paperclip-runner/scripts/public-eval-chat.mjs

@@ -149,7 +149,8 @@ export function publicChatView(artifact, evalCase) {
       items,
     };
   });
-  if (!turns.length) {
+  if (!turns.some((turn) => turn.items.length > 0)) {
+    turns.length = 0;
     turns.push({
       id: "public-turn-1",
       ordinal: 1,

packages/paperclip-runner/scripts/public-eval-chat.test.mjs

@@ -122,6 +122,45 @@ test("scrubs credentials and private references before truncating text", () => {
   assert.match(publicText("a".repeat(41_000)), /\[truncated\]$/);
 });
 
+test("turns containing only withheld items still show the missing-recording notice", () => {
+  const source = artifact();
+  source.issueThread.turns = [
+    { items: [{ kind: "thinking", body: "private-reasoning" }] },
+    { items: [] },
+  ];
+  const view = publicChatView(source, { id: "missing" });
+  assert.equal(view.turns.length, 1);
+  assert.equal(view.turns[0].items[0].kind, "system_notice");
+  assert.match(view.turns[0].items[0].text, /No publishable conversation/);
+});
+
+test("trusted viewer rejects index and assets symlinks outside its root", async () => {
+  const root = await mkdtemp(join(tmpdir(), "eval-viewer-symlinks-"));
+  try {
+    const viewer = join(root, "viewer");
+    const outside = join(root, "outside");
+    await mkdir(viewer);
+    await mkdir(outside);
+    await writeFile(
+      join(outside, "index.html"),
+      '<script type="module"></script>',
+    );
+    await mkdir(join(outside, "assets"));
+    await writeFile(join(outside, "assets/app.js"), "// outside canary");
+    await symlink(join(outside, "index.html"), join(viewer, "index.html"));
+    await symlink(join(outside, "assets"), join(viewer, "assets"));
+    await assert.rejects(trustedViewerFiles(viewer), /symlinks/);
+    await rm(join(viewer, "index.html"));
+    await writeFile(
+      join(viewer, "index.html"),
+      '<script type="module"></script>',
+    );
+    await assert.rejects(trustedViewerFiles(viewer), /symlinks/);
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});
+
 test("fails closed on unknown fields, raw tools, private evidence and identities", () => {
   for (const mutate of [
     (p) => {

packages/paperclip-runner/scripts/public-eval-viewer.mjs

@@ -22,13 +22,18 @@ export function publicViewerShell(index, encodedPayload) {
 }
 
 export async function trustedViewerFiles(viewerRoot) {
-  if (!viewerRoot || !(await lstat(viewerRoot)).isDirectory())
+  const rootStat = viewerRoot ? await lstat(viewerRoot) : null;
+  if (!rootStat || rootStat.isSymbolicLink() || !rootStat.isDirectory())
     throw new Error(
       "A trusted viewer build is required for public chat reports",
     );
+  const indexStat = await lstat(join(viewerRoot, "index.html"));
+  const assetsStat = await lstat(join(viewerRoot, "assets"));
   if (
-    !(await lstat(join(viewerRoot, "index.html"))).isFile() ||
-    !(await lstat(join(viewerRoot, "assets"))).isDirectory()
+    indexStat.isSymbolicLink() ||
+    !indexStat.isFile() ||
+    assetsStat.isSymbolicLink() ||
+    !assetsStat.isDirectory()
   )
     throw new Error("Trusted viewer must not use symlinks");
   const index = await readFile(join(viewerRoot, "index.html"), "utf8");

packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.mjs

@@ -321,22 +321,28 @@ export function mergeProtocolEvalHistory(history, record) {
     );
   }
   const campaigns = existing
-    ? history.campaigns
+    ? [...history.campaigns]
     : [...history.campaigns, record];
-  campaigns.sort((left, right) =>
-    right.generatedAt.localeCompare(left.generatedAt),
-  );
-  const latest = campaigns[0] ?? null;
+  const activityAt = (campaign) =>
+    campaign.reportRevision?.renderedAt ?? campaign.generatedAt;
+  const activityOrder = (left, right) =>
+    activityAt(right).localeCompare(activityAt(left));
+  campaigns.sort(activityOrder);
+  // Report revisions are discoverable history entries, never qualification runs.
+  const qualifications = campaigns
+    .filter((campaign) => !campaign.reportRevision)
+    .sort((left, right) => right.generatedAt.localeCompare(left.generatedAt));
+  const latest = qualifications[0] ?? null;
   const latestGreen =
-    campaigns.find((campaign) => campaign.complete && campaign.allPassed) ??
-    null;
-  const retained = campaigns.slice(0, MAX_HISTORY_CAMPAIGNS);
-  if (
-    latestGreen &&
-    !retained.some((campaign) => campaign.campaignId === latestGreen.campaignId)
-  ) {
-    retained[retained.length - 1] = latestGreen;
-  }
+    qualifications.find(
+      (campaign) => campaign.complete && campaign.allPassed,
+    ) ?? null;
+  const pointers = [...new Set([latest, latestGreen].filter(Boolean))];
+  const retained = campaigns
+    .filter((campaign) => !pointers.includes(campaign))
+    .slice(0, MAX_HISTORY_CAMPAIGNS - pointers.length)
+    .concat(pointers)
+    .sort(activityOrder);
   return {
     schema: history.schema,
     updatedAt: new Date().toISOString(),

packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.test.mjs

@@ -189,6 +189,60 @@ test("retains immutable history and independent latest-green pointers", () => {
   );
 });
 
+test("report refreshes never replace qualification pointers, including after retention", () => {
+  const record = (value) =>
+    protocolEvalHistoryRecord(
+      value,
+      "https://reports.example/runner-protocol-evals",
+    );
+  let history = mergeProtocolEvalHistory(
+    emptyProtocolEvalHistory(),
+    record(campaign()),
+  );
+  history = mergeProtocolEvalHistory(
+    history,
+    record(
+      campaign({
+        campaignId: "gha-43-1",
+        generatedAt: "2026-09-06T00:00:00.000Z",
+        allPassed: false,
+      }),
+    ),
+  );
+  for (let index = 0; index < 205; index++) {
+    history = mergeProtocolEvalHistory(
+      history,
+      record(
+        campaign({
+          campaignId: `gha-42-1-report-refresh-${index}`,
+          // Defend even against an incorrectly timestamped refresh producer.
+          generatedAt: "2026-09-07T00:00:00.000Z",
+          reportRevision: {
+            sourceCampaignId: "gha-42-1",
+            renderedAt: "2026-09-07T00:00:00.000Z",
+            providerCalls: 0,
+          },
+        }),
+      ),
+    );
+  }
+  assert.equal(history.latestCampaignId, "gha-43-1");
+  assert.equal(history.latestGreenCampaignId, "gha-42-1");
+  assert.equal(
+    buildProtocolEvalPointers(history).latest.campaign.campaignId,
+    "gha-43-1",
+  );
+  assert.equal(
+    buildProtocolEvalPointers(history).latestGreen.campaign.campaignId,
+    "gha-42-1",
+  );
+  assert.equal(history.campaigns.length, 200);
+  assert.match(
+    renderProtocolEvalHistoryIndex(history),
+    /Report refresh · no new model calls/,
+  );
+});
+
 test("retains the latest green pointer outside the 200 newest campaigns", () => {
   const green = protocolEvalHistoryRecord(
     campaign(),

packages/paperclip-runner/scripts/refresh-runner-protocol-eval-report.mjs

@@ -60,7 +60,8 @@ export async function refreshProtocolEvalReport({
   const refreshed = {
     ...campaign,
     campaignId: `${campaign.campaignId}-report-${revision}`,
-    generatedAt: renderedAt,
+    // A presentation refresh is not a new model measurement.
+    generatedAt: campaign.generatedAt,
     reportRevision: {
       sourceCampaignId: campaign.campaignId,
       sourceGeneratedAt: campaign.generatedAt,