diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index 9019b78983..fd00a9ac8d 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -59,6 +59,10 @@ adds a labeled refresh and retains the source campaign, original measurement timestamp, renderer digest, and `providerCalls: 0`. Scores and evaluated source revisions do not change. This is not a new model qualification run. Future live runs create chat reports automatically. +Refreshes are ordered by their render time in the history list, but keep the +original measurement timestamp and never replace the latest or latest-green +qualification pointers. Those two real-run records remain retained even when +many report revisions fill the history window. This is the provider-backed, one-turn protocol qualification layer in `paperclipai/paperclip-evals/evals/paperclip-runner`. It is intentionally diff --git a/packages/paperclip-runner/scripts/public-eval-chat.mjs b/packages/paperclip-runner/scripts/public-eval-chat.mjs index cde0286df8..d20f998020 100644 --- a/packages/paperclip-runner/scripts/public-eval-chat.mjs +++ b/packages/paperclip-runner/scripts/public-eval-chat.mjs @@ -149,7 +149,8 @@ export function publicChatView(artifact, evalCase) { items, }; }); - if (!turns.length) { + if (!turns.some((turn) => turn.items.length > 0)) { + turns.length = 0; turns.push({ id: "public-turn-1", ordinal: 1, diff --git a/packages/paperclip-runner/scripts/public-eval-chat.test.mjs b/packages/paperclip-runner/scripts/public-eval-chat.test.mjs index 7de7f957da..96f8c02fbc 100644 --- a/packages/paperclip-runner/scripts/public-eval-chat.test.mjs +++ b/packages/paperclip-runner/scripts/public-eval-chat.test.mjs @@ -122,6 +122,45 @@ test("scrubs credentials and private references before truncating text", () => { assert.match(publicText("a".repeat(41_000)), /\[truncated\]$/); }); +test("turns containing only withheld items still show the missing-recording notice", () => { + const source = artifact(); + source.issueThread.turns = [ + { items: [{ kind: "thinking", body: "private-reasoning" }] }, + { items: [] }, + ]; + const view = publicChatView(source, { id: "missing" }); + assert.equal(view.turns.length, 1); + assert.equal(view.turns[0].items[0].kind, "system_notice"); + assert.match(view.turns[0].items[0].text, /No publishable conversation/); +}); + +test("trusted viewer rejects index and assets symlinks outside its root", async () => { + const root = await mkdtemp(join(tmpdir(), "eval-viewer-symlinks-")); + try { + const viewer = join(root, "viewer"); + const outside = join(root, "outside"); + await mkdir(viewer); + await mkdir(outside); + await writeFile( + join(outside, "index.html"), + '', + ); + await mkdir(join(outside, "assets")); + await writeFile(join(outside, "assets/app.js"), "// outside canary"); + await symlink(join(outside, "index.html"), join(viewer, "index.html")); + await symlink(join(outside, "assets"), join(viewer, "assets")); + await assert.rejects(trustedViewerFiles(viewer), /symlinks/); + await rm(join(viewer, "index.html")); + await writeFile( + join(viewer, "index.html"), + '', + ); + await assert.rejects(trustedViewerFiles(viewer), /symlinks/); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + test("fails closed on unknown fields, raw tools, private evidence and identities", () => { for (const mutate of [ (p) => { diff --git a/packages/paperclip-runner/scripts/public-eval-viewer.mjs b/packages/paperclip-runner/scripts/public-eval-viewer.mjs index a228736b9f..702f69862d 100644 --- a/packages/paperclip-runner/scripts/public-eval-viewer.mjs +++ b/packages/paperclip-runner/scripts/public-eval-viewer.mjs @@ -22,13 +22,18 @@ export function publicViewerShell(index, encodedPayload) { } export async function trustedViewerFiles(viewerRoot) { - if (!viewerRoot || !(await lstat(viewerRoot)).isDirectory()) + const rootStat = viewerRoot ? await lstat(viewerRoot) : null; + if (!rootStat || rootStat.isSymbolicLink() || !rootStat.isDirectory()) throw new Error( "A trusted viewer build is required for public chat reports", ); + const indexStat = await lstat(join(viewerRoot, "index.html")); + const assetsStat = await lstat(join(viewerRoot, "assets")); if ( - !(await lstat(join(viewerRoot, "index.html"))).isFile() || - !(await lstat(join(viewerRoot, "assets"))).isDirectory() + indexStat.isSymbolicLink() || + !indexStat.isFile() || + assetsStat.isSymbolicLink() || + !assetsStat.isDirectory() ) throw new Error("Trusted viewer must not use symlinks"); const index = await readFile(join(viewerRoot, "index.html"), "utf8"); diff --git a/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.mjs b/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.mjs index b97760c61d..067c23b996 100644 --- a/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.mjs +++ b/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.mjs @@ -321,22 +321,28 @@ export function mergeProtocolEvalHistory(history, record) { ); } const campaigns = existing - ? history.campaigns + ? [...history.campaigns] : [...history.campaigns, record]; - campaigns.sort((left, right) => - right.generatedAt.localeCompare(left.generatedAt), - ); - const latest = campaigns[0] ?? null; + const activityAt = (campaign) => + campaign.reportRevision?.renderedAt ?? campaign.generatedAt; + const activityOrder = (left, right) => + activityAt(right).localeCompare(activityAt(left)); + campaigns.sort(activityOrder); + // Report revisions are discoverable history entries, never qualification runs. + const qualifications = campaigns + .filter((campaign) => !campaign.reportRevision) + .sort((left, right) => right.generatedAt.localeCompare(left.generatedAt)); + const latest = qualifications[0] ?? null; const latestGreen = - campaigns.find((campaign) => campaign.complete && campaign.allPassed) ?? - null; - const retained = campaigns.slice(0, MAX_HISTORY_CAMPAIGNS); - if ( - latestGreen && - !retained.some((campaign) => campaign.campaignId === latestGreen.campaignId) - ) { - retained[retained.length - 1] = latestGreen; - } + qualifications.find( + (campaign) => campaign.complete && campaign.allPassed, + ) ?? null; + const pointers = [...new Set([latest, latestGreen].filter(Boolean))]; + const retained = campaigns + .filter((campaign) => !pointers.includes(campaign)) + .slice(0, MAX_HISTORY_CAMPAIGNS - pointers.length) + .concat(pointers) + .sort(activityOrder); return { schema: history.schema, updatedAt: new Date().toISOString(), diff --git a/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.test.mjs b/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.test.mjs index 0e9c17b91f..ee919d09cb 100644 --- a/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.test.mjs +++ b/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.test.mjs @@ -189,6 +189,60 @@ test("retains immutable history and independent latest-green pointers", () => { ); }); +test("report refreshes never replace qualification pointers, including after retention", () => { + const record = (value) => + protocolEvalHistoryRecord( + value, + "https://reports.example/runner-protocol-evals", + ); + let history = mergeProtocolEvalHistory( + emptyProtocolEvalHistory(), + record(campaign()), + ); + history = mergeProtocolEvalHistory( + history, + record( + campaign({ + campaignId: "gha-43-1", + generatedAt: "2026-09-06T00:00:00.000Z", + allPassed: false, + }), + ), + ); + for (let index = 0; index < 205; index++) { + history = mergeProtocolEvalHistory( + history, + record( + campaign({ + campaignId: `gha-42-1-report-refresh-${index}`, + // Defend even against an incorrectly timestamped refresh producer. + generatedAt: "2026-09-07T00:00:00.000Z", + reportRevision: { + sourceCampaignId: "gha-42-1", + renderedAt: "2026-09-07T00:00:00.000Z", + providerCalls: 0, + }, + }), + ), + ); + } + assert.equal(history.latestCampaignId, "gha-43-1"); + assert.equal(history.latestGreenCampaignId, "gha-42-1"); + assert.equal( + buildProtocolEvalPointers(history).latest.campaign.campaignId, + "gha-43-1", + ); + assert.equal( + buildProtocolEvalPointers(history).latestGreen.campaign.campaignId, + "gha-42-1", + ); + assert.equal(history.campaigns.length, 200); + assert.match( + renderProtocolEvalHistoryIndex(history), + /Report refresh ยท no new model calls/, + ); +}); + test("retains the latest green pointer outside the 200 newest campaigns", () => { const green = protocolEvalHistoryRecord( campaign(), diff --git a/packages/paperclip-runner/scripts/refresh-runner-protocol-eval-report.mjs b/packages/paperclip-runner/scripts/refresh-runner-protocol-eval-report.mjs index 9fb57aae0b..19c91d2a29 100644 --- a/packages/paperclip-runner/scripts/refresh-runner-protocol-eval-report.mjs +++ b/packages/paperclip-runner/scripts/refresh-runner-protocol-eval-report.mjs @@ -60,7 +60,8 @@ export async function refreshProtocolEvalReport({ const refreshed = { ...campaign, campaignId: `${campaign.campaignId}-report-${revision}`, - generatedAt: renderedAt, + // A presentation refresh is not a new model measurement. + generatedAt: campaign.generatedAt, reportRevision: { sourceCampaignId: campaign.campaignId, sourceGeneratedAt: campaign.generatedAt,