From dfb35a3ac2cdba4056582d1907b8ff0f8b78f6f6 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 6 Sep 2026 18:14:47 -0500 Subject: [PATCH] fix(evals): publish the canonical chat viewer on every direct run Use a closed public replay projection and verify exact viewer assets. Keep read-only reports usable, preserve immutable campaign history, and add a no-provider report refresh command. Co-Authored-By: Paperclip --- .../workflows/runner-protocol-live-evals.yml | 24 +- .../devtools/issue-thread/src/App.tsx | 19 +- .../issue-thread/src/DevtoolsInspector.tsx | 269 ++++++++++-------- .../issue-thread/src/EvidencePanel.tsx | 4 +- .../devtools/issue-thread/src/main.tsx | 14 +- .../docs/runner-protocol-live-evals.md | 49 +++- packages/paperclip-runner/package.json | 2 +- .../scripts/public-eval-chat.mjs | 216 ++++++++++++++ .../scripts/public-eval-chat.test.mjs | 234 +++++++++++++++ .../scripts/public-eval-viewer.mjs | 229 +++++++++++++++ .../publish-runner-protocol-eval-history.mjs | 109 +++++-- ...lish-runner-protocol-eval-history.test.mjs | 3 +- .../refresh-runner-protocol-eval-report.mjs | 99 +++++++ .../render-runner-workflow-evalbook.mjs | 11 + .../scripts/runner-protocol-eval-campaign.mjs | 53 +++- .../runner-protocol-eval-campaign.test.mjs | 6 +- ...r-protocol-eval-workflow-security.test.mjs | 7 + 17 files changed, 1174 insertions(+), 174 deletions(-) create mode 100644 packages/paperclip-runner/scripts/public-eval-chat.mjs create mode 100644 packages/paperclip-runner/scripts/public-eval-chat.test.mjs create mode 100644 packages/paperclip-runner/scripts/public-eval-viewer.mjs create mode 100644 packages/paperclip-runner/scripts/refresh-runner-protocol-eval-report.mjs diff --git a/.github/workflows/runner-protocol-live-evals.yml b/.github/workflows/runner-protocol-live-evals.yml index f659f5ec65..a33abf393d 100644 --- a/.github/workflows/runner-protocol-live-evals.yml +++ b/.github/workflows/runner-protocol-live-evals.yml @@ -213,6 +213,11 @@ jobs: --max-parallel "$MAX_PARALLEL" \ --output runner-protocol-eval-catalog.json + - name: Require the chat-report renderer before paid execution + run: | + set -euo pipefail + python3 .paperclip-evals/evals/paperclip-runner/tools/eval_program.py report --help | grep -q -- --public-viewer + - name: Upload immutable campaign catalog uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: @@ -290,6 +295,14 @@ jobs: compression-level: 0 if-no-files-found: error + - name: Upload canonical viewer for publisher byte verification + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: runner-protocol-viewer-${{ github.run_id }}-${{ github.run_attempt }} + path: packages/paperclip-runner/dist-issue-thread/ + retention-days: 30 + if-no-files-found: error + eval_shard_0: name: Direct eval ${{ matrix.rosterId }} / ${{ matrix.caseId }} needs: [authorize, catalog, build_runner] @@ -595,6 +608,8 @@ jobs: python3 .paperclip-evals/evals/paperclip-runner/tools/eval_program.py report \ --runs-root runner-protocol-merged/public-runs \ --output runner-protocol-merged/public-report \ + --viewer-root runner-protocol-build/extracted/dist-issue-thread \ + --public-viewer \ --inventory .paperclip-evals/evals/paperclip-runner/inventory.json \ --coverage-matrix .paperclip-evals/evals/paperclip-runner/coverage-matrix.json cp runner-protocol-merged/campaign.json runner-protocol-merged/public-report/campaign.json @@ -602,7 +617,7 @@ jobs: - name: Enforce the static public allowlist id: public_report run: | - node --input-type=module -e 'import { validatePublicProtocolEvalReport } from "./packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.mjs"; await validatePublicProtocolEvalReport("runner-protocol-merged/public-report");' + node --input-type=module -e 'import { validatePublicProtocolEvalReport } from "./packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.mjs"; await validatePublicProtocolEvalReport("runner-protocol-merged/public-report", { viewerRoot: "runner-protocol-build/extracted/dist-issue-thread" });' echo "ready=true" >> "$GITHUB_OUTPUT" - name: Add campaign result to the workflow summary @@ -666,6 +681,12 @@ jobs: name: runner-protocol-eval-public-${{ github.run_id }}-${{ github.run_attempt }} path: runner-protocol-public-report + - name: Download the same-run canonical viewer for byte verification + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: runner-protocol-viewer-${{ github.run_id }}-${{ github.run_attempt }} + path: runner-protocol-trusted-viewer + - name: Exchange GitHub OIDC identity for scoped AWS credentials uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6 with: @@ -675,6 +696,7 @@ jobs: - name: Publish versioned report and refresh the root index env: PAPERCLIP_RUNNER_PROTOCOL_EVAL_PUBLIC_REPORT_DIR: ${{ github.workspace }}/runner-protocol-public-report + PAPERCLIP_RUNNER_PROTOCOL_EVAL_VIEWER_DIR: ${{ github.workspace }}/runner-protocol-trusted-viewer RUNNER_PROTOCOL_EVAL_HISTORY_S3_BUCKET: ${{ vars.RUNNER_PROTOCOL_EVAL_HISTORY_S3_BUCKET || vars.RUNNER_E2E_HISTORY_S3_BUCKET }} RUNNER_PROTOCOL_EVAL_HISTORY_PREFIX: ${{ vars.RUNNER_PROTOCOL_EVAL_HISTORY_PREFIX || 'runner-protocol-evals' }} RUNNER_PROTOCOL_EVAL_HISTORY_PUBLIC_BASE_URL: ${{ vars.RUNNER_PROTOCOL_EVAL_HISTORY_PUBLIC_BASE_URL || vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL }} diff --git a/packages/paperclip-runner/devtools/issue-thread/src/App.tsx b/packages/paperclip-runner/devtools/issue-thread/src/App.tsx index 5ebeca9d49..3cf8d53440 100644 --- a/packages/paperclip-runner/devtools/issue-thread/src/App.tsx +++ b/packages/paperclip-runner/devtools/issue-thread/src/App.tsx @@ -78,6 +78,7 @@ interface EmbeddedEvalCheck { } interface EmbeddedEvalReport { + publication?: { schema: string; notice: string }; attemptId: string; caseId: string; disposition: string; @@ -101,7 +102,7 @@ interface EmbeddedEvalReport { runnerBuild: string; startedAt: string; finishedAt: string; - durationMs: number; + durationMs: number | null; initialRevision: number; finalRevision: number; usage: { @@ -117,7 +118,7 @@ interface EmbeddedEvalReport { } | null; }; view: CapabilityIssueThreadSnapshot; - devtools: CapabilityDevtoolsSnapshot; + devtools: CapabilityDevtoolsSnapshot | null; navigation: { suiteHref: string; previous: { label: string; href: string } | null; next: { label: string; href: string } | null }; } @@ -491,7 +492,7 @@ export function App() { await document.fonts.ready; } if (cancelled) return; - if (scroller !== null) scroller.scrollTop = scroller.scrollHeight; + if (scroller !== null) scroller.scrollTop = embeddedEval === null ? scroller.scrollHeight : 0; await new Promise((resolve) => requestAnimationFrame(() => resolve())); if (!cancelled) setSettled(true); })(); @@ -501,14 +502,14 @@ export function App() { }, [snapshot]); useEffect(() => { - if (!panelOpen || snapshot === null || route.mode !== "live") return; + if (embeddedEval !== null || !panelOpen || snapshot === null || route.mode !== "live") return; if (historicSessionId !== null) return; let cancelled = false; void capabilityLiveClient.devtools(snapshot.sessionId) .then((next) => { if (!cancelled) setDevtools(next); }) .catch((cause) => { if (!cancelled) setActionError(describe(cause)); }); return () => { cancelled = true; }; - }, [historicSessionId, panelOpen, route.mode, snapshot?.renderedAt, snapshot?.sessionId]); + }, [embeddedEval, historicSessionId, panelOpen, route.mode, snapshot?.renderedAt, snapshot?.sessionId]); useEffect(() => { if (!chat || snapshot === null || historicSessionId !== null) return; @@ -1363,7 +1364,7 @@ export function App() { >
{embeddedEval !== null ? ( -
Eval execution
+
Eval execution{embeddedEval.publication ? {embeddedEval.publication.notice} : null}
) : null} {snapshot.turns.length === 0 ? (
@@ -1404,7 +1405,7 @@ export function App() { {embeddedEval !== null ? (
Post-run state - Final mock control-plane revision {embeddedEval.run.finalRevision} + {embeddedEval.publication ? "Company-state details withheld from public replay" : `Final mock control-plane revision ${embeddedEval.run.finalRevision}`} check.anchor.kind === "run")} />
) : null} @@ -1425,7 +1426,7 @@ export function App() { ) : null}
- + /> : null} {showPanel && layout === "side" ? ( diff --git a/packages/paperclip-runner/devtools/issue-thread/src/DevtoolsInspector.tsx b/packages/paperclip-runner/devtools/issue-thread/src/DevtoolsInspector.tsx index acfd180775..ee057682b2 100644 --- a/packages/paperclip-runner/devtools/issue-thread/src/DevtoolsInspector.tsx +++ b/packages/paperclip-runner/devtools/issue-thread/src/DevtoolsInspector.tsx @@ -23,6 +23,7 @@ export interface EvalInspectorReport { disposition: string; passed: boolean; checks: EvalAssertion[]; + publication?: { schema: string; notice: string }; run: { model: string; provider: string; @@ -41,7 +42,7 @@ export interface EvalInspectorReport { runnerBuild: string; startedAt: string; finishedAt: string; - durationMs: number; + durationMs: number | null; initialRevision: number; finalRevision: number; usage: { @@ -53,7 +54,7 @@ export interface EvalInspectorReport { reasoningTokens: number; providerReportedCostNanodollars?: number; estimatedCostNanodollars: number; - pricingVersion: string; + pricingVersion?: string; } | null; }; } @@ -197,6 +198,149 @@ function documentsOf(state: Json): Array<{ }); } +export function EvalReportInspector({ + evalReport, +}: { + evalReport: EvalInspectorReport; +}) { + const isPublic = Boolean(evalReport.publication); + return ( +
+
+ ← Eval suite + + {evalReport.passed + ? "PASS" + : evalReport.disposition.replaceAll("_", " ").toUpperCase()} + + {evalReport.attemptId} +
+
+
+
Model
+
+ {evalReport.run.model.startsWith(`${evalReport.run.provider}/`) + ? evalReport.run.model + : `${evalReport.run.provider}/${evalReport.run.model}`} +
+
+
+
Configuration
+
{evalReport.run.configuration}
+
+
+
Session
+
+ {isPublic + ? "Withheld from public replay" + : evalReport.run.sessionId} +
+
+
+
Provider session
+
+ {isPublic + ? "Withheld from public replay" + : (evalReport.run.providerSessionId ?? "unavailable")} +
+
+
+
Driver
+
+ {evalReport.run.driver} + {evalReport.run.providerVersion + ? ` · ${evalReport.run.providerVersion}` + : ""} +
+
+ {evalReport.run.agentVersion ? ( +
+
Agent version
+
{evalReport.run.agentVersion}
+
+ ) : null} +
+
Retained session
+
+ {isPublic + ? "Withheld from public replay" + : evalReport.run.retainedSession === true + ? (evalReport.run.retainedSessionStatus ?? "retained") + : "not applicable"} +
+
+
+
Duration
+
+ {evalReport.run.durationMs == null + ? "unavailable" + : `${evalReport.run.durationMs} ms`} +
+
+
+
Fixture
+
{evalReport.run.fixtureDigest}
+
+
+
State
+
+ {isPublic + ? "Withheld from public replay" + : `r${evalReport.run.initialRevision} → r${evalReport.run.finalRevision}`} +
+
+
+
Tokens
+
+ {evalReport.run.usage === null + ? "unknown" + : `${evalReport.run.usage.inputTokens} in · ${evalReport.run.usage.outputTokens} out · ${evalReport.run.usage.cachedInputTokens} cached`} +
+
+
+
Agent turns
+
{evalReport.run.usage?.agentTurns ?? "unknown"}
+
+
+
Provider requests
+
{evalReport.run.usage?.providerRequests ?? "unavailable"}
+
+
+
Estimated cost
+
+ {evalReport.run.usage === null + ? "unknown" + : `$${(evalReport.run.usage.estimatedCostNanodollars / 1_000_000_000).toFixed(6)}${evalReport.run.usage.pricingVersion ? ` · ${evalReport.run.usage.pricingVersion}` : ""}`} +
+
+
+
Provider list cost
+
+ {typeof evalReport.run.usage?.providerReportedCostNanodollars !== + "number" + ? "unknown" + : `$${(evalReport.run.usage.providerReportedCostNanodollars / 1_000_000_000).toFixed(6)}`} +
+
+
+
Runner
+
{evalReport.run.runnerPackageDigest}
+
+
+
Runner build
+
{evalReport.run.runnerBuild}
+
+
+
runnerd
+
{evalReport.run.runnerdDigest}
+
+
+

Assertions

+ +
+ ); +} + export function DevtoolsInspector({ snapshot, onFork, @@ -291,6 +435,8 @@ export function DevtoolsInspector({ className="pit-button" type="button" onClick={() => onFork(revision)} + disabled={evalReport != null} + title={evalReport ? "Eval reports are read-only" : undefined} > Fork r{revision} @@ -328,124 +474,7 @@ export function DevtoolsInspector({ ))} {tab === "eval" && evalReport ? ( -
-
- ← Eval suite - - {evalReport.passed - ? "PASS" - : evalReport.disposition.replaceAll("_", " ").toUpperCase()} - - {evalReport.attemptId} -
-
-
-
Model
-
- {evalReport.run.model.startsWith(`${evalReport.run.provider}/`) - ? evalReport.run.model - : `${evalReport.run.provider}/${evalReport.run.model}`} -
-
-
-
Configuration
-
{evalReport.run.configuration}
-
-
-
Session
-
{evalReport.run.sessionId}
-
-
-
Provider session
-
{evalReport.run.providerSessionId ?? "unavailable"}
-
-
-
Driver
-
- {evalReport.run.driver} - {evalReport.run.providerVersion - ? ` · ${evalReport.run.providerVersion}` - : ""} -
-
- {evalReport.run.agentVersion ? ( -
-
Agent version
-
{evalReport.run.agentVersion}
-
- ) : null} -
-
Retained session
-
- {evalReport.run.retainedSession === true - ? (evalReport.run.retainedSessionStatus ?? "retained") - : "not applicable"} -
-
-
-
Duration
-
{evalReport.run.durationMs} ms
-
-
-
Fixture
-
{evalReport.run.fixtureDigest}
-
-
-
State
-
- r{evalReport.run.initialRevision} → r - {evalReport.run.finalRevision} -
-
-
-
Tokens
-
- {evalReport.run.usage === null - ? "unknown" - : `${evalReport.run.usage.inputTokens} in · ${evalReport.run.usage.outputTokens} out · ${evalReport.run.usage.cachedInputTokens} cached`} -
-
-
-
Agent turns
-
{evalReport.run.usage?.agentTurns ?? "unknown"}
-
-
-
Provider requests
-
{evalReport.run.usage?.providerRequests ?? "unavailable"}
-
-
-
Estimated cost
-
- {evalReport.run.usage === null - ? "unknown" - : `$${(evalReport.run.usage.estimatedCostNanodollars / 1_000_000_000).toFixed(6)} · ${evalReport.run.usage.pricingVersion}`} -
-
-
-
Provider list cost
-
- {typeof evalReport.run.usage - ?.providerReportedCostNanodollars !== "number" - ? "unknown" - : `$${(evalReport.run.usage.providerReportedCostNanodollars / 1_000_000_000).toFixed(6)}`} -
-
-
-
Runner
-
{evalReport.run.runnerPackageDigest}
-
-
-
Runner build
-
{evalReport.run.runnerBuild}
-
-
-
runnerd
-
{evalReport.run.runnerdDigest}
-
-
-

Assertions

- -
+ ) : null} {tab === "timeline" ? (
diff --git a/packages/paperclip-runner/devtools/issue-thread/src/EvidencePanel.tsx b/packages/paperclip-runner/devtools/issue-thread/src/EvidencePanel.tsx index 61aa2d7373..8c5bed14ee 100644 --- a/packages/paperclip-runner/devtools/issue-thread/src/EvidencePanel.tsx +++ b/packages/paperclip-runner/devtools/issue-thread/src/EvidencePanel.tsx @@ -10,7 +10,7 @@ import type { CapabilityToolDisposition, } from "../../../src/issue-thread/types"; import type { CapabilityDevtoolsSnapshot } from "../../../src/devtools"; -import { DevtoolsInspector, type CapabilityDevtoolsTab, type EvalInspectorReport } from "./DevtoolsInspector"; +import { DevtoolsInspector, EvalReportInspector, type CapabilityDevtoolsTab, type EvalInspectorReport } from "./DevtoolsInspector"; import { Icon } from "./Icons"; import { capabilitySemanticToolDescriptor } from "../../../src/semantic-tools/catalog"; import { @@ -392,7 +392,7 @@ export function EvidencePanel(props: EvidencePanelProps) { {devtools !== undefined ? ( <> {devtools === null ? ( -

Loading company state…

+ evalReport ? :

Loading company state…

) : ( )} diff --git a/packages/paperclip-runner/devtools/issue-thread/src/main.tsx b/packages/paperclip-runner/devtools/issue-thread/src/main.tsx index 0a62fa2ef8..e524b52fd4 100644 --- a/packages/paperclip-runner/devtools/issue-thread/src/main.tsx +++ b/packages/paperclip-runner/devtools/issue-thread/src/main.tsx @@ -4,11 +4,23 @@ import { createRoot } from "react-dom/client"; import { App } from "./App"; import "./issue-thread.css"; +// Hosted Evalbooks use inert JSON plus the same trusted viewer bundle. No +// inline executable script or network fetch is needed to load an attempt. +const reportData = document.getElementById("paperclip-eval-report"); +if (reportData !== null) { + window.__PAPERCLIP_EVAL_REPORT__ = JSON.parse( + reportData.textContent ?? "null", + ); +} + // `?capture=1` freezes animation, caret, and smooth scrolling so the // screenshot matrix is byte-stable across runs (contract §10.1). const params = new URLSearchParams(window.location.search); const hashQuery = window.location.hash.split("?")[1] ?? ""; -if (params.get("capture") === "1" || new URLSearchParams(hashQuery).get("capture") === "1") { +if ( + params.get("capture") === "1" || + new URLSearchParams(hashQuery).get("capture") === "1" +) { document.documentElement.dataset.capture = "true"; } diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index 809ef86ebd..9904217445 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -1,5 +1,48 @@ # Direct live Runner protocol evals +## One Evalbook presentation + +Every new report uses the canonical Evalbook grid and the existing Runner Lab +chat viewer for attempt drill-downs. There is no plain-HTML attempt fallback. +Missing recordings show a notice in the same viewer; missing viewer builds +fail generation. Build with +`pnpm --filter @paperclipai/paperclip-runner build:issue-thread` and provide +`--viewer-root` or `PAPERCLIP_EVAL_VIEWER_ROOT` to the canonical Python renderer. + +The Actions artifact contains full evidence. S3 uses the same viewer with a +closed public DTO: only isolated mock-run conversation text, scrubbed private +references, named tool outcomes, and checks. Tool arguments/results, reasoning, +provider identities, and company snapshots stay private. The public notice +explains these redactions. An unverified isolation boundary yields no public +conversation, not a guessed reconstruction. + +Public attempts use inert JSON and one shared viewer asset directory. The +publisher verifies each shell and asset against the exact same-run viewer build, +checks the public payload contract and local links, and rejects other scripts. +The CSP prohibits network calls, forms and external resources. Supply +`PAPERCLIP_RUNNER_PROTOCOL_EVAL_VIEWER_DIR` to the publisher. The workflow sends +a viewer-only artifact to that job; raw attempts and provider secrets stay out. + +### Refresh a completed report without calling models + +Download the aggregate Actions artifact, then: + +```sh +node packages/paperclip-runner/scripts/refresh-runner-protocol-eval-report.mjs \ + --source /path/to/downloaded-aggregate \ + --evals-root /path/to/paperclip-evals \ + --viewer-root packages/paperclip-runner/dist-issue-thread \ + --output /path/to/new-refresh-directory \ + --revision chat-v1 +``` + +Publish the returned `reportRoot` with the normal history publisher. The new ID +is `gha-RUN-ATTEMPT-report-chat-v1`. Original reports remain immutable; history +adds a labeled refresh and retains the source campaign, original measurement +timestamp, renderer digest, and `providerCalls: 0`. Scores and evaluated source +revisions do not change. This is not a new model qualification run. Future live +runs create chat reports automatically. + This is the provider-backed, one-turn protocol qualification layer in `paperclipai/paperclip-evals/evals/paperclip-runner`. It is intentionally separate from both the browser full-stack model E2E and the stress-derived @@ -149,11 +192,11 @@ read-only Runner issue-thread attempt pages, and raw immutable run records. Public publishing uses a separate projection and a separate trusted OIDC job. The projection retains model/config identity, status, usage totals, and check -outcomes but removes provider session identifiers, transcripts, semantic-tool +outcomes and scrubbed mock conversation, but removes provider session identifiers, semantic-tool payloads, state revisions, traces, remote profile identities, and raw failure text. The same Evalbook `report` command renders that projection, so the public -grid and test pages have the standard Evalbook layout. The publisher rejects -scripts, remote resources, symlinks, unknown paths, broken links, raw session +grid, test pages and chat viewer have the standard Evalbook layout. The publisher rejects +untrusted scripts, remote resources, symlinks, unknown paths, broken links, raw session fields, and credential-shaped values. S3 publication is additive: diff --git a/packages/paperclip-runner/package.json b/packages/paperclip-runner/package.json index a133b72024..74e717232e 100644 --- a/packages/paperclip-runner/package.json +++ b/packages/paperclip-runner/package.json @@ -115,7 +115,7 @@ "test:capability-evals": "vitest run src/conformance/capability-eval-suite.test.ts", "test:eval-slice": "pnpm run ensure:eval-build-deps && vitest run src/eval", "test:runner-workflow-evals": "pnpm run ensure:eval-build-deps && node --test scripts/render-runner-workflow-evalbook.test.mjs && vitest run src/eval/workflow-evals.test.ts src/eval/live-workflow-executor.test.ts", - "test:runner-protocol-eval-publish": "node --test scripts/runner-protocol-eval-campaign.test.mjs scripts/publish-runner-protocol-eval-history.test.mjs scripts/runner-protocol-eval-workflow-security.test.mjs", + "test:runner-protocol-eval-publish": "node --test scripts/runner-protocol-eval-campaign.test.mjs scripts/publish-runner-protocol-eval-history.test.mjs scripts/runner-protocol-eval-workflow-security.test.mjs scripts/public-eval-chat.test.mjs", "check:runner-workflow-traceability": "pnpm run build:typescript && node scripts/check-runner-workflow-traceability.mjs", "report:capability-evals": "pnpm run build:typescript && node scripts/run-capability-eval-suite.mjs", "report:capability-live-evals": "pnpm run build:typescript && cargo build --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --bin paperclip-runnerd && node scripts/run-capability-live-eval-matrix.mjs", diff --git a/packages/paperclip-runner/scripts/public-eval-chat.mjs b/packages/paperclip-runner/scripts/public-eval-chat.mjs new file mode 100644 index 0000000000..cde0286df8 --- /dev/null +++ b/packages/paperclip-runner/scripts/public-eval-chat.mjs @@ -0,0 +1,216 @@ +// Public replay is a new DTO, never a recursive copy of a provider artifact. +export const PUBLIC_CHAT_SCHEMA = + "paperclip.runner-protocol-eval.public-chat/v1"; +export const PUBLIC_CHAT_NOTICE = + "Public replay of an isolated mock eval. Conversation text is scrubbed; provider identities, tool payloads, traces, and company-state snapshots are withheld. Full evidence remains in the access-controlled Actions artifact."; + +export const SECRET_TEXT = [ + /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/gu, + /\bsk-[A-Za-z0-9_-]{16,}\b/gu, + /\b(?:gh[pousr]_|github_pat_)[A-Za-z0-9_]{16,}\b/gu, + /\bBearer\s+[A-Za-z0-9._~+\/-]{8,}=*/giu, + /-----BEGIN [^-]*PRIVATE KEY-----[\s\S]*?-----END [^-]*PRIVATE KEY-----/gu, + /\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/gu, + /\b(?:https?|file|s3):\/\/[^\s<>"')]+/giu, + /\barn:aws[^\s<>"')]+/gu, + /(?:\/(?:Users|home|tmp|private|var)\/|[A-Z]:\\)[^\s<>"')]+/gu, + /\b(?:api[_-]?key|access[_-]?token|secret|password|authorization|cookie)\s*[=:]\s*[^\s,;]+/giu, +]; + +export function publicText(value, privateValues = []) { + let text = typeof value === "string" ? value : ""; + for (const secret of privateValues) { + if (typeof secret === "string" && secret.length >= 8) + text = text.replaceAll(secret, "[redacted]"); + } + for (const pattern of SECRET_TEXT) text = text.replace(pattern, "[redacted]"); + return text.length > 40_000 ? `${text.slice(0, 40_000)}\n[truncated]` : text; +} + +function privateIdentities(value, found = new Set()) { + if (value && typeof value === "object") { + for (const [key, child] of Object.entries(value)) { + if ( + /(?:session|profile|account|runtime|endpoint|memory|agentversion).*id$|arn$|token$|secret$|password$/i.test( + key, + ) && + typeof child === "string" + ) + found.add(child); + else if (child && typeof child === "object") + privateIdentities(child, found); + } + } + return [...found]; +} + +function timestamp(value) { + return typeof value === "string" && /^\d{4}-\d\d-\d\dT[\d:.]+Z$/.test(value) + ? value + : "1970-01-01T00:00:00.000Z"; +} + +function operation(value) { + return typeof value === "string" && /^[a-z][a-z_]{0,79}$/.test(value) + ? value + : "unknown_operation"; +} + +export function publicChatView(artifact, evalCase) { + const privateValues = privateIdentities(artifact); + const scrub = (value) => publicText(value, privateValues); + const network = artifact.snapshot?.networkEvidence; + // Only the dedicated mock eval boundary can publish recorded conversation. + // Early infrastructure failures still receive a viewer with an honest notice. + const isolated = + network?.realPaperclipRequests === 0 && + Array.isArray(network?.childPaperclipEnvironmentKeys) && + network.childPaperclipEnvironmentKeys.length === 0; + const source = + isolated && + artifact.issueThread?.schema === "paperclip.capability.issue-thread-view.v1" + ? artifact.issueThread + : null; + const evidence = Object.fromEntries( + [ + "tools", + "calls", + "authorization", + "control_plane", + "runner", + "state", + "traceability", + "parity", + ].map((key) => [key, []]), + ); + const turns = (source?.turns ?? []).map((turn, turnIndex) => { + const turnId = `public-turn-${turnIndex + 1}`; + const items = []; + for (const item of turn.items ?? []) { + const base = { + id: `public-item-${turnIndex + 1}-${items.length + 1}`, + at: timestamp(item.at), + }; + if ( + ["user_message", "agent_message", "durable_comment"].includes(item.kind) + ) { + items.push({ + ...base, + kind: item.kind === "user_message" ? "user_message" : "agent_message", + author: item.kind === "user_message" ? "You (eval prompt)" : "Agent", + body: scrub(item.body), + streaming: false, + }); + } else if (item.kind === "tool_activity") { + const operationId = operation(item.operationId); + const status = ["ok", "denied", "running"].includes(item.status) + ? item.status + : "running"; + const result = { + outcome: status, + detail: "Tool payload withheld from public replay.", + }; + const recordId = `public-call-${turnIndex + 1}-${items.length + 1}`; + items.push({ + ...base, + kind: "tool_activity", + operationId, + status, + summary: `${operationId}: ${status}`, + input: { detail: "Arguments withheld from public replay." }, + result, + evidenceRef: { section: "calls", recordId }, + }); + if (status !== "running") + evidence.calls.push({ + id: recordId, + turnId, + operationId, + version: 1, + providerRequest: operationId, + dispatchedCommand: operationId, + outcome: status, + result, + redactions: ["arguments", "result payload", "provider identities"], + threadAnchorId: base.id, + }); + } + // Provider activity, reasoning, raw events, file refs and unrecognized + // future item kinds are deliberately not part of the public contract. + } + return { + id: turnId, + ordinal: turnIndex + 1, + mode: "replay", + toolCallCount: items.filter((item) => item.kind === "tool_activity") + .length, + at: timestamp(turn.at), + stoppedByUser: turn.stoppedByUser === true, + items, + }; + }); + if (!turns.length) { + turns.push({ + id: "public-turn-1", + ordinal: 1, + mode: "replay", + toolCallCount: 0, + at: timestamp(artifact.snapshot?.createdAt), + stoppedByUser: false, + items: [ + { + id: "public-notice", + at: timestamp(artifact.snapshot?.createdAt), + kind: "system_notice", + glyph: "–", + text: "No publishable conversation was recorded for this attempt. See the checks and the access-controlled artifact for diagnostics.", + evidenceRef: { section: "runner", recordId: "public-notice" }, + }, + ], + }); + } + return { + schema: "paperclip.capability.issue-thread-view.v1", + sessionId: "public-report", + mode: "replay", + identity: { + agentLabel: "Recorded agent", + runnerLabel: "Recorded runner", + runnerAttached: false, + controlPlaneLabel: "Mock Paperclip", + controlPlaneTooltip: PUBLIC_CHAT_NOTICE, + replaySource: "live", + }, + issue: { + identifier: "EVAL", + title: scrub(evalCase.title || evalCase.id), + status: [ + "backlog", + "todo", + "in_progress", + "in_review", + "done", + "blocked", + "cancelled", + ].includes(source?.issue?.status) + ? source.issue.status + : "in_review", + priority: "medium", + assignee: null, + runState: "Read-only public replay", + scenarioId: evalCase.id, + fixtureProfile: evalCase.id, + }, + turns, + composer: { + state: "disabled", + helper: null, + reason: "Read-only eval report", + pendingInteractionId: null, + }, + evidence, + connection: { state: "closed", attempt: 0 }, + replay: null, + renderedAt: timestamp(source?.renderedAt || artifact.snapshot?.createdAt), + }; +} diff --git a/packages/paperclip-runner/scripts/public-eval-chat.test.mjs b/packages/paperclip-runner/scripts/public-eval-chat.test.mjs new file mode 100644 index 0000000000..7de7f957da --- /dev/null +++ b/packages/paperclip-runner/scripts/public-eval-chat.test.mjs @@ -0,0 +1,234 @@ +import assert from "node:assert/strict"; +import { mkdtemp, mkdir, writeFile, cp, rm, symlink } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { + publicChatView, + publicText, + PUBLIC_CHAT_SCHEMA, + PUBLIC_CHAT_NOTICE, +} from "./public-eval-chat.mjs"; +import { + publicViewerShell, + validatePublicChatPayload, + trustedViewerFiles, +} from "./public-eval-viewer.mjs"; +import { validatePublicProtocolEvalReport } from "./publish-runner-protocol-eval-history.mjs"; + +function artifact() { + return { + providerSessionId: "private-session-canary", + snapshot: { + networkEvidence: { + realPaperclipRequests: 0, + childPaperclipEnvironmentKeys: [], + }, + }, + issueThread: { + schema: "paperclip.capability.issue-thread-view.v1", + issue: { status: "blocked" }, + turns: [ + { + items: [ + { kind: "user_message", body: "Please block this task." }, + { + kind: "agent_message", + body: "Blocked. private-session-canary https://private.example/path", + privateField: "private-field-canary", + }, + { + kind: "tool_activity", + operationId: "block_task", + status: "ok", + input: { secret: "argument-canary" }, + result: { token: "result-canary" }, + }, + { kind: "thinking", body: "reasoning-canary" }, + { kind: "future_kind", body: "future-canary" }, + ], + }, + ], + }, + }; +} + +function payload() { + return { + attemptId: "attempt-01", + caseId: "block-task", + disposition: "pass", + passed: true, + checks: [], + publication: { schema: PUBLIC_CHAT_SCHEMA, notice: PUBLIC_CHAT_NOTICE }, + view: publicChatView(artifact(), { id: "block-task" }), + devtools: null, + navigation: { suiteHref: "../../index.html", previous: null, next: null }, + run: { + model: "test", + provider: "test", + sessionId: "public-report", + effectiveModelHistory: [], + managedProfile: null, + acpxProfile: null, + usage: null, + }, + }; +} + +test("projects only isolated recorded messages and bounded tool facts", () => { + const view = publicChatView(artifact(), { id: "block-task" }); + assert.equal(view.issue.status, "blocked"); + assert.deepEqual( + view.turns[0].items.map((item) => item.kind), + ["user_message", "agent_message", "tool_activity"], + ); + assert.match(view.turns[0].items[1].body, /Blocked/); + assert.doesNotMatch(JSON.stringify(view), /canary|private\.example/); + assert.deepEqual(view.turns[0].items[2].input, { + detail: "Arguments withheld from public replay.", + }); + validatePublicChatPayload(payload()); + for (const networkEvidence of [ + undefined, + { realPaperclipRequests: 1, childPaperclipEnvironmentKeys: [] }, + { + realPaperclipRequests: 0, + childPaperclipEnvironmentKeys: ["PAPERCLIP_API_KEY"], + }, + ]) { + const source = artifact(); + source.snapshot.networkEvidence = networkEvidence; + const unavailable = publicChatView(source, { id: "block-task" }); + assert.equal(unavailable.turns[0].items[0].kind, "system_notice"); + assert.doesNotMatch(JSON.stringify(unavailable), /Please block|Blocked\./); + } +}); + +test("scrubs credentials and private references before truncating text", () => { + for (const secret of [ + "sk-" + "a".repeat(32), + "ghp_" + "b".repeat(32), + "Bearer abcdef123456", + "password=secret-canary", + "/Users/someone/private.txt", + "arn:aws:service:region:account:resource", + "-----BEGIN PRIVATE KEY-----\n" + + "x".repeat(41_000) + + "\n-----END PRIVATE KEY-----", + ]) { + assert.equal(publicText(secret), "[redacted]"); + } + assert.match(publicText("a".repeat(41_000)), /\[truncated\]$/); +}); + +test("fails closed on unknown fields, raw tools, private evidence and identities", () => { + for (const mutate of [ + (p) => { + p.view.turns[0].items[0].extra = "unprojected"; + }, + (p) => { + p.view.turns[0].items[2].input = { password: "oops" }; + }, + (p) => { + p.view.evidence.calls[0].result.detail = "raw-result"; + }, + (p) => { + p.view.evidence.state.push({ secret: "raw-state" }); + }, + (p) => { + p.run.providerSessionId = "private-session"; + }, + (p) => { + p.view.turns[0].items[1].body = "sk-" + "x".repeat(30); + }, + (p) => { + p.devtools = {}; + }, + (p) => { + p.view.composer.state = "ready"; + }, + ]) { + const value = payload(); + mutate(value); + assert.throws(() => validatePublicChatPayload(value)); + } +}); + +test("publisher permits only the exact trusted shell/assets and valid local navigation", async () => { + const root = await mkdtemp(join(tmpdir(), "eval-chat-contract-")); + try { + const viewer = join(root, "trusted"); + const report = join(root, "report"); + await mkdir(join(viewer, "assets"), { recursive: true }); + await mkdir(join(report, "attempts/attempt-01"), { recursive: true }); + const index = + '
'; + await writeFile(join(viewer, "index.html"), index); + await writeFile(join(viewer, "assets/app.js"), "// trusted build"); + await writeFile(join(viewer, "assets/app.css"), ":root {}"); + await cp(join(viewer, "assets"), join(report, "viewer/assets"), { + recursive: true, + }); + await writeFile( + join(report, "index.html"), + 'PASS', + ); + await writeFile( + join(report, "campaign.json"), + JSON.stringify({ + schema: "paperclip.runner-protocol-eval.campaign/v1", + campaignId: "gha-42-1", + }), + ); + const page = join(report, "attempts/attempt-01/index.html"); + const writePayload = async (value) => + writeFile( + page, + publicViewerShell( + index, + JSON.stringify(value).replaceAll("<", "\\u003c"), + ), + ); + await writePayload(payload()); + await validatePublicProtocolEvalReport(report, { viewerRoot: viewer }); + await assert.rejects(validatePublicProtocolEvalReport(report)); + await writeFile( + join(report, "viewer/assets/app.js"), + "// substituted build", + ); + await assert.rejects( + validatePublicProtocolEvalReport(report, { viewerRoot: viewer }), + /trusted/, + ); + await writeFile(join(report, "viewer/assets/app.js"), "// trusted build"); + await writeFile( + page, + publicViewerShell(index, JSON.stringify(payload())) + + "", + ); + await assert.rejects( + validatePublicProtocolEvalReport(report, { viewerRoot: viewer }), + /trusted shell/, + ); + const escaped = payload(); + escaped.view.turns[0].items[0].body = + ''; + await writePayload(escaped); + await validatePublicProtocolEvalReport(report, { viewerRoot: viewer }); + const broken = payload(); + broken.navigation.next = { + label: "Next attempt", + href: "../missing/index.html", + }; + await writePayload(broken); + await assert.rejects( + validatePublicProtocolEvalReport(report, { viewerRoot: viewer }), + /link|reference/i, + ); + await symlink(viewer, join(root, "symlink")); + await assert.rejects(trustedViewerFiles(join(root, "symlink"))); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/packages/paperclip-runner/scripts/public-eval-viewer.mjs b/packages/paperclip-runner/scripts/public-eval-viewer.mjs new file mode 100644 index 0000000000..a228736b9f --- /dev/null +++ b/packages/paperclip-runner/scripts/public-eval-viewer.mjs @@ -0,0 +1,229 @@ +import { readFile, readdir, lstat } from "node:fs/promises"; +import { join } from "node:path"; +import { PUBLIC_CHAT_SCHEMA, SECRET_TEXT } from "./public-eval-chat.mjs"; + +export const PUBLIC_VIEWER_CSP = + "default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; font-src 'self'; img-src 'none'; connect-src 'none'; form-action 'none'; base-uri 'none'"; +export const PUBLIC_VIEWER_DATA = + /\n