diff --git a/.github/workflows/runner-protocol-live-evals.yml b/.github/workflows/runner-protocol-live-evals.yml index f659f5ec65..e3eafa2da5 100644 --- a/.github/workflows/runner-protocol-live-evals.yml +++ b/.github/workflows/runner-protocol-live-evals.yml @@ -213,6 +213,11 @@ jobs: --max-parallel "$MAX_PARALLEL" \ --output runner-protocol-eval-catalog.json + - name: Require the chat-report renderer before paid execution + run: | + set -euo pipefail + python3 .paperclip-evals/evals/paperclip-runner/tools/eval_program.py report --help | grep -q -- --public-viewer + - name: Upload immutable campaign catalog uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: @@ -259,6 +264,9 @@ jobs: pnpm --filter @paperclipai/paperclip-runner build:typescript pnpm --filter @paperclipai/paperclip-runner build:runner-binaries pnpm --filter @paperclipai/paperclip-runner build:issue-thread + # Older target refs must fail before paid cells, not publish an empty viewer. + grep -q 'paperclip-eval-report' packages/paperclip-runner/dist-issue-thread/assets/*.js + grep -q 'evalbook-site' packages/paperclip-runner/dist-issue-thread/assets/*.css - name: Package a portable provider runtime run: | @@ -290,6 +298,14 @@ jobs: compression-level: 0 if-no-files-found: error + - name: Upload canonical viewer for publisher byte verification + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: runner-protocol-viewer-${{ github.run_id }}-${{ github.run_attempt }} + path: packages/paperclip-runner/dist-issue-thread/ + retention-days: 30 + if-no-files-found: error + eval_shard_0: name: Direct eval ${{ matrix.rosterId }} / ${{ matrix.caseId }} needs: [authorize, catalog, build_runner] @@ -595,14 +611,28 @@ jobs: python3 .paperclip-evals/evals/paperclip-runner/tools/eval_program.py report \ --runs-root runner-protocol-merged/public-runs \ --output runner-protocol-merged/public-report \ + --viewer-root runner-protocol-build/extracted/dist-issue-thread \ + --public-viewer \ --inventory .paperclip-evals/evals/paperclip-runner/inventory.json \ --coverage-matrix .paperclip-evals/evals/paperclip-runner/coverage-matrix.json cp runner-protocol-merged/campaign.json runner-protocol-merged/public-report/campaign.json + - name: Set up report browser verification + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 + with: + version: 9.15.4 + + - name: Verify the actual chat viewer before publication + run: | + pnpm install --frozen-lockfile --ignore-scripts + pnpm --filter @paperclipai/paperclip-runner exec playwright install --with-deps chromium + node packages/paperclip-runner/scripts/verify-runner-evalbook-viewer.mjs --report-root runner-protocol-merged/public-report --screenshots runner-protocol-merged/viewer-proof + node packages/paperclip-runner/scripts/verify-runner-evalbook-viewer.mjs --report-root runner-protocol-merged/report + - name: Enforce the static public allowlist id: public_report run: | - node --input-type=module -e 'import { validatePublicProtocolEvalReport } from "./packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.mjs"; await validatePublicProtocolEvalReport("runner-protocol-merged/public-report");' + node --input-type=module -e 'import { validatePublicProtocolEvalReport } from "./packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.mjs"; await validatePublicProtocolEvalReport("runner-protocol-merged/public-report", { viewerRoot: "runner-protocol-build/extracted/dist-issue-thread" });' echo "ready=true" >> "$GITHUB_OUTPUT" - name: Add campaign result to the workflow summary @@ -649,6 +679,7 @@ jobs: id-token: write environment: name: runner-e2e-history + url: ${{ steps.publish.outputs.report_url }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: @@ -666,6 +697,12 @@ jobs: name: runner-protocol-eval-public-${{ github.run_id }}-${{ github.run_attempt }} path: runner-protocol-public-report + - name: Download the same-run canonical viewer for byte verification + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: runner-protocol-viewer-${{ github.run_id }}-${{ github.run_attempt }} + path: runner-protocol-trusted-viewer + - name: Exchange GitHub OIDC identity for scoped AWS credentials uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6 with: @@ -673,8 +710,10 @@ jobs: aws-region: ${{ vars.RUNNER_PROTOCOL_EVAL_HISTORY_AWS_REGION || vars.RUNNER_E2E_HISTORY_AWS_REGION }} - name: Publish versioned report and refresh the root index + id: publish env: PAPERCLIP_RUNNER_PROTOCOL_EVAL_PUBLIC_REPORT_DIR: ${{ github.workspace }}/runner-protocol-public-report + PAPERCLIP_RUNNER_PROTOCOL_EVAL_VIEWER_DIR: ${{ github.workspace }}/runner-protocol-trusted-viewer RUNNER_PROTOCOL_EVAL_HISTORY_S3_BUCKET: ${{ vars.RUNNER_PROTOCOL_EVAL_HISTORY_S3_BUCKET || vars.RUNNER_E2E_HISTORY_S3_BUCKET }} RUNNER_PROTOCOL_EVAL_HISTORY_PREFIX: ${{ vars.RUNNER_PROTOCOL_EVAL_HISTORY_PREFIX || 'runner-protocol-evals' }} RUNNER_PROTOCOL_EVAL_HISTORY_PUBLIC_BASE_URL: ${{ vars.RUNNER_PROTOCOL_EVAL_HISTORY_PUBLIC_BASE_URL || vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL }} diff --git a/packages/paperclip-runner/devtools/issue-thread/src/App.tsx b/packages/paperclip-runner/devtools/issue-thread/src/App.tsx index 5ebeca9d49..c511c631fc 100644 --- a/packages/paperclip-runner/devtools/issue-thread/src/App.tsx +++ b/packages/paperclip-runner/devtools/issue-thread/src/App.tsx @@ -12,6 +12,7 @@ import type { CapabilityDevtoolsSnapshot } from "../../../src/devtools"; import { capabilityDenialCount } from "../../../src/issue-thread/types"; import { Composer } from "./Composer"; import { EvidencePanel } from "./EvidencePanel"; +import type { CapabilityDevtoolsTab } from "./DevtoolsInspector"; import { Icon } from "./Icons"; import { IssueHeader } from "./IssueHeader"; import { applyFakeInteractionResponse } from "./fake-store"; @@ -78,6 +79,7 @@ interface EmbeddedEvalCheck { } interface EmbeddedEvalReport { + publication?: { schema: string; notice: string }; attemptId: string; caseId: string; disposition: string; @@ -101,7 +103,7 @@ interface EmbeddedEvalReport { runnerBuild: string; startedAt: string; finishedAt: string; - durationMs: number; + durationMs: number | null; initialRevision: number; finalRevision: number; usage: { @@ -117,7 +119,7 @@ interface EmbeddedEvalReport { } | null; }; view: CapabilityIssueThreadSnapshot; - devtools: CapabilityDevtoolsSnapshot; + devtools: CapabilityDevtoolsSnapshot | null; navigation: { suiteHref: string; previous: { label: string; href: string } | null; next: { label: string; href: string } | null }; } @@ -332,6 +334,7 @@ export function App() { const [panelOpen, setPanelOpen] = useState(() => embeddedEval !== null || readStoredFlag(route.surface === "chat" ? CHAT_PANEL_OPEN_KEY : PANEL_OPEN_KEY, false)); const [panelWidth, setPanelWidth] = useState(() => readStoredNumber(PANEL_WIDTH_KEY, 384)); + const [devtoolsTab, setDevtoolsTab] = useState(embeddedEval !== null ? "eval" : "evidence"); const [segment, setSegment] = useState<"thread" | "evidence">(route.segment); const [openSections, setOpenSections] = useState(["tools"]); const [selectedTurnId, setSelectedTurnId] = useState("all"); @@ -491,7 +494,7 @@ export function App() { await document.fonts.ready; } if (cancelled) return; - if (scroller !== null) scroller.scrollTop = scroller.scrollHeight; + if (scroller !== null) scroller.scrollTop = embeddedEval === null ? scroller.scrollHeight : 0; await new Promise((resolve) => requestAnimationFrame(() => resolve())); if (!cancelled) setSettled(true); })(); @@ -501,14 +504,14 @@ export function App() { }, [snapshot]); useEffect(() => { - if (!panelOpen || snapshot === null || route.mode !== "live") return; + if (embeddedEval !== null || !panelOpen || snapshot === null || route.mode !== "live") return; if (historicSessionId !== null) return; let cancelled = false; void capabilityLiveClient.devtools(snapshot.sessionId) .then((next) => { if (!cancelled) setDevtools(next); }) .catch((cause) => { if (!cancelled) setActionError(describe(cause)); }); return () => { cancelled = true; }; - }, [historicSessionId, panelOpen, route.mode, snapshot?.renderedAt, snapshot?.sessionId]); + }, [embeddedEval, historicSessionId, panelOpen, route.mode, snapshot?.renderedAt, snapshot?.sessionId]); useEffect(() => { if (!chat || snapshot === null || historicSessionId !== null) return; @@ -571,6 +574,7 @@ export function App() { const openEvidence = useCallback( (section: CapabilityEvidenceSectionId, recordId: string) => { setPanelOpen(true); + setDevtoolsTab("evidence"); setSegment("evidence"); setSelectedTurnId("all"); setOpenSections((current) => (current.includes(section) ? current : [...current, section])); @@ -1363,7 +1367,7 @@ export function App() { >
{embeddedEval !== null ? ( -
Eval execution
+
Eval execution{embeddedEval.publication ? {embeddedEval.publication.notice} : null}
) : null} {snapshot.turns.length === 0 ? (
@@ -1404,7 +1408,7 @@ export function App() { {embeddedEval !== null ? (
Post-run state - Final mock control-plane revision {embeddedEval.run.finalRevision} + {embeddedEval.publication ? "Company-state details withheld from public replay" : `Final mock control-plane revision ${embeddedEval.run.finalRevision}`} check.anchor.kind === "run")} />
) : null} @@ -1425,7 +1429,7 @@ export function App() { ) : null}
- + /> : null} {showPanel && layout === "side" ? ( @@ -1496,6 +1500,8 @@ export function App() { {showPanel ? ( { diff --git a/packages/paperclip-runner/devtools/issue-thread/src/DevtoolsInspector.tsx b/packages/paperclip-runner/devtools/issue-thread/src/DevtoolsInspector.tsx index acfd180775..ee057682b2 100644 --- a/packages/paperclip-runner/devtools/issue-thread/src/DevtoolsInspector.tsx +++ b/packages/paperclip-runner/devtools/issue-thread/src/DevtoolsInspector.tsx @@ -23,6 +23,7 @@ export interface EvalInspectorReport { disposition: string; passed: boolean; checks: EvalAssertion[]; + publication?: { schema: string; notice: string }; run: { model: string; provider: string; @@ -41,7 +42,7 @@ export interface EvalInspectorReport { runnerBuild: string; startedAt: string; finishedAt: string; - durationMs: number; + durationMs: number | null; initialRevision: number; finalRevision: number; usage: { @@ -53,7 +54,7 @@ export interface EvalInspectorReport { reasoningTokens: number; providerReportedCostNanodollars?: number; estimatedCostNanodollars: number; - pricingVersion: string; + pricingVersion?: string; } | null; }; } @@ -197,6 +198,149 @@ function documentsOf(state: Json): Array<{ }); } +export function EvalReportInspector({ + evalReport, +}: { + evalReport: EvalInspectorReport; +}) { + const isPublic = Boolean(evalReport.publication); + return ( +
+
+ ← Eval suite + + {evalReport.passed + ? "PASS" + : evalReport.disposition.replaceAll("_", " ").toUpperCase()} + + {evalReport.attemptId} +
+
+
+
Model
+
+ {evalReport.run.model.startsWith(`${evalReport.run.provider}/`) + ? evalReport.run.model + : `${evalReport.run.provider}/${evalReport.run.model}`} +
+
+
+
Configuration
+
{evalReport.run.configuration}
+
+
+
Session
+
+ {isPublic + ? "Withheld from public replay" + : evalReport.run.sessionId} +
+
+
+
Provider session
+
+ {isPublic + ? "Withheld from public replay" + : (evalReport.run.providerSessionId ?? "unavailable")} +
+
+
+
Driver
+
+ {evalReport.run.driver} + {evalReport.run.providerVersion + ? ` · ${evalReport.run.providerVersion}` + : ""} +
+
+ {evalReport.run.agentVersion ? ( +
+
Agent version
+
{evalReport.run.agentVersion}
+
+ ) : null} +
+
Retained session
+
+ {isPublic + ? "Withheld from public replay" + : evalReport.run.retainedSession === true + ? (evalReport.run.retainedSessionStatus ?? "retained") + : "not applicable"} +
+
+
+
Duration
+
+ {evalReport.run.durationMs == null + ? "unavailable" + : `${evalReport.run.durationMs} ms`} +
+
+
+
Fixture
+
{evalReport.run.fixtureDigest}
+
+
+
State
+
+ {isPublic + ? "Withheld from public replay" + : `r${evalReport.run.initialRevision} → r${evalReport.run.finalRevision}`} +
+
+
+
Tokens
+
+ {evalReport.run.usage === null + ? "unknown" + : `${evalReport.run.usage.inputTokens} in · ${evalReport.run.usage.outputTokens} out · ${evalReport.run.usage.cachedInputTokens} cached`} +
+
+
+
Agent turns
+
{evalReport.run.usage?.agentTurns ?? "unknown"}
+
+
+
Provider requests
+
{evalReport.run.usage?.providerRequests ?? "unavailable"}
+
+
+
Estimated cost
+
+ {evalReport.run.usage === null + ? "unknown" + : `$${(evalReport.run.usage.estimatedCostNanodollars / 1_000_000_000).toFixed(6)}${evalReport.run.usage.pricingVersion ? ` · ${evalReport.run.usage.pricingVersion}` : ""}`} +
+
+
+
Provider list cost
+
+ {typeof evalReport.run.usage?.providerReportedCostNanodollars !== + "number" + ? "unknown" + : `$${(evalReport.run.usage.providerReportedCostNanodollars / 1_000_000_000).toFixed(6)}`} +
+
+
+
Runner
+
{evalReport.run.runnerPackageDigest}
+
+
+
Runner build
+
{evalReport.run.runnerBuild}
+
+
+
runnerd
+
{evalReport.run.runnerdDigest}
+
+
+

Assertions

+ +
+ ); +} + export function DevtoolsInspector({ snapshot, onFork, @@ -291,6 +435,8 @@ export function DevtoolsInspector({ className="pit-button" type="button" onClick={() => onFork(revision)} + disabled={evalReport != null} + title={evalReport ? "Eval reports are read-only" : undefined} > Fork r{revision} @@ -328,124 +474,7 @@ export function DevtoolsInspector({ ))} {tab === "eval" && evalReport ? ( -
-
- ← Eval suite - - {evalReport.passed - ? "PASS" - : evalReport.disposition.replaceAll("_", " ").toUpperCase()} - - {evalReport.attemptId} -
-
-
-
Model
-
- {evalReport.run.model.startsWith(`${evalReport.run.provider}/`) - ? evalReport.run.model - : `${evalReport.run.provider}/${evalReport.run.model}`} -
-
-
-
Configuration
-
{evalReport.run.configuration}
-
-
-
Session
-
{evalReport.run.sessionId}
-
-
-
Provider session
-
{evalReport.run.providerSessionId ?? "unavailable"}
-
-
-
Driver
-
- {evalReport.run.driver} - {evalReport.run.providerVersion - ? ` · ${evalReport.run.providerVersion}` - : ""} -
-
- {evalReport.run.agentVersion ? ( -
-
Agent version
-
{evalReport.run.agentVersion}
-
- ) : null} -
-
Retained session
-
- {evalReport.run.retainedSession === true - ? (evalReport.run.retainedSessionStatus ?? "retained") - : "not applicable"} -
-
-
-
Duration
-
{evalReport.run.durationMs} ms
-
-
-
Fixture
-
{evalReport.run.fixtureDigest}
-
-
-
State
-
- r{evalReport.run.initialRevision} → r - {evalReport.run.finalRevision} -
-
-
-
Tokens
-
- {evalReport.run.usage === null - ? "unknown" - : `${evalReport.run.usage.inputTokens} in · ${evalReport.run.usage.outputTokens} out · ${evalReport.run.usage.cachedInputTokens} cached`} -
-
-
-
Agent turns
-
{evalReport.run.usage?.agentTurns ?? "unknown"}
-
-
-
Provider requests
-
{evalReport.run.usage?.providerRequests ?? "unavailable"}
-
-
-
Estimated cost
-
- {evalReport.run.usage === null - ? "unknown" - : `$${(evalReport.run.usage.estimatedCostNanodollars / 1_000_000_000).toFixed(6)} · ${evalReport.run.usage.pricingVersion}`} -
-
-
-
Provider list cost
-
- {typeof evalReport.run.usage - ?.providerReportedCostNanodollars !== "number" - ? "unknown" - : `$${(evalReport.run.usage.providerReportedCostNanodollars / 1_000_000_000).toFixed(6)}`} -
-
-
-
Runner
-
{evalReport.run.runnerPackageDigest}
-
-
-
Runner build
-
{evalReport.run.runnerBuild}
-
-
-
runnerd
-
{evalReport.run.runnerdDigest}
-
-
-

Assertions

- -
+ ) : null} {tab === "timeline" ? (
diff --git a/packages/paperclip-runner/devtools/issue-thread/src/EvidencePanel.tsx b/packages/paperclip-runner/devtools/issue-thread/src/EvidencePanel.tsx index 61aa2d7373..7c7dd485c4 100644 --- a/packages/paperclip-runner/devtools/issue-thread/src/EvidencePanel.tsx +++ b/packages/paperclip-runner/devtools/issue-thread/src/EvidencePanel.tsx @@ -10,7 +10,7 @@ import type { CapabilityToolDisposition, } from "../../../src/issue-thread/types"; import type { CapabilityDevtoolsSnapshot } from "../../../src/devtools"; -import { DevtoolsInspector, type CapabilityDevtoolsTab, type EvalInspectorReport } from "./DevtoolsInspector"; +import { DevtoolsInspector, EvalReportInspector, type CapabilityDevtoolsTab, type EvalInspectorReport } from "./DevtoolsInspector"; import { Icon } from "./Icons"; import { capabilitySemanticToolDescriptor } from "../../../src/semantic-tools/catalog"; import { @@ -114,6 +114,8 @@ export interface EvidencePanelProps { snapshot: CapabilityIssueThreadSnapshot; devtools?: CapabilityDevtoolsSnapshot | null; evalReport?: EvalInspectorReport | null; + devtoolsTab: CapabilityDevtoolsTab; + onDevtoolsTabChange: (tab: CapabilityDevtoolsTab) => void; onForkRevision?: (revision: number) => void; layout: "side" | "overlay" | "segment"; width: number; @@ -331,11 +333,12 @@ function Section({ } export function EvidencePanel(props: EvidencePanelProps) { - const [devtoolsTab, setDevtoolsTab] = useState(props.evalReport ? "eval" : "evidence"); const { snapshot, devtools, evalReport, + devtoolsTab, + onDevtoolsTabChange, onForkRevision = () => undefined, layout, width, @@ -392,9 +395,21 @@ export function EvidencePanel(props: EvidencePanelProps) { {devtools !== undefined ? ( <> {devtools === null ? ( -

Loading company state…

+ evalReport ?
+
+ {(["eval", "evidence"] as const).map((tab) => )} +
+ {devtoolsTab === "eval" ? : null} +
:

Loading company state…

) : ( - + )} ) : null} diff --git a/packages/paperclip-runner/devtools/issue-thread/src/evalbook-site.css b/packages/paperclip-runner/devtools/issue-thread/src/evalbook-site.css new file mode 100644 index 0000000000..cdba55acb8 --- /dev/null +++ b/packages/paperclip-runner/devtools/issue-thread/src/evalbook-site.css @@ -0,0 +1,87 @@ +/* Static Evalbook pages share the Runner Lab token/font layer and built asset. + Keep every selector scoped so report navigation cannot restyle live chat. */ +.evalbook-site { color-scheme: dark; } +.evalbook-site main { width: 100%; max-width: 112rem; margin: auto; padding: var(--pit-space-6); } +.evalbook-site a { color: var(--pit-live); text-underline-offset: var(--pit-space-1); } +.evalbook-site a:hover { color: var(--pit-foreground); } +.evalbook-site :is(a, summary, [tabindex]):focus-visible { outline: 2px solid var(--pit-ring); outline-offset: 3px; } +.evalbook-site .top { display: flex; justify-content: space-between; align-items: center; gap: var(--pit-space-4); padding-bottom: var(--pit-space-4); border-bottom: 1px solid var(--pit-border); } +.evalbook-site .top > a { color: var(--pit-foreground); font-weight: 700; text-decoration: none; } +.evalbook-site h1 { margin: var(--pit-space-6) 0 var(--pit-space-3); font-size: var(--pit-text-xl); line-height: var(--pit-leading-tight); overflow-wrap: anywhere; } +.evalbook-site h2 { margin: var(--pit-space-6) 0 var(--pit-space-3); font-size: var(--pit-text-base); line-height: var(--pit-leading-tight); } +.evalbook-site p { margin: var(--pit-space-3) 0; } +.evalbook-site .badge { padding: var(--pit-space-1) var(--pit-space-2); color: var(--pit-live); background: var(--pit-live-surface); border: 1px solid var(--pit-live); border-radius: var(--pit-radius-sm); font-size: var(--pit-text-xs); white-space: nowrap; } +.evalbook-site :is(.tabs, .pointers, .breadcrumbs) { display: flex; align-items: center; gap: var(--pit-space-4); margin: var(--pit-space-4) 0; overflow-x: auto; } +.evalbook-site .tabs { border-bottom: 1px solid var(--pit-border); gap: 0; } +.evalbook-site .tabs a { flex: 0 0 auto; padding: var(--pit-space-3) var(--pit-space-4); color: var(--pit-muted-foreground); text-decoration: none; border-bottom: 2px solid transparent; } +.evalbook-site .tabs a:hover { background: var(--pit-surface); color: var(--pit-foreground); } +.evalbook-site .tabs a.active { background: var(--pit-accent-surface); border-color: var(--pit-accent); color: var(--pit-foreground); } +.evalbook-site .breadcrumbs span { color: var(--pit-muted-foreground); } +.evalbook-site .spend-summary { display: flex; flex-wrap: wrap; gap: var(--pit-space-6); margin: var(--pit-space-5) 0; } +.evalbook-site .spend-summary > div { display: flex; flex-direction: column; gap: var(--pit-space-1); } +.evalbook-site .spend-summary span { color: var(--pit-muted-foreground); font-size: var(--pit-text-xs); } +.evalbook-site .spend-summary strong { font-size: var(--pit-text-base); font-variant-numeric: tabular-nums; } +.evalbook-site .lede { color: var(--pit-muted-foreground); margin-bottom: var(--pit-space-5); } +.evalbook-site :is(.card, .message, .tool, .check) { margin: var(--pit-space-3) 0; padding: var(--pit-space-4); background: var(--pit-surface); border: 1px solid var(--pit-border); border-radius: var(--pit-radius-sm); overflow-wrap: anywhere; } +.evalbook-site .pass { border-left: 3px solid var(--pit-success); } +.evalbook-site .fail { border-left: 3px solid var(--pit-danger); } +.evalbook-site .infra { border-left: 3px solid var(--pit-warning); } +.evalbook-site .grid { display: grid; grid-template-columns: minmax(0, 1fr) minmax(0, 1.3fr); gap: var(--pit-space-6); } +.evalbook-site .grid > section { min-width: 0; } +.evalbook-site :is(.table-shell, .table) { width: 100%; overflow: auto; border: 1px solid var(--pit-border); border-radius: var(--pit-radius-sm); background: var(--pit-surface-sunken); overscroll-behavior-inline: contain; scrollbar-color: var(--pit-border-strong) transparent; } +.evalbook-site table { width: 100%; border-collapse: separate; border-spacing: 0; font-size: var(--pit-text-xs); } +.evalbook-site :is(th, td) { padding: var(--pit-space-3); border-bottom: 1px solid var(--pit-border); text-align: left; vertical-align: top; } +.evalbook-site thead th { background: var(--pit-surface); color: var(--pit-muted-foreground); font-weight: 600; } +.evalbook-site tbody tr:last-child > :is(th, td) { border-bottom: 0; } +.evalbook-site tbody tr:hover > :is(th, td) { background: var(--pit-surface-raised); } +.evalbook-site .matrix { min-width: max(100%, var(--matrix-min-width, 1500px)); } +.evalbook-site .matrix :is(th, td):not(:first-child) { width: 10rem; text-align: center; } +.evalbook-site .matrix thead th { position: sticky; top: 0; z-index: 2; } +.evalbook-site .matrix thead th > div { color: var(--pit-foreground); font-size: var(--pit-text-sm); overflow-wrap: anywhere; } +.evalbook-site .matrix thead small { display: block; max-width: 11rem; margin: var(--pit-space-1) auto 0; font-size: var(--pit-text-xs); font-weight: 400; overflow-wrap: anywhere; } +.evalbook-site .matrix th:first-child { position: sticky; left: 0; z-index: 1; width: 18rem; min-width: 14rem; background: var(--pit-surface-sunken); } +.evalbook-site .matrix thead th:first-child { z-index: 3; background: var(--pit-surface); } +.evalbook-site .matrix tbody th:first-child a { display: block; color: var(--pit-foreground); font-weight: 500; overflow-wrap: anywhere; text-decoration: none; } +.evalbook-site .matrix tbody th:first-child a:hover { color: var(--pit-live); text-decoration: underline; } +.evalbook-site .result { display: flex; align-items: center; justify-content: center; width: 100%; max-width: 10rem; min-height: 2rem; margin-inline: auto; padding: var(--pit-space-1) var(--pit-space-2); border: 1px solid currentColor; border-radius: var(--pit-radius-sm); font-size: var(--pit-text-xs); font-weight: 700; text-decoration: none; } +.evalbook-site :is(.result-pass, .passed) { background: var(--pit-success-surface); color: var(--pit-success); } +.evalbook-site :is(.result-fail, .failed) { background: var(--pit-danger-surface); color: var(--pit-danger); } +.evalbook-site .result-infra { background: var(--pit-warning-surface); color: var(--pit-warning); } +.evalbook-site :is(.muted, .empty, small, footer) { color: var(--pit-muted-foreground); } +.evalbook-site .empty { text-align: center; } +.evalbook-site code { font-family: var(--pit-font-mono); font-size: var(--pit-text-xs); overflow-wrap: anywhere; } +.evalbook-site pre { max-height: 32rem; padding: var(--pit-space-3); overflow: auto; white-space: pre-wrap; overflow-wrap: anywhere; background: var(--pit-surface-sunken); color: var(--pit-foreground); border: 1px solid var(--pit-border); border-radius: var(--pit-radius-sm); font-family: var(--pit-font-mono); font-size: var(--pit-text-xs); } +.evalbook-site details { margin: var(--pit-space-4) 0; } +.evalbook-site summary { cursor: pointer; color: var(--pit-muted-foreground); } +.evalbook-site .status { display: inline-block; padding: var(--pit-space-1) var(--pit-space-2); border-radius: var(--pit-radius-sm); font-weight: 600; text-transform: uppercase; } +.evalbook-site small { display: block; margin-top: var(--pit-space-1); } +.evalbook-site footer { margin-top: var(--pit-space-5); font-size: var(--pit-text-xs); } +.evalbook-site .trend-group { padding: var(--pit-space-4); border: 1px solid var(--pit-border); border-radius: var(--pit-radius-sm); background: var(--pit-surface-sunken); } +.evalbook-site .trend-group > summary { color: var(--pit-foreground); font-weight: 600; } +.evalbook-site .trend-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: var(--pit-space-5); } +.evalbook-site figure { min-width: 0; margin: var(--pit-space-4) 0; } +.evalbook-site figcaption { color: var(--pit-muted-foreground); font-size: var(--pit-text-xs); } +.evalbook-site .trend-chart { width: 100%; overflow: visible; } +.evalbook-site .trend-chart text { fill: var(--pit-muted-foreground); font: var(--pit-text-xs) var(--pit-font-mono); } +.evalbook-site .trend-axis { fill: none; stroke: var(--pit-border); } +.evalbook-site .trend-line { fill: none; stroke-width: 2; } +.evalbook-site :is(.trend-line, .trend-point).pass { stroke: var(--pit-success); } +.evalbook-site :is(.trend-line, .trend-point).all_attempts { stroke: var(--pit-live); } +.evalbook-site :is(.trend-line, .trend-point).final_attempts { stroke: var(--pit-accent); stroke-dasharray: 4 3; } +.evalbook-site .trend-point { fill: var(--pit-surface); stroke-width: 2; } +.evalbook-site .trend-point:hover { fill: var(--pit-foreground); } +.evalbook-site .pass-rate { display: block; margin-top: var(--pit-space-2); white-space: nowrap; } +.evalbook-site .history-table table { min-width: 68rem; } +.evalbook-site .history-table td:first-child { max-width: 22rem; } +.evalbook-site .history-table details { margin: var(--pit-space-2) 0; } +.evalbook-site .history-table li { margin: var(--pit-space-2) 0; } +.evalbook-site .incomplete { background: var(--pit-warning-surface); color: var(--pit-warning); } +@media (max-width: 760px) { + .evalbook-site main { padding: var(--pit-space-4); } + .evalbook-site .grid { grid-template-columns: 1fr; } + .evalbook-site .trend-grid { grid-template-columns: 1fr; } + .evalbook-site .trend-grid figure { overflow-x: auto; } + .evalbook-site .trend-chart { min-width: 30rem; } + .evalbook-site .matrix th:first-child { width: 12rem; min-width: 10rem; } + .evalbook-site .badge { white-space: normal; text-align: center; } +} diff --git a/packages/paperclip-runner/devtools/issue-thread/src/main.tsx b/packages/paperclip-runner/devtools/issue-thread/src/main.tsx index 0a62fa2ef8..ebe93e307a 100644 --- a/packages/paperclip-runner/devtools/issue-thread/src/main.tsx +++ b/packages/paperclip-runner/devtools/issue-thread/src/main.tsx @@ -3,12 +3,25 @@ import { createRoot } from "react-dom/client"; import { App } from "./App"; import "./issue-thread.css"; +import "./evalbook-site.css"; + +// Hosted Evalbooks use inert JSON plus the same trusted viewer bundle. No +// inline executable script or network fetch is needed to load an attempt. +const reportData = document.getElementById("paperclip-eval-report"); +if (reportData !== null) { + window.__PAPERCLIP_EVAL_REPORT__ = JSON.parse( + reportData.textContent ?? "null", + ); +} // `?capture=1` freezes animation, caret, and smooth scrolling so the // screenshot matrix is byte-stable across runs (contract §10.1). const params = new URLSearchParams(window.location.search); const hashQuery = window.location.hash.split("?")[1] ?? ""; -if (params.get("capture") === "1" || new URLSearchParams(hashQuery).get("capture") === "1") { +if ( + params.get("capture") === "1" || + new URLSearchParams(hashQuery).get("capture") === "1" +) { document.documentElement.dataset.capture = "true"; } diff --git a/packages/paperclip-runner/docs/images/evalbook-chat/failed.png b/packages/paperclip-runner/docs/images/evalbook-chat/failed.png new file mode 100644 index 0000000000..21d4be7419 Binary files /dev/null and b/packages/paperclip-runner/docs/images/evalbook-chat/failed.png differ diff --git a/packages/paperclip-runner/docs/images/evalbook-chat/missing-recording.png b/packages/paperclip-runner/docs/images/evalbook-chat/missing-recording.png new file mode 100644 index 0000000000..57eba71894 Binary files /dev/null and b/packages/paperclip-runner/docs/images/evalbook-chat/missing-recording.png differ diff --git a/packages/paperclip-runner/docs/images/evalbook-chat/passed.png b/packages/paperclip-runner/docs/images/evalbook-chat/passed.png new file mode 100644 index 0000000000..1c28f8f565 Binary files /dev/null and b/packages/paperclip-runner/docs/images/evalbook-chat/passed.png differ diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index 809ef86ebd..fe9ee06f6e 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -1,5 +1,84 @@ # Direct live Runner protocol evals +## One Evalbook presentation + +Every new report uses the canonical Evalbook grid and the existing Runner Lab +chat viewer for attempt drill-downs. There is no plain-HTML attempt fallback. +The grid, Latest, test-design, inventory and server-gate pages load the same +built stylesheet and fonts as the chat viewer. The long-term S3 history index +references that stylesheet inside an immutable campaign, so every page keeps +the same dark theme. Static site styles live in +`devtools/issue-thread/src/evalbook-site.css`, scoped to `.evalbook-site`; +colors and typography come from the Runner Lab token layer. +Missing recordings show a notice in the same viewer; missing viewer builds +fail generation. Build with +`pnpm --filter @paperclipai/paperclip-runner build:issue-thread` and provide +`--viewer-root` or `PAPERCLIP_EVAL_VIEWER_ROOT` to the canonical Python renderer. + +The Actions artifact contains full evidence. S3 uses the same viewer with a +closed public DTO: only isolated mock-run conversation text, scrubbed private +references, named tool outcomes, and checks. Tool arguments/results, reasoning, +provider identities, and company snapshots stay private. The public notice +explains these redactions. An unverified isolation boundary yields no public +conversation, not a guessed reconstruction. + +Public attempts use inert JSON and one shared viewer asset directory. The +publisher verifies each shell and asset against the exact same-run viewer build, +checks the public payload contract and local links, and rejects other scripts. +The CSP prohibits network calls, forms and external resources. Supply +`PAPERCLIP_RUNNER_PROTOCOL_EVAL_VIEWER_DIR` to the publisher. The workflow sends +a viewer-only artifact to that job; raw attempts and provider secrets stay out. +After publication succeeds, the publishing job writes **Open this run's +Evalbook** and **All eval runs** links to the Actions summary. Its deployment +URL also points to the exact immutable report, not to the downloadable ZIP. +Failed publication does not advertise a successful deployment. + +Before uploading, the report job runs the actual built application in Chromium +against representative passing, failing and missing-recording pages in both +reports. It checks initial rendering, tool expansion, read-only controls, +navigation, reload, a narrow viewport and the public no-API-request boundary. +In full-evidence reports, a tool's **View in Evidence** link selects the Evidence +tab and highlights its record, including when reopening that same record after +switching tabs. The browser check exercises this cross-link too. +Run the same check locally with +`node packages/paperclip-runner/scripts/verify-runner-evalbook-viewer.mjs --report-root /path/to/report`. +Add `--screenshots /path/to/proof` for visual evidence. Public screenshots are +retained in the aggregate Actions artifact, not added to the public report's +closed file allowlist. + +These screenshots are generated acceptance-test evidence from the scrubbed +completed campaign, not a separate report design: + +![Passing chat replay with expanded tool evidence](images/evalbook-chat/passed.png) +![Failed chat replay with visible assertions](images/evalbook-chat/failed.png) +![Missing recording shown in the same viewer](images/evalbook-chat/missing-recording.png) + +### Refresh a completed report without calling models + +Download the aggregate Actions artifact, then: + +```sh +node packages/paperclip-runner/scripts/refresh-runner-protocol-eval-report.mjs \ + --source /path/to/downloaded-aggregate \ + --evals-root /path/to/paperclip-evals \ + --viewer-root packages/paperclip-runner/dist-issue-thread \ + --output /path/to/new-refresh-directory \ + --revision chat-v1 +``` + +Publish the returned `reportRoot` with the normal history publisher. The new ID +is `gha-RUN-ATTEMPT-report-chat-v1`. Original reports remain immutable; history +adds a labeled refresh and retains the source campaign, original measurement +timestamp, renderer digest, and `providerCalls: 0`. Scores and evaluated source +revisions do not change. This is not a new model qualification run. Future live +runs create chat reports automatically. +Refreshes are ordered by their render time in the history list, but keep the +original measurement timestamp and never replace the latest or latest-green +qualification identities. The HTML history links each measurement to its newest +presentation, while retaining every original bundle and listing refreshes separately. +Refreshes also recover retry-inclusive estimated and provider-list costs from the +retained attempt records. They never add another model-cost measurement. + This is the provider-backed, one-turn protocol qualification layer in `paperclipai/paperclip-evals/evals/paperclip-runner`. It is intentionally separate from both the browser full-stack model E2E and the stress-derived @@ -149,11 +228,11 @@ read-only Runner issue-thread attempt pages, and raw immutable run records. Public publishing uses a separate projection and a separate trusted OIDC job. The projection retains model/config identity, status, usage totals, and check -outcomes but removes provider session identifiers, transcripts, semantic-tool +outcomes and scrubbed mock conversation, but removes provider session identifiers, semantic-tool payloads, state revisions, traces, remote profile identities, and raw failure text. The same Evalbook `report` command renders that projection, so the public -grid and test pages have the standard Evalbook layout. The publisher rejects -scripts, remote resources, symlinks, unknown paths, broken links, raw session +grid, test pages and chat viewer have the standard Evalbook layout. The publisher rejects +untrusted scripts, remote resources, symlinks, unknown paths, broken links, raw session fields, and credential-shaped values. S3 publication is additive: @@ -178,9 +257,27 @@ runner-protocol-evals/ Campaign files use immutable cache headers and a digest manifest. Reusing a campaign ID with different bytes fails closed. Only the root history and pointer files are mutable, and the publisher never deletes objects. The root -history retains at most 200 records, reserving one record for the latest green -campaign when it would otherwise fall outside that window so its pointer stays -valid. +history retains **all** run records; it no longer drops entries after 200 campaigns. + +The history index includes: + +- Pass-rate and cost timelines, grouped by identical cell/model/driver membership + and eval-suite SHA. Different suites and subsets cannot silently share a baseline. +- Regression and recovery lists against the previous matching run, linking to the + affected tests. Infrastructure failures stay distinct from behavior failures. +- Estimated cost and provider-reported list cost, shown separately, never added. + New campaigns include all retained attempts (including retries). Backfilled old + campaigns with only final-cell usage are labeled **historical final attempts only**. + Missing usage is unknown, not zero; partial totals use `≥` and display coverage. +- Exact Paperclip and eval-suite commit links (full SHA on hover), the source ref, + and the GitHub Actions run. These identify the code **evaluated**, not merely + the commit used to render an old report. +- A separate report-refresh list, excluded from trend points and model-spend totals. + +`history.json` stores a versioned, derived `analytics` projection separately from +immutable campaign records. The publisher backfills missing analytics from each +original `campaign.json`; a refresh may enrich costs using retained raw attempts +only when its source metadata and scores still match the original record. The publishing job uses dedicated `RUNNER_PROTOCOL_EVAL_HISTORY_*` variables when present and falls back to the existing Runner E2E history role, region, diff --git a/packages/paperclip-runner/package.json b/packages/paperclip-runner/package.json index a133b72024..030fc51505 100644 --- a/packages/paperclip-runner/package.json +++ b/packages/paperclip-runner/package.json @@ -115,7 +115,7 @@ "test:capability-evals": "vitest run src/conformance/capability-eval-suite.test.ts", "test:eval-slice": "pnpm run ensure:eval-build-deps && vitest run src/eval", "test:runner-workflow-evals": "pnpm run ensure:eval-build-deps && node --test scripts/render-runner-workflow-evalbook.test.mjs && vitest run src/eval/workflow-evals.test.ts src/eval/live-workflow-executor.test.ts", - "test:runner-protocol-eval-publish": "node --test scripts/runner-protocol-eval-campaign.test.mjs scripts/publish-runner-protocol-eval-history.test.mjs scripts/runner-protocol-eval-workflow-security.test.mjs", + "test:runner-protocol-eval-publish": "node --test scripts/runner-protocol-eval-campaign.test.mjs scripts/publish-runner-protocol-eval-history.test.mjs scripts/runner-protocol-eval-metrics.test.mjs scripts/runner-protocol-eval-workflow-security.test.mjs scripts/public-eval-chat.test.mjs", "check:runner-workflow-traceability": "pnpm run build:typescript && node scripts/check-runner-workflow-traceability.mjs", "report:capability-evals": "pnpm run build:typescript && node scripts/run-capability-eval-suite.mjs", "report:capability-live-evals": "pnpm run build:typescript && cargo build --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --bin paperclip-runnerd && node scripts/run-capability-live-eval-matrix.mjs", diff --git a/packages/paperclip-runner/scripts/public-eval-chat.mjs b/packages/paperclip-runner/scripts/public-eval-chat.mjs new file mode 100644 index 0000000000..d20f998020 --- /dev/null +++ b/packages/paperclip-runner/scripts/public-eval-chat.mjs @@ -0,0 +1,217 @@ +// Public replay is a new DTO, never a recursive copy of a provider artifact. +export const PUBLIC_CHAT_SCHEMA = + "paperclip.runner-protocol-eval.public-chat/v1"; +export const PUBLIC_CHAT_NOTICE = + "Public replay of an isolated mock eval. Conversation text is scrubbed; provider identities, tool payloads, traces, and company-state snapshots are withheld. Full evidence remains in the access-controlled Actions artifact."; + +export const SECRET_TEXT = [ + /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/gu, + /\bsk-[A-Za-z0-9_-]{16,}\b/gu, + /\b(?:gh[pousr]_|github_pat_)[A-Za-z0-9_]{16,}\b/gu, + /\bBearer\s+[A-Za-z0-9._~+\/-]{8,}=*/giu, + /-----BEGIN [^-]*PRIVATE KEY-----[\s\S]*?-----END [^-]*PRIVATE KEY-----/gu, + /\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/gu, + /\b(?:https?|file|s3):\/\/[^\s<>"')]+/giu, + /\barn:aws[^\s<>"')]+/gu, + /(?:\/(?:Users|home|tmp|private|var)\/|[A-Z]:\\)[^\s<>"')]+/gu, + /\b(?:api[_-]?key|access[_-]?token|secret|password|authorization|cookie)\s*[=:]\s*[^\s,;]+/giu, +]; + +export function publicText(value, privateValues = []) { + let text = typeof value === "string" ? value : ""; + for (const secret of privateValues) { + if (typeof secret === "string" && secret.length >= 8) + text = text.replaceAll(secret, "[redacted]"); + } + for (const pattern of SECRET_TEXT) text = text.replace(pattern, "[redacted]"); + return text.length > 40_000 ? `${text.slice(0, 40_000)}\n[truncated]` : text; +} + +function privateIdentities(value, found = new Set()) { + if (value && typeof value === "object") { + for (const [key, child] of Object.entries(value)) { + if ( + /(?:session|profile|account|runtime|endpoint|memory|agentversion).*id$|arn$|token$|secret$|password$/i.test( + key, + ) && + typeof child === "string" + ) + found.add(child); + else if (child && typeof child === "object") + privateIdentities(child, found); + } + } + return [...found]; +} + +function timestamp(value) { + return typeof value === "string" && /^\d{4}-\d\d-\d\dT[\d:.]+Z$/.test(value) + ? value + : "1970-01-01T00:00:00.000Z"; +} + +function operation(value) { + return typeof value === "string" && /^[a-z][a-z_]{0,79}$/.test(value) + ? value + : "unknown_operation"; +} + +export function publicChatView(artifact, evalCase) { + const privateValues = privateIdentities(artifact); + const scrub = (value) => publicText(value, privateValues); + const network = artifact.snapshot?.networkEvidence; + // Only the dedicated mock eval boundary can publish recorded conversation. + // Early infrastructure failures still receive a viewer with an honest notice. + const isolated = + network?.realPaperclipRequests === 0 && + Array.isArray(network?.childPaperclipEnvironmentKeys) && + network.childPaperclipEnvironmentKeys.length === 0; + const source = + isolated && + artifact.issueThread?.schema === "paperclip.capability.issue-thread-view.v1" + ? artifact.issueThread + : null; + const evidence = Object.fromEntries( + [ + "tools", + "calls", + "authorization", + "control_plane", + "runner", + "state", + "traceability", + "parity", + ].map((key) => [key, []]), + ); + const turns = (source?.turns ?? []).map((turn, turnIndex) => { + const turnId = `public-turn-${turnIndex + 1}`; + const items = []; + for (const item of turn.items ?? []) { + const base = { + id: `public-item-${turnIndex + 1}-${items.length + 1}`, + at: timestamp(item.at), + }; + if ( + ["user_message", "agent_message", "durable_comment"].includes(item.kind) + ) { + items.push({ + ...base, + kind: item.kind === "user_message" ? "user_message" : "agent_message", + author: item.kind === "user_message" ? "You (eval prompt)" : "Agent", + body: scrub(item.body), + streaming: false, + }); + } else if (item.kind === "tool_activity") { + const operationId = operation(item.operationId); + const status = ["ok", "denied", "running"].includes(item.status) + ? item.status + : "running"; + const result = { + outcome: status, + detail: "Tool payload withheld from public replay.", + }; + const recordId = `public-call-${turnIndex + 1}-${items.length + 1}`; + items.push({ + ...base, + kind: "tool_activity", + operationId, + status, + summary: `${operationId}: ${status}`, + input: { detail: "Arguments withheld from public replay." }, + result, + evidenceRef: { section: "calls", recordId }, + }); + if (status !== "running") + evidence.calls.push({ + id: recordId, + turnId, + operationId, + version: 1, + providerRequest: operationId, + dispatchedCommand: operationId, + outcome: status, + result, + redactions: ["arguments", "result payload", "provider identities"], + threadAnchorId: base.id, + }); + } + // Provider activity, reasoning, raw events, file refs and unrecognized + // future item kinds are deliberately not part of the public contract. + } + return { + id: turnId, + ordinal: turnIndex + 1, + mode: "replay", + toolCallCount: items.filter((item) => item.kind === "tool_activity") + .length, + at: timestamp(turn.at), + stoppedByUser: turn.stoppedByUser === true, + items, + }; + }); + if (!turns.some((turn) => turn.items.length > 0)) { + turns.length = 0; + turns.push({ + id: "public-turn-1", + ordinal: 1, + mode: "replay", + toolCallCount: 0, + at: timestamp(artifact.snapshot?.createdAt), + stoppedByUser: false, + items: [ + { + id: "public-notice", + at: timestamp(artifact.snapshot?.createdAt), + kind: "system_notice", + glyph: "–", + text: "No publishable conversation was recorded for this attempt. See the checks and the access-controlled artifact for diagnostics.", + evidenceRef: { section: "runner", recordId: "public-notice" }, + }, + ], + }); + } + return { + schema: "paperclip.capability.issue-thread-view.v1", + sessionId: "public-report", + mode: "replay", + identity: { + agentLabel: "Recorded agent", + runnerLabel: "Recorded runner", + runnerAttached: false, + controlPlaneLabel: "Mock Paperclip", + controlPlaneTooltip: PUBLIC_CHAT_NOTICE, + replaySource: "live", + }, + issue: { + identifier: "EVAL", + title: scrub(evalCase.title || evalCase.id), + status: [ + "backlog", + "todo", + "in_progress", + "in_review", + "done", + "blocked", + "cancelled", + ].includes(source?.issue?.status) + ? source.issue.status + : "in_review", + priority: "medium", + assignee: null, + runState: "Read-only public replay", + scenarioId: evalCase.id, + fixtureProfile: evalCase.id, + }, + turns, + composer: { + state: "disabled", + helper: null, + reason: "Read-only eval report", + pendingInteractionId: null, + }, + evidence, + connection: { state: "closed", attempt: 0 }, + replay: null, + renderedAt: timestamp(source?.renderedAt || artifact.snapshot?.createdAt), + }; +} diff --git a/packages/paperclip-runner/scripts/public-eval-chat.test.mjs b/packages/paperclip-runner/scripts/public-eval-chat.test.mjs new file mode 100644 index 0000000000..96f8c02fbc --- /dev/null +++ b/packages/paperclip-runner/scripts/public-eval-chat.test.mjs @@ -0,0 +1,273 @@ +import assert from "node:assert/strict"; +import { mkdtemp, mkdir, writeFile, cp, rm, symlink } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { + publicChatView, + publicText, + PUBLIC_CHAT_SCHEMA, + PUBLIC_CHAT_NOTICE, +} from "./public-eval-chat.mjs"; +import { + publicViewerShell, + validatePublicChatPayload, + trustedViewerFiles, +} from "./public-eval-viewer.mjs"; +import { validatePublicProtocolEvalReport } from "./publish-runner-protocol-eval-history.mjs"; + +function artifact() { + return { + providerSessionId: "private-session-canary", + snapshot: { + networkEvidence: { + realPaperclipRequests: 0, + childPaperclipEnvironmentKeys: [], + }, + }, + issueThread: { + schema: "paperclip.capability.issue-thread-view.v1", + issue: { status: "blocked" }, + turns: [ + { + items: [ + { kind: "user_message", body: "Please block this task." }, + { + kind: "agent_message", + body: "Blocked. private-session-canary https://private.example/path", + privateField: "private-field-canary", + }, + { + kind: "tool_activity", + operationId: "block_task", + status: "ok", + input: { secret: "argument-canary" }, + result: { token: "result-canary" }, + }, + { kind: "thinking", body: "reasoning-canary" }, + { kind: "future_kind", body: "future-canary" }, + ], + }, + ], + }, + }; +} + +function payload() { + return { + attemptId: "attempt-01", + caseId: "block-task", + disposition: "pass", + passed: true, + checks: [], + publication: { schema: PUBLIC_CHAT_SCHEMA, notice: PUBLIC_CHAT_NOTICE }, + view: publicChatView(artifact(), { id: "block-task" }), + devtools: null, + navigation: { suiteHref: "../../index.html", previous: null, next: null }, + run: { + model: "test", + provider: "test", + sessionId: "public-report", + effectiveModelHistory: [], + managedProfile: null, + acpxProfile: null, + usage: null, + }, + }; +} + +test("projects only isolated recorded messages and bounded tool facts", () => { + const view = publicChatView(artifact(), { id: "block-task" }); + assert.equal(view.issue.status, "blocked"); + assert.deepEqual( + view.turns[0].items.map((item) => item.kind), + ["user_message", "agent_message", "tool_activity"], + ); + assert.match(view.turns[0].items[1].body, /Blocked/); + assert.doesNotMatch(JSON.stringify(view), /canary|private\.example/); + assert.deepEqual(view.turns[0].items[2].input, { + detail: "Arguments withheld from public replay.", + }); + validatePublicChatPayload(payload()); + for (const networkEvidence of [ + undefined, + { realPaperclipRequests: 1, childPaperclipEnvironmentKeys: [] }, + { + realPaperclipRequests: 0, + childPaperclipEnvironmentKeys: ["PAPERCLIP_API_KEY"], + }, + ]) { + const source = artifact(); + source.snapshot.networkEvidence = networkEvidence; + const unavailable = publicChatView(source, { id: "block-task" }); + assert.equal(unavailable.turns[0].items[0].kind, "system_notice"); + assert.doesNotMatch(JSON.stringify(unavailable), /Please block|Blocked\./); + } +}); + +test("scrubs credentials and private references before truncating text", () => { + for (const secret of [ + "sk-" + "a".repeat(32), + "ghp_" + "b".repeat(32), + "Bearer abcdef123456", + "password=secret-canary", + "/Users/someone/private.txt", + "arn:aws:service:region:account:resource", + "-----BEGIN PRIVATE KEY-----\n" + + "x".repeat(41_000) + + "\n-----END PRIVATE KEY-----", + ]) { + assert.equal(publicText(secret), "[redacted]"); + } + assert.match(publicText("a".repeat(41_000)), /\[truncated\]$/); +}); + +test("turns containing only withheld items still show the missing-recording notice", () => { + const source = artifact(); + source.issueThread.turns = [ + { items: [{ kind: "thinking", body: "private-reasoning" }] }, + { items: [] }, + ]; + const view = publicChatView(source, { id: "missing" }); + assert.equal(view.turns.length, 1); + assert.equal(view.turns[0].items[0].kind, "system_notice"); + assert.match(view.turns[0].items[0].text, /No publishable conversation/); +}); + +test("trusted viewer rejects index and assets symlinks outside its root", async () => { + const root = await mkdtemp(join(tmpdir(), "eval-viewer-symlinks-")); + try { + const viewer = join(root, "viewer"); + const outside = join(root, "outside"); + await mkdir(viewer); + await mkdir(outside); + await writeFile( + join(outside, "index.html"), + '', + ); + await mkdir(join(outside, "assets")); + await writeFile(join(outside, "assets/app.js"), "// outside canary"); + await symlink(join(outside, "index.html"), join(viewer, "index.html")); + await symlink(join(outside, "assets"), join(viewer, "assets")); + await assert.rejects(trustedViewerFiles(viewer), /symlinks/); + await rm(join(viewer, "index.html")); + await writeFile( + join(viewer, "index.html"), + '', + ); + await assert.rejects(trustedViewerFiles(viewer), /symlinks/); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("fails closed on unknown fields, raw tools, private evidence and identities", () => { + for (const mutate of [ + (p) => { + p.view.turns[0].items[0].extra = "unprojected"; + }, + (p) => { + p.view.turns[0].items[2].input = { password: "oops" }; + }, + (p) => { + p.view.evidence.calls[0].result.detail = "raw-result"; + }, + (p) => { + p.view.evidence.state.push({ secret: "raw-state" }); + }, + (p) => { + p.run.providerSessionId = "private-session"; + }, + (p) => { + p.view.turns[0].items[1].body = "sk-" + "x".repeat(30); + }, + (p) => { + p.devtools = {}; + }, + (p) => { + p.view.composer.state = "ready"; + }, + ]) { + const value = payload(); + mutate(value); + assert.throws(() => validatePublicChatPayload(value)); + } +}); + +test("publisher permits only the exact trusted shell/assets and valid local navigation", async () => { + const root = await mkdtemp(join(tmpdir(), "eval-chat-contract-")); + try { + const viewer = join(root, "trusted"); + const report = join(root, "report"); + await mkdir(join(viewer, "assets"), { recursive: true }); + await mkdir(join(report, "attempts/attempt-01"), { recursive: true }); + const index = + '
'; + await writeFile(join(viewer, "index.html"), index); + await writeFile(join(viewer, "assets/app.js"), "// trusted build"); + await writeFile(join(viewer, "assets/app.css"), ":root {}"); + await cp(join(viewer, "assets"), join(report, "viewer/assets"), { + recursive: true, + }); + await writeFile( + join(report, "index.html"), + 'PASS', + ); + await writeFile( + join(report, "campaign.json"), + JSON.stringify({ + schema: "paperclip.runner-protocol-eval.campaign/v1", + campaignId: "gha-42-1", + }), + ); + const page = join(report, "attempts/attempt-01/index.html"); + const writePayload = async (value) => + writeFile( + page, + publicViewerShell( + index, + JSON.stringify(value).replaceAll("<", "\\u003c"), + ), + ); + await writePayload(payload()); + await validatePublicProtocolEvalReport(report, { viewerRoot: viewer }); + await assert.rejects(validatePublicProtocolEvalReport(report)); + await writeFile( + join(report, "viewer/assets/app.js"), + "// substituted build", + ); + await assert.rejects( + validatePublicProtocolEvalReport(report, { viewerRoot: viewer }), + /trusted/, + ); + await writeFile(join(report, "viewer/assets/app.js"), "// trusted build"); + await writeFile( + page, + publicViewerShell(index, JSON.stringify(payload())) + + "", + ); + await assert.rejects( + validatePublicProtocolEvalReport(report, { viewerRoot: viewer }), + /trusted shell/, + ); + const escaped = payload(); + escaped.view.turns[0].items[0].body = + ''; + await writePayload(escaped); + await validatePublicProtocolEvalReport(report, { viewerRoot: viewer }); + const broken = payload(); + broken.navigation.next = { + label: "Next attempt", + href: "../missing/index.html", + }; + await writePayload(broken); + await assert.rejects( + validatePublicProtocolEvalReport(report, { viewerRoot: viewer }), + /link|reference/i, + ); + await symlink(viewer, join(root, "symlink")); + await assert.rejects(trustedViewerFiles(join(root, "symlink"))); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/packages/paperclip-runner/scripts/public-eval-viewer.mjs b/packages/paperclip-runner/scripts/public-eval-viewer.mjs new file mode 100644 index 0000000000..702f69862d --- /dev/null +++ b/packages/paperclip-runner/scripts/public-eval-viewer.mjs @@ -0,0 +1,234 @@ +import { readFile, readdir, lstat } from "node:fs/promises"; +import { join } from "node:path"; +import { PUBLIC_CHAT_SCHEMA, SECRET_TEXT } from "./public-eval-chat.mjs"; + +export const PUBLIC_VIEWER_CSP = + "default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; font-src 'self'; img-src 'none'; connect-src 'none'; form-action 'none'; base-uri 'none'"; +export const PUBLIC_VIEWER_DATA = + /\n