diff --git a/.github/workflows/runner-protocol-live-evals.yml b/.github/workflows/runner-protocol-live-evals.yml index f7ce6d5c39..e3eafa2da5 100644 --- a/.github/workflows/runner-protocol-live-evals.yml +++ b/.github/workflows/runner-protocol-live-evals.yml @@ -266,6 +266,7 @@ jobs: pnpm --filter @paperclipai/paperclip-runner build:issue-thread # Older target refs must fail before paid cells, not publish an empty viewer. grep -q 'paperclip-eval-report' packages/paperclip-runner/dist-issue-thread/assets/*.js + grep -q 'evalbook-site' packages/paperclip-runner/dist-issue-thread/assets/*.css - name: Package a portable provider runtime run: | @@ -678,6 +679,7 @@ jobs: id-token: write environment: name: runner-e2e-history + url: ${{ steps.publish.outputs.report_url }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: @@ -708,6 +710,7 @@ jobs: aws-region: ${{ vars.RUNNER_PROTOCOL_EVAL_HISTORY_AWS_REGION || vars.RUNNER_E2E_HISTORY_AWS_REGION }} - name: Publish versioned report and refresh the root index + id: publish env: PAPERCLIP_RUNNER_PROTOCOL_EVAL_PUBLIC_REPORT_DIR: ${{ github.workspace }}/runner-protocol-public-report PAPERCLIP_RUNNER_PROTOCOL_EVAL_VIEWER_DIR: ${{ github.workspace }}/runner-protocol-trusted-viewer diff --git a/packages/paperclip-runner/devtools/issue-thread/src/evalbook-site.css b/packages/paperclip-runner/devtools/issue-thread/src/evalbook-site.css new file mode 100644 index 0000000000..6567eafaaf --- /dev/null +++ b/packages/paperclip-runner/devtools/issue-thread/src/evalbook-site.css @@ -0,0 +1,64 @@ +/* Static Evalbook pages share the Runner Lab token/font layer and built asset. + Keep every selector scoped so report navigation cannot restyle live chat. */ +.evalbook-site { color-scheme: dark; } +.evalbook-site main { width: 100%; max-width: 112rem; margin: auto; padding: var(--pit-space-6); } +.evalbook-site a { color: var(--pit-live); text-underline-offset: var(--pit-space-1); } +.evalbook-site a:hover { color: var(--pit-foreground); } +.evalbook-site :is(a, summary, [tabindex]):focus-visible { outline: 2px solid var(--pit-ring); outline-offset: 3px; } +.evalbook-site .top { display: flex; justify-content: space-between; align-items: center; gap: var(--pit-space-4); padding-bottom: var(--pit-space-4); border-bottom: 1px solid var(--pit-border); } +.evalbook-site .top > a { color: var(--pit-foreground); font-weight: 700; text-decoration: none; } +.evalbook-site h1 { margin: var(--pit-space-6) 0 var(--pit-space-3); font-size: var(--pit-text-xl); line-height: var(--pit-leading-tight); overflow-wrap: anywhere; } +.evalbook-site h2 { margin: var(--pit-space-6) 0 var(--pit-space-3); font-size: var(--pit-text-base); line-height: var(--pit-leading-tight); } +.evalbook-site p { margin: var(--pit-space-3) 0; } +.evalbook-site .badge { padding: var(--pit-space-1) var(--pit-space-2); color: var(--pit-live); background: var(--pit-live-surface); border: 1px solid var(--pit-live); border-radius: var(--pit-radius-sm); font-size: var(--pit-text-xs); white-space: nowrap; } +.evalbook-site :is(.tabs, .pointers, .breadcrumbs) { display: flex; align-items: center; gap: var(--pit-space-4); margin: var(--pit-space-4) 0; overflow-x: auto; } +.evalbook-site .tabs { border-bottom: 1px solid var(--pit-border); gap: 0; } +.evalbook-site .tabs a { flex: 0 0 auto; padding: var(--pit-space-3) var(--pit-space-4); color: var(--pit-muted-foreground); text-decoration: none; border-bottom: 2px solid transparent; } +.evalbook-site .tabs a:hover { background: var(--pit-surface); color: var(--pit-foreground); } +.evalbook-site .tabs a.active { background: var(--pit-accent-surface); border-color: var(--pit-accent); color: var(--pit-foreground); } +.evalbook-site .breadcrumbs span { color: var(--pit-muted-foreground); } +.evalbook-site .spend-summary { display: flex; flex-wrap: wrap; gap: var(--pit-space-6); margin: var(--pit-space-5) 0; } +.evalbook-site .spend-summary > div { display: flex; flex-direction: column; gap: var(--pit-space-1); } +.evalbook-site .spend-summary span { color: var(--pit-muted-foreground); font-size: var(--pit-text-xs); } +.evalbook-site .spend-summary strong { font-size: var(--pit-text-base); font-variant-numeric: tabular-nums; } +.evalbook-site .lede { color: var(--pit-muted-foreground); margin-bottom: var(--pit-space-5); } +.evalbook-site :is(.card, .message, .tool, .check) { margin: var(--pit-space-3) 0; padding: var(--pit-space-4); background: var(--pit-surface); border: 1px solid var(--pit-border); border-radius: var(--pit-radius-sm); overflow-wrap: anywhere; } +.evalbook-site .pass { border-left: 3px solid var(--pit-success); } +.evalbook-site .fail { border-left: 3px solid var(--pit-danger); } +.evalbook-site .infra { border-left: 3px solid var(--pit-warning); } +.evalbook-site .grid { display: grid; grid-template-columns: minmax(0, 1fr) minmax(0, 1.3fr); gap: var(--pit-space-6); } +.evalbook-site .grid > section { min-width: 0; } +.evalbook-site :is(.table-shell, .table) { width: 100%; overflow: auto; border: 1px solid var(--pit-border); border-radius: var(--pit-radius-sm); background: var(--pit-surface-sunken); overscroll-behavior-inline: contain; scrollbar-color: var(--pit-border-strong) transparent; } +.evalbook-site table { width: 100%; border-collapse: separate; border-spacing: 0; font-size: var(--pit-text-xs); } +.evalbook-site :is(th, td) { padding: var(--pit-space-3); border-bottom: 1px solid var(--pit-border); text-align: left; vertical-align: top; } +.evalbook-site thead th { background: var(--pit-surface); color: var(--pit-muted-foreground); font-weight: 600; } +.evalbook-site tbody tr:last-child > :is(th, td) { border-bottom: 0; } +.evalbook-site tbody tr:hover > :is(th, td) { background: var(--pit-surface-raised); } +.evalbook-site .matrix { min-width: max(100%, var(--matrix-min-width, 1500px)); } +.evalbook-site .matrix :is(th, td):not(:first-child) { width: 10rem; text-align: center; } +.evalbook-site .matrix thead th { position: sticky; top: 0; z-index: 2; } +.evalbook-site .matrix thead th > div { color: var(--pit-foreground); font-size: var(--pit-text-sm); overflow-wrap: anywhere; } +.evalbook-site .matrix thead small { display: block; max-width: 11rem; margin: var(--pit-space-1) auto 0; font-size: var(--pit-text-xs); font-weight: 400; overflow-wrap: anywhere; } +.evalbook-site .matrix th:first-child { position: sticky; left: 0; z-index: 1; width: 18rem; min-width: 14rem; background: var(--pit-surface-sunken); } +.evalbook-site .matrix thead th:first-child { z-index: 3; background: var(--pit-surface); } +.evalbook-site .matrix tbody th:first-child a { display: block; color: var(--pit-foreground); font-weight: 500; overflow-wrap: anywhere; text-decoration: none; } +.evalbook-site .matrix tbody th:first-child a:hover { color: var(--pit-live); text-decoration: underline; } +.evalbook-site .result { display: flex; align-items: center; justify-content: center; width: 100%; max-width: 10rem; min-height: 2rem; margin-inline: auto; padding: var(--pit-space-1) var(--pit-space-2); border: 1px solid currentColor; border-radius: var(--pit-radius-sm); font-size: var(--pit-text-xs); font-weight: 700; text-decoration: none; } +.evalbook-site :is(.result-pass, .passed) { background: var(--pit-success-surface); color: var(--pit-success); } +.evalbook-site :is(.result-fail, .failed) { background: var(--pit-danger-surface); color: var(--pit-danger); } +.evalbook-site .result-infra { background: var(--pit-warning-surface); color: var(--pit-warning); } +.evalbook-site :is(.muted, .empty, small, footer) { color: var(--pit-muted-foreground); } +.evalbook-site .empty { text-align: center; } +.evalbook-site code { font-family: var(--pit-font-mono); font-size: var(--pit-text-xs); overflow-wrap: anywhere; } +.evalbook-site pre { max-height: 32rem; padding: var(--pit-space-3); overflow: auto; white-space: pre-wrap; overflow-wrap: anywhere; background: var(--pit-surface-sunken); color: var(--pit-foreground); border: 1px solid var(--pit-border); border-radius: var(--pit-radius-sm); font-family: var(--pit-font-mono); font-size: var(--pit-text-xs); } +.evalbook-site details { margin: var(--pit-space-4) 0; } +.evalbook-site summary { cursor: pointer; color: var(--pit-muted-foreground); } +.evalbook-site .status { display: inline-block; padding: var(--pit-space-1) var(--pit-space-2); border-radius: var(--pit-radius-sm); font-weight: 600; text-transform: uppercase; } +.evalbook-site small { display: block; margin-top: var(--pit-space-1); } +.evalbook-site footer { margin-top: var(--pit-space-5); font-size: var(--pit-text-xs); } +@media (max-width: 760px) { + .evalbook-site main { padding: var(--pit-space-4); } + .evalbook-site .grid { grid-template-columns: 1fr; } + .evalbook-site .matrix th:first-child { width: 12rem; min-width: 10rem; } + .evalbook-site .badge { white-space: normal; text-align: center; } +} diff --git a/packages/paperclip-runner/devtools/issue-thread/src/main.tsx b/packages/paperclip-runner/devtools/issue-thread/src/main.tsx index e524b52fd4..ebe93e307a 100644 --- a/packages/paperclip-runner/devtools/issue-thread/src/main.tsx +++ b/packages/paperclip-runner/devtools/issue-thread/src/main.tsx @@ -3,6 +3,7 @@ import { createRoot } from "react-dom/client"; import { App } from "./App"; import "./issue-thread.css"; +import "./evalbook-site.css"; // Hosted Evalbooks use inert JSON plus the same trusted viewer bundle. No // inline executable script or network fetch is needed to load an attempt. diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index 9ff718f324..9585aaea57 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -4,6 +4,12 @@ Every new report uses the canonical Evalbook grid and the existing Runner Lab chat viewer for attempt drill-downs. There is no plain-HTML attempt fallback. +The grid, Latest, test-design, inventory and server-gate pages load the same +built stylesheet and fonts as the chat viewer. The long-term S3 history index +references that stylesheet inside an immutable campaign, so every page keeps +the same dark theme. Static site styles live in +`devtools/issue-thread/src/evalbook-site.css`, scoped to `.evalbook-site`; +colors and typography come from the Runner Lab token layer. Missing recordings show a notice in the same viewer; missing viewer builds fail generation. Build with `pnpm --filter @paperclipai/paperclip-runner build:issue-thread` and provide @@ -22,6 +28,10 @@ checks the public payload contract and local links, and rejects other scripts. The CSP prohibits network calls, forms and external resources. Supply `PAPERCLIP_RUNNER_PROTOCOL_EVAL_VIEWER_DIR` to the publisher. The workflow sends a viewer-only artifact to that job; raw attempts and provider secrets stay out. +After publication succeeds, the publishing job writes **Open this run's +Evalbook** and **All eval runs** links to the Actions summary. Its deployment +URL also points to the exact immutable report, not to the downloadable ZIP. +Failed publication does not advertise a successful deployment. Before uploading, the report job runs the actual built application in Chromium against representative passing, failing and missing-recording pages in both diff --git a/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.mjs b/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.mjs index 067c23b996..8e16d10639 100644 --- a/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.mjs +++ b/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.mjs @@ -2,6 +2,7 @@ import { createHash } from "node:crypto"; import { execFile } from "node:child_process"; import { + appendFile, lstat, mkdtemp, mkdir, @@ -390,7 +391,9 @@ function date(value) { }).format(new Date(value)); } -export function renderProtocolEvalHistoryIndex(history) { +export function renderProtocolEvalHistoryIndex(history, stylesheetHref) { + if (!/^campaigns\/gha-[a-z0-9-]+\/viewer\/assets\/[A-Za-z0-9._-]+\.css$/.test(stylesheetHref ?? "")) + throw new Error("History requires an immutable campaign's Runner Lab stylesheet"); const rows = history.campaigns.length ? history.campaigns .map((campaign) => { @@ -413,9 +416,9 @@ export function renderProtocolEvalHistoryIndex(history) { (campaign) => campaign.campaignId === history.latestGreenCampaignId, ); return ` -Runner protocol eval campaigns · Paperclip - -
Paperclip quality engineering

Runner protocol eval campaigns

Versioned direct live-runner Evalbook reports. Full provider transcripts, session identifiers, state, and raw tool evidence remain in access-controlled workflow artifacts.

+Runner protocol eval campaigns · Paperclip + +
paperclip-runner evalsRun history

Runner protocol eval campaigns

Versioned live-runner reports. Full provider evidence remains in access-controlled workflow artifacts.

${rows}
CampaignStatusCellsModels / rostersSource
Updated ${html(date(history.updatedAt))} UTC · Immutable campaign bundles · Canonical Evalbook layout with public-safe evidence projections
`; } @@ -504,6 +507,11 @@ export async function publishProtocolEvalHistory({ const { campaign } = await validatePublicProtocolEvalReport(reportRoot, { viewerRoot, }); + const viewer = await trustedViewerFiles(viewerRoot); + const stylesheet = viewer.index.match(//)?.[1]; + if (!stylesheet || !viewer.files.get(`viewer/${stylesheet}`)?.includes(".evalbook-site")) + throw new Error("Published history requires the same-run Runner Lab site theme"); + const stylesheetHref = `campaigns/${campaign.campaignId}/viewer/${stylesheet}`; const manifest = await createProtocolEvalBundleManifest( reportRoot, campaign.campaignId, @@ -568,7 +576,7 @@ export async function publishProtocolEvalHistory({ ); } const index = join(temporary, "index.html"); - await writeFile(index, renderProtocolEvalHistoryIndex(history)); + await writeFile(index, renderProtocolEvalHistoryIndex(history, stylesheetHref)); await uploadFile( validatedDestination.bucket, `${validatedDestination.prefix}/index.html`, @@ -579,9 +587,28 @@ export async function publishProtocolEvalHistory({ campaignId: campaign.campaignId, bundleDigest: manifest.bundleDigest, historySize: history.campaigns.length, + reportUrl: `${validatedDestination.publicBaseUrl}/${campaignPrefix}/index.html`, + historyUrl: `${validatedDestination.publicBaseUrl}/${validatedDestination.prefix}/index.html`, }; } +export async function writeProtocolEvalPublicationLinks(result, environment = process.env) { + const { campaignId, reportUrl, historyUrl } = result; + if (!SAFE_CAMPAIGN.test(campaignId)) throw new Error("Invalid published campaign ID"); + const safeUrl = (value) => { + const url = new URL(value); + if (url.protocol !== "https:" || url.username || url.password || /[\r\n<>]/.test(value)) + throw new Error("Invalid published report URL"); + return url.href; + }; + const report = safeUrl(reportUrl); + const history = safeUrl(historyUrl); + if (environment.GITHUB_OUTPUT) + await appendFile(environment.GITHUB_OUTPUT, `report_url=${report}\nhistory_url=${history}\n`); + if (environment.GITHUB_STEP_SUMMARY) + await appendFile(environment.GITHUB_STEP_SUMMARY, `## Published Runner Evalbook\n\n[Open this run's Evalbook](<${report}>) · [All eval runs](<${history}>)\n\nCampaign: \`${campaignId}\`\n\nPublic replay uses the Runner Lab theme; full evidence is in the workflow artifact.\n`); +} + async function main() { const result = await publishProtocolEvalHistory({ viewerRoot: process.env.PAPERCLIP_RUNNER_PROTOCOL_EVAL_VIEWER_DIR, @@ -598,9 +625,11 @@ async function main() { process.env.RUNNER_PROTOCOL_EVAL_HISTORY_PUBLIC_BASE_URL ?? "", }, }); + await writeProtocolEvalPublicationLinks(result); console.log( `Published immutable protocol eval campaign ${result.campaignId} (${result.bundleDigest}) and ${result.historySize} history record(s)`, ); + console.log(`Evalbook: ${result.reportUrl}\nRun history: ${result.historyUrl}`); } if ( diff --git a/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.test.mjs b/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.test.mjs index ee919d09cb..7856f4742e 100644 --- a/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.test.mjs +++ b/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.test.mjs @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import test from "node:test"; @@ -14,9 +14,28 @@ import { renderProtocolEvalHistoryIndex, validateProtocolEvalHistoryDestination, validatePublicProtocolEvalReport, + writeProtocolEvalPublicationLinks, } from "./publish-runner-protocol-eval-history.mjs"; const roots = []; +test("successful publication exposes exact report and history links to Actions", async () => { + const root = await mkdtemp(join(tmpdir(), "evalbook-publication-links-")); + roots.push(root); + const environment = { GITHUB_OUTPUT: join(root, "outputs"), GITHUB_STEP_SUMMARY: join(root, "summary") }; + const result = { + campaignId: "gha-42-2", + reportUrl: "https://reports.example/runner-protocol-evals/campaigns/gha-42-2/index.html", + historyUrl: "https://reports.example/runner-protocol-evals/index.html", + }; + await writeProtocolEvalPublicationLinks(result, environment); + assert.equal(await readFile(environment.GITHUB_OUTPUT, "utf8"), `report_url=${result.reportUrl}\nhistory_url=${result.historyUrl}\n`); + const summary = await readFile(environment.GITHUB_STEP_SUMMARY, "utf8"); + assert.ok(summary.includes(`[Open this run's Evalbook](<${result.reportUrl}>)`)); + assert.ok(summary.includes(`[All eval runs](<${result.historyUrl}>)`)); + await assert.rejects(writeProtocolEvalPublicationLinks({ ...result, reportUrl: "https://example.test/\nreport_url=bad" }, environment)); + assert.throws(() => renderProtocolEvalHistoryIndex(emptyProtocolEvalHistory(), "https://untrusted.example/style.css")); +}); + test.afterEach(async () => { await Promise.all( roots.splice(0).map((root) => rm(root, { recursive: true, force: true })), @@ -178,7 +197,10 @@ test("retains immutable history and independent latest-green pointers", () => { const pointers = buildProtocolEvalPointers(history); assert.equal(pointers.latest.campaign.campaignId, "gha-43-1"); assert.equal(pointers.latestGreen.campaign.campaignId, "gha-42-1"); - const index = renderProtocolEvalHistoryIndex(history); + const index = renderProtocolEvalHistoryIndex(history, "campaigns/gha-43-1/viewer/assets/index.css"); + assert.match(index, /class="evalbook-site"/); + assert.match(index, /href="campaigns\/gha-43-1\/viewer\/assets\/index.css"/); + assert.doesNotMatch(index, /