fix(evals): unify Evalbook site theme and link published runs from Actions
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Dotta
parent
99697c2c51
commit
bdd5b51cae
|
|
@ -266,6 +266,7 @@ jobs:
|
|||
pnpm --filter @paperclipai/paperclip-runner build:issue-thread
|
||||
# Older target refs must fail before paid cells, not publish an empty viewer.
|
||||
grep -q 'paperclip-eval-report' packages/paperclip-runner/dist-issue-thread/assets/*.js
|
||||
grep -q 'evalbook-site' packages/paperclip-runner/dist-issue-thread/assets/*.css
|
||||
|
||||
- name: Package a portable provider runtime
|
||||
run: |
|
||||
|
|
@ -678,6 +679,7 @@ jobs:
|
|||
id-token: write
|
||||
environment:
|
||||
name: runner-e2e-history
|
||||
url: ${{ steps.publish.outputs.report_url }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
|
|
@ -708,6 +710,7 @@ jobs:
|
|||
aws-region: ${{ vars.RUNNER_PROTOCOL_EVAL_HISTORY_AWS_REGION || vars.RUNNER_E2E_HISTORY_AWS_REGION }}
|
||||
|
||||
- name: Publish versioned report and refresh the root index
|
||||
id: publish
|
||||
env:
|
||||
PAPERCLIP_RUNNER_PROTOCOL_EVAL_PUBLIC_REPORT_DIR: ${{ github.workspace }}/runner-protocol-public-report
|
||||
PAPERCLIP_RUNNER_PROTOCOL_EVAL_VIEWER_DIR: ${{ github.workspace }}/runner-protocol-trusted-viewer
|
||||
|
|
|
|||
|
|
@ -0,0 +1,64 @@
|
|||
/* Static Evalbook pages share the Runner Lab token/font layer and built asset.
|
||||
Keep every selector scoped so report navigation cannot restyle live chat. */
|
||||
.evalbook-site { color-scheme: dark; }
|
||||
.evalbook-site main { width: 100%; max-width: 112rem; margin: auto; padding: var(--pit-space-6); }
|
||||
.evalbook-site a { color: var(--pit-live); text-underline-offset: var(--pit-space-1); }
|
||||
.evalbook-site a:hover { color: var(--pit-foreground); }
|
||||
.evalbook-site :is(a, summary, [tabindex]):focus-visible { outline: 2px solid var(--pit-ring); outline-offset: 3px; }
|
||||
.evalbook-site .top { display: flex; justify-content: space-between; align-items: center; gap: var(--pit-space-4); padding-bottom: var(--pit-space-4); border-bottom: 1px solid var(--pit-border); }
|
||||
.evalbook-site .top > a { color: var(--pit-foreground); font-weight: 700; text-decoration: none; }
|
||||
.evalbook-site h1 { margin: var(--pit-space-6) 0 var(--pit-space-3); font-size: var(--pit-text-xl); line-height: var(--pit-leading-tight); overflow-wrap: anywhere; }
|
||||
.evalbook-site h2 { margin: var(--pit-space-6) 0 var(--pit-space-3); font-size: var(--pit-text-base); line-height: var(--pit-leading-tight); }
|
||||
.evalbook-site p { margin: var(--pit-space-3) 0; }
|
||||
.evalbook-site .badge { padding: var(--pit-space-1) var(--pit-space-2); color: var(--pit-live); background: var(--pit-live-surface); border: 1px solid var(--pit-live); border-radius: var(--pit-radius-sm); font-size: var(--pit-text-xs); white-space: nowrap; }
|
||||
.evalbook-site :is(.tabs, .pointers, .breadcrumbs) { display: flex; align-items: center; gap: var(--pit-space-4); margin: var(--pit-space-4) 0; overflow-x: auto; }
|
||||
.evalbook-site .tabs { border-bottom: 1px solid var(--pit-border); gap: 0; }
|
||||
.evalbook-site .tabs a { flex: 0 0 auto; padding: var(--pit-space-3) var(--pit-space-4); color: var(--pit-muted-foreground); text-decoration: none; border-bottom: 2px solid transparent; }
|
||||
.evalbook-site .tabs a:hover { background: var(--pit-surface); color: var(--pit-foreground); }
|
||||
.evalbook-site .tabs a.active { background: var(--pit-accent-surface); border-color: var(--pit-accent); color: var(--pit-foreground); }
|
||||
.evalbook-site .breadcrumbs span { color: var(--pit-muted-foreground); }
|
||||
.evalbook-site .spend-summary { display: flex; flex-wrap: wrap; gap: var(--pit-space-6); margin: var(--pit-space-5) 0; }
|
||||
.evalbook-site .spend-summary > div { display: flex; flex-direction: column; gap: var(--pit-space-1); }
|
||||
.evalbook-site .spend-summary span { color: var(--pit-muted-foreground); font-size: var(--pit-text-xs); }
|
||||
.evalbook-site .spend-summary strong { font-size: var(--pit-text-base); font-variant-numeric: tabular-nums; }
|
||||
.evalbook-site .lede { color: var(--pit-muted-foreground); margin-bottom: var(--pit-space-5); }
|
||||
.evalbook-site :is(.card, .message, .tool, .check) { margin: var(--pit-space-3) 0; padding: var(--pit-space-4); background: var(--pit-surface); border: 1px solid var(--pit-border); border-radius: var(--pit-radius-sm); overflow-wrap: anywhere; }
|
||||
.evalbook-site .pass { border-left: 3px solid var(--pit-success); }
|
||||
.evalbook-site .fail { border-left: 3px solid var(--pit-danger); }
|
||||
.evalbook-site .infra { border-left: 3px solid var(--pit-warning); }
|
||||
.evalbook-site .grid { display: grid; grid-template-columns: minmax(0, 1fr) minmax(0, 1.3fr); gap: var(--pit-space-6); }
|
||||
.evalbook-site .grid > section { min-width: 0; }
|
||||
.evalbook-site :is(.table-shell, .table) { width: 100%; overflow: auto; border: 1px solid var(--pit-border); border-radius: var(--pit-radius-sm); background: var(--pit-surface-sunken); overscroll-behavior-inline: contain; scrollbar-color: var(--pit-border-strong) transparent; }
|
||||
.evalbook-site table { width: 100%; border-collapse: separate; border-spacing: 0; font-size: var(--pit-text-xs); }
|
||||
.evalbook-site :is(th, td) { padding: var(--pit-space-3); border-bottom: 1px solid var(--pit-border); text-align: left; vertical-align: top; }
|
||||
.evalbook-site thead th { background: var(--pit-surface); color: var(--pit-muted-foreground); font-weight: 600; }
|
||||
.evalbook-site tbody tr:last-child > :is(th, td) { border-bottom: 0; }
|
||||
.evalbook-site tbody tr:hover > :is(th, td) { background: var(--pit-surface-raised); }
|
||||
.evalbook-site .matrix { min-width: max(100%, var(--matrix-min-width, 1500px)); }
|
||||
.evalbook-site .matrix :is(th, td):not(:first-child) { width: 10rem; text-align: center; }
|
||||
.evalbook-site .matrix thead th { position: sticky; top: 0; z-index: 2; }
|
||||
.evalbook-site .matrix thead th > div { color: var(--pit-foreground); font-size: var(--pit-text-sm); overflow-wrap: anywhere; }
|
||||
.evalbook-site .matrix thead small { display: block; max-width: 11rem; margin: var(--pit-space-1) auto 0; font-size: var(--pit-text-xs); font-weight: 400; overflow-wrap: anywhere; }
|
||||
.evalbook-site .matrix th:first-child { position: sticky; left: 0; z-index: 1; width: 18rem; min-width: 14rem; background: var(--pit-surface-sunken); }
|
||||
.evalbook-site .matrix thead th:first-child { z-index: 3; background: var(--pit-surface); }
|
||||
.evalbook-site .matrix tbody th:first-child a { display: block; color: var(--pit-foreground); font-weight: 500; overflow-wrap: anywhere; text-decoration: none; }
|
||||
.evalbook-site .matrix tbody th:first-child a:hover { color: var(--pit-live); text-decoration: underline; }
|
||||
.evalbook-site .result { display: flex; align-items: center; justify-content: center; width: 100%; max-width: 10rem; min-height: 2rem; margin-inline: auto; padding: var(--pit-space-1) var(--pit-space-2); border: 1px solid currentColor; border-radius: var(--pit-radius-sm); font-size: var(--pit-text-xs); font-weight: 700; text-decoration: none; }
|
||||
.evalbook-site :is(.result-pass, .passed) { background: var(--pit-success-surface); color: var(--pit-success); }
|
||||
.evalbook-site :is(.result-fail, .failed) { background: var(--pit-danger-surface); color: var(--pit-danger); }
|
||||
.evalbook-site .result-infra { background: var(--pit-warning-surface); color: var(--pit-warning); }
|
||||
.evalbook-site :is(.muted, .empty, small, footer) { color: var(--pit-muted-foreground); }
|
||||
.evalbook-site .empty { text-align: center; }
|
||||
.evalbook-site code { font-family: var(--pit-font-mono); font-size: var(--pit-text-xs); overflow-wrap: anywhere; }
|
||||
.evalbook-site pre { max-height: 32rem; padding: var(--pit-space-3); overflow: auto; white-space: pre-wrap; overflow-wrap: anywhere; background: var(--pit-surface-sunken); color: var(--pit-foreground); border: 1px solid var(--pit-border); border-radius: var(--pit-radius-sm); font-family: var(--pit-font-mono); font-size: var(--pit-text-xs); }
|
||||
.evalbook-site details { margin: var(--pit-space-4) 0; }
|
||||
.evalbook-site summary { cursor: pointer; color: var(--pit-muted-foreground); }
|
||||
.evalbook-site .status { display: inline-block; padding: var(--pit-space-1) var(--pit-space-2); border-radius: var(--pit-radius-sm); font-weight: 600; text-transform: uppercase; }
|
||||
.evalbook-site small { display: block; margin-top: var(--pit-space-1); }
|
||||
.evalbook-site footer { margin-top: var(--pit-space-5); font-size: var(--pit-text-xs); }
|
||||
@media (max-width: 760px) {
|
||||
.evalbook-site main { padding: var(--pit-space-4); }
|
||||
.evalbook-site .grid { grid-template-columns: 1fr; }
|
||||
.evalbook-site .matrix th:first-child { width: 12rem; min-width: 10rem; }
|
||||
.evalbook-site .badge { white-space: normal; text-align: center; }
|
||||
}
|
||||
|
|
@ -3,6 +3,7 @@ import { createRoot } from "react-dom/client";
|
|||
|
||||
import { App } from "./App";
|
||||
import "./issue-thread.css";
|
||||
import "./evalbook-site.css";
|
||||
|
||||
// Hosted Evalbooks use inert JSON plus the same trusted viewer bundle. No
|
||||
// inline executable script or network fetch is needed to load an attempt.
|
||||
|
|
|
|||
|
|
@ -4,6 +4,12 @@
|
|||
|
||||
Every new report uses the canonical Evalbook grid and the existing Runner Lab
|
||||
chat viewer for attempt drill-downs. There is no plain-HTML attempt fallback.
|
||||
The grid, Latest, test-design, inventory and server-gate pages load the same
|
||||
built stylesheet and fonts as the chat viewer. The long-term S3 history index
|
||||
references that stylesheet inside an immutable campaign, so every page keeps
|
||||
the same dark theme. Static site styles live in
|
||||
`devtools/issue-thread/src/evalbook-site.css`, scoped to `.evalbook-site`;
|
||||
colors and typography come from the Runner Lab token layer.
|
||||
Missing recordings show a notice in the same viewer; missing viewer builds
|
||||
fail generation. Build with
|
||||
`pnpm --filter @paperclipai/paperclip-runner build:issue-thread` and provide
|
||||
|
|
@ -22,6 +28,10 @@ checks the public payload contract and local links, and rejects other scripts.
|
|||
The CSP prohibits network calls, forms and external resources. Supply
|
||||
`PAPERCLIP_RUNNER_PROTOCOL_EVAL_VIEWER_DIR` to the publisher. The workflow sends
|
||||
a viewer-only artifact to that job; raw attempts and provider secrets stay out.
|
||||
After publication succeeds, the publishing job writes **Open this run's
|
||||
Evalbook** and **All eval runs** links to the Actions summary. Its deployment
|
||||
URL also points to the exact immutable report, not to the downloadable ZIP.
|
||||
Failed publication does not advertise a successful deployment.
|
||||
|
||||
Before uploading, the report job runs the actual built application in Chromium
|
||||
against representative passing, failing and missing-recording pages in both
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
import { createHash } from "node:crypto";
|
||||
import { execFile } from "node:child_process";
|
||||
import {
|
||||
appendFile,
|
||||
lstat,
|
||||
mkdtemp,
|
||||
mkdir,
|
||||
|
|
@ -390,7 +391,9 @@ function date(value) {
|
|||
}).format(new Date(value));
|
||||
}
|
||||
|
||||
export function renderProtocolEvalHistoryIndex(history) {
|
||||
export function renderProtocolEvalHistoryIndex(history, stylesheetHref) {
|
||||
if (!/^campaigns\/gha-[a-z0-9-]+\/viewer\/assets\/[A-Za-z0-9._-]+\.css$/.test(stylesheetHref ?? ""))
|
||||
throw new Error("History requires an immutable campaign's Runner Lab stylesheet");
|
||||
const rows = history.campaigns.length
|
||||
? history.campaigns
|
||||
.map((campaign) => {
|
||||
|
|
@ -413,9 +416,9 @@ export function renderProtocolEvalHistoryIndex(history) {
|
|||
(campaign) => campaign.campaignId === history.latestGreenCampaignId,
|
||||
);
|
||||
return `<!doctype html>
|
||||
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width"><meta name="color-scheme" content="light dark"><title>Runner protocol eval campaigns · Paperclip</title>
|
||||
<style>:root{color-scheme:light dark;--bg:#fff;--fg:#172019;--muted:#667069;--line:#dfe3dc;--raised:#f8f9f6;--pass:#17603a;--pass-bg:#e5f3e9;--fail:#942f2b;--fail-bg:#f9e5e3} @media(prefers-color-scheme:dark){:root{--bg:#141413;--fg:#fafafa;--muted:#aaa;--line:#ffffff1f;--raised:#1c1c1b;--pass:#65d58c;--pass-bg:#22c55e1f;--fail:#ff7770;--fail-bg:#dc26262e}} *{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--fg);font:14px/1.5 ui-sans-serif,system-ui,sans-serif}main{width:min(1560px,calc(100% - 48px));margin:48px auto 72px}h1{margin:0;font-size:clamp(34px,4vw,56px);line-height:1.05;letter-spacing:-.04em}p{max-width:760px;color:var(--muted);font-size:16px}a{color:inherit;text-underline-offset:3px}.pointers{display:flex;gap:10px;margin:28px 0 18px}.pointers a{padding:8px 11px;border:1px solid var(--line);border-radius:8px;background:var(--raised);text-decoration:none}.table{overflow:auto;border:1px solid var(--line);border-radius:12px}table{width:100%;border-collapse:collapse}th,td{padding:13px 14px;border-bottom:1px solid var(--line);text-align:left;vertical-align:top}th{background:var(--raised);color:var(--muted);font-size:10px;text-transform:uppercase;letter-spacing:.06em}tr:last-child td{border:0}small{display:block;margin-top:4px;color:var(--muted);font-size:10px}.status{display:inline-block;padding:3px 8px;border-radius:99px;font-size:10px;font-weight:750;text-transform:uppercase}.passed{color:var(--pass);background:var(--pass-bg)}.failed{color:var(--fail);background:var(--fail-bg)}.empty{padding:48px;text-align:center;color:var(--muted)}footer{margin-top:18px;color:var(--muted);font-size:11px}@media(max-width:700px){main{width:calc(100% - 28px);margin-top:28px}}</style></head>
|
||||
<body><main><div><small>Paperclip quality engineering</small><h1>Runner protocol eval campaigns</h1><p>Versioned direct live-runner Evalbook reports. Full provider transcripts, session identifiers, state, and raw tool evidence remain in access-controlled workflow artifacts.</p></div>
|
||||
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width"><title>Runner protocol eval campaigns · Paperclip</title>
|
||||
<link rel="stylesheet" href="${html(stylesheetHref)}"></head>
|
||||
<body class="evalbook-site"><main><header class="top"><a href="index.html">paperclip-runner evals</a><span class="badge">Run history</span></header><h1>Runner protocol eval campaigns</h1><p class="muted">Versioned live-runner reports. Full provider evidence remains in access-controlled workflow artifacts.</p>
|
||||
<nav class="pointers">${latest ? `<a href="${html(latest.publicUrl)}">Latest · ${html(latest.campaignId)}</a>` : ""}${latestGreen ? `<a href="${html(latestGreen.publicUrl)}">Latest green · ${html(latestGreen.campaignId)}</a>` : ""}</nav>
|
||||
<div class="table"><table><thead><tr><th>Campaign</th><th>Status</th><th>Cells</th><th>Models / rosters</th><th>Source</th><th></th></tr></thead><tbody>${rows}</tbody></table></div><footer>Updated ${html(date(history.updatedAt))} UTC · Immutable campaign bundles · Canonical Evalbook layout with public-safe evidence projections</footer></main></body></html>`;
|
||||
}
|
||||
|
|
@ -504,6 +507,11 @@ export async function publishProtocolEvalHistory({
|
|||
const { campaign } = await validatePublicProtocolEvalReport(reportRoot, {
|
||||
viewerRoot,
|
||||
});
|
||||
const viewer = await trustedViewerFiles(viewerRoot);
|
||||
const stylesheet = viewer.index.match(/<link rel="stylesheet" crossorigin href="\.\/(assets\/[A-Za-z0-9._-]+\.css)">/)?.[1];
|
||||
if (!stylesheet || !viewer.files.get(`viewer/${stylesheet}`)?.includes(".evalbook-site"))
|
||||
throw new Error("Published history requires the same-run Runner Lab site theme");
|
||||
const stylesheetHref = `campaigns/${campaign.campaignId}/viewer/${stylesheet}`;
|
||||
const manifest = await createProtocolEvalBundleManifest(
|
||||
reportRoot,
|
||||
campaign.campaignId,
|
||||
|
|
@ -568,7 +576,7 @@ export async function publishProtocolEvalHistory({
|
|||
);
|
||||
}
|
||||
const index = join(temporary, "index.html");
|
||||
await writeFile(index, renderProtocolEvalHistoryIndex(history));
|
||||
await writeFile(index, renderProtocolEvalHistoryIndex(history, stylesheetHref));
|
||||
await uploadFile(
|
||||
validatedDestination.bucket,
|
||||
`${validatedDestination.prefix}/index.html`,
|
||||
|
|
@ -579,9 +587,28 @@ export async function publishProtocolEvalHistory({
|
|||
campaignId: campaign.campaignId,
|
||||
bundleDigest: manifest.bundleDigest,
|
||||
historySize: history.campaigns.length,
|
||||
reportUrl: `${validatedDestination.publicBaseUrl}/${campaignPrefix}/index.html`,
|
||||
historyUrl: `${validatedDestination.publicBaseUrl}/${validatedDestination.prefix}/index.html`,
|
||||
};
|
||||
}
|
||||
|
||||
export async function writeProtocolEvalPublicationLinks(result, environment = process.env) {
|
||||
const { campaignId, reportUrl, historyUrl } = result;
|
||||
if (!SAFE_CAMPAIGN.test(campaignId)) throw new Error("Invalid published campaign ID");
|
||||
const safeUrl = (value) => {
|
||||
const url = new URL(value);
|
||||
if (url.protocol !== "https:" || url.username || url.password || /[\r\n<>]/.test(value))
|
||||
throw new Error("Invalid published report URL");
|
||||
return url.href;
|
||||
};
|
||||
const report = safeUrl(reportUrl);
|
||||
const history = safeUrl(historyUrl);
|
||||
if (environment.GITHUB_OUTPUT)
|
||||
await appendFile(environment.GITHUB_OUTPUT, `report_url=${report}\nhistory_url=${history}\n`);
|
||||
if (environment.GITHUB_STEP_SUMMARY)
|
||||
await appendFile(environment.GITHUB_STEP_SUMMARY, `## Published Runner Evalbook\n\n[Open this run's Evalbook](<${report}>) · [All eval runs](<${history}>)\n\nCampaign: \`${campaignId}\`\n\nPublic replay uses the Runner Lab theme; full evidence is in the workflow artifact.\n`);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const result = await publishProtocolEvalHistory({
|
||||
viewerRoot: process.env.PAPERCLIP_RUNNER_PROTOCOL_EVAL_VIEWER_DIR,
|
||||
|
|
@ -598,9 +625,11 @@ async function main() {
|
|||
process.env.RUNNER_PROTOCOL_EVAL_HISTORY_PUBLIC_BASE_URL ?? "",
|
||||
},
|
||||
});
|
||||
await writeProtocolEvalPublicationLinks(result);
|
||||
console.log(
|
||||
`Published immutable protocol eval campaign ${result.campaignId} (${result.bundleDigest}) and ${result.historySize} history record(s)`,
|
||||
);
|
||||
console.log(`Evalbook: ${result.reportUrl}\nRun history: ${result.historyUrl}`);
|
||||
}
|
||||
|
||||
if (
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
|
||||
import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
|
|
@ -14,9 +14,28 @@ import {
|
|||
renderProtocolEvalHistoryIndex,
|
||||
validateProtocolEvalHistoryDestination,
|
||||
validatePublicProtocolEvalReport,
|
||||
writeProtocolEvalPublicationLinks,
|
||||
} from "./publish-runner-protocol-eval-history.mjs";
|
||||
|
||||
const roots = [];
|
||||
test("successful publication exposes exact report and history links to Actions", async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "evalbook-publication-links-"));
|
||||
roots.push(root);
|
||||
const environment = { GITHUB_OUTPUT: join(root, "outputs"), GITHUB_STEP_SUMMARY: join(root, "summary") };
|
||||
const result = {
|
||||
campaignId: "gha-42-2",
|
||||
reportUrl: "https://reports.example/runner-protocol-evals/campaigns/gha-42-2/index.html",
|
||||
historyUrl: "https://reports.example/runner-protocol-evals/index.html",
|
||||
};
|
||||
await writeProtocolEvalPublicationLinks(result, environment);
|
||||
assert.equal(await readFile(environment.GITHUB_OUTPUT, "utf8"), `report_url=${result.reportUrl}\nhistory_url=${result.historyUrl}\n`);
|
||||
const summary = await readFile(environment.GITHUB_STEP_SUMMARY, "utf8");
|
||||
assert.ok(summary.includes(`[Open this run's Evalbook](<${result.reportUrl}>)`));
|
||||
assert.ok(summary.includes(`[All eval runs](<${result.historyUrl}>)`));
|
||||
await assert.rejects(writeProtocolEvalPublicationLinks({ ...result, reportUrl: "https://example.test/\nreport_url=bad" }, environment));
|
||||
assert.throws(() => renderProtocolEvalHistoryIndex(emptyProtocolEvalHistory(), "https://untrusted.example/style.css"));
|
||||
});
|
||||
|
||||
test.afterEach(async () => {
|
||||
await Promise.all(
|
||||
roots.splice(0).map((root) => rm(root, { recursive: true, force: true })),
|
||||
|
|
@ -178,7 +197,10 @@ test("retains immutable history and independent latest-green pointers", () => {
|
|||
const pointers = buildProtocolEvalPointers(history);
|
||||
assert.equal(pointers.latest.campaign.campaignId, "gha-43-1");
|
||||
assert.equal(pointers.latestGreen.campaign.campaignId, "gha-42-1");
|
||||
const index = renderProtocolEvalHistoryIndex(history);
|
||||
const index = renderProtocolEvalHistoryIndex(history, "campaigns/gha-43-1/viewer/assets/index.css");
|
||||
assert.match(index, /class="evalbook-site"/);
|
||||
assert.match(index, /href="campaigns\/gha-43-1\/viewer\/assets\/index.css"/);
|
||||
assert.doesNotMatch(index, /<style>|color-scheme:light/);
|
||||
assert.match(index, /Runner protocol eval campaigns/);
|
||||
assert.match(index, /Open Evalbook/);
|
||||
assert.match(index, /34\/35/);
|
||||
|
|
@ -238,7 +260,7 @@ test("report refreshes never replace qualification pointers, including after ret
|
|||
);
|
||||
assert.equal(history.campaigns.length, 200);
|
||||
assert.match(
|
||||
renderProtocolEvalHistoryIndex(history),
|
||||
renderProtocolEvalHistoryIndex(history, "campaigns/gha-42-1-report-chat-v1/viewer/assets/index.css"),
|
||||
/Report refresh · no new model calls/,
|
||||
);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -158,6 +158,8 @@ test("publishes only the separately sanitized Evalbook through trusted OIDC code
|
|||
assert.match(publisher, /runner-protocol-evals/u);
|
||||
assert.match(publisher, /runner-protocol-viewer-/u);
|
||||
assert.match(publisher, /PAPERCLIP_RUNNER_PROTOCOL_EVAL_VIEWER_DIR/u);
|
||||
assert.match(publisher, /url: \$\{\{ steps\.publish\.outputs\.report_url \}\}/u);
|
||||
assert.match(publisher, /Publish versioned report and refresh the root index\n\s+id: publish/u);
|
||||
assert.doesNotMatch(publisher, /(?:OPENAI|ANTHROPIC|OPENROUTER)_API_KEY/u);
|
||||
assert.doesNotMatch(publisher, /paperclipai\/paperclip-evals/u);
|
||||
assert.doesNotMatch(publisher, /downloaded-runner-protocol-evals/u);
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ try {
|
|||
page.on("request", (request) => requests.push(request.url()));
|
||||
await page.goto(`${origin}/${route}`, { waitUntil: "networkidle" });
|
||||
await page.locator(".pit-thread").waitFor({ state: "visible" });
|
||||
const chatBackground = await page.evaluate(() => getComputedStyle(document.body).backgroundColor);
|
||||
assert.equal(
|
||||
await page.locator("h1").textContent(),
|
||||
payload.view.issue.title,
|
||||
|
|
@ -149,6 +150,23 @@ try {
|
|||
.getByRole("link", { name: "← All results", exact: true })
|
||||
.click();
|
||||
assert.equal(new URL(page.url()).pathname, "/index.html");
|
||||
await expect(page.locator("body")).toHaveClass("evalbook-site");
|
||||
const assertSiteTheme = async () => {
|
||||
assert.equal(await page.evaluate(() => getComputedStyle(document.body).backgroundColor), chatBackground);
|
||||
assert.equal(await page.evaluate(() => getComputedStyle(document.documentElement).colorScheme), "dark");
|
||||
};
|
||||
await assertSiteTheme();
|
||||
if (screenshots) await page.screenshot({ path: join(screenshots, "index.png"), fullPage: true });
|
||||
await page.locator(".matrix tbody th a").first().click();
|
||||
await expect(page.getByText("Test design", { exact: true })).toBeVisible();
|
||||
await assertSiteTheme();
|
||||
await page.getByText("Full authored test definition", { exact: true }).click();
|
||||
if (screenshots) await page.screenshot({ path: join(screenshots, "test-design.png"), fullPage: true });
|
||||
await page.getByRole("link", { name: "← All results", exact: true }).click();
|
||||
await page.getByRole("link", { name: "Latest", exact: true }).click();
|
||||
await assertSiteTheme();
|
||||
await page.getByRole("link", { name: "Overview", exact: true }).click();
|
||||
await assertSiteTheme();
|
||||
await page.goto(`${origin}/${route}`, { waitUntil: "networkidle" });
|
||||
await page.reload({ waitUntil: "networkidle" });
|
||||
await page.locator(".pit-thread").waitFor({ state: "visible" });
|
||||
|
|
|
|||
Loading…
Reference in New Issue