import { compareCampaignAnalytics } from "./runner-protocol-eval-metrics.mjs";
const escape = (value) => String(value ?? "").replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll('"', """).replaceAll("'", "'");
const date = (value) => new Intl.DateTimeFormat("en-US", { dateStyle: "medium", timeStyle: "short", timeZone: "UTC" }).format(new Date(value));
const percentage = (run) => run.totals.selected ? 100 * run.totals.passed / run.totals.selected : null;
const dollars = (value) => `$${(value / 1e9).toFixed(6)}`;
export function costLabel(costs, field = "estimated") {
const metric = costs?.[field];
if (!Number.isFinite(metric?.nanodollars) || metric.nanodollars < 0) return "Unknown";
const incomplete = metric.recordedAttempts < costs.attempts;
return `${incomplete ? "≥ " : ""}${dollars(metric.nanodollars)}`;
}
function costCell(costs) {
const coverage = costs?.estimated?.recordedAttempts ?? 0;
const attempts = costs?.attempts ?? 0;
return `${escape(costLabel(costs))} Estimated · ${coverage}/${attempts} entries Provider list: ${escape(costLabel(costs, "providerReported"))} ${costs?.scope === "all_attempts" ? "All attempts, including retries" : "Historical final attempts only"} `;
}
function commit(repository, sha, label) {
if (!/^[a-f0-9]{40}$/.test(sha ?? "")) return `${label}: unknown `;
return `${label}: ${sha.slice(0, 8)} `;
}
function sourceLinks(run) {
const url = run.source?.workflowRunUrl;
const workflow = /^https:\/\/github\.com\/paperclipai\/paperclip\/actions\/runs\/[1-9][0-9]*$/.test(url ?? "")
? `GitHub Actions ↗ ` : "";
return `${commit("paperclipai/paperclip", run.source?.paperclip?.sha, "Paperclip")}${commit("paperclipai/paperclip-evals", run.source?.evals?.sha, "Evals")}${workflow}${escape(run.source?.paperclip?.ref ?? "")} `;
}
function chart(runs, analytics, kind, id) {
const cost = kind === "cost";
const label = cost ? "Estimated cost over time (USD)" : "Pass rate over time (%)";
const values = runs.map((run) => cost ? analytics[run.campaignId]?.costs?.estimated?.nanodollars : percentage(run));
const max = cost ? Math.max(1, ...values.filter(Number.isFinite)) : 100;
const timestamps = runs.map((run) => Date.parse(run.generatedAt));
const elapsed = timestamps.at(-1) - timestamps[0];
const x = (index) => elapsed > 0 ? 48 + 504 * (timestamps[index] - timestamps[0]) / elapsed : 300;
const y = (value) => 145 - value / max * 116;
const scopes = cost ? ["all_attempts", "final_attempts"] : ["pass"];
const series = scopes.map((scope) => {
let drawing = false;
const path = values.map((value, index) => {
const present = Number.isFinite(value) && value >= 0 && (!cost || analytics[runs[index].campaignId]?.costs?.scope === scope);
if (!present) { drawing = false; return ""; }
const point = `${drawing ? "L" : "M"}${x(index).toFixed(2)},${y(value).toFixed(2)}`;
drawing = true;
return point;
}).join(" ");
return ` `;
}).join("");
const points = values.map((value, index) => {
if (!Number.isFinite(value) || value < 0) return "";
const run = runs[index];
const metric = analytics[run.campaignId];
const description = `${date(run.generatedAt)} UTC · ${run.campaignId} · ${cost ? `${costLabel(metric?.costs)} (${metric?.costs?.scope})` : `${value.toFixed(1)}% (${run.totals.passed}/${run.totals.selected})`}`;
return `${escape(description)} `;
}).join("");
const tick = (run) => `${run.generatedAt.slice(5, 16).replace("T", " ")} UTC`;
return `${label} ${label}. Each point links to its recorded run. ${cost ? `$${(max / 1e9).toFixed(2)}` : "100%"} 0 ${series}${points}${escape(tick(runs[0]))} ${escape(tick(runs.at(-1)))} `;
}
function changeCell(run, analytics, previous) {
if (run.reportRevision) return 'Presentation only ';
if (!previous) return 'No matching baseline ';
const change = compareCampaignAnalytics(analytics[run.campaignId], analytics[previous.campaignId]);
if (!change) return 'Suite changed ';
const failures = (cells, destination) => cells.map((cell) => `
${escape(cell.caseId)} · ${escape(cell.rosterId)}${cell.disposition === "infrastructure_failure" ? " · infrastructure" : ""} `).join("");
const counts = `${change.regressions.length} regressions · ${change.recoveries.length} recoveries`;
return `${counts} Versus ${escape(previous.campaignId)} ${change.regressions.length ? `Previously passing → failing ${failures(change.regressions, run)} ` : ""}${change.recoveries.length ? `Previously failing → passing ${failures(change.recoveries, run)} ` : ""} `;
}
export function renderProtocolEvalHistoryIndex(history, stylesheetHref) {
if (!/^campaigns\/gha-[a-z0-9-]+\/viewer\/assets\/[A-Za-z0-9._-]+\.css$/.test(stylesheetHref ?? ""))
throw new Error("History requires an immutable campaign's Runner Lab stylesheet");
for (const run of history.campaigns) {
if (!["selected", "passed", "behaviorFailures", "infrastructureFailures"].every((field) => Number.isSafeInteger(run.totals?.[field]) && run.totals[field] >= 0)
|| !run.rosters.every((roster) => Number.isSafeInteger(roster.selected) && Number.isSafeInteger(roster.passed)))
throw new Error("History requires numeric recorded counts");
const url = new URL(run.publicUrl);
if (url.protocol !== "https:" || url.username || url.password)
throw new Error("History report links must be credential-free HTTPS URLs");
}
const analytics = history.analytics ?? {};
const presentations = new Map();
for (const refresh of history.campaigns.filter((run) => run.reportRevision)
.sort((a, b) => a.reportRevision.renderedAt.localeCompare(b.reportRevision.renderedAt)))
presentations.set(refresh.reportRevision.sourceCampaignId, refresh);
const measurements = history.campaigns.filter((run) => !run.reportRevision)
.map((run) => presentations.has(run.campaignId)
? { ...run, originalPublicUrl: run.publicUrl, publicUrl: presentations.get(run.campaignId).publicUrl }
: run)
.sort((a, b) => a.generatedAt.localeCompare(b.generatedAt) || a.campaignId.localeCompare(b.campaignId));
const groups = new Map();
const baselines = new Map();
for (const run of measurements) {
const key = analytics[run.campaignId]?.suiteKey;
if (!key || !run.complete) continue;
const group = groups.get(key) ?? [];
baselines.set(run.campaignId, group.at(-1));
group.push(run);
groups.set(key, group);
}
const ordered = [...groups.entries()].sort(([, a], [, b]) => b.at(-1).totals.selected - a.at(-1).totals.selected || b.at(-1).generatedAt.localeCompare(a.at(-1).generatedAt));
const trends = ordered.map(([key, runs], index) => {
const latest = runs.at(-1);
return `${latest.totals.selected} cells · ${latest.rosters.length} configurations · ${runs.length} recorded runs · evals ${escape(latest.source.evals.sha.slice(0, 8))} ${chart(runs, analytics, "pass", `pass-${key}`)}${chart(runs, analytics, "cost", `cost-${key}`)}
Cost: solid cyan includes retries; dashed purple is historical final-attempt cost. Missing cost is a gap, not zero. Hover or select a point to inspect its run.
`;
}).join("") || 'Comparable run metadata has not been recorded yet.
';
const row = (run) => {
const metric = analytics[run.reportRevision?.sourceCampaignId ?? run.campaignId];
const status = !run.complete ? "incomplete" : run.allPassed ? "passed" : "failed";
const models = run.rosters.map((roster) => `${escape(roster.model)} · ${roster.passed}/${roster.selected}${escape(roster.driver)} · ${escape(roster.rosterId)} `).join("");
const scope = metric?.selection === "maintained_full" ? "Full maintained suite" : metric?.selection === "subset" ? "Selected subset" : `${run.totals.selected} recorded cells`;
return `${escape(run.campaignId)}${escape(date(run.generatedAt))} UTC ${run.reportRevision ? `Report refresh · no new model calls · source ${escape(run.reportRevision.sourceCampaignId)} ` : `${scope} `}Open Evalbook → ${status} ${run.totals.passed}/${run.totals.selected} · ${percentage(run)?.toFixed(1) ?? "—"}% ${run.totals.behaviorFailures} behavior · ${run.totals.infrastructureFailures} infrastructure ${run.rosters.length} model configurations ${changeCell(run, analytics, baselines.get(run.campaignId))} ${run.reportRevision ? 'No additional model cost ' : costCell(metric?.costs)} ${sourceLinks(run)} `;
};
const table = (runs) => `Run Pass / fail Change vs matching suite Cost (USD) Exact source ${runs.map(row).join("") || 'No campaigns have been published yet. '}
`;
const refreshes = history.campaigns.filter((run) => run.reportRevision);
const latest = measurements.find((run) => run.campaignId === history.latestCampaignId);
const green = measurements.find((run) => run.campaignId === history.latestGreenCampaignId);
return `Runner protocol eval campaigns · Paperclip Runner protocol eval campaigns Every recorded campaign, its cost, and its exact source. Public chat replays are linked below; full provider evidence remains in access-controlled workflow artifacts.
${latest ? `Latest · ${escape(latest.campaignId)} ` : ""}${green ? `Latest green · ${escape(green.campaignId)} ` : ""}Download history JSON Like-for-like trends Only identical cells, model configurations, and eval-suite SHAs are compared. A changed suite starts a separate series. Report refreshes never count as new measurements. Regressions distinguish model behavior from infrastructure failures.
${trends}All runs · ${measurements.length} Estimated cost and provider list cost are alternatives, not additive. ≥ means some entries lack usage. Historical final-only cost excludes retry spending. Commit links expose the full SHA on hover.
${table([...measurements].reverse())}${refreshes.length ? `Report refreshes · ${refreshes.length} (no new measurements) ${table(refreshes)} ` : ""}Updated ${escape(date(history.updatedAt))} UTC · All run records retained · Immutable campaign bundles `;
}