import { compareCampaignAnalytics } from "./runner-protocol-eval-metrics.mjs"; const escape = (value) => String(value ?? "").replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll('"', """).replaceAll("'", "'"); const date = (value) => new Intl.DateTimeFormat("en-US", { dateStyle: "medium", timeStyle: "short", timeZone: "UTC" }).format(new Date(value)); const percentage = (run) => run.totals.selected ? 100 * run.totals.passed / run.totals.selected : null; const dollars = (value) => `$${(value / 1e9).toFixed(6)}`; export function costLabel(costs, field = "estimated") { const metric = costs?.[field]; if (!Number.isFinite(metric?.nanodollars) || metric.nanodollars < 0) return "Unknown"; const incomplete = metric.recordedAttempts < costs.attempts; return `${incomplete ? "≥ " : ""}${dollars(metric.nanodollars)}`; } function costCell(costs) { const coverage = costs?.estimated?.recordedAttempts ?? 0; const attempts = costs?.attempts ?? 0; return `${escape(costLabel(costs))}Estimated · ${coverage}/${attempts} entriesProvider list: ${escape(costLabel(costs, "providerReported"))}${costs?.scope === "all_attempts" ? "All attempts, including retries" : "Historical final attempts only"}`; } function commit(repository, sha, label) { if (!/^[a-f0-9]{40}$/.test(sha ?? "")) return `${label}: unknown`; return `${label}: ${sha.slice(0, 8)}`; } function sourceLinks(run) { const url = run.source?.workflowRunUrl; const workflow = /^https:\/\/github\.com\/paperclipai\/paperclip\/actions\/runs\/[1-9][0-9]*$/.test(url ?? "") ? `GitHub Actions ↗` : ""; return `${commit("paperclipai/paperclip", run.source?.paperclip?.sha, "Paperclip")}${commit("paperclipai/paperclip-evals", run.source?.evals?.sha, "Evals")}${workflow}${escape(run.source?.paperclip?.ref ?? "")}`; } function chart(runs, analytics, kind, id) { const cost = kind === "cost"; const label = cost ? "Estimated cost over time (USD)" : "Pass rate over time (%)"; const values = runs.map((run) => cost ? analytics[run.campaignId]?.costs?.estimated?.nanodollars : percentage(run)); const max = cost ? Math.max(1, ...values.filter(Number.isFinite)) : 100; const timestamps = runs.map((run) => Date.parse(run.generatedAt)); const elapsed = timestamps.at(-1) - timestamps[0]; const x = (index) => elapsed > 0 ? 48 + 504 * (timestamps[index] - timestamps[0]) / elapsed : 300; const y = (value) => 145 - value / max * 116; const scopes = cost ? ["all_attempts", "final_attempts"] : ["pass"]; const series = scopes.map((scope) => { let drawing = false; const path = values.map((value, index) => { const present = Number.isFinite(value) && value >= 0 && (!cost || analytics[runs[index].campaignId]?.costs?.scope === scope); if (!present) { drawing = false; return ""; } const point = `${drawing ? "L" : "M"}${x(index).toFixed(2)},${y(value).toFixed(2)}`; drawing = true; return point; }).join(" "); return ``; }).join(""); const points = values.map((value, index) => { if (!Number.isFinite(value) || value < 0) return ""; const run = runs[index]; const metric = analytics[run.campaignId]; const description = `${date(run.generatedAt)} UTC · ${run.campaignId} · ${cost ? `${costLabel(metric?.costs)} (${metric?.costs?.scope})` : `${value.toFixed(1)}% (${run.totals.passed}/${run.totals.selected})`}`; return `${escape(description)}`; }).join(""); const tick = (run) => `${run.generatedAt.slice(5, 16).replace("T", " ")} UTC`; return `
${label}
${label}. Each point links to its recorded run.${cost ? `$${(max / 1e9).toFixed(2)}` : "100%"}0${series}${points}${escape(tick(runs[0]))}${escape(tick(runs.at(-1)))}
`; } function changeCell(run, analytics, previous) { if (run.reportRevision) return 'Presentation only'; if (!previous) return 'No matching baseline'; const change = compareCampaignAnalytics(analytics[run.campaignId], analytics[previous.campaignId]); if (!change) return 'Suite changed'; const failures = (cells, destination) => cells.map((cell) => `
  • ${escape(cell.caseId)} · ${escape(cell.rosterId)}${cell.disposition === "infrastructure_failure" ? " · infrastructure" : ""}
  • `).join(""); const counts = `${change.regressions.length} regressions · ${change.recoveries.length} recoveries`; return `
    ${counts}Versus ${escape(previous.campaignId)}${change.regressions.length ? `Previously passing → failing` : ""}${change.recoveries.length ? `Previously failing → passing` : ""}
    `; } export function renderProtocolEvalHistoryIndex(history, stylesheetHref) { if (!/^campaigns\/gha-[a-z0-9-]+\/viewer\/assets\/[A-Za-z0-9._-]+\.css$/.test(stylesheetHref ?? "")) throw new Error("History requires an immutable campaign's Runner Lab stylesheet"); for (const run of history.campaigns) { if (!["selected", "passed", "behaviorFailures", "infrastructureFailures"].every((field) => Number.isSafeInteger(run.totals?.[field]) && run.totals[field] >= 0) || !run.rosters.every((roster) => Number.isSafeInteger(roster.selected) && Number.isSafeInteger(roster.passed))) throw new Error("History requires numeric recorded counts"); const url = new URL(run.publicUrl); if (url.protocol !== "https:" || url.username || url.password) throw new Error("History report links must be credential-free HTTPS URLs"); } const analytics = history.analytics ?? {}; const presentations = new Map(); for (const refresh of history.campaigns.filter((run) => run.reportRevision) .sort((a, b) => a.reportRevision.renderedAt.localeCompare(b.reportRevision.renderedAt))) presentations.set(refresh.reportRevision.sourceCampaignId, refresh); const measurements = history.campaigns.filter((run) => !run.reportRevision) .map((run) => presentations.has(run.campaignId) ? { ...run, originalPublicUrl: run.publicUrl, publicUrl: presentations.get(run.campaignId).publicUrl } : run) .sort((a, b) => a.generatedAt.localeCompare(b.generatedAt) || a.campaignId.localeCompare(b.campaignId)); const groups = new Map(); const baselines = new Map(); for (const run of measurements) { const key = analytics[run.campaignId]?.suiteKey; if (!key || !run.complete) continue; const group = groups.get(key) ?? []; baselines.set(run.campaignId, group.at(-1)); group.push(run); groups.set(key, group); } const ordered = [...groups.entries()].sort(([, a], [, b]) => b.at(-1).totals.selected - a.at(-1).totals.selected || b.at(-1).generatedAt.localeCompare(a.at(-1).generatedAt)); const trends = ordered.map(([key, runs], index) => { const latest = runs.at(-1); return `
    ${latest.totals.selected} cells · ${latest.rosters.length} configurations · ${runs.length} recorded runs · evals ${escape(latest.source.evals.sha.slice(0, 8))}
    ${chart(runs, analytics, "pass", `pass-${key}`)}${chart(runs, analytics, "cost", `cost-${key}`)}

    Cost: solid cyan includes retries; dashed purple is historical final-attempt cost. Missing cost is a gap, not zero. Hover or select a point to inspect its run.

    `; }).join("") || '

    Comparable run metadata has not been recorded yet.

    '; const row = (run) => { const metric = analytics[run.reportRevision?.sourceCampaignId ?? run.campaignId]; const status = !run.complete ? "incomplete" : run.allPassed ? "passed" : "failed"; const models = run.rosters.map((roster) => `
  • ${escape(roster.model)} · ${roster.passed}/${roster.selected}${escape(roster.driver)} · ${escape(roster.rosterId)}
  • `).join(""); const scope = metric?.selection === "maintained_full" ? "Full maintained suite" : metric?.selection === "subset" ? "Selected subset" : `${run.totals.selected} recorded cells`; return `${escape(run.campaignId)}${escape(date(run.generatedAt))} UTC${run.reportRevision ? `Report refresh · no new model calls · source ${escape(run.reportRevision.sourceCampaignId)}` : `${scope}`}Open Evalbook →${status}${run.totals.passed}/${run.totals.selected} · ${percentage(run)?.toFixed(1) ?? "—"}%${run.totals.behaviorFailures} behavior · ${run.totals.infrastructureFailures} infrastructure
    ${run.rosters.length} model configurations
    ${changeCell(run, analytics, baselines.get(run.campaignId))}${run.reportRevision ? 'No additional model cost' : costCell(metric?.costs)}${sourceLinks(run)}`; }; const table = (runs) => `
    ${runs.map(row).join("") || ''}
    RunPass / failChange vs matching suiteCost (USD)Exact source
    No campaigns have been published yet.
    `; const refreshes = history.campaigns.filter((run) => run.reportRevision); const latest = measurements.find((run) => run.campaignId === history.latestCampaignId); const green = measurements.find((run) => run.campaignId === history.latestGreenCampaignId); return `Runner protocol eval campaigns · Paperclip
    paperclip-runner evalsRun history

    Runner protocol eval campaigns

    Every recorded campaign, its cost, and its exact source. Public chat replays are linked below; full provider evidence remains in access-controlled workflow artifacts.

    Like-for-like trends

    Only identical cells, model configurations, and eval-suite SHAs are compared. A changed suite starts a separate series. Report refreshes never count as new measurements. Regressions distinguish model behavior from infrastructure failures.

    ${trends}

    All runs · ${measurements.length}

    Estimated cost and provider list cost are alternatives, not additive. ≥ means some entries lack usage. Historical final-only cost excludes retry spending. Commit links expose the full SHA on hover.

    ${table([...measurements].reverse())}${refreshes.length ? `
    Report refreshes · ${refreshes.length} (no new measurements)${table(refreshes)}
    ` : ""}
    `; }