import { discoverReportCatalog, type ReportExecution, } from "./report-catalog.js"; import type { RunnerE2ECampaign, RunnerE2EHistoryIndex, RunnerE2EResult, RunnerE2ESuiteSummary, } from "./types.js"; import { aggregateCampaignBilling, summarizeExecutionBilling, } from "./billing.js"; export interface RunnerDashboardEntry { result: RunnerE2EResult; valid: boolean; errors: readonly string[]; evidenceBaseHref?: string; evidenceFiles?: readonly string[]; } export interface RunnerDashboardInput { title: string; generatedAt: string; expected: readonly string[]; catalog: readonly ReportExecution[]; entries: readonly RunnerDashboardEntry[]; campaign?: RunnerE2ECampaign; history?: RunnerE2EHistoryIndex; publicSummaryImageHref?: string; } interface ResolvedScreenshot { id: string; label: string; file: string; href: string; } function html(value: unknown) { return String(value ?? "") .replaceAll("&", "&") .replaceAll("<", "<") .replaceAll(">", ">") .replaceAll('"', """) .replaceAll("'", "'"); } function durationLabel(durationMs: number) { if (durationMs < 1_000) return `${durationMs}ms`; const seconds = Math.round(durationMs / 1_000); if (seconds < 60) return `${seconds}s`; return `${Math.floor(seconds / 60)}m ${seconds % 60}s`; } function tokenLabel(value: number) { return new Intl.NumberFormat("en-US").format(value); } function usdLabel(value: number) { return `$${value.toFixed(value < 0.01 ? 6 : 4)}`; } function safeEvidenceHref(base: string | undefined, relative: string) { if (!base || /^(?:[a-z]+:|\/\/)/i.test(base)) return null; const cleanBase = base .split("/") .filter((segment) => segment && segment !== "." && segment !== "..") .map(encodeURIComponent) .join("/"); const cleanRelative = relative .split("/") .filter((segment) => segment && segment !== "." && segment !== "..") .map(encodeURIComponent) .join("/"); return `${cleanBase}/${cleanRelative}`; } function safePublicAssetHref(relative: string | undefined) { if (!relative || /^(?:[a-z]+:|\/\/|\/)/i.test(relative)) return null; const segments = relative.split("/"); if ( segments.some((segment) => !segment || segment === "." || segment === "..") ) { return null; } return segments.map(encodeURIComponent).join("/"); } function compactJson(value: unknown) { const serialized = JSON.stringify(value); return serialized && serialized.length > 1_200 ? `${serialized.slice(0, 1_200)}…` : serialized; } function resolveScreenshots( entry: RunnerDashboardEntry | undefined, ): ResolvedScreenshot[] { const declaredScreenshots = entry?.result.screenshots?.length ? entry.result.screenshots : entry ? [ { id: "final-state", label: "Final visible task state", file: "final-state.png", }, ] : []; const availableFiles = entry?.evidenceFiles ? new Set(entry.evidenceFiles) : null; const screenshots = declaredScreenshots .filter((item) => !availableFiles || availableFiles.has(item.file)) .flatMap((item) => { const href = safeEvidenceHref(entry?.evidenceBaseHref, item.file); return href ? [{ ...item, href }] : []; }); if ( entry?.result.status === "failed" && !screenshots.some((item) => item.file === "failure.png") && (!availableFiles || availableFiles.has("failure.png")) ) { const href = safeEvidenceHref(entry.evidenceBaseHref, "failure.png"); if (href) { screenshots.push({ id: "failure", label: "Failure state", file: "failure.png", href, }); } } return screenshots; } function renderCase( execution: ReportExecution, expected: ReadonlySet, entryById: ReadonlyMap, ) { const selected = expected.has(execution.id); const entry = entryById.get(execution.id); const state = !selected ? "not-selected" : !entry ? "missing" : entry.valid ? "passed" : "failed"; const label = state.replace("-", " "); const detail = entry?.errors.join("; ") || (entry?.valid ? "All invariants passed" : selected ? "No result artifact was uploaded" : "Not selected"); const screenshots = resolveScreenshots(entry); const billing = entry ? summarizeExecutionBilling(entry.result) : null; const matcherResults = entry?.result.matcherResults ?? []; const passedMatchers = matcherResults.filter( (result) => result.passed, ).length; const searchText = [ execution.id, execution.task.label, execution.profile.label, execution.profile.generation, execution.profile.provider, execution.profile.model, execution.environment.label, execution.environment.provider, execution.suite.label, label, ] .join(" ") .toLowerCase(); const availableFiles = entry?.evidenceFiles ? new Set(entry.evidenceFiles) : null; const playwright = !availableFiles || availableFiles.has("html-report/index.html") ? safeEvidenceHref(entry?.evidenceBaseHref, "html-report/index.html") : null; const links = screenshots.length > 0 || playwright ? `` : ""; const matcherRows = (entry?.result.matcherResults ?? []) .map( (result) => ` ${result.passed ? "Pass" : "Fail"} ${html(result.matcher.kind)} ${html(compactJson(result.matcher))} ${html(result.detail)} `, ) .join(""); const turnTimingRows = (entry?.result.turnTimings ?? []) .map( (timing) => ` ${timing.turn} ${html(timing.runId)} ${html(timing.leaseAcquisitionOutcome)} ${html(timing.schedulerLatencyMs === null ? "unavailable" : durationLabel(timing.schedulerLatencyMs))} ${html(timing.runDurationMs === null ? "unavailable" : durationLabel(timing.runDurationMs))} ${html(timing.responseLatencyMs === null ? "unavailable" : durationLabel(timing.responseLatencyMs))} `, ) .join(""); const gallery = screenshots.length ? `` : ""; const billingStrip = billing ? `
Tokens${html(tokenLabel(billing.llm.inputTokens))} in · ${html(tokenLabel(billing.llm.outputTokens))} out${html(tokenLabel(billing.llm.cachedInputTokens))} cached · ${billing.llm.runsWithTokenUsage}/${billing.llm.runCount} runs covered
LLM spend${billing.llm.runsWithReportedCost > 0 ? html(usdLabel(billing.reportedCostUsd)) : html(billing.llm.costStatus)}${billing.llm.runsWithReportedCost}/${billing.llm.runCount} runs provider-priced
Execution${billing.runtime.estimatedListCostUsd === undefined ? html(billing.runtime.costStatus === "not_metered" ? "Local · not metered" : "Cost unavailable") : `${html(usdLabel(billing.runtime.estimatedListCostUsd))} est.`}${html(durationLabel(billing.runtime.agentRunDurationMs))} agent${billing.runtime.leaseDurationMs === null ? "" : ` · ${html(durationLabel(billing.runtime.leaseDurationMs))} lease`}
` : ""; return `
${html(execution.task.label)} ${html(label)}
${gallery} ${billingStrip} ${html(execution.id)}
Matchers and test context ${ entry ? `
Attempt
${entry.result.attempt}
Duration
${html(durationLabel(entry.result.durationMs))}
Agent runtime
${html(durationLabel(billing!.runtime.agentRunDurationMs))}
${billing!.runtime.leaseDurationMs === null ? "" : `
Environment lease
${html(durationLabel(billing!.runtime.leaseDurationMs))}
`}
Runtime
${html(entry.result.runtimeMode)}
Provider
${html(entry.result.provider)}
Model
${html(entry.result.model)}
${entry.result.issueIdentifier ? `
Issue
${html(entry.result.issueIdentifier)}
` : ""}
` : "" }

${html(detail)}

${turnTimingRows ? `
${turnTimingRows}
TurnRunLeaseSchedulerRun durationResponse
` : ""} ${matcherRows ? `
${matcherRows}
ResultMatcherExpectationDetail
` : `

No matcher result was recorded.

`} ${entry ? `
Usage and billing metadata
${html(JSON.stringify({ billing, rawUsage: entry.result.usage ?? null }, null, 2))}
` : ""} ${links}
`; } function renderTrendChart(input: { history: RunnerE2EHistoryIndex; label: string; value(campaign: RunnerE2EHistoryIndex["campaigns"][number]): number; format(value: number): string; include?(campaign: RunnerE2EHistoryIndex["campaigns"][number]): boolean; fingerprint?(campaign: RunnerE2EHistoryIndex["campaigns"][number]): string; }) { const campaigns = input.history.campaigns .filter(input.include ?? ((campaign) => campaign.complete)) .slice(0, 20) .reverse(); if (campaigns.length === 0) { return `
${html(input.label)}No complete campaigns
`; } const values = campaigns.map(input.value); const maximum = Math.max(...values, 1); const pointRows = values.map((value, index) => { const x = campaigns.length === 1 ? 50 : (index / (campaigns.length - 1)) * 100; const y = 96 - (value / maximum) * 88; return { point: `${x.toFixed(2)},${y.toFixed(2)}`, x, y, fingerprint: input.fingerprint?.(campaigns[index]!) ?? "stable", }; }); const segments = pointRows.reduce>( (groups, point) => { const current = groups.at(-1); if (!current || current.at(-1)?.fingerprint !== point.fingerprint) { groups.push([point]); } else { current.push(point); } return groups; }, [], ); const definitionCount = new Set(pointRows.map((point) => point.fingerprint)) .size; const latest = values.at(-1) ?? 0; return `
${html(input.label)} ${html(input.format(latest))} ${segments.map((segment) => ``).join("")} ${pointRows.map((point) => ``).join("")} ${campaigns.length} complete campaign${campaigns.length === 1 ? "" : "s"} · ${definitionCount} definition${definitionCount === 1 ? "" : "s"}
`; } function suiteSummaryFor( campaign: RunnerE2EHistoryIndex["campaigns"][number], suiteId: string, ) { return campaign.suites.find((suite) => suite.suiteId === suiteId); } function renderHistory(history: RunnerE2EHistoryIndex | undefined) { if (!history || history.campaigns.length === 0) { return `

History

Campaign trends

No historical campaigns have been published yet.

`; } const suiteIds = [ ...new Set( history.campaigns.flatMap((campaign) => campaign.suites.map((suite) => suite.suiteId), ), ), ]; const charts = [ renderTrendChart({ history, label: "Observed + estimated cost", value: (campaign) => campaign.billing.observedAndEstimatedCostUsd, format: usdLabel, fingerprint: (campaign) => campaign.suites .map((suite) => `${suite.suiteId}:${suite.suiteDefinitionHash}`) .sort() .join("|"), }), renderTrendChart({ history, label: "Total tokens", value: (campaign) => campaign.billing.llm.totalTokens, format: tokenLabel, fingerprint: (campaign) => campaign.suites .map((suite) => `${suite.suiteId}:${suite.suiteDefinitionHash}`) .sort() .join("|"), }), renderTrendChart({ history, label: "Agent execution time", value: (campaign) => campaign.billing.agentRunDurationMs, format: durationLabel, fingerprint: (campaign) => campaign.suites .map((suite) => `${suite.suiteId}:${suite.suiteDefinitionHash}`) .sort() .join("|"), }), renderTrendChart({ history, label: "Daytona lease time", value: (campaign) => campaign.billing.leaseDurationMs, format: durationLabel, fingerprint: (campaign) => campaign.suites .map((suite) => `${suite.suiteId}:${suite.suiteDefinitionHash}`) .sort() .join("|"), }), renderTrendChart({ history, label: "Pass rate", value: (campaign) => campaign.selected > 0 ? (campaign.passed / campaign.selected) * 100 : 0, format: (value) => `${value.toFixed(1)}%`, fingerprint: (campaign) => campaign.suites .map((suite) => `${suite.suiteId}:${suite.suiteDefinitionHash}`) .sort() .join("|"), }), ].join(""); const suiteCharts = suiteIds .map((suiteId) => { const include = (campaign: RunnerE2EHistoryIndex["campaigns"][number]) => suiteSummaryFor(campaign, suiteId)?.complete === true; const value = ( campaign: RunnerE2EHistoryIndex["campaigns"][number], metric: "cost" | "tokens" | "agent" | "lease" | "passRate", ) => { const suite = suiteSummaryFor(campaign, suiteId); if (!suite) return 0; if (metric === "cost") return suite.billing.observedAndEstimatedCostUsd; if (metric === "tokens") return suite.billing.llm.totalTokens; if (metric === "agent") return suite.billing.agentRunDurationMs; if (metric === "lease") return suite.billing.leaseDurationMs; return suite.selected > 0 ? (suite.passed / suite.selected) * 100 : 0; }; const fingerprint = ( campaign: RunnerE2EHistoryIndex["campaigns"][number], ) => suiteSummaryFor(campaign, suiteId)?.suiteDefinitionHash ?? "unknown"; return ``; }) .join(""); const rows = history.campaigns .map((campaign) => { const status = campaign.failed === 0 ? "passed" : "failed"; const sha = campaign.source.sha; const searchable = [ campaign.campaignId, sha, campaign.source.ref, ...campaign.executions.flatMap((execution) => [ execution.suiteId, execution.profileId, execution.model, execution.environmentId, execution.caseId, execution.status, ]), ] .filter(Boolean) .join(" ") .toLowerCase(); return ` ${html(campaign.campaignId)}${html(new Date(campaign.generatedAt).toLocaleString("en-US", { timeZone: "UTC" }))} UTC ${sha ? `${html(sha.slice(0, 10))}` : "Unknown"}${html(campaign.source.ref ?? "unknown ref")} ${status}${campaign.passed}/${campaign.passed + campaign.failed} passed · ${campaign.complete ? "complete" : "partial"} ${html(tokenLabel(campaign.billing.llm.inputTokens))} / ${html(tokenLabel(campaign.billing.llm.outputTokens))}input / output · ${html(tokenLabel(campaign.billing.llm.cachedInputTokens))} cached ${html(usdLabel(campaign.billing.reportedLlmCostUsd))}${html(usdLabel(campaign.billing.estimatedRuntimeCostUsd))} runtime estimate ${html(durationLabel(campaign.billing.agentRunDurationMs))}${html(durationLabel(campaign.billing.leaseDurationMs))} lease `; }) .join(""); const latest = history.campaigns.find( (campaign) => campaign.campaignId === history.latestCampaignId, ); const latestGreen = history.campaigns.find( (campaign) => campaign.campaignId === history.latestGreenCampaignId, ); return `

History

Campaign trends

Complete campaigns are compared by default. Partial smoke runs remain searchable and are labeled explicitly.

${charts}
${suiteCharts}
${rows}
CampaignPaperclip SHAResultTokensCostExecution
`; } function renderSuiteMatrix(input: { suiteCatalog: readonly ReportExecution[]; expected: ReadonlySet; entryById: ReadonlyMap; summary?: RunnerE2ESuiteSummary; }) { const suite = input.suiteCatalog[0]?.suite; if (!suite) return ""; const profiles = [ ...new Map( input.suiteCatalog.map((execution) => [ execution.profile.id, execution.profile, ]), ).values(), ]; const environments = [ ...new Map( input.suiteCatalog.map((execution) => [ execution.environment.id, execution.environment, ]), ).values(), ]; const rows = profiles .map((profile, profileIndex) => { const columns = environments .map((environment) => { const executions = input.suiteCatalog.filter( (execution) => execution.profile.id === profile.id && execution.environment.id === environment.id, ); return `
${html(environment.label)}${html(environment.provider)} · ${html(environment.expectedExecutionTarget.kind)}
${executions.map((execution) => renderCase(execution, input.expected, input.entryById)).join("")}
`; }) .join(""); return `
${html(profile.label)}${html(profile.generation)}${html(profile.provider)} · ${html(profile.model)}
${columns}`; }) .join(""); const environmentHeaders = environments .map( (environment) => `${html(environment.label)}${html(environment.provider)} · ${html(environment.expectedExecutionTarget.kind)}`, ) .join(""); const selected = input.suiteCatalog.filter((execution) => input.expected.has(execution.id), ).length; const summary = input.summary; const summaryHtml = summary ? `
Pass rate${summary.selected > 0 ? ((summary.passed / summary.selected) * 100).toFixed(1) : "0.0"}%${summary.passed}/${summary.selected} passed
Tokens${html(tokenLabel(summary.billing.llm.totalTokens))}${html(tokenLabel(summary.billing.llm.inputTokens))} input · ${html(tokenLabel(summary.billing.llm.outputTokens))} output
Cost${html(usdLabel(summary.billing.observedAndEstimatedCostUsd))}reported LLM + runtime estimate
Agent time${html(durationLabel(summary.billing.agentRunDurationMs))}${html(durationLabel(summary.billing.leaseDurationMs))} lease
Execution${summary.executed}/${summary.selected}${summary.retries} retries · cleanup ${summary.cleanupPassed ? "passed" : "failed"}
` : ""; return `

Test suite

${html(suite.label)}

${html(suite.description)}

${summaryHtml}
Configuration matrix${profiles.length} profiles · ${environments.length} environments · ${selected} selected
${environments.map(() => '').join("")}${environmentHeaders}${rows}
Agent profile
`; } export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { const catalog = discoverReportCatalog({ catalog: input.catalog, expected: input.expected, results: input.entries.map((entry) => entry.result), }); const expected = new Set(input.expected); const entryById = new Map( input.entries.map((entry) => [entry.result.executionId, entry]), ); const selectedEntries = input.entries.filter((entry) => expected.has(entry.result.executionId), ); const passed = selectedEntries.filter((entry) => entry.valid).length; const failed = input.expected.length - passed; const totalDuration = selectedEntries.reduce( (total, entry) => total + entry.result.durationMs, 0, ); const screenshotCount = selectedEntries.reduce( (total, entry) => total + resolveScreenshots(entry).length, 0, ); const campaignBilling = aggregateCampaignBilling( selectedEntries.map((entry) => entry.result), ); const suites = [ ...new Map( catalog.map((execution) => [execution.suite.id, execution.suite]), ).values(), ]; const suiteSections = suites .map((suite) => renderSuiteMatrix({ suiteCatalog: catalog.filter( (execution) => execution.suite.id === suite.id, ), expected, entryById, summary: input.campaign?.suites.find( (summary) => summary.suiteId === suite.id, ), }), ) .join(""); const historySection = renderHistory(input.history); const publicSummaryImageHref = safePublicAssetHref( input.publicSummaryImageHref, ); const filterProfiles = [ ...new Map( catalog.map((execution) => [execution.profile.id, execution.profile]), ).values(), ]; const filterEnvironments = [ ...new Map( catalog.map((execution) => [ execution.environment.id, execution.environment, ]), ).values(), ]; return ` ${html(input.title)} · Paperclip
Paperclip Quality engineering · Runner acceptance

Full-stack acceptance campaign

${html(input.title)}

A browser-verified matrix of runner profiles, execution environments, and deterministic task contracts. Declared PNG screenshots and sanitized structured evidence are retained with every published campaign; additional diagnostic evidence remains in the access-controlled workflow artifact.

${passed}/${input.expected.length}Passed
${failed}Failed
${html(durationLabel(totalDuration))}Test time
${publicSummaryImageHref ? `
Runner E2E campaign status summary
` : ""}
${html(tokenLabel(campaignBilling.llm.inputTokens))}Input tokens
${html(tokenLabel(campaignBilling.llm.outputTokens))}Output tokens
${html(tokenLabel(campaignBilling.llm.cachedInputTokens))}Cached tokens
${html(usdLabel(campaignBilling.reportedLlmCostUsd))}LLM reported subtotal
${html(usdLabel(campaignBilling.estimatedRuntimeCostUsd))}Daytona list estimate
${html(durationLabel(campaignBilling.agentRunDurationMs))}Agent execution time
${html(durationLabel(campaignBilling.leaseDurationMs))}Daytona lease time
${campaignBilling.llm.runsWithReportedCost}/${campaignBilling.llm.runCount}Runs provider-priced

Model spend is the provider-reported subtotal; unpriced or unavailable runs are excluded, never counted as free. Daytona runtime is a public-list-price estimate from captured lease time and pinned resources, before credits, discounts, storage allowance, or invoice adjustments. Local execution has no external runtime meter.

${suiteSections} ${historySection}
Generated ${html(input.generatedAt)}${catalog.length} catalog executions · Declared screenshots and sanitized structured evidence published
`; }