98 lines
5.2 KiB
JavaScript
98 lines
5.2 KiB
JavaScript
import { createHash } from "node:crypto";
|
|
import { isDeepStrictEqual } from "node:util";
|
|
|
|
const SHA = /^[a-f0-9]{40}$/;
|
|
const ID = /^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$/;
|
|
|
|
// Provider list cost and our pricing estimate are alternative measurements,
|
|
// never additive. Missing usage is unknown, not a zero-dollar call.
|
|
export function sumAttemptCosts(usages, scope = "all_attempts") {
|
|
const metric = (field) => {
|
|
const values = usages.map((usage) => usage?.[field]).filter((value) =>
|
|
typeof value === "number" && Number.isFinite(value) && value >= 0);
|
|
return {
|
|
nanodollars: values.length ? values.reduce((sum, value) => sum + value, 0) : null,
|
|
recordedAttempts: values.length,
|
|
};
|
|
};
|
|
return {
|
|
scope,
|
|
attempts: usages.length,
|
|
estimated: metric("estimatedCostNanodollars"),
|
|
providerReported: metric("providerReportedCostNanodollars"),
|
|
};
|
|
}
|
|
|
|
export function campaignAnalytics(campaign) {
|
|
const results = campaign.results ?? [];
|
|
const cells = results.map((result) => [result.cellId, result.model, result.provider, result.driver]);
|
|
cells.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b)));
|
|
const comparable = SHA.test(campaign.source?.evals?.sha ?? "")
|
|
&& cells.length > 0 && cells.length === campaign.totals?.selected
|
|
&& results.every((result) => typeof result.passed === "boolean")
|
|
&& results.filter((result) => result.passed).length === campaign.totals.passed
|
|
&& new Set(cells.map(([id]) => id)).size === cells.length
|
|
&& cells.every((cell) => cell.every((value) => typeof value === "string" && value.length > 0));
|
|
const suiteKey = comparable ? createHash("sha256")
|
|
.update(JSON.stringify([campaign.source.evals.sha, cells])).digest("hex") : null;
|
|
const suppliedCosts = campaign.costs;
|
|
const validCosts = suppliedCosts && ["all_attempts", "final_attempts"].includes(suppliedCosts.scope)
|
|
&& Number.isSafeInteger(suppliedCosts.attempts) && suppliedCosts.attempts >= 0
|
|
&& ["estimated", "providerReported"].every((field) => {
|
|
const metric = suppliedCosts[field];
|
|
return Number.isSafeInteger(metric?.recordedAttempts) && metric.recordedAttempts >= 0
|
|
&& metric.recordedAttempts <= suppliedCosts.attempts
|
|
&& (metric.nanodollars === null || typeof metric.nanodollars === "number" && Number.isFinite(metric.nanodollars) && metric.nanodollars >= 0);
|
|
});
|
|
return {
|
|
schema: "paperclip.runner-protocol-eval.analytics/v1",
|
|
suiteKey,
|
|
selection: campaign.selection?.kind === "maintained_full" ? "maintained_full"
|
|
: campaign.selection?.kind === "subset" ? "subset" : "recorded",
|
|
costs: validCosts ? {
|
|
scope: suppliedCosts.scope, attempts: suppliedCosts.attempts,
|
|
estimated: { nanodollars: suppliedCosts.estimated.nanodollars, recordedAttempts: suppliedCosts.estimated.recordedAttempts },
|
|
providerReported: { nanodollars: suppliedCosts.providerReported.nanodollars, recordedAttempts: suppliedCosts.providerReported.recordedAttempts },
|
|
} : sumAttemptCosts(results.map((result) => result.usage), "final_attempts"),
|
|
failedCells: results.filter((result) => !result.passed).map((result) => ({
|
|
cellId: ID.test(result.cellId) ? result.cellId : "unknown",
|
|
caseId: ID.test(result.caseId) ? result.caseId : "unknown",
|
|
rosterId: ID.test(result.rosterId) ? result.rosterId : "unknown",
|
|
disposition: result.disposition === "behavior_failure" ? "behavior_failure" : "infrastructure_failure",
|
|
})),
|
|
};
|
|
}
|
|
|
|
export function compareCampaignAnalytics(current, previous) {
|
|
if (!current?.suiteKey || current.suiteKey !== previous?.suiteKey) return null;
|
|
const before = new Set(previous.failedCells.map((cell) => cell.cellId));
|
|
const after = new Set(current.failedCells.map((cell) => cell.cellId));
|
|
return {
|
|
regressions: current.failedCells.filter((cell) => !before.has(cell.cellId)),
|
|
recoveries: previous.failedCells.filter((cell) => !after.has(cell.cellId)),
|
|
};
|
|
}
|
|
|
|
// History records and campaign bundles stay immutable. This separate, derived
|
|
// projection can be backfilled from old bundles or enriched by a report refresh.
|
|
export async function enrichProtocolEvalHistory(history, { currentCampaign, loadCampaign }) {
|
|
const analytics = { ...history.analytics };
|
|
const sourceId = currentCampaign.reportRevision?.sourceCampaignId ?? currentCampaign.campaignId;
|
|
for (const record of history.campaigns.filter((item) => !item.reportRevision)) {
|
|
const refresh = record.campaignId === sourceId ? currentCampaign : null;
|
|
if (!refresh && analytics[record.campaignId]?.schema === "paperclip.runner-protocol-eval.analytics/v1") continue;
|
|
const campaign = refresh ?? await loadCampaign(record.campaignId);
|
|
if (!campaign) continue;
|
|
if (!isDeepStrictEqual(campaign.source, record.source)
|
|
|| !isDeepStrictEqual(campaign.totals, record.totals))
|
|
throw new Error(`Campaign analytics do not match immutable history: ${record.campaignId}`);
|
|
const projected = campaignAnalytics(campaign);
|
|
const previous = analytics[record.campaignId];
|
|
// Never downgrade an all-attempt measurement with a historical final-only one.
|
|
if (previous?.costs?.scope === "all_attempts" && projected.costs.scope !== "all_attempts")
|
|
projected.costs = previous.costs;
|
|
analytics[record.campaignId] = projected;
|
|
}
|
|
return { ...history, analytics };
|
|
}
|