import { useEffect, useMemo, useState } from "react"; import type { CapabilityDevtoolsSnapshot } from "../../../src/devtools"; import { Icon, type IconName } from "./Icons"; import { EvalAssertions, type EvalAssertion } from "./ThreadItems"; export type CapabilityDevtoolsTab = | "eval" | "evidence" | "timeline" | "state" | "diff" | "documents" | "protocol" | "trace" | "runtime" | "authority"; type Json = null | boolean | number | string | Json[] | { [key: string]: Json }; export interface EvalInspectorReport { attemptId: string; caseId: string; disposition: string; passed: boolean; checks: EvalAssertion[]; run: { model: string; provider: string; driver: string; providerVersion: string | null; runnerProvider: string; configuration: string; sessionId: string; providerSessionId: string | null; agentVersion: string | null; retainedSession: boolean | null; retainedSessionStatus: string | null; fixtureDigest: string; runnerPackageDigest: string; runnerdDigest: string; runnerBuild: string; startedAt: string; finishedAt: string; durationMs: number; initialRevision: number; finalRevision: number; usage: { agentTurns: number; providerRequests: number | null; inputTokens: number; outputTokens: number; cachedInputTokens: number; reasoningTokens: number; providerReportedCostNanodollars?: number; estimatedCostNanodollars: number; pricingVersion: string; } | null; }; } function JsonTree({ value, name = "root" }: { value: Json; name?: string }) { if (value === null || typeof value !== "object") { return {JSON.stringify(value)}; } const entries = Array.isArray(value) ? value.map((entry, index) => [String(index), entry] as const) : Object.entries(value); return (
{name}{" "} {Array.isArray(value) ? `[${entries.length}]` : `{${entries.length}}`}
{entries.map(([key, entry]) => (
{key} {entry !== null && typeof entry === "object" ? ( ) : ( )}
))}
); } interface DiffRow { path: string; before: Json | undefined; after: Json | undefined; } function diff( before: Json | undefined, after: Json | undefined, path = "state", ): DiffRow[] { if (JSON.stringify(before) === JSON.stringify(after)) return []; if ( before !== null && after !== null && typeof before === "object" && typeof after === "object" && !Array.isArray(before) && !Array.isArray(after) ) { const keys = new Set([...Object.keys(before), ...Object.keys(after)]); return [...keys].flatMap((key) => diff(before[key], after[key], `${path}.${key}`), ); } return [{ path, before, after }]; } function download(snapshot: CapabilityDevtoolsSnapshot) { const blob = new Blob([JSON.stringify(snapshot, null, 2)], { type: "application/json", }); const url = URL.createObjectURL(blob); const anchor = document.createElement("a"); anchor.href = url; anchor.download = `paperclip-devtools-r${snapshot.currentRevision}.json`; anchor.click(); URL.revokeObjectURL(url); } const TABS: ReadonlyArray<{ id: CapabilityDevtoolsTab; icon: IconName; label: string; }> = [ { id: "evidence", icon: "evidence", label: "Evidence" }, { id: "timeline", icon: "timeline", label: "Timeline" }, { id: "state", icon: "state", label: "State" }, { id: "diff", icon: "diff", label: "Diff" }, { id: "documents", icon: "documents", label: "Documents" }, { id: "protocol", icon: "protocol", label: "Protocol" }, { id: "trace", icon: "protocol", label: "Provider trace" }, { id: "runtime", icon: "terminal", label: "Runtime" }, { id: "authority", icon: "authority", label: "Authority" }, ]; function documentsOf(state: Json): Array<{ id: string; key: string; title: string; revisions: Array<{ id: string; revision: number; body: string; changeSummary: string | null; createdAt: string; }>; }> { if (state === null || typeof state !== "object" || Array.isArray(state)) return []; const documents = state.documents; if (!Array.isArray(documents)) return []; return documents.flatMap((value) => { if (value === null || typeof value !== "object" || Array.isArray(value)) return []; const revisions = Array.isArray(value.revisions) ? value.revisions.flatMap((candidate) => { if ( candidate === null || typeof candidate !== "object" || Array.isArray(candidate) ) return []; return [ { id: String(candidate.id ?? ""), revision: Number(candidate.revision ?? 0), body: String(candidate.body ?? ""), changeSummary: candidate.changeSummary === null ? null : String(candidate.changeSummary ?? ""), createdAt: String(candidate.createdAt ?? ""), }, ]; }) : []; return [ { id: String(value.id ?? ""), key: String(value.key ?? ""), title: String(value.title ?? "Untitled"), revisions, }, ]; }); } export function DevtoolsInspector({ snapshot, onFork, tab, onTabChange, evalReport, }: { snapshot: CapabilityDevtoolsSnapshot; onFork: (revision: number) => void; tab: CapabilityDevtoolsTab; onTabChange: (tab: CapabilityDevtoolsTab) => void; evalReport?: EvalInspectorReport | null; }) { const [query, setQuery] = useState(""); const latest = snapshot.revisions.at(-1)!; const [revision, setRevision] = useState(latest.revision); const [following, setFollowing] = useState(true); const [compareRevision, setCompareRevision] = useState( snapshot.revisions[0]!.revision, ); const selected = snapshot.revisions.find((entry) => entry.revision === revision) ?? latest; const compared = snapshot.revisions.find((entry) => entry.revision === compareRevision) ?? snapshot.revisions[0]!; const rows = useMemo( () => diff(compared.state as Json, selected.state as Json), [compared.state, selected.state], ); const normalized = query.trim().toLowerCase(); const revisions = snapshot.revisions.filter( (entry) => normalized.length === 0 || JSON.stringify(entry).toLowerCase().includes(normalized), ); const protocol = snapshot.protocol.filter( (entry) => normalized.length === 0 || JSON.stringify(entry).toLowerCase().includes(normalized), ); const documents = useMemo( () => documentsOf(selected.state as Json), [selected.state], ); const [documentId, setDocumentId] = useState(""); const document = documents.find((entry) => entry.id === documentId) ?? documents[0] ?? null; const [documentRevision, setDocumentRevision] = useState(0); const visibleDocumentRevision = document?.revisions.find((entry) => entry.revision === documentRevision) ?? document?.revisions.at(-1) ?? null; useEffect(() => { if (following) setRevision(latest.revision); }, [following, latest.revision]); return (
State monitor setQuery(event.target.value)} placeholder="operation, entity, turn…" />
{[ ...(evalReport ? [ { id: "eval" as const, icon: "evidence" as const, label: "Eval", }, ] : []), ...TABS, ].map(({ id, icon, label }) => ( ))}
{tab === "eval" && evalReport ? (
← Eval suite {evalReport.passed ? "PASS" : evalReport.disposition.replaceAll("_", " ").toUpperCase()} {evalReport.attemptId}
Model
{evalReport.run.model.startsWith(`${evalReport.run.provider}/`) ? evalReport.run.model : `${evalReport.run.provider}/${evalReport.run.model}`}
Configuration
{evalReport.run.configuration}
Session
{evalReport.run.sessionId}
Provider session
{evalReport.run.providerSessionId ?? "unavailable"}
Driver
{evalReport.run.driver} {evalReport.run.providerVersion ? ` · ${evalReport.run.providerVersion}` : ""}
{evalReport.run.agentVersion ? (
Agent version
{evalReport.run.agentVersion}
) : null}
Retained session
{evalReport.run.retainedSession === true ? (evalReport.run.retainedSessionStatus ?? "retained") : "not applicable"}
Duration
{evalReport.run.durationMs} ms
Fixture
{evalReport.run.fixtureDigest}
State
r{evalReport.run.initialRevision} → r {evalReport.run.finalRevision}
Tokens
{evalReport.run.usage === null ? "unknown" : `${evalReport.run.usage.inputTokens} in · ${evalReport.run.usage.outputTokens} out · ${evalReport.run.usage.cachedInputTokens} cached`}
Agent turns
{evalReport.run.usage?.agentTurns ?? "unknown"}
Provider requests
{evalReport.run.usage?.providerRequests ?? "unavailable"}
Estimated cost
{evalReport.run.usage === null ? "unknown" : `$${(evalReport.run.usage.estimatedCostNanodollars / 1_000_000_000).toFixed(6)} · ${evalReport.run.usage.pricingVersion}`}
Provider list cost
{typeof evalReport.run.usage ?.providerReportedCostNanodollars !== "number" ? "unknown" : `$${(evalReport.run.usage.providerReportedCostNanodollars / 1_000_000_000).toFixed(6)}`}
Runner
{evalReport.run.runnerPackageDigest}
Runner build
{evalReport.run.runnerBuild}
runnerd
{evalReport.run.runnerdDigest}

Assertions

) : null} {tab === "timeline" ? (
{revisions.map((entry) => ( ))}
) : null} {tab === "state" ? (
) : null} {tab === "diff" ? (
{rows.map((row) => (
{row.path} {JSON.stringify(row.before)} {JSON.stringify(row.after)}
))} {rows.length === 0 ? (

No changes between these revisions.

) : null}
) : null} {tab === "documents" ? ( documents.length === 0 ? (

No documents exist at r{selected.revision}.

) : (
{documents.map((entry) => ( ))}
{document !== null && visibleDocumentRevision !== null ? (
{document.key}

{document.title}

{visibleDocumentRevision.changeSummary ? (

{visibleDocumentRevision.changeSummary}

) : null}
                  {visibleDocumentRevision.body}
                
) : null}
) ) : null} {tab === "protocol" ? (
{protocol.map((entry) => (
{entry.boundary} · {entry.event}
))}
) : null} {tab === "trace" ? ( snapshot.providerTrace ? (

{snapshot.providerTrace.status} · expires{" "} {snapshot.providerTrace.expiresAt}

Raw frame → Interpretation → PRP events → Presentation

) : (

Raw provider capture was off for this run. Canonical protocol records remain available in Protocol.

) ) : null} {tab === "runtime" ? (
) : null} {tab === "authority" ? (
) : null}
); }