fix(evals): publish the canonical chat viewer on every direct run
Use a closed public replay projection and verify exact viewer assets. Keep read-only reports usable, preserve immutable campaign history, and add a no-provider report refresh command. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Dotta
parent
856813ba3a
commit
dfb35a3ac2
|
|
@ -213,6 +213,11 @@ jobs:
|
|||
--max-parallel "$MAX_PARALLEL" \
|
||||
--output runner-protocol-eval-catalog.json
|
||||
|
||||
- name: Require the chat-report renderer before paid execution
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 .paperclip-evals/evals/paperclip-runner/tools/eval_program.py report --help | grep -q -- --public-viewer
|
||||
|
||||
- name: Upload immutable campaign catalog
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
|
|
@ -290,6 +295,14 @@ jobs:
|
|||
compression-level: 0
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Upload canonical viewer for publisher byte verification
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: runner-protocol-viewer-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: packages/paperclip-runner/dist-issue-thread/
|
||||
retention-days: 30
|
||||
if-no-files-found: error
|
||||
|
||||
eval_shard_0:
|
||||
name: Direct eval ${{ matrix.rosterId }} / ${{ matrix.caseId }}
|
||||
needs: [authorize, catalog, build_runner]
|
||||
|
|
@ -595,6 +608,8 @@ jobs:
|
|||
python3 .paperclip-evals/evals/paperclip-runner/tools/eval_program.py report \
|
||||
--runs-root runner-protocol-merged/public-runs \
|
||||
--output runner-protocol-merged/public-report \
|
||||
--viewer-root runner-protocol-build/extracted/dist-issue-thread \
|
||||
--public-viewer \
|
||||
--inventory .paperclip-evals/evals/paperclip-runner/inventory.json \
|
||||
--coverage-matrix .paperclip-evals/evals/paperclip-runner/coverage-matrix.json
|
||||
cp runner-protocol-merged/campaign.json runner-protocol-merged/public-report/campaign.json
|
||||
|
|
@ -602,7 +617,7 @@ jobs:
|
|||
- name: Enforce the static public allowlist
|
||||
id: public_report
|
||||
run: |
|
||||
node --input-type=module -e 'import { validatePublicProtocolEvalReport } from "./packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.mjs"; await validatePublicProtocolEvalReport("runner-protocol-merged/public-report");'
|
||||
node --input-type=module -e 'import { validatePublicProtocolEvalReport } from "./packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.mjs"; await validatePublicProtocolEvalReport("runner-protocol-merged/public-report", { viewerRoot: "runner-protocol-build/extracted/dist-issue-thread" });'
|
||||
echo "ready=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Add campaign result to the workflow summary
|
||||
|
|
@ -666,6 +681,12 @@ jobs:
|
|||
name: runner-protocol-eval-public-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: runner-protocol-public-report
|
||||
|
||||
- name: Download the same-run canonical viewer for byte verification
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
||||
with:
|
||||
name: runner-protocol-viewer-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: runner-protocol-trusted-viewer
|
||||
|
||||
- name: Exchange GitHub OIDC identity for scoped AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6
|
||||
with:
|
||||
|
|
@ -675,6 +696,7 @@ jobs:
|
|||
- name: Publish versioned report and refresh the root index
|
||||
env:
|
||||
PAPERCLIP_RUNNER_PROTOCOL_EVAL_PUBLIC_REPORT_DIR: ${{ github.workspace }}/runner-protocol-public-report
|
||||
PAPERCLIP_RUNNER_PROTOCOL_EVAL_VIEWER_DIR: ${{ github.workspace }}/runner-protocol-trusted-viewer
|
||||
RUNNER_PROTOCOL_EVAL_HISTORY_S3_BUCKET: ${{ vars.RUNNER_PROTOCOL_EVAL_HISTORY_S3_BUCKET || vars.RUNNER_E2E_HISTORY_S3_BUCKET }}
|
||||
RUNNER_PROTOCOL_EVAL_HISTORY_PREFIX: ${{ vars.RUNNER_PROTOCOL_EVAL_HISTORY_PREFIX || 'runner-protocol-evals' }}
|
||||
RUNNER_PROTOCOL_EVAL_HISTORY_PUBLIC_BASE_URL: ${{ vars.RUNNER_PROTOCOL_EVAL_HISTORY_PUBLIC_BASE_URL || vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL }}
|
||||
|
|
|
|||
|
|
@ -78,6 +78,7 @@ interface EmbeddedEvalCheck {
|
|||
}
|
||||
|
||||
interface EmbeddedEvalReport {
|
||||
publication?: { schema: string; notice: string };
|
||||
attemptId: string;
|
||||
caseId: string;
|
||||
disposition: string;
|
||||
|
|
@ -101,7 +102,7 @@ interface EmbeddedEvalReport {
|
|||
runnerBuild: string;
|
||||
startedAt: string;
|
||||
finishedAt: string;
|
||||
durationMs: number;
|
||||
durationMs: number | null;
|
||||
initialRevision: number;
|
||||
finalRevision: number;
|
||||
usage: {
|
||||
|
|
@ -117,7 +118,7 @@ interface EmbeddedEvalReport {
|
|||
} | null;
|
||||
};
|
||||
view: CapabilityIssueThreadSnapshot;
|
||||
devtools: CapabilityDevtoolsSnapshot;
|
||||
devtools: CapabilityDevtoolsSnapshot | null;
|
||||
navigation: { suiteHref: string; previous: { label: string; href: string } | null; next: { label: string; href: string } | null };
|
||||
}
|
||||
|
||||
|
|
@ -491,7 +492,7 @@ export function App() {
|
|||
await document.fonts.ready;
|
||||
}
|
||||
if (cancelled) return;
|
||||
if (scroller !== null) scroller.scrollTop = scroller.scrollHeight;
|
||||
if (scroller !== null) scroller.scrollTop = embeddedEval === null ? scroller.scrollHeight : 0;
|
||||
await new Promise<void>((resolve) => requestAnimationFrame(() => resolve()));
|
||||
if (!cancelled) setSettled(true);
|
||||
})();
|
||||
|
|
@ -501,14 +502,14 @@ export function App() {
|
|||
}, [snapshot]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!panelOpen || snapshot === null || route.mode !== "live") return;
|
||||
if (embeddedEval !== null || !panelOpen || snapshot === null || route.mode !== "live") return;
|
||||
if (historicSessionId !== null) return;
|
||||
let cancelled = false;
|
||||
void capabilityLiveClient.devtools(snapshot.sessionId)
|
||||
.then((next) => { if (!cancelled) setDevtools(next); })
|
||||
.catch((cause) => { if (!cancelled) setActionError(describe(cause)); });
|
||||
return () => { cancelled = true; };
|
||||
}, [historicSessionId, panelOpen, route.mode, snapshot?.renderedAt, snapshot?.sessionId]);
|
||||
}, [embeddedEval, historicSessionId, panelOpen, route.mode, snapshot?.renderedAt, snapshot?.sessionId]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!chat || snapshot === null || historicSessionId !== null) return;
|
||||
|
|
@ -1363,7 +1364,7 @@ export function App() {
|
|||
>
|
||||
<div className="pit-thread">
|
||||
{embeddedEval !== null ? (
|
||||
<div className="pit-eval-boundary" data-phase="execution"><strong>Eval execution</strong></div>
|
||||
<div className="pit-eval-boundary" data-phase="execution"><strong>Eval execution</strong>{embeddedEval.publication ? <span>{embeddedEval.publication.notice}</span> : null}</div>
|
||||
) : null}
|
||||
{snapshot.turns.length === 0 ? (
|
||||
<section className="pit-empty-thread" data-testid="clean-room-empty">
|
||||
|
|
@ -1404,7 +1405,7 @@ export function App() {
|
|||
{embeddedEval !== null ? (
|
||||
<div className="pit-eval-boundary" data-phase="post-run">
|
||||
<strong>Post-run state</strong>
|
||||
<span>Final mock control-plane revision {embeddedEval.run.finalRevision}</span>
|
||||
<span>{embeddedEval.publication ? "Company-state details withheld from public replay" : `Final mock control-plane revision ${embeddedEval.run.finalRevision}`}</span>
|
||||
<EvalAssertions assertions={embeddedEval.checks.filter((check) => check.anchor.kind === "run")} />
|
||||
</div>
|
||||
) : null}
|
||||
|
|
@ -1425,7 +1426,7 @@ export function App() {
|
|||
) : null}
|
||||
</div>
|
||||
|
||||
<Composer
|
||||
{embeddedEval === null ? <Composer
|
||||
model={snapshot.composer}
|
||||
sessionId={snapshot.sessionId}
|
||||
onSend={send}
|
||||
|
|
@ -1439,7 +1440,7 @@ export function App() {
|
|||
.getElementById(`interaction-${interactionId}`)
|
||||
?.scrollIntoView({ block: "center" });
|
||||
}}
|
||||
/>
|
||||
/> : null}
|
||||
</main>
|
||||
|
||||
{showPanel && layout === "side" ? (
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ export interface EvalInspectorReport {
|
|||
disposition: string;
|
||||
passed: boolean;
|
||||
checks: EvalAssertion[];
|
||||
publication?: { schema: string; notice: string };
|
||||
run: {
|
||||
model: string;
|
||||
provider: string;
|
||||
|
|
@ -41,7 +42,7 @@ export interface EvalInspectorReport {
|
|||
runnerBuild: string;
|
||||
startedAt: string;
|
||||
finishedAt: string;
|
||||
durationMs: number;
|
||||
durationMs: number | null;
|
||||
initialRevision: number;
|
||||
finalRevision: number;
|
||||
usage: {
|
||||
|
|
@ -53,7 +54,7 @@ export interface EvalInspectorReport {
|
|||
reasoningTokens: number;
|
||||
providerReportedCostNanodollars?: number;
|
||||
estimatedCostNanodollars: number;
|
||||
pricingVersion: string;
|
||||
pricingVersion?: string;
|
||||
} | null;
|
||||
};
|
||||
}
|
||||
|
|
@ -197,6 +198,149 @@ function documentsOf(state: Json): Array<{
|
|||
});
|
||||
}
|
||||
|
||||
export function EvalReportInspector({
|
||||
evalReport,
|
||||
}: {
|
||||
evalReport: EvalInspectorReport;
|
||||
}) {
|
||||
const isPublic = Boolean(evalReport.publication);
|
||||
return (
|
||||
<div className="pit-devtools-pane pit-eval-inspector">
|
||||
<div className="pit-eval-summary-head">
|
||||
<a href="../../index.html">← Eval suite</a>
|
||||
<strong>
|
||||
{evalReport.passed
|
||||
? "PASS"
|
||||
: evalReport.disposition.replaceAll("_", " ").toUpperCase()}
|
||||
</strong>
|
||||
<code>{evalReport.attemptId}</code>
|
||||
</div>
|
||||
<dl className="pit-eval-run-facts">
|
||||
<div>
|
||||
<dt>Model</dt>
|
||||
<dd>
|
||||
{evalReport.run.model.startsWith(`${evalReport.run.provider}/`)
|
||||
? evalReport.run.model
|
||||
: `${evalReport.run.provider}/${evalReport.run.model}`}
|
||||
</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Configuration</dt>
|
||||
<dd>{evalReport.run.configuration}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Session</dt>
|
||||
<dd>
|
||||
{isPublic
|
||||
? "Withheld from public replay"
|
||||
: evalReport.run.sessionId}
|
||||
</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Provider session</dt>
|
||||
<dd>
|
||||
{isPublic
|
||||
? "Withheld from public replay"
|
||||
: (evalReport.run.providerSessionId ?? "unavailable")}
|
||||
</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Driver</dt>
|
||||
<dd>
|
||||
{evalReport.run.driver}
|
||||
{evalReport.run.providerVersion
|
||||
? ` · ${evalReport.run.providerVersion}`
|
||||
: ""}
|
||||
</dd>
|
||||
</div>
|
||||
{evalReport.run.agentVersion ? (
|
||||
<div>
|
||||
<dt>Agent version</dt>
|
||||
<dd>{evalReport.run.agentVersion}</dd>
|
||||
</div>
|
||||
) : null}
|
||||
<div>
|
||||
<dt>Retained session</dt>
|
||||
<dd>
|
||||
{isPublic
|
||||
? "Withheld from public replay"
|
||||
: evalReport.run.retainedSession === true
|
||||
? (evalReport.run.retainedSessionStatus ?? "retained")
|
||||
: "not applicable"}
|
||||
</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Duration</dt>
|
||||
<dd>
|
||||
{evalReport.run.durationMs == null
|
||||
? "unavailable"
|
||||
: `${evalReport.run.durationMs} ms`}
|
||||
</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Fixture</dt>
|
||||
<dd>{evalReport.run.fixtureDigest}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>State</dt>
|
||||
<dd>
|
||||
{isPublic
|
||||
? "Withheld from public replay"
|
||||
: `r${evalReport.run.initialRevision} → r${evalReport.run.finalRevision}`}
|
||||
</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Tokens</dt>
|
||||
<dd>
|
||||
{evalReport.run.usage === null
|
||||
? "unknown"
|
||||
: `${evalReport.run.usage.inputTokens} in · ${evalReport.run.usage.outputTokens} out · ${evalReport.run.usage.cachedInputTokens} cached`}
|
||||
</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Agent turns</dt>
|
||||
<dd>{evalReport.run.usage?.agentTurns ?? "unknown"}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Provider requests</dt>
|
||||
<dd>{evalReport.run.usage?.providerRequests ?? "unavailable"}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Estimated cost</dt>
|
||||
<dd>
|
||||
{evalReport.run.usage === null
|
||||
? "unknown"
|
||||
: `$${(evalReport.run.usage.estimatedCostNanodollars / 1_000_000_000).toFixed(6)}${evalReport.run.usage.pricingVersion ? ` · ${evalReport.run.usage.pricingVersion}` : ""}`}
|
||||
</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Provider list cost</dt>
|
||||
<dd>
|
||||
{typeof evalReport.run.usage?.providerReportedCostNanodollars !==
|
||||
"number"
|
||||
? "unknown"
|
||||
: `$${(evalReport.run.usage.providerReportedCostNanodollars / 1_000_000_000).toFixed(6)}`}
|
||||
</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Runner</dt>
|
||||
<dd>{evalReport.run.runnerPackageDigest}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Runner build</dt>
|
||||
<dd>{evalReport.run.runnerBuild}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>runnerd</dt>
|
||||
<dd>{evalReport.run.runnerdDigest}</dd>
|
||||
</div>
|
||||
</dl>
|
||||
<h3>Assertions</h3>
|
||||
<EvalAssertions assertions={evalReport.checks} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function DevtoolsInspector({
|
||||
snapshot,
|
||||
onFork,
|
||||
|
|
@ -291,6 +435,8 @@ export function DevtoolsInspector({
|
|||
className="pit-button"
|
||||
type="button"
|
||||
onClick={() => onFork(revision)}
|
||||
disabled={evalReport != null}
|
||||
title={evalReport ? "Eval reports are read-only" : undefined}
|
||||
>
|
||||
<Icon name="branch" /> Fork r{revision}
|
||||
</button>
|
||||
|
|
@ -328,124 +474,7 @@ export function DevtoolsInspector({
|
|||
))}
|
||||
</div>
|
||||
{tab === "eval" && evalReport ? (
|
||||
<div className="pit-devtools-pane pit-eval-inspector">
|
||||
<div className="pit-eval-summary-head">
|
||||
<a href="../../index.html">← Eval suite</a>
|
||||
<strong>
|
||||
{evalReport.passed
|
||||
? "PASS"
|
||||
: evalReport.disposition.replaceAll("_", " ").toUpperCase()}
|
||||
</strong>
|
||||
<code>{evalReport.attemptId}</code>
|
||||
</div>
|
||||
<dl className="pit-eval-run-facts">
|
||||
<div>
|
||||
<dt>Model</dt>
|
||||
<dd>
|
||||
{evalReport.run.model.startsWith(`${evalReport.run.provider}/`)
|
||||
? evalReport.run.model
|
||||
: `${evalReport.run.provider}/${evalReport.run.model}`}
|
||||
</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Configuration</dt>
|
||||
<dd>{evalReport.run.configuration}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Session</dt>
|
||||
<dd>{evalReport.run.sessionId}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Provider session</dt>
|
||||
<dd>{evalReport.run.providerSessionId ?? "unavailable"}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Driver</dt>
|
||||
<dd>
|
||||
{evalReport.run.driver}
|
||||
{evalReport.run.providerVersion
|
||||
? ` · ${evalReport.run.providerVersion}`
|
||||
: ""}
|
||||
</dd>
|
||||
</div>
|
||||
{evalReport.run.agentVersion ? (
|
||||
<div>
|
||||
<dt>Agent version</dt>
|
||||
<dd>{evalReport.run.agentVersion}</dd>
|
||||
</div>
|
||||
) : null}
|
||||
<div>
|
||||
<dt>Retained session</dt>
|
||||
<dd>
|
||||
{evalReport.run.retainedSession === true
|
||||
? (evalReport.run.retainedSessionStatus ?? "retained")
|
||||
: "not applicable"}
|
||||
</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Duration</dt>
|
||||
<dd>{evalReport.run.durationMs} ms</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Fixture</dt>
|
||||
<dd>{evalReport.run.fixtureDigest}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>State</dt>
|
||||
<dd>
|
||||
r{evalReport.run.initialRevision} → r
|
||||
{evalReport.run.finalRevision}
|
||||
</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Tokens</dt>
|
||||
<dd>
|
||||
{evalReport.run.usage === null
|
||||
? "unknown"
|
||||
: `${evalReport.run.usage.inputTokens} in · ${evalReport.run.usage.outputTokens} out · ${evalReport.run.usage.cachedInputTokens} cached`}
|
||||
</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Agent turns</dt>
|
||||
<dd>{evalReport.run.usage?.agentTurns ?? "unknown"}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Provider requests</dt>
|
||||
<dd>{evalReport.run.usage?.providerRequests ?? "unavailable"}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Estimated cost</dt>
|
||||
<dd>
|
||||
{evalReport.run.usage === null
|
||||
? "unknown"
|
||||
: `$${(evalReport.run.usage.estimatedCostNanodollars / 1_000_000_000).toFixed(6)} · ${evalReport.run.usage.pricingVersion}`}
|
||||
</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Provider list cost</dt>
|
||||
<dd>
|
||||
{typeof evalReport.run.usage
|
||||
?.providerReportedCostNanodollars !== "number"
|
||||
? "unknown"
|
||||
: `$${(evalReport.run.usage.providerReportedCostNanodollars / 1_000_000_000).toFixed(6)}`}
|
||||
</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Runner</dt>
|
||||
<dd>{evalReport.run.runnerPackageDigest}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Runner build</dt>
|
||||
<dd>{evalReport.run.runnerBuild}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>runnerd</dt>
|
||||
<dd>{evalReport.run.runnerdDigest}</dd>
|
||||
</div>
|
||||
</dl>
|
||||
<h3>Assertions</h3>
|
||||
<EvalAssertions assertions={evalReport.checks} />
|
||||
</div>
|
||||
<EvalReportInspector evalReport={evalReport} />
|
||||
) : null}
|
||||
{tab === "timeline" ? (
|
||||
<div className="pit-devtools-list">
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import type {
|
|||
CapabilityToolDisposition,
|
||||
} from "../../../src/issue-thread/types";
|
||||
import type { CapabilityDevtoolsSnapshot } from "../../../src/devtools";
|
||||
import { DevtoolsInspector, type CapabilityDevtoolsTab, type EvalInspectorReport } from "./DevtoolsInspector";
|
||||
import { DevtoolsInspector, EvalReportInspector, type CapabilityDevtoolsTab, type EvalInspectorReport } from "./DevtoolsInspector";
|
||||
import { Icon } from "./Icons";
|
||||
import { capabilitySemanticToolDescriptor } from "../../../src/semantic-tools/catalog";
|
||||
import {
|
||||
|
|
@ -392,7 +392,7 @@ export function EvidencePanel(props: EvidencePanelProps) {
|
|||
{devtools !== undefined ? (
|
||||
<>
|
||||
{devtools === null ? (
|
||||
<p className="pit-muted pit-devtools-loading">Loading company state…</p>
|
||||
evalReport ? <EvalReportInspector evalReport={evalReport} /> : <p className="pit-muted pit-devtools-loading">Loading company state…</p>
|
||||
) : (
|
||||
<DevtoolsInspector snapshot={devtools} onFork={onForkRevision} tab={devtoolsTab} onTabChange={setDevtoolsTab} evalReport={evalReport} />
|
||||
)}
|
||||
|
|
|
|||
|
|
@ -4,11 +4,23 @@ import { createRoot } from "react-dom/client";
|
|||
import { App } from "./App";
|
||||
import "./issue-thread.css";
|
||||
|
||||
// Hosted Evalbooks use inert JSON plus the same trusted viewer bundle. No
|
||||
// inline executable script or network fetch is needed to load an attempt.
|
||||
const reportData = document.getElementById("paperclip-eval-report");
|
||||
if (reportData !== null) {
|
||||
window.__PAPERCLIP_EVAL_REPORT__ = JSON.parse(
|
||||
reportData.textContent ?? "null",
|
||||
);
|
||||
}
|
||||
|
||||
// `?capture=1` freezes animation, caret, and smooth scrolling so the
|
||||
// screenshot matrix is byte-stable across runs (contract §10.1).
|
||||
const params = new URLSearchParams(window.location.search);
|
||||
const hashQuery = window.location.hash.split("?")[1] ?? "";
|
||||
if (params.get("capture") === "1" || new URLSearchParams(hashQuery).get("capture") === "1") {
|
||||
if (
|
||||
params.get("capture") === "1" ||
|
||||
new URLSearchParams(hashQuery).get("capture") === "1"
|
||||
) {
|
||||
document.documentElement.dataset.capture = "true";
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,48 @@
|
|||
# Direct live Runner protocol evals
|
||||
|
||||
## One Evalbook presentation
|
||||
|
||||
Every new report uses the canonical Evalbook grid and the existing Runner Lab
|
||||
chat viewer for attempt drill-downs. There is no plain-HTML attempt fallback.
|
||||
Missing recordings show a notice in the same viewer; missing viewer builds
|
||||
fail generation. Build with
|
||||
`pnpm --filter @paperclipai/paperclip-runner build:issue-thread` and provide
|
||||
`--viewer-root` or `PAPERCLIP_EVAL_VIEWER_ROOT` to the canonical Python renderer.
|
||||
|
||||
The Actions artifact contains full evidence. S3 uses the same viewer with a
|
||||
closed public DTO: only isolated mock-run conversation text, scrubbed private
|
||||
references, named tool outcomes, and checks. Tool arguments/results, reasoning,
|
||||
provider identities, and company snapshots stay private. The public notice
|
||||
explains these redactions. An unverified isolation boundary yields no public
|
||||
conversation, not a guessed reconstruction.
|
||||
|
||||
Public attempts use inert JSON and one shared viewer asset directory. The
|
||||
publisher verifies each shell and asset against the exact same-run viewer build,
|
||||
checks the public payload contract and local links, and rejects other scripts.
|
||||
The CSP prohibits network calls, forms and external resources. Supply
|
||||
`PAPERCLIP_RUNNER_PROTOCOL_EVAL_VIEWER_DIR` to the publisher. The workflow sends
|
||||
a viewer-only artifact to that job; raw attempts and provider secrets stay out.
|
||||
|
||||
### Refresh a completed report without calling models
|
||||
|
||||
Download the aggregate Actions artifact, then:
|
||||
|
||||
```sh
|
||||
node packages/paperclip-runner/scripts/refresh-runner-protocol-eval-report.mjs \
|
||||
--source /path/to/downloaded-aggregate \
|
||||
--evals-root /path/to/paperclip-evals \
|
||||
--viewer-root packages/paperclip-runner/dist-issue-thread \
|
||||
--output /path/to/new-refresh-directory \
|
||||
--revision chat-v1
|
||||
```
|
||||
|
||||
Publish the returned `reportRoot` with the normal history publisher. The new ID
|
||||
is `gha-RUN-ATTEMPT-report-chat-v1`. Original reports remain immutable; history
|
||||
adds a labeled refresh and retains the source campaign, original measurement
|
||||
timestamp, renderer digest, and `providerCalls: 0`. Scores and evaluated source
|
||||
revisions do not change. This is not a new model qualification run. Future live
|
||||
runs create chat reports automatically.
|
||||
|
||||
This is the provider-backed, one-turn protocol qualification layer in
|
||||
`paperclipai/paperclip-evals/evals/paperclip-runner`. It is intentionally
|
||||
separate from both the browser full-stack model E2E and the stress-derived
|
||||
|
|
@ -149,11 +192,11 @@ read-only Runner issue-thread attempt pages, and raw immutable run records.
|
|||
|
||||
Public publishing uses a separate projection and a separate trusted OIDC job.
|
||||
The projection retains model/config identity, status, usage totals, and check
|
||||
outcomes but removes provider session identifiers, transcripts, semantic-tool
|
||||
outcomes and scrubbed mock conversation, but removes provider session identifiers, semantic-tool
|
||||
payloads, state revisions, traces, remote profile identities, and raw failure
|
||||
text. The same Evalbook `report` command renders that projection, so the public
|
||||
grid and test pages have the standard Evalbook layout. The publisher rejects
|
||||
scripts, remote resources, symlinks, unknown paths, broken links, raw session
|
||||
grid, test pages and chat viewer have the standard Evalbook layout. The publisher rejects
|
||||
untrusted scripts, remote resources, symlinks, unknown paths, broken links, raw session
|
||||
fields, and credential-shaped values.
|
||||
|
||||
S3 publication is additive:
|
||||
|
|
|
|||
|
|
@ -115,7 +115,7 @@
|
|||
"test:capability-evals": "vitest run src/conformance/capability-eval-suite.test.ts",
|
||||
"test:eval-slice": "pnpm run ensure:eval-build-deps && vitest run src/eval",
|
||||
"test:runner-workflow-evals": "pnpm run ensure:eval-build-deps && node --test scripts/render-runner-workflow-evalbook.test.mjs && vitest run src/eval/workflow-evals.test.ts src/eval/live-workflow-executor.test.ts",
|
||||
"test:runner-protocol-eval-publish": "node --test scripts/runner-protocol-eval-campaign.test.mjs scripts/publish-runner-protocol-eval-history.test.mjs scripts/runner-protocol-eval-workflow-security.test.mjs",
|
||||
"test:runner-protocol-eval-publish": "node --test scripts/runner-protocol-eval-campaign.test.mjs scripts/publish-runner-protocol-eval-history.test.mjs scripts/runner-protocol-eval-workflow-security.test.mjs scripts/public-eval-chat.test.mjs",
|
||||
"check:runner-workflow-traceability": "pnpm run build:typescript && node scripts/check-runner-workflow-traceability.mjs",
|
||||
"report:capability-evals": "pnpm run build:typescript && node scripts/run-capability-eval-suite.mjs",
|
||||
"report:capability-live-evals": "pnpm run build:typescript && cargo build --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --bin paperclip-runnerd && node scripts/run-capability-live-eval-matrix.mjs",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,216 @@
|
|||
// Public replay is a new DTO, never a recursive copy of a provider artifact.
|
||||
export const PUBLIC_CHAT_SCHEMA =
|
||||
"paperclip.runner-protocol-eval.public-chat/v1";
|
||||
export const PUBLIC_CHAT_NOTICE =
|
||||
"Public replay of an isolated mock eval. Conversation text is scrubbed; provider identities, tool payloads, traces, and company-state snapshots are withheld. Full evidence remains in the access-controlled Actions artifact.";
|
||||
|
||||
export const SECRET_TEXT = [
|
||||
/\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/gu,
|
||||
/\bsk-[A-Za-z0-9_-]{16,}\b/gu,
|
||||
/\b(?:gh[pousr]_|github_pat_)[A-Za-z0-9_]{16,}\b/gu,
|
||||
/\bBearer\s+[A-Za-z0-9._~+\/-]{8,}=*/giu,
|
||||
/-----BEGIN [^-]*PRIVATE KEY-----[\s\S]*?-----END [^-]*PRIVATE KEY-----/gu,
|
||||
/\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/gu,
|
||||
/\b(?:https?|file|s3):\/\/[^\s<>"')]+/giu,
|
||||
/\barn:aws[^\s<>"')]+/gu,
|
||||
/(?:\/(?:Users|home|tmp|private|var)\/|[A-Z]:\\)[^\s<>"')]+/gu,
|
||||
/\b(?:api[_-]?key|access[_-]?token|secret|password|authorization|cookie)\s*[=:]\s*[^\s,;]+/giu,
|
||||
];
|
||||
|
||||
export function publicText(value, privateValues = []) {
|
||||
let text = typeof value === "string" ? value : "";
|
||||
for (const secret of privateValues) {
|
||||
if (typeof secret === "string" && secret.length >= 8)
|
||||
text = text.replaceAll(secret, "[redacted]");
|
||||
}
|
||||
for (const pattern of SECRET_TEXT) text = text.replace(pattern, "[redacted]");
|
||||
return text.length > 40_000 ? `${text.slice(0, 40_000)}\n[truncated]` : text;
|
||||
}
|
||||
|
||||
function privateIdentities(value, found = new Set()) {
|
||||
if (value && typeof value === "object") {
|
||||
for (const [key, child] of Object.entries(value)) {
|
||||
if (
|
||||
/(?:session|profile|account|runtime|endpoint|memory|agentversion).*id$|arn$|token$|secret$|password$/i.test(
|
||||
key,
|
||||
) &&
|
||||
typeof child === "string"
|
||||
)
|
||||
found.add(child);
|
||||
else if (child && typeof child === "object")
|
||||
privateIdentities(child, found);
|
||||
}
|
||||
}
|
||||
return [...found];
|
||||
}
|
||||
|
||||
function timestamp(value) {
|
||||
return typeof value === "string" && /^\d{4}-\d\d-\d\dT[\d:.]+Z$/.test(value)
|
||||
? value
|
||||
: "1970-01-01T00:00:00.000Z";
|
||||
}
|
||||
|
||||
function operation(value) {
|
||||
return typeof value === "string" && /^[a-z][a-z_]{0,79}$/.test(value)
|
||||
? value
|
||||
: "unknown_operation";
|
||||
}
|
||||
|
||||
export function publicChatView(artifact, evalCase) {
|
||||
const privateValues = privateIdentities(artifact);
|
||||
const scrub = (value) => publicText(value, privateValues);
|
||||
const network = artifact.snapshot?.networkEvidence;
|
||||
// Only the dedicated mock eval boundary can publish recorded conversation.
|
||||
// Early infrastructure failures still receive a viewer with an honest notice.
|
||||
const isolated =
|
||||
network?.realPaperclipRequests === 0 &&
|
||||
Array.isArray(network?.childPaperclipEnvironmentKeys) &&
|
||||
network.childPaperclipEnvironmentKeys.length === 0;
|
||||
const source =
|
||||
isolated &&
|
||||
artifact.issueThread?.schema === "paperclip.capability.issue-thread-view.v1"
|
||||
? artifact.issueThread
|
||||
: null;
|
||||
const evidence = Object.fromEntries(
|
||||
[
|
||||
"tools",
|
||||
"calls",
|
||||
"authorization",
|
||||
"control_plane",
|
||||
"runner",
|
||||
"state",
|
||||
"traceability",
|
||||
"parity",
|
||||
].map((key) => [key, []]),
|
||||
);
|
||||
const turns = (source?.turns ?? []).map((turn, turnIndex) => {
|
||||
const turnId = `public-turn-${turnIndex + 1}`;
|
||||
const items = [];
|
||||
for (const item of turn.items ?? []) {
|
||||
const base = {
|
||||
id: `public-item-${turnIndex + 1}-${items.length + 1}`,
|
||||
at: timestamp(item.at),
|
||||
};
|
||||
if (
|
||||
["user_message", "agent_message", "durable_comment"].includes(item.kind)
|
||||
) {
|
||||
items.push({
|
||||
...base,
|
||||
kind: item.kind === "user_message" ? "user_message" : "agent_message",
|
||||
author: item.kind === "user_message" ? "You (eval prompt)" : "Agent",
|
||||
body: scrub(item.body),
|
||||
streaming: false,
|
||||
});
|
||||
} else if (item.kind === "tool_activity") {
|
||||
const operationId = operation(item.operationId);
|
||||
const status = ["ok", "denied", "running"].includes(item.status)
|
||||
? item.status
|
||||
: "running";
|
||||
const result = {
|
||||
outcome: status,
|
||||
detail: "Tool payload withheld from public replay.",
|
||||
};
|
||||
const recordId = `public-call-${turnIndex + 1}-${items.length + 1}`;
|
||||
items.push({
|
||||
...base,
|
||||
kind: "tool_activity",
|
||||
operationId,
|
||||
status,
|
||||
summary: `${operationId}: ${status}`,
|
||||
input: { detail: "Arguments withheld from public replay." },
|
||||
result,
|
||||
evidenceRef: { section: "calls", recordId },
|
||||
});
|
||||
if (status !== "running")
|
||||
evidence.calls.push({
|
||||
id: recordId,
|
||||
turnId,
|
||||
operationId,
|
||||
version: 1,
|
||||
providerRequest: operationId,
|
||||
dispatchedCommand: operationId,
|
||||
outcome: status,
|
||||
result,
|
||||
redactions: ["arguments", "result payload", "provider identities"],
|
||||
threadAnchorId: base.id,
|
||||
});
|
||||
}
|
||||
// Provider activity, reasoning, raw events, file refs and unrecognized
|
||||
// future item kinds are deliberately not part of the public contract.
|
||||
}
|
||||
return {
|
||||
id: turnId,
|
||||
ordinal: turnIndex + 1,
|
||||
mode: "replay",
|
||||
toolCallCount: items.filter((item) => item.kind === "tool_activity")
|
||||
.length,
|
||||
at: timestamp(turn.at),
|
||||
stoppedByUser: turn.stoppedByUser === true,
|
||||
items,
|
||||
};
|
||||
});
|
||||
if (!turns.length) {
|
||||
turns.push({
|
||||
id: "public-turn-1",
|
||||
ordinal: 1,
|
||||
mode: "replay",
|
||||
toolCallCount: 0,
|
||||
at: timestamp(artifact.snapshot?.createdAt),
|
||||
stoppedByUser: false,
|
||||
items: [
|
||||
{
|
||||
id: "public-notice",
|
||||
at: timestamp(artifact.snapshot?.createdAt),
|
||||
kind: "system_notice",
|
||||
glyph: "–",
|
||||
text: "No publishable conversation was recorded for this attempt. See the checks and the access-controlled artifact for diagnostics.",
|
||||
evidenceRef: { section: "runner", recordId: "public-notice" },
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
return {
|
||||
schema: "paperclip.capability.issue-thread-view.v1",
|
||||
sessionId: "public-report",
|
||||
mode: "replay",
|
||||
identity: {
|
||||
agentLabel: "Recorded agent",
|
||||
runnerLabel: "Recorded runner",
|
||||
runnerAttached: false,
|
||||
controlPlaneLabel: "Mock Paperclip",
|
||||
controlPlaneTooltip: PUBLIC_CHAT_NOTICE,
|
||||
replaySource: "live",
|
||||
},
|
||||
issue: {
|
||||
identifier: "EVAL",
|
||||
title: scrub(evalCase.title || evalCase.id),
|
||||
status: [
|
||||
"backlog",
|
||||
"todo",
|
||||
"in_progress",
|
||||
"in_review",
|
||||
"done",
|
||||
"blocked",
|
||||
"cancelled",
|
||||
].includes(source?.issue?.status)
|
||||
? source.issue.status
|
||||
: "in_review",
|
||||
priority: "medium",
|
||||
assignee: null,
|
||||
runState: "Read-only public replay",
|
||||
scenarioId: evalCase.id,
|
||||
fixtureProfile: evalCase.id,
|
||||
},
|
||||
turns,
|
||||
composer: {
|
||||
state: "disabled",
|
||||
helper: null,
|
||||
reason: "Read-only eval report",
|
||||
pendingInteractionId: null,
|
||||
},
|
||||
evidence,
|
||||
connection: { state: "closed", attempt: 0 },
|
||||
replay: null,
|
||||
renderedAt: timestamp(source?.renderedAt || artifact.snapshot?.createdAt),
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,234 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { mkdtemp, mkdir, writeFile, cp, rm, symlink } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
import {
|
||||
publicChatView,
|
||||
publicText,
|
||||
PUBLIC_CHAT_SCHEMA,
|
||||
PUBLIC_CHAT_NOTICE,
|
||||
} from "./public-eval-chat.mjs";
|
||||
import {
|
||||
publicViewerShell,
|
||||
validatePublicChatPayload,
|
||||
trustedViewerFiles,
|
||||
} from "./public-eval-viewer.mjs";
|
||||
import { validatePublicProtocolEvalReport } from "./publish-runner-protocol-eval-history.mjs";
|
||||
|
||||
function artifact() {
|
||||
return {
|
||||
providerSessionId: "private-session-canary",
|
||||
snapshot: {
|
||||
networkEvidence: {
|
||||
realPaperclipRequests: 0,
|
||||
childPaperclipEnvironmentKeys: [],
|
||||
},
|
||||
},
|
||||
issueThread: {
|
||||
schema: "paperclip.capability.issue-thread-view.v1",
|
||||
issue: { status: "blocked" },
|
||||
turns: [
|
||||
{
|
||||
items: [
|
||||
{ kind: "user_message", body: "Please block this task." },
|
||||
{
|
||||
kind: "agent_message",
|
||||
body: "Blocked. private-session-canary https://private.example/path",
|
||||
privateField: "private-field-canary",
|
||||
},
|
||||
{
|
||||
kind: "tool_activity",
|
||||
operationId: "block_task",
|
||||
status: "ok",
|
||||
input: { secret: "argument-canary" },
|
||||
result: { token: "result-canary" },
|
||||
},
|
||||
{ kind: "thinking", body: "reasoning-canary" },
|
||||
{ kind: "future_kind", body: "future-canary" },
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function payload() {
|
||||
return {
|
||||
attemptId: "attempt-01",
|
||||
caseId: "block-task",
|
||||
disposition: "pass",
|
||||
passed: true,
|
||||
checks: [],
|
||||
publication: { schema: PUBLIC_CHAT_SCHEMA, notice: PUBLIC_CHAT_NOTICE },
|
||||
view: publicChatView(artifact(), { id: "block-task" }),
|
||||
devtools: null,
|
||||
navigation: { suiteHref: "../../index.html", previous: null, next: null },
|
||||
run: {
|
||||
model: "test",
|
||||
provider: "test",
|
||||
sessionId: "public-report",
|
||||
effectiveModelHistory: [],
|
||||
managedProfile: null,
|
||||
acpxProfile: null,
|
||||
usage: null,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test("projects only isolated recorded messages and bounded tool facts", () => {
|
||||
const view = publicChatView(artifact(), { id: "block-task" });
|
||||
assert.equal(view.issue.status, "blocked");
|
||||
assert.deepEqual(
|
||||
view.turns[0].items.map((item) => item.kind),
|
||||
["user_message", "agent_message", "tool_activity"],
|
||||
);
|
||||
assert.match(view.turns[0].items[1].body, /Blocked/);
|
||||
assert.doesNotMatch(JSON.stringify(view), /canary|private\.example/);
|
||||
assert.deepEqual(view.turns[0].items[2].input, {
|
||||
detail: "Arguments withheld from public replay.",
|
||||
});
|
||||
validatePublicChatPayload(payload());
|
||||
for (const networkEvidence of [
|
||||
undefined,
|
||||
{ realPaperclipRequests: 1, childPaperclipEnvironmentKeys: [] },
|
||||
{
|
||||
realPaperclipRequests: 0,
|
||||
childPaperclipEnvironmentKeys: ["PAPERCLIP_API_KEY"],
|
||||
},
|
||||
]) {
|
||||
const source = artifact();
|
||||
source.snapshot.networkEvidence = networkEvidence;
|
||||
const unavailable = publicChatView(source, { id: "block-task" });
|
||||
assert.equal(unavailable.turns[0].items[0].kind, "system_notice");
|
||||
assert.doesNotMatch(JSON.stringify(unavailable), /Please block|Blocked\./);
|
||||
}
|
||||
});
|
||||
|
||||
test("scrubs credentials and private references before truncating text", () => {
|
||||
for (const secret of [
|
||||
"sk-" + "a".repeat(32),
|
||||
"ghp_" + "b".repeat(32),
|
||||
"Bearer abcdef123456",
|
||||
"password=secret-canary",
|
||||
"/Users/someone/private.txt",
|
||||
"arn:aws:service:region:account:resource",
|
||||
"-----BEGIN PRIVATE KEY-----\n" +
|
||||
"x".repeat(41_000) +
|
||||
"\n-----END PRIVATE KEY-----",
|
||||
]) {
|
||||
assert.equal(publicText(secret), "[redacted]");
|
||||
}
|
||||
assert.match(publicText("a".repeat(41_000)), /\[truncated\]$/);
|
||||
});
|
||||
|
||||
test("fails closed on unknown fields, raw tools, private evidence and identities", () => {
|
||||
for (const mutate of [
|
||||
(p) => {
|
||||
p.view.turns[0].items[0].extra = "unprojected";
|
||||
},
|
||||
(p) => {
|
||||
p.view.turns[0].items[2].input = { password: "oops" };
|
||||
},
|
||||
(p) => {
|
||||
p.view.evidence.calls[0].result.detail = "raw-result";
|
||||
},
|
||||
(p) => {
|
||||
p.view.evidence.state.push({ secret: "raw-state" });
|
||||
},
|
||||
(p) => {
|
||||
p.run.providerSessionId = "private-session";
|
||||
},
|
||||
(p) => {
|
||||
p.view.turns[0].items[1].body = "sk-" + "x".repeat(30);
|
||||
},
|
||||
(p) => {
|
||||
p.devtools = {};
|
||||
},
|
||||
(p) => {
|
||||
p.view.composer.state = "ready";
|
||||
},
|
||||
]) {
|
||||
const value = payload();
|
||||
mutate(value);
|
||||
assert.throws(() => validatePublicChatPayload(value));
|
||||
}
|
||||
});
|
||||
|
||||
test("publisher permits only the exact trusted shell/assets and valid local navigation", async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "eval-chat-contract-"));
|
||||
try {
|
||||
const viewer = join(root, "trusted");
|
||||
const report = join(root, "report");
|
||||
await mkdir(join(viewer, "assets"), { recursive: true });
|
||||
await mkdir(join(report, "attempts/attempt-01"), { recursive: true });
|
||||
const index =
|
||||
'<!doctype html><html><head><script type="module" src="./assets/app.js"></script><link rel="stylesheet" href="./assets/app.css"></head><body><div id="root"></div></body></html>';
|
||||
await writeFile(join(viewer, "index.html"), index);
|
||||
await writeFile(join(viewer, "assets/app.js"), "// trusted build");
|
||||
await writeFile(join(viewer, "assets/app.css"), ":root {}");
|
||||
await cp(join(viewer, "assets"), join(report, "viewer/assets"), {
|
||||
recursive: true,
|
||||
});
|
||||
await writeFile(
|
||||
join(report, "index.html"),
|
||||
'<a href="attempts/attempt-01/index.html">PASS</a>',
|
||||
);
|
||||
await writeFile(
|
||||
join(report, "campaign.json"),
|
||||
JSON.stringify({
|
||||
schema: "paperclip.runner-protocol-eval.campaign/v1",
|
||||
campaignId: "gha-42-1",
|
||||
}),
|
||||
);
|
||||
const page = join(report, "attempts/attempt-01/index.html");
|
||||
const writePayload = async (value) =>
|
||||
writeFile(
|
||||
page,
|
||||
publicViewerShell(
|
||||
index,
|
||||
JSON.stringify(value).replaceAll("<", "\\u003c"),
|
||||
),
|
||||
);
|
||||
await writePayload(payload());
|
||||
await validatePublicProtocolEvalReport(report, { viewerRoot: viewer });
|
||||
await assert.rejects(validatePublicProtocolEvalReport(report));
|
||||
await writeFile(
|
||||
join(report, "viewer/assets/app.js"),
|
||||
"// substituted build",
|
||||
);
|
||||
await assert.rejects(
|
||||
validatePublicProtocolEvalReport(report, { viewerRoot: viewer }),
|
||||
/trusted/,
|
||||
);
|
||||
await writeFile(join(report, "viewer/assets/app.js"), "// trusted build");
|
||||
await writeFile(
|
||||
page,
|
||||
publicViewerShell(index, JSON.stringify(payload())) +
|
||||
"<script>alert(1)</script>",
|
||||
);
|
||||
await assert.rejects(
|
||||
validatePublicProtocolEvalReport(report, { viewerRoot: viewer }),
|
||||
/trusted shell/,
|
||||
);
|
||||
const escaped = payload();
|
||||
escaped.view.turns[0].items[0].body =
|
||||
'</script><script>alert("not executable")</script>';
|
||||
await writePayload(escaped);
|
||||
await validatePublicProtocolEvalReport(report, { viewerRoot: viewer });
|
||||
const broken = payload();
|
||||
broken.navigation.next = {
|
||||
label: "Next attempt",
|
||||
href: "../missing/index.html",
|
||||
};
|
||||
await writePayload(broken);
|
||||
await assert.rejects(
|
||||
validatePublicProtocolEvalReport(report, { viewerRoot: viewer }),
|
||||
/link|reference/i,
|
||||
);
|
||||
await symlink(viewer, join(root, "symlink"));
|
||||
await assert.rejects(trustedViewerFiles(join(root, "symlink")));
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
|
@ -0,0 +1,229 @@
|
|||
import { readFile, readdir, lstat } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { PUBLIC_CHAT_SCHEMA, SECRET_TEXT } from "./public-eval-chat.mjs";
|
||||
|
||||
export const PUBLIC_VIEWER_CSP =
|
||||
"default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; font-src 'self'; img-src 'none'; connect-src 'none'; form-action 'none'; base-uri 'none'";
|
||||
export const PUBLIC_VIEWER_DATA =
|
||||
/<script type="application\/json" id="paperclip-eval-report">([^<]*)<\/script>/u;
|
||||
const ASSET = /^[A-Za-z0-9][A-Za-z0-9._-]*\.(?:js|css|woff2)$/;
|
||||
|
||||
export function publicViewerShell(index, encodedPayload) {
|
||||
return index
|
||||
.replaceAll('"./assets/', '"../../viewer/assets/')
|
||||
.replace(
|
||||
"<head>",
|
||||
`<head>\n <meta http-equiv="Content-Security-Policy" content="${PUBLIC_VIEWER_CSP}">`,
|
||||
)
|
||||
.replace(
|
||||
'<script type="module"',
|
||||
`<script type="application/json" id="paperclip-eval-report">${encodedPayload}</script>\n <script type="module"`,
|
||||
);
|
||||
}
|
||||
|
||||
export async function trustedViewerFiles(viewerRoot) {
|
||||
if (!viewerRoot || !(await lstat(viewerRoot)).isDirectory())
|
||||
throw new Error(
|
||||
"A trusted viewer build is required for public chat reports",
|
||||
);
|
||||
if (
|
||||
!(await lstat(join(viewerRoot, "index.html"))).isFile() ||
|
||||
!(await lstat(join(viewerRoot, "assets"))).isDirectory()
|
||||
)
|
||||
throw new Error("Trusted viewer must not use symlinks");
|
||||
const index = await readFile(join(viewerRoot, "index.html"), "utf8");
|
||||
const files = new Map();
|
||||
for (const entry of await readdir(join(viewerRoot, "assets"), {
|
||||
withFileTypes: true,
|
||||
})) {
|
||||
if (!entry.isFile() || entry.isSymbolicLink() || !ASSET.test(entry.name))
|
||||
throw new Error("Unexpected trusted viewer asset");
|
||||
files.set(
|
||||
`viewer/assets/${entry.name}`,
|
||||
await readFile(join(viewerRoot, "assets", entry.name)),
|
||||
);
|
||||
}
|
||||
if (
|
||||
![...files.keys()].some((name) => name.endsWith(".js")) ||
|
||||
!index.includes('<script type="module"')
|
||||
)
|
||||
throw new Error("Incomplete trusted viewer build");
|
||||
return { index, files };
|
||||
}
|
||||
|
||||
export function validatePublicChatPayload(payload) {
|
||||
if (
|
||||
payload?.publication?.schema !== PUBLIC_CHAT_SCHEMA ||
|
||||
payload.view?.sessionId !== "public-report" ||
|
||||
payload.view?.composer?.state !== "disabled" ||
|
||||
payload.view?.connection?.state !== "closed" ||
|
||||
payload.devtools !== null
|
||||
)
|
||||
throw new Error(
|
||||
"Public attempt must contain the read-only public chat projection",
|
||||
);
|
||||
const allowed = new Set([
|
||||
"attemptId",
|
||||
"caseId",
|
||||
"disposition",
|
||||
"passed",
|
||||
"checks",
|
||||
"view",
|
||||
"devtools",
|
||||
"navigation",
|
||||
"run",
|
||||
"publication",
|
||||
]);
|
||||
if (Object.keys(payload).some((key) => !allowed.has(key)))
|
||||
throw new Error("Unknown public chat payload field");
|
||||
const fields = (value, names) => {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== "object" ||
|
||||
Array.isArray(value) ||
|
||||
Object.keys(value).some((key) => !names.split(" ").includes(key))
|
||||
)
|
||||
throw new Error("Unknown public chat projection field");
|
||||
};
|
||||
fields(payload.publication, "schema notice");
|
||||
fields(payload.navigation, "suiteHref previous next");
|
||||
for (const link of [payload.navigation.previous, payload.navigation.next])
|
||||
if (link !== null) fields(link, "label href");
|
||||
fields(
|
||||
payload.run,
|
||||
"model provider driver providerVersion runnerProvider acpxAgent acpxProfile requestedModel effectiveModelHistory configuration sessionId providerSessionId agentVersion managedProfile retainedSession retainedSessionStatus fixtureDigest runnerPackageDigest runnerdDigest startedAt finishedAt durationMs runnerBuild initialRevision finalRevision usage",
|
||||
);
|
||||
if (
|
||||
payload.run.effectiveModelHistory?.length ||
|
||||
payload.run.managedProfile != null ||
|
||||
payload.run.acpxProfile != null
|
||||
)
|
||||
throw new Error("Public replay contains private provider metadata");
|
||||
if (payload.run.usage !== null)
|
||||
fields(
|
||||
payload.run.usage,
|
||||
"agentTurns providerRequests inputTokens outputTokens cachedInputTokens reasoningTokens providerReportedCostNanodollars estimatedCostNanodollars pricingVersion",
|
||||
);
|
||||
fields(
|
||||
payload.view,
|
||||
"schema sessionId mode identity issue turns composer evidence connection replay renderedAt",
|
||||
);
|
||||
fields(
|
||||
payload.view.identity,
|
||||
"agentLabel runnerLabel runnerAttached controlPlaneLabel controlPlaneTooltip replaySource",
|
||||
);
|
||||
fields(
|
||||
payload.view.issue,
|
||||
"identifier title status priority assignee runState scenarioId fixtureProfile",
|
||||
);
|
||||
fields(payload.view.composer, "state helper reason pendingInteractionId");
|
||||
fields(payload.view.connection, "state attempt");
|
||||
fields(
|
||||
payload.view.evidence,
|
||||
"tools calls authorization control_plane runner state traceability parity",
|
||||
);
|
||||
for (const check of payload.checks) {
|
||||
fields(
|
||||
check,
|
||||
"id kind passed detail evidenceRefs title description definition anchor",
|
||||
);
|
||||
fields(check.definition, "id kind");
|
||||
fields(check.anchor, "kind id");
|
||||
if (check.evidenceRefs.length)
|
||||
throw new Error("Public replay contains raw evidence references");
|
||||
}
|
||||
const visit = (value, key = "") => {
|
||||
if (typeof value === "string") {
|
||||
if (
|
||||
/(?:sessionId|providerSessionId)$/i.test(key) &&
|
||||
!["public-report", "unknown", "redacted"].includes(value)
|
||||
)
|
||||
throw new Error("Public replay contains a private session identity");
|
||||
for (const pattern of SECRET_TEXT) {
|
||||
pattern.lastIndex = 0;
|
||||
if (pattern.test(value))
|
||||
throw new Error(
|
||||
"Public replay contains credential or private reference material",
|
||||
);
|
||||
}
|
||||
} else if (value && typeof value === "object") {
|
||||
for (const [name, child] of Object.entries(value)) {
|
||||
if (
|
||||
/^(?:managedProfile|acpxProfile|providerTrace|mockState|stateHistory|trace|environment|env|apiKey|accessToken|password|secret)$/i.test(
|
||||
name,
|
||||
) &&
|
||||
child != null
|
||||
)
|
||||
throw new Error("Public replay contains a private field");
|
||||
visit(child, name);
|
||||
}
|
||||
}
|
||||
};
|
||||
visit(payload);
|
||||
for (const section of [
|
||||
"tools",
|
||||
"authorization",
|
||||
"control_plane",
|
||||
"runner",
|
||||
"state",
|
||||
"traceability",
|
||||
"parity",
|
||||
]) {
|
||||
if (
|
||||
!Array.isArray(payload.view.evidence?.[section]) ||
|
||||
payload.view.evidence[section].length
|
||||
)
|
||||
throw new Error("Public replay contains unprojected evidence");
|
||||
}
|
||||
for (const call of payload.view.evidence.calls) {
|
||||
fields(
|
||||
call,
|
||||
"id turnId operationId version providerRequest dispatchedCommand outcome result redactions threadAnchorId",
|
||||
);
|
||||
fields(call.result, "outcome detail");
|
||||
if (call.result.detail !== "Tool payload withheld from public replay.")
|
||||
throw new Error("Public replay contains raw call evidence");
|
||||
}
|
||||
for (const turn of payload.view.turns ?? []) {
|
||||
fields(turn, "id ordinal mode toolCallCount at stoppedByUser items");
|
||||
for (const item of turn.items ?? []) {
|
||||
if (
|
||||
![
|
||||
"user_message",
|
||||
"agent_message",
|
||||
"tool_activity",
|
||||
"system_notice",
|
||||
].includes(item.kind)
|
||||
)
|
||||
throw new Error("Public replay contains an unprojected item");
|
||||
const shapes = {
|
||||
user_message: "kind id at author body streaming",
|
||||
agent_message: "kind id at author body streaming",
|
||||
tool_activity:
|
||||
"kind id at operationId status summary input result evidenceRef",
|
||||
system_notice: "kind id at glyph text evidenceRef",
|
||||
};
|
||||
fields(item, shapes[item.kind]);
|
||||
if (item.evidenceRef) fields(item.evidenceRef, "section recordId");
|
||||
if (
|
||||
item.kind === "tool_activity" &&
|
||||
(JSON.stringify(item.input) !==
|
||||
JSON.stringify({
|
||||
detail: "Arguments withheld from public replay.",
|
||||
}) ||
|
||||
Object.keys(item.result).sort().join(",") !== "detail,outcome" ||
|
||||
item.result.detail !== "Tool payload withheld from public replay.")
|
||||
)
|
||||
throw new Error("Public replay contains a raw tool payload");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function validatePublicViewerPage(content, trustedIndex) {
|
||||
const match = content.match(PUBLIC_VIEWER_DATA);
|
||||
if (!match || publicViewerShell(trustedIndex, match[1]) !== content)
|
||||
throw new Error("Public viewer page differs from the trusted shell");
|
||||
const payload = JSON.parse(match[1]);
|
||||
validatePublicChatPayload(payload);
|
||||
return payload;
|
||||
}
|
||||
|
|
@ -13,13 +13,20 @@ import {
|
|||
import { tmpdir } from "node:os";
|
||||
import { extname, join, relative, resolve, sep } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import {
|
||||
trustedViewerFiles,
|
||||
validatePublicViewerPage,
|
||||
} from "./public-eval-viewer.mjs";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const SAFE_CAMPAIGN = /^gha-[1-9][0-9]*-[1-9][0-9]*$/;
|
||||
const SAFE_CAMPAIGN =
|
||||
/^gha-[1-9][0-9]*-[1-9][0-9]*(?:-report-[a-z0-9][a-z0-9-]{0,39})?$/;
|
||||
const SAFE_REPORT_PATHS = [
|
||||
/^(?:index|latest|inventory|real-server)\.html$/,
|
||||
/^tests\/[A-Za-z0-9][A-Za-z0-9._-]{0,199}\.html$/,
|
||||
/^attempts\/[A-Za-z0-9][A-Za-z0-9._-]{0,199}\.html$/,
|
||||
/^attempts\/[A-Za-z0-9][A-Za-z0-9._-]{0,199}\/index\.html$/,
|
||||
/^viewer\/assets\/[A-Za-z0-9][A-Za-z0-9._-]*\.(?:js|css|woff2)$/,
|
||||
/^campaign\.json$/,
|
||||
];
|
||||
const CREDENTIAL_PATTERNS = [
|
||||
|
|
@ -136,9 +143,16 @@ function internalHtmlHrefs(content) {
|
|||
.filter((href) => href && !href.startsWith("#"));
|
||||
}
|
||||
|
||||
export async function validatePublicProtocolEvalReport(reportRoot) {
|
||||
export async function validatePublicProtocolEvalReport(
|
||||
reportRoot,
|
||||
{ viewerRoot } = {},
|
||||
) {
|
||||
const root = resolve(reportRoot);
|
||||
const files = await relativeFiles(root);
|
||||
const hasChat = files.some((file) =>
|
||||
/^attempts\/[^/]+\/index\.html$/.test(file),
|
||||
);
|
||||
const viewer = hasChat ? await trustedViewerFiles(viewerRoot) : null;
|
||||
if (!files.includes("index.html") || !files.includes("campaign.json")) {
|
||||
throw new Error(
|
||||
"Public protocol eval report requires index.html and campaign.json",
|
||||
|
|
@ -157,21 +171,44 @@ export async function validatePublicProtocolEvalReport(reportRoot) {
|
|||
`Public protocol eval file exceeds its size boundary: ${file}`,
|
||||
);
|
||||
}
|
||||
if (file.startsWith("viewer/")) {
|
||||
const expected = viewer?.files.get(file);
|
||||
if (!expected || !expected.equals(await readFile(absolute)))
|
||||
throw new Error(
|
||||
`Public viewer asset differs from trusted build: ${file}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const content = await readFile(absolute, "utf8");
|
||||
for (const pattern of CREDENTIAL_PATTERNS) {
|
||||
if (pattern.test(content))
|
||||
throw new Error(
|
||||
`Public report contains credential/session material: ${file}`,
|
||||
);
|
||||
const richAttempt = /^attempts\/[^/]+\/index\.html$/.test(file);
|
||||
const payload = richAttempt
|
||||
? validatePublicViewerPage(content, viewer.index)
|
||||
: null;
|
||||
if (!richAttempt) {
|
||||
for (const pattern of CREDENTIAL_PATTERNS) {
|
||||
if (pattern.test(content))
|
||||
throw new Error(
|
||||
`Public report contains credential/session material: ${file}`,
|
||||
);
|
||||
}
|
||||
if (extname(file) !== ".html") continue;
|
||||
for (const pattern of ACTIVE_HTML_PATTERNS) {
|
||||
if (pattern.test(content))
|
||||
throw new Error(
|
||||
`Public report contains active or remote HTML: ${file}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (extname(file) !== ".html") continue;
|
||||
for (const pattern of ACTIVE_HTML_PATTERNS) {
|
||||
if (pattern.test(content))
|
||||
throw new Error(
|
||||
`Public report contains active or remote HTML: ${file}`,
|
||||
);
|
||||
}
|
||||
for (const href of internalHtmlHrefs(content)) {
|
||||
const navigation = payload
|
||||
? [
|
||||
payload.navigation?.suiteHref,
|
||||
payload.navigation?.previous?.href,
|
||||
payload.navigation?.next?.href,
|
||||
].filter(Boolean)
|
||||
: [];
|
||||
for (const href of [...internalHtmlHrefs(content), ...navigation]) {
|
||||
if (typeof href !== "string" || /[?:\\]|^\/|^[a-z]+:/i.test(href))
|
||||
throw new Error(`Unsafe report navigation in ${file}`);
|
||||
const clean = href.split("#", 1)[0].split("?", 1)[0];
|
||||
const target = resolve(
|
||||
root,
|
||||
|
|
@ -189,6 +226,15 @@ export async function validatePublicProtocolEvalReport(reportRoot) {
|
|||
}
|
||||
}
|
||||
}
|
||||
if (viewer) {
|
||||
for (const file of viewer.files.keys())
|
||||
if (!files.includes(file))
|
||||
throw new Error(`Missing public viewer asset: ${file}`);
|
||||
if (files.some((file) => /^attempts\/[^/]+\.html$/.test(file)))
|
||||
throw new Error(
|
||||
"Chat Evalbook must not mix in legacy plain attempt pages",
|
||||
);
|
||||
}
|
||||
const campaign = await loadObject(join(root, "campaign.json"));
|
||||
if (
|
||||
campaign.schema !== "paperclip.runner-protocol-eval.campaign/v1" ||
|
||||
|
|
@ -199,11 +245,17 @@ export async function validatePublicProtocolEvalReport(reportRoot) {
|
|||
return { files, campaign };
|
||||
}
|
||||
|
||||
export async function createProtocolEvalBundleManifest(reportRoot, campaignId) {
|
||||
export async function createProtocolEvalBundleManifest(
|
||||
reportRoot,
|
||||
campaignId,
|
||||
{ viewerRoot } = {},
|
||||
) {
|
||||
if (!SAFE_CAMPAIGN.test(campaignId))
|
||||
throw new Error("Unsafe protocol eval campaign ID");
|
||||
const { files, campaign } =
|
||||
await validatePublicProtocolEvalReport(reportRoot);
|
||||
const { files, campaign } = await validatePublicProtocolEvalReport(
|
||||
reportRoot,
|
||||
{ viewerRoot },
|
||||
);
|
||||
if (campaign.campaignId !== campaignId)
|
||||
throw new Error("Report campaign ID does not match publication target");
|
||||
const entries = await Promise.all(
|
||||
|
|
@ -250,6 +302,9 @@ export function protocolEvalHistoryRecord(campaign, publicRoot) {
|
|||
totals: campaign.totals,
|
||||
rosters: campaign.rosters,
|
||||
source: campaign.source,
|
||||
...(campaign.reportRevision
|
||||
? { reportRevision: campaign.reportRevision }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -278,9 +333,7 @@ export function mergeProtocolEvalHistory(history, record) {
|
|||
const retained = campaigns.slice(0, MAX_HISTORY_CAMPAIGNS);
|
||||
if (
|
||||
latestGreen &&
|
||||
!retained.some(
|
||||
(campaign) => campaign.campaignId === latestGreen.campaignId,
|
||||
)
|
||||
!retained.some((campaign) => campaign.campaignId === latestGreen.campaignId)
|
||||
) {
|
||||
retained[retained.length - 1] = latestGreen;
|
||||
}
|
||||
|
|
@ -343,7 +396,7 @@ export function renderProtocolEvalHistoryIndex(history) {
|
|||
`${html(roster.model)} · ${roster.passed}/${roster.selected}`,
|
||||
)
|
||||
.join("<br>");
|
||||
return `<tr><td><a href="${html(campaign.publicUrl)}"><code>${html(campaign.campaignId)}</code></a><small>${html(date(campaign.generatedAt))} UTC</small></td><td><span class="status ${status}">${status}</span></td><td><strong>${html(campaign.totals.passed)}/${html(campaign.totals.selected)}</strong><small>${html(campaign.totals.behaviorFailures)} behavior · ${html(campaign.totals.infrastructureFailures)} infrastructure</small></td><td>${rosters}</td><td><code>${html(campaign.source?.paperclip?.sha?.slice(0, 8) ?? "unknown")}</code><small>evals ${html(campaign.source?.evals?.sha?.slice(0, 8) ?? "unknown")}</small></td><td><a href="${html(campaign.publicUrl)}">Open Evalbook →</a></td></tr>`;
|
||||
return `<tr><td><a href="${html(campaign.publicUrl)}"><code>${html(campaign.campaignId)}</code></a><small>${html(date(campaign.generatedAt))} UTC</small>${campaign.reportRevision ? `<small>Report refresh · no new model calls · source ${html(campaign.reportRevision.sourceCampaignId)}</small>` : ""}</td><td><span class="status ${status}">${status}</span></td><td><strong>${html(campaign.totals.passed)}/${html(campaign.totals.selected)}</strong><small>${html(campaign.totals.behaviorFailures)} behavior · ${html(campaign.totals.infrastructureFailures)} infrastructure</small></td><td>${rosters}</td><td><code>${html(campaign.source?.paperclip?.sha?.slice(0, 8) ?? "unknown")}</code><small>evals ${html(campaign.source?.evals?.sha?.slice(0, 8) ?? "unknown")}</small></td><td><a href="${html(campaign.publicUrl)}">Open Evalbook →</a></td></tr>`;
|
||||
})
|
||||
.join("")
|
||||
: '<tr><td colspan="6" class="empty">No campaigns have been published yet.</td></tr>';
|
||||
|
|
@ -435,13 +488,20 @@ async function uploadImmutableReport(bucket, prefix, reportRoot) {
|
|||
);
|
||||
}
|
||||
|
||||
export async function publishProtocolEvalHistory({ reportRoot, destination }) {
|
||||
export async function publishProtocolEvalHistory({
|
||||
reportRoot,
|
||||
destination,
|
||||
viewerRoot,
|
||||
}) {
|
||||
const validatedDestination =
|
||||
validateProtocolEvalHistoryDestination(destination);
|
||||
const { campaign } = await validatePublicProtocolEvalReport(reportRoot);
|
||||
const { campaign } = await validatePublicProtocolEvalReport(reportRoot, {
|
||||
viewerRoot,
|
||||
});
|
||||
const manifest = await createProtocolEvalBundleManifest(
|
||||
reportRoot,
|
||||
campaign.campaignId,
|
||||
{ viewerRoot },
|
||||
);
|
||||
const temporary = await mkdtemp(
|
||||
join(tmpdir(), "runner-protocol-eval-history-"),
|
||||
|
|
@ -518,6 +578,7 @@ export async function publishProtocolEvalHistory({ reportRoot, destination }) {
|
|||
|
||||
async function main() {
|
||||
const result = await publishProtocolEvalHistory({
|
||||
viewerRoot: process.env.PAPERCLIP_RUNNER_PROTOCOL_EVAL_VIEWER_DIR,
|
||||
reportRoot: resolve(
|
||||
process.env.PAPERCLIP_RUNNER_PROTOCOL_EVAL_PUBLIC_REPORT_DIR ??
|
||||
"runner-protocol-eval-public-report",
|
||||
|
|
|
|||
|
|
@ -111,12 +111,13 @@ test("keeps publication to the canonical static Evalbook surface", () => {
|
|||
"inventory.html",
|
||||
"tests/get-context.html",
|
||||
"attempts/attempt-01.html",
|
||||
"attempts/attempt-01/index.html",
|
||||
"viewer/assets/index-build.js",
|
||||
"campaign.json",
|
||||
]) {
|
||||
assert.equal(isPublicProtocolEvalPath(file), true, file);
|
||||
}
|
||||
for (const file of [
|
||||
"attempts/attempt-01/index.html",
|
||||
"runs/attempt/artifact.json",
|
||||
"provider-trace.log",
|
||||
"../secret",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,99 @@
|
|||
#!/usr/bin/env node
|
||||
// Re-render immutable recorded evidence; this command never invokes a model.
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdir, readFile, writeFile, lstat } from "node:fs/promises";
|
||||
import { join, resolve } from "node:path";
|
||||
import { sanitizeProtocolEvalRuns } from "./runner-protocol-eval-campaign.mjs";
|
||||
import { validatePublicProtocolEvalReport } from "./publish-runner-protocol-eval-history.mjs";
|
||||
|
||||
export async function refreshProtocolEvalReport({
|
||||
sourceRoot,
|
||||
evalsRoot,
|
||||
viewerRoot,
|
||||
outputRoot,
|
||||
revision,
|
||||
renderedAt = new Date().toISOString(),
|
||||
}) {
|
||||
if (!/^[a-z0-9][a-z0-9-]{0,39}$/.test(revision ?? ""))
|
||||
throw new Error("A safe, unique --revision is required");
|
||||
if (await lstat(outputRoot).catch(() => null))
|
||||
throw new Error("Report refresh output must be a new directory");
|
||||
const campaign = JSON.parse(
|
||||
await readFile(join(sourceRoot, "campaign.json"), "utf8"),
|
||||
);
|
||||
if (!/^gha-[1-9][0-9]*-[1-9][0-9]*$/.test(campaign.campaignId ?? ""))
|
||||
throw new Error("Expected an original Actions campaign");
|
||||
const program = join(
|
||||
evalsRoot,
|
||||
"evals/paperclip-runner/tools/eval_program.py",
|
||||
);
|
||||
const rendererDigest = createHash("sha256")
|
||||
.update(await readFile(program))
|
||||
.digest("hex");
|
||||
await mkdir(outputRoot, { recursive: true });
|
||||
const runsRoot = join(outputRoot, "public-runs");
|
||||
const reportRoot = join(outputRoot, "report");
|
||||
await sanitizeProtocolEvalRuns({
|
||||
runsRoot: join(sourceRoot, "runs"),
|
||||
publicRunsRoot: runsRoot,
|
||||
});
|
||||
execFileSync(
|
||||
"python3",
|
||||
[
|
||||
program,
|
||||
"report",
|
||||
"--runs-root",
|
||||
runsRoot,
|
||||
"--output",
|
||||
reportRoot,
|
||||
"--viewer-root",
|
||||
viewerRoot,
|
||||
"--public-viewer",
|
||||
"--inventory",
|
||||
join(evalsRoot, "evals/paperclip-runner/inventory.json"),
|
||||
"--coverage-matrix",
|
||||
join(evalsRoot, "evals/paperclip-runner/coverage-matrix.json"),
|
||||
],
|
||||
{ stdio: "inherit" },
|
||||
);
|
||||
const refreshed = {
|
||||
...campaign,
|
||||
campaignId: `${campaign.campaignId}-report-${revision}`,
|
||||
generatedAt: renderedAt,
|
||||
reportRevision: {
|
||||
sourceCampaignId: campaign.campaignId,
|
||||
sourceGeneratedAt: campaign.generatedAt,
|
||||
renderedAt,
|
||||
rendererDigest,
|
||||
providerCalls: 0,
|
||||
},
|
||||
};
|
||||
await writeFile(
|
||||
join(reportRoot, "campaign.json"),
|
||||
`${JSON.stringify(refreshed, null, 2)}\n`,
|
||||
);
|
||||
await validatePublicProtocolEvalReport(reportRoot, { viewerRoot });
|
||||
return { reportRoot, campaignId: refreshed.campaignId, providerCalls: 0 };
|
||||
}
|
||||
|
||||
if (
|
||||
process.argv[1] &&
|
||||
resolve(process.argv[1]) === resolve(import.meta.filename)
|
||||
) {
|
||||
const arg = (name) => {
|
||||
const index = process.argv.indexOf(name);
|
||||
if (index < 0 || !process.argv[index + 1])
|
||||
throw new Error(`Missing ${name}`);
|
||||
return process.argv[index + 1];
|
||||
};
|
||||
console.log(
|
||||
await refreshProtocolEvalReport({
|
||||
sourceRoot: resolve(arg("--source")),
|
||||
evalsRoot: resolve(arg("--evals-root")),
|
||||
viewerRoot: resolve(arg("--viewer-root")),
|
||||
outputRoot: resolve(arg("--output")),
|
||||
revision: arg("--revision"),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
|
@ -309,6 +309,15 @@ export async function renderRunnerWorkflowWithCanonicalEvalbook({
|
|||
environment = process.env,
|
||||
}) {
|
||||
const program = await resolveCanonicalEvalProgram(packageRoot, environment);
|
||||
const viewerRoot = resolve(
|
||||
environment.PAPERCLIP_EVAL_VIEWER_ROOT ??
|
||||
resolve(packageRoot, "dist-issue-thread"),
|
||||
);
|
||||
await access(resolve(viewerRoot, "index.html")).catch(() => {
|
||||
throw new Error(
|
||||
"Evalbook requires the chat viewer. Run pnpm --filter @paperclipai/paperclip-runner build:issue-thread first.",
|
||||
);
|
||||
});
|
||||
const runsRoot = resolve(outputDirectory, "evalbook-runs");
|
||||
await rm(runsRoot, { recursive: true, force: true });
|
||||
const attempts = await writeRunnerWorkflowEvalbookAttempts({
|
||||
|
|
@ -331,6 +340,8 @@ export async function renderRunnerWorkflowWithCanonicalEvalbook({
|
|||
runsRoot,
|
||||
"--output",
|
||||
outputDirectory,
|
||||
"--viewer-root",
|
||||
viewerRoot,
|
||||
]);
|
||||
const programBytes = await readFile(program);
|
||||
const manifest = {
|
||||
|
|
|
|||
|
|
@ -11,6 +11,11 @@ import {
|
|||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { basename, dirname, join, relative, resolve, sep } from "node:path";
|
||||
import {
|
||||
publicChatView,
|
||||
PUBLIC_CHAT_SCHEMA,
|
||||
PUBLIC_CHAT_NOTICE,
|
||||
} from "./public-eval-chat.mjs";
|
||||
|
||||
const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$/;
|
||||
const ATTEMPT_FILES = new Set([
|
||||
|
|
@ -97,7 +102,9 @@ async function maintainedRosterSelection(programRoot) {
|
|||
return basename(rosterPath);
|
||||
});
|
||||
if (selected.length === 0 || new Set(selected).size !== selected.length) {
|
||||
throw new Error("Maintained live campaign must contain unique enabled rosters");
|
||||
throw new Error(
|
||||
"Maintained live campaign must contain unique enabled rosters",
|
||||
);
|
||||
}
|
||||
return new Set(selected);
|
||||
}
|
||||
|
|
@ -120,8 +127,7 @@ export async function buildProtocolEvalCatalog({
|
|||
const programRoot = resolve(evalsRoot, "evals/paperclip-runner");
|
||||
const rosterRoot = resolve(programRoot, "rosters");
|
||||
const requested = parseRosterSelection(rosterSelection);
|
||||
const selected =
|
||||
requested ?? (await maintainedRosterSelection(programRoot));
|
||||
const selected = requested ?? (await maintainedRosterSelection(programRoot));
|
||||
const rosterFiles = (await readdir(rosterRoot, { withFileTypes: true }))
|
||||
.filter(
|
||||
(entry) =>
|
||||
|
|
@ -509,7 +515,8 @@ export async function aggregateProtocolEvalCampaign({
|
|||
return campaign;
|
||||
}
|
||||
|
||||
function publicArtifact(artifact) {
|
||||
function publicArtifact(artifact, evalCase) {
|
||||
const issueThread = publicChatView(artifact, evalCase);
|
||||
const model = artifact.snapshot?.providerModel ?? {};
|
||||
const infrastructure = artifact.infrastructureFailure;
|
||||
const providerVersion =
|
||||
|
|
@ -525,12 +532,29 @@ function publicArtifact(artifact) {
|
|||
provider: artifact.provider,
|
||||
driver: artifact.driver,
|
||||
providerVersion,
|
||||
retainedSession: false,
|
||||
retainedSession: null,
|
||||
retainedSessionStatus: "redacted from the public report",
|
||||
usage: safeUsage(artifact.usage),
|
||||
turn: { status: artifact.turn?.status ?? "failed" },
|
||||
timing: {
|
||||
startedAt:
|
||||
typeof artifact.timing?.startedAt === "string"
|
||||
? artifact.timing.startedAt
|
||||
: null,
|
||||
finishedAt:
|
||||
typeof artifact.timing?.finishedAt === "string"
|
||||
? artifact.timing.finishedAt
|
||||
: null,
|
||||
durationMs: Number.isFinite(artifact.timing?.durationMs)
|
||||
? artifact.timing.durationMs
|
||||
: null,
|
||||
},
|
||||
turn: {
|
||||
status: artifact.turn?.status ?? "failed",
|
||||
turnId: issueThread.turns.at(-1).id,
|
||||
},
|
||||
snapshot: {
|
||||
createdAt: artifact.snapshot?.createdAt ?? artifact.createdAt,
|
||||
sessionId: "public-report",
|
||||
providerModel: {
|
||||
id: model.id ?? artifact.requestedModel,
|
||||
provider: model.provider ?? artifact.provider,
|
||||
|
|
@ -539,6 +563,8 @@ function publicArtifact(artifact) {
|
|||
evidence: [],
|
||||
},
|
||||
devtools: { revisions: [] },
|
||||
publication: { schema: PUBLIC_CHAT_SCHEMA, notice: PUBLIC_CHAT_NOTICE },
|
||||
issueThread,
|
||||
...(infrastructure && typeof infrastructure === "object"
|
||||
? {
|
||||
infrastructureFailure: {
|
||||
|
|
@ -622,15 +648,22 @@ export async function sanitizeProtocolEvalRuns({ runsRoot, publicRunsRoot }) {
|
|||
await Promise.all([
|
||||
writeFile(
|
||||
join(destination, "artifact.json"),
|
||||
json(publicArtifact(artifact)),
|
||||
json(publicArtifact(artifact, evalCase)),
|
||||
{ mode: 0o600 },
|
||||
),
|
||||
writeFile(join(destination, "score.json"), json(publicScore(score)), {
|
||||
mode: 0o600,
|
||||
}),
|
||||
writeFile(join(destination, "case.json"), json(evalCase), {
|
||||
mode: 0o600,
|
||||
}),
|
||||
writeFile(
|
||||
join(destination, "case.json"),
|
||||
json({
|
||||
id: evalCase.id,
|
||||
checks: (evalCase.checks ?? []).map(({ id, kind }) => ({ id, kind })),
|
||||
}),
|
||||
{
|
||||
mode: 0o600,
|
||||
},
|
||||
),
|
||||
writeFile(join(destination, "config.json"), json(publicConfig(config)), {
|
||||
mode: 0o600,
|
||||
}),
|
||||
|
|
|
|||
|
|
@ -286,7 +286,7 @@ test("rejects downloaded cells that were not declared by the immutable catalog",
|
|||
);
|
||||
});
|
||||
|
||||
test("public run projection removes provider sessions, traces, transcripts, evidence, and state", async () => {
|
||||
test("public run projection removes raw evidence and gives unverified recordings an empty chat view", async () => {
|
||||
const { root, config, evalCase } = await fixture();
|
||||
const attemptId = "get-task-context-opencode-gha-42-1-attempt-01";
|
||||
const source = join(root, "raw-runs", attemptId);
|
||||
|
|
@ -357,12 +357,14 @@ test("public run projection removes provider sessions, traces, transcripts, evid
|
|||
);
|
||||
assert.doesNotMatch(
|
||||
serialized,
|
||||
/private-session|private transcript|private-turn|issueThread|trace/,
|
||||
/private-session|private transcript|private-turn|"trace":/,
|
||||
);
|
||||
const artifact = JSON.parse(serialized);
|
||||
assert.deepEqual(artifact.snapshot.transcript, []);
|
||||
assert.deepEqual(artifact.snapshot.evidence, []);
|
||||
assert.deepEqual(artifact.devtools.revisions, []);
|
||||
assert.equal(artifact.issueThread.composer.state, "disabled");
|
||||
assert.equal(artifact.issueThread.turns[0].items[0].kind, "system_notice");
|
||||
const score = JSON.parse(
|
||||
await readFile(join(root, "public-runs", attemptId, "score.json"), "utf8"),
|
||||
);
|
||||
|
|
|
|||
|
|
@ -136,6 +136,11 @@ test("publishes only the separately sanitized Evalbook through trusted OIDC code
|
|||
/Upload access-controlled canonical Evalbook and raw attempts/u,
|
||||
);
|
||||
assert.match(report, /Upload publisher-only sanitized Evalbook/u);
|
||||
assert.match(
|
||||
report,
|
||||
/--viewer-root runner-protocol-build\/extracted\/dist-issue-thread\s*\\\n\s*--public-viewer/u,
|
||||
);
|
||||
assert.equal([...report.matchAll(/--viewer-root /gu)].length, 2);
|
||||
|
||||
const publisher = workflow.slice(workflow.indexOf(" publish_history:"));
|
||||
assert.match(publisher, /ref: \$\{\{ github\.sha \}\}/u);
|
||||
|
|
@ -143,6 +148,8 @@ test("publishes only the separately sanitized Evalbook through trusted OIDC code
|
|||
assert.match(publisher, /runner-protocol-eval-public-/u);
|
||||
assert.match(publisher, /publish-runner-protocol-eval-history\.mjs/u);
|
||||
assert.match(publisher, /runner-protocol-evals/u);
|
||||
assert.match(publisher, /runner-protocol-viewer-/u);
|
||||
assert.match(publisher, /PAPERCLIP_RUNNER_PROTOCOL_EVAL_VIEWER_DIR/u);
|
||||
assert.doesNotMatch(publisher, /(?:OPENAI|ANTHROPIC|OPENROUTER)_API_KEY/u);
|
||||
assert.doesNotMatch(publisher, /paperclipai\/paperclip-evals/u);
|
||||
assert.doesNotMatch(publisher, /downloaded-runner-protocol-evals/u);
|
||||
|
|
|
|||
Loading…
Reference in New Issue