fix(runner-e2e): retain branch-only suites and chat screenshots in reports
This commit is contained in:
Dotta
parent
d691c01abf
commit
e713692095
|
|
@ -275,9 +275,20 @@ requested three-hour repair window. No acceptance assertions were disabled.
|
|||
- [Exact campaign results](https://d1p6rlowie26tp.cloudfront.net/runner-e2e/campaigns/gha-34648511170-1/summary.md)
|
||||
- [GitHub run and retained evidence](https://github.com/paperclipai/paperclip/actions/runs/34648511170)
|
||||
|
||||
Use the Markdown results for this branch-only suite: the trusted default-branch
|
||||
HTML dashboard still uses its older catalog, while the normalized results and
|
||||
Markdown report include all 24 chat cells.
|
||||
The [HTML dashboard](https://d1p6rlowie26tp.cloudfront.net/runner-e2e/campaigns/gha-34648511170-1/index.html?report=agent-chat#suite-agent-chat)
|
||||
was repaired from retained evidence after its older trusted catalog omitted the
|
||||
branch-only suite. It now includes the chat suite and 32 screenshots, including
|
||||
eight draft/revised plan captures recovered from their original Playwright
|
||||
attachments. No paid cells were rerun; result records, tested SHA, timestamps,
|
||||
usage, billing, attempts, and cleanup outcomes remain unchanged.
|
||||
|
||||
Reporting now discovers validated display-only entries for unknown selected
|
||||
execution IDs, and publication rejects missing declared screenshots. The exact
|
||||
chat plan filenames are included in packaged evidence. All 165 runner unit tests
|
||||
and runner TypeScript checks passed. Browser verification covered suite
|
||||
filtering, restored plan images, and gallery navigation. This explicitly
|
||||
authorized repair replaces only this campaign's report objects; normal
|
||||
immutable-publication protections remain unchanged.
|
||||
|
||||
The published summary and normalized results were verified after publication:
|
||||
exactly 24 unique expected cells, all passed on attempt 1, all cleanup checks
|
||||
|
|
|
|||
|
|
@ -256,6 +256,12 @@ usage is labeled `unavailable` or `unpriced`; it is never presented as zero
|
|||
cost. The CI report job stages the same portable site at
|
||||
`normalized/index.html` inside the access-controlled merged report artifact.
|
||||
|
||||
The trusted publisher discovers display-only entries for selected execution IDs
|
||||
absent from its local catalog, so branch-only suites remain visible in the
|
||||
dashboard, filters, gallery, and summary image. It validates execution identity
|
||||
and escapes display text without loading target-branch executable code. Unknown
|
||||
suite cardinality is not treated as proof of full-suite coverage.
|
||||
|
||||
Permanent publication uses two explicit bundles. Both retain only normalized
|
||||
result PNG files with the explicit `public-runner-fixture` publication marker,
|
||||
including marked `failure.png` captures, so every campaign dashboard has its
|
||||
|
|
@ -274,6 +280,10 @@ retains allowlisted inert per-attempt evidence (`.json`, `.log`, `.md`, and
|
|||
redaction. The GitHub Pages bundle is regenerated separately with the same
|
||||
declared-screenshot boundary.
|
||||
|
||||
Publication fails if any declared public screenshot is missing from the bundle.
|
||||
The evidence packager explicitly retains `chat-plan-draft.png` and
|
||||
`chat-plan-revised.png`; arbitrary chat-prefixed files remain excluded.
|
||||
|
||||
Both public bundles exclude video, archives, raw/unallowlisted logs, SVG or
|
||||
other active content, generated Playwright/blob/HTML report trees, and
|
||||
undeclared PNG files, and per-attempt XML. The root `junit.xml` remains public
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
import {
|
||||
discoverReportCatalog,
|
||||
type ReportExecution,
|
||||
} from "./report-catalog.js";
|
||||
import type {
|
||||
MatrixExecution,
|
||||
RunnerE2ECampaign,
|
||||
RunnerE2EHistoryIndex,
|
||||
RunnerE2EResult,
|
||||
|
|
@ -22,7 +25,7 @@ export interface RunnerDashboardInput {
|
|||
title: string;
|
||||
generatedAt: string;
|
||||
expected: readonly string[];
|
||||
catalog: readonly MatrixExecution[];
|
||||
catalog: readonly ReportExecution[];
|
||||
entries: readonly RunnerDashboardEntry[];
|
||||
campaign?: RunnerE2ECampaign;
|
||||
history?: RunnerE2EHistoryIndex;
|
||||
|
|
@ -135,7 +138,7 @@ function resolveScreenshots(
|
|||
}
|
||||
|
||||
function renderCase(
|
||||
execution: MatrixExecution,
|
||||
execution: ReportExecution,
|
||||
expected: ReadonlySet<string>,
|
||||
entryById: ReadonlyMap<string, RunnerDashboardEntry>,
|
||||
) {
|
||||
|
|
@ -151,7 +154,11 @@ function renderCase(
|
|||
const label = state.replace("-", " ");
|
||||
const detail =
|
||||
entry?.errors.join("; ") ||
|
||||
(entry?.valid ? "All invariants passed" : "Not selected");
|
||||
(entry?.valid
|
||||
? "All invariants passed"
|
||||
: selected
|
||||
? "No result artifact was uploaded"
|
||||
: "Not selected");
|
||||
const screenshots = resolveScreenshots(entry);
|
||||
const billing = entry ? summarizeExecutionBilling(entry.result) : null;
|
||||
const matcherResults = entry?.result.matcherResults ?? [];
|
||||
|
|
@ -508,7 +515,7 @@ function renderHistory(history: RunnerE2EHistoryIndex | undefined) {
|
|||
}
|
||||
|
||||
function renderSuiteMatrix(input: {
|
||||
suiteCatalog: readonly MatrixExecution[];
|
||||
suiteCatalog: readonly ReportExecution[];
|
||||
expected: ReadonlySet<string>;
|
||||
entryById: ReadonlyMap<string, RunnerDashboardEntry>;
|
||||
summary?: RunnerE2ESuiteSummary;
|
||||
|
|
@ -580,6 +587,11 @@ function renderSuiteMatrix(input: {
|
|||
}
|
||||
|
||||
export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {
|
||||
const catalog = discoverReportCatalog({
|
||||
catalog: input.catalog,
|
||||
expected: input.expected,
|
||||
results: input.entries.map((entry) => entry.result),
|
||||
});
|
||||
const expected = new Set(input.expected);
|
||||
const entryById = new Map(
|
||||
input.entries.map((entry) => [entry.result.executionId, entry]),
|
||||
|
|
@ -602,13 +614,13 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {
|
|||
);
|
||||
const suites = [
|
||||
...new Map(
|
||||
input.catalog.map((execution) => [execution.suite.id, execution.suite]),
|
||||
catalog.map((execution) => [execution.suite.id, execution.suite]),
|
||||
).values(),
|
||||
];
|
||||
const suiteSections = suites
|
||||
.map((suite) =>
|
||||
renderSuiteMatrix({
|
||||
suiteCatalog: input.catalog.filter(
|
||||
suiteCatalog: catalog.filter(
|
||||
(execution) => execution.suite.id === suite.id,
|
||||
),
|
||||
expected,
|
||||
|
|
@ -625,15 +637,12 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {
|
|||
);
|
||||
const filterProfiles = [
|
||||
...new Map(
|
||||
input.catalog.map((execution) => [
|
||||
execution.profile.id,
|
||||
execution.profile,
|
||||
]),
|
||||
catalog.map((execution) => [execution.profile.id, execution.profile]),
|
||||
).values(),
|
||||
];
|
||||
const filterEnvironments = [
|
||||
...new Map(
|
||||
input.catalog.map((execution) => [
|
||||
catalog.map((execution) => [
|
||||
execution.environment.id,
|
||||
execution.environment,
|
||||
]),
|
||||
|
|
@ -1034,7 +1043,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {
|
|||
</section>
|
||||
${suiteSections}
|
||||
${historySection}
|
||||
<footer><span>Generated ${html(input.generatedAt)}</span><span>${input.catalog.length} catalog executions · Declared screenshots and sanitized structured evidence published</span></footer>
|
||||
<footer><span>Generated ${html(input.generatedAt)}</span><span>${catalog.length} catalog executions · Declared screenshots and sanitized structured evidence published</span></footer>
|
||||
</main>
|
||||
<dialog class="gallery-dialog" data-gallery-dialog aria-labelledby="gallery-title">
|
||||
<div class="gallery-shell">
|
||||
|
|
|
|||
|
|
@ -38,6 +38,8 @@ const ALLOWED_ROOT_FILES = new Set([
|
|||
"result.json",
|
||||
"final-state.png",
|
||||
"failure.png",
|
||||
"chat-plan-draft.png",
|
||||
"chat-plan-revised.png",
|
||||
"server.log",
|
||||
"playwright.log",
|
||||
"junit.xml",
|
||||
|
|
|
|||
|
|
@ -313,10 +313,14 @@ export async function createBundleManifest(
|
|||
if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$/.test(campaignId)) {
|
||||
throw new Error("Campaign ID is unsafe for immutable object storage");
|
||||
}
|
||||
const retainedPaths = await relativeFiles(root, root, allowPublicSummary, publicScreenshots);
|
||||
const retained = new Set(retainedPaths);
|
||||
const missingScreenshots = [...publicScreenshots].filter((relative) => !retained.has(relative));
|
||||
if (missingScreenshots.length > 0) {
|
||||
throw new Error(`Declared public screenshots are missing from the historical bundle: ${missingScreenshots.sort().join(", ")}`);
|
||||
}
|
||||
const files = await Promise.all(
|
||||
(await relativeFiles(root, root, allowPublicSummary, publicScreenshots))
|
||||
.sort()
|
||||
.map(async (relative) => {
|
||||
retainedPaths.sort().map(async (relative) => {
|
||||
const absolute = path.join(root, ...relative.split("/"));
|
||||
if (
|
||||
relative === "public-images/campaign-summary.png" ||
|
||||
|
|
|
|||
|
|
@ -501,6 +501,22 @@ describe("historical publication security", () => {
|
|||
).rejects.toThrow("does not match its raster file type");
|
||||
});
|
||||
|
||||
it("refuses a public bundle when any declared screenshot is absent", async () => {
|
||||
const root = await mkdtemp(path.join(os.tmpdir(), "runner-missing-screenshot-"));
|
||||
temporaryDirectories.push(root);
|
||||
const directory = path.join(root, "evidence", "agent-chat.runner-codex.local.plan-handoff", "attempt-1");
|
||||
await mkdir(directory, { recursive: true });
|
||||
const base = "evidence/agent-chat.runner-codex.local.plan-handoff/attempt-1";
|
||||
const declared = new Set([`${base}/final-state.png`, `${base}/chat-plan-draft.png`]);
|
||||
const png = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", "base64");
|
||||
await writeFile(path.join(directory, "final-state.png"), png);
|
||||
await expect(createBundleManifest(root, "campaign-1", false, declared))
|
||||
.rejects.toThrow(`Declared public screenshots are missing from the historical bundle: ${base}/chat-plan-draft.png`);
|
||||
await writeFile(path.join(directory, "chat-plan-draft.png"), png);
|
||||
const manifest = await createBundleManifest(root, "campaign-1", false, declared);
|
||||
expect(new Set(manifest.files.map((file) => file.path))).toEqual(declared);
|
||||
});
|
||||
|
||||
it("requires trusted-fixture opt-in and rejects unsafe screenshot paths", () => {
|
||||
const execution = runnerMatrix[0]!;
|
||||
const campaign = buildRunnerCampaign({
|
||||
|
|
|
|||
|
|
@ -1,3 +1,8 @@
|
|||
import {
|
||||
discoverReportCatalog,
|
||||
parseReportExecutionId,
|
||||
type ReportExecution,
|
||||
} from "./report-catalog.js";
|
||||
import { runnerMatrix, runnerSuites } from "./catalog.js";
|
||||
import {
|
||||
aggregateCampaignBilling,
|
||||
|
|
@ -24,7 +29,12 @@ export function upgradeRunnerResult(result: RunnerE2EResult): RunnerE2EResult {
|
|||
const execution = runnerMatrix.find(
|
||||
(candidate) => candidate.id === executionId,
|
||||
);
|
||||
if (!execution) return result;
|
||||
if (!execution)
|
||||
return {
|
||||
...result,
|
||||
executionId,
|
||||
suiteId: result.suiteId ?? parseReportExecutionId(executionId).suiteId,
|
||||
};
|
||||
return {
|
||||
...result,
|
||||
executionId,
|
||||
|
|
@ -49,25 +59,39 @@ export function buildRunnerCampaign(input: {
|
|||
expected: readonly string[];
|
||||
results: readonly RunnerE2EResult[];
|
||||
eventName?: string | null;
|
||||
catalog?: readonly ReportExecution[];
|
||||
}): RunnerE2ECampaign {
|
||||
const expected = input.expected.map(canonicalExecutionId);
|
||||
const results = input.results.map((result) => ({
|
||||
...upgradeRunnerResult(result),
|
||||
billing: result.billing ?? summarizeExecutionBilling(result),
|
||||
}));
|
||||
const knownCatalog = input.catalog ?? runnerMatrix;
|
||||
const catalog = discoverReportCatalog({
|
||||
catalog: knownCatalog,
|
||||
expected,
|
||||
results,
|
||||
});
|
||||
const discoveredSuites = [
|
||||
...new Map(
|
||||
catalog.map((execution) => [execution.suite.id, execution.suite]),
|
||||
).values(),
|
||||
];
|
||||
const resultSource = results.find((result) => result.source)?.source;
|
||||
const source = {
|
||||
...resolveRunnerE2ESource(resultSource),
|
||||
eventName: input.eventName ?? process.env.GITHUB_EVENT_NAME ?? null,
|
||||
};
|
||||
const suites = runnerSuites
|
||||
const suites = discoveredSuites
|
||||
.map((suite) => {
|
||||
const suiteExpected = expected.filter((id) =>
|
||||
id.startsWith(`${suite.id}.`),
|
||||
);
|
||||
if (suiteExpected.length === 0) return null;
|
||||
const suiteResults = results.filter(
|
||||
(result) => result.suiteId === suite.id,
|
||||
(result) =>
|
||||
expected.includes(result.executionId) &&
|
||||
result.executionId.startsWith(`${suite.id}.`),
|
||||
);
|
||||
const passed = suiteResults.filter(
|
||||
(result) => result.status === "passed" && result.cleanup === "passed",
|
||||
|
|
@ -79,7 +103,7 @@ export function buildRunnerCampaign(input: {
|
|||
suiteId: suite.id,
|
||||
suiteDefinitionHash:
|
||||
suiteResults[0]?.suiteDefinitionHash ??
|
||||
runnerMatrix.find((execution) => execution.suite.id === suite.id)!
|
||||
catalog.find((execution) => execution.suite.id === suite.id)!
|
||||
.suiteDefinitionHash,
|
||||
expected: suite.expectedMatrixSize,
|
||||
selected: suiteExpected.length,
|
||||
|
|
@ -93,7 +117,9 @@ export function buildRunnerCampaign(input: {
|
|||
cleanupPassed: suiteResults.every(
|
||||
(result) => result.cleanup === "passed",
|
||||
),
|
||||
complete: suiteExpected.length === suite.expectedMatrixSize,
|
||||
complete:
|
||||
knownCatalog.some((execution) => execution.suite.id === suite.id) &&
|
||||
suiteExpected.length === suite.expectedMatrixSize,
|
||||
durationMs: suiteResults.reduce(
|
||||
(total, result) => total + result.durationMs,
|
||||
0,
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { discoverReportCatalog } from "./report-catalog.js";
|
||||
import path from "node:path";
|
||||
import { mkdir } from "node:fs/promises";
|
||||
import { chromium } from "@playwright/test";
|
||||
|
|
@ -24,8 +25,13 @@ function durationLabel(durationMs: number) {
|
|||
}
|
||||
|
||||
export function renderPublicCampaignSummary(campaign: RunnerE2ECampaign) {
|
||||
const catalog = discoverReportCatalog({
|
||||
catalog: runnerMatrix,
|
||||
expected: campaign.expected,
|
||||
results: campaign.results,
|
||||
});
|
||||
const catalogById = new Map(
|
||||
runnerMatrix.map((execution) => [execution.id, execution]),
|
||||
catalog.map((execution) => [execution.id, execution]),
|
||||
);
|
||||
const expectedIds = [
|
||||
...new Set(
|
||||
|
|
@ -53,10 +59,7 @@ export function renderPublicCampaignSummary(campaign: RunnerE2ECampaign) {
|
|||
);
|
||||
const suites = [
|
||||
...new Map(
|
||||
runnerMatrix.map((execution) => [
|
||||
execution.suite.id,
|
||||
execution.suite.label,
|
||||
]),
|
||||
catalog.map((execution) => [execution.suite.id, execution.suite.label]),
|
||||
),
|
||||
].map(([suiteId, label]) => {
|
||||
const selectedIds = expectedIds.filter(
|
||||
|
|
@ -111,12 +114,12 @@ export function renderPublicCampaignSummary(campaign: RunnerE2ECampaign) {
|
|||
<p class="eyebrow">Runner full-stack E2E</p>
|
||||
<h1>Campaign summary</h1>
|
||||
<section class="metrics">
|
||||
<div class="metric"><strong>${passed}/${expectedIds.length}</strong><span>Known executions passed</span></div>
|
||||
<div class="metric"><strong>${passed}/${expectedIds.length}</strong><span>Selected executions passed</span></div>
|
||||
<div class="metric"><strong>${expectedIds.length - passed}</strong><span>Failed or incomplete</span></div>
|
||||
<div class="metric"><strong>${html(durationLabel(durationMs))}</strong><span>Total test time</span></div>
|
||||
</section>
|
||||
<section class="suites">${rows}</section>
|
||||
<footer>Generated from fixed catalog labels and sanitized numeric/status fields. Provider output is never rendered here.</footer>
|
||||
<footer>Generated from trusted catalog labels, validated execution identities, and sanitized numeric/status fields. Provider output is never rendered here.</footer>
|
||||
</main>
|
||||
</body>
|
||||
</html>`;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,242 @@
|
|||
import { renderPublicCampaignSummary } from "./public-summary-image.js";
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { runnerMatrix } from "./catalog.js";
|
||||
import { discoverReportCatalog } from "./report-catalog.js";
|
||||
import { buildRunnerCampaign } from "./history.js";
|
||||
import { renderRunnerE2EDashboard } from "./dashboard.js";
|
||||
import { regenerateRunnerDashboard } from "./dashboard-regenerate.js";
|
||||
import type { RunnerE2EResult } from "./types.js";
|
||||
|
||||
const temporary: string[] = [];
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
temporary.splice(0).map((dir) => rm(dir, { recursive: true, force: true })),
|
||||
);
|
||||
});
|
||||
const id = "branch-only-chat.branch-provider.local.conversation";
|
||||
const missingId = "branch-only-chat.branch-provider.local.reset";
|
||||
function result(): RunnerE2EResult {
|
||||
return {
|
||||
schema: "paperclip.runner-e2e.result/v2",
|
||||
executionId: id,
|
||||
suiteId: "branch-only-chat",
|
||||
suiteDefinitionHash: "branch-definition",
|
||||
profileId: "branch-provider",
|
||||
environmentId: "local",
|
||||
caseId: "conversation",
|
||||
provider: '<img src=x onerror="alert(1)">',
|
||||
model: "branch-model",
|
||||
runtimeMode: "native",
|
||||
attempt: 1,
|
||||
status: "passed",
|
||||
cleanup: "passed",
|
||||
durationMs: 1000,
|
||||
startedAt: "2026-09-11T21:00:00.000Z",
|
||||
finishedAt: "2026-09-11T21:00:01.000Z",
|
||||
screenshots: [
|
||||
{
|
||||
id: "final-state",
|
||||
label: '<script>alert("label")</script>',
|
||||
file: "final-state.png",
|
||||
publication: "public-runner-fixture",
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
function campaign(results: RunnerE2EResult[] = [result()]) {
|
||||
return buildRunnerCampaign({
|
||||
campaignId: "branch-campaign",
|
||||
generatedAt: "2026-09-11T21:01:00.000Z",
|
||||
expected: [id, missingId],
|
||||
results,
|
||||
});
|
||||
}
|
||||
|
||||
describe("trusted report catalog discovery", () => {
|
||||
it("aggregates unknown suites and renders their cases, missing results, and screenshot gallery", () => {
|
||||
const summary = campaign();
|
||||
expect(summary.suites).toEqual([
|
||||
expect.objectContaining({
|
||||
suiteId: "branch-only-chat",
|
||||
selected: 2,
|
||||
executed: 1,
|
||||
passed: 1,
|
||||
failed: 1,
|
||||
complete: false,
|
||||
}),
|
||||
]);
|
||||
const page = renderRunnerE2EDashboard({
|
||||
title: "Report",
|
||||
generatedAt: summary.generatedAt,
|
||||
expected: summary.expected,
|
||||
catalog: [],
|
||||
campaign: summary,
|
||||
entries: [
|
||||
{
|
||||
result: result(),
|
||||
valid: true,
|
||||
errors: [],
|
||||
evidenceBaseHref: `evidence/${id}/attempt-1`,
|
||||
evidenceFiles: ["final-state.png"],
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(page).toContain('id="suite-branch-only-chat"');
|
||||
expect(page).toContain('class="suite-summary"');
|
||||
expect(page).toContain(`data-execution-id="${id}"`);
|
||||
expect(page).toContain(`data-execution-id="${missingId}"`);
|
||||
expect(page).toContain('class="case case-missing"');
|
||||
expect(page).toContain("No result artifact was uploaded");
|
||||
expect(page).toContain(
|
||||
`data-gallery-href="evidence/${id}/attempt-1/final-state.png"`,
|
||||
);
|
||||
expect(page).toContain("View gallery · 1");
|
||||
expect(page).toContain("<img src=x onerror="alert(1)">");
|
||||
expect(page).not.toContain("<img src=x");
|
||||
expect(page).not.toContain('<script>alert("label")</script>');
|
||||
});
|
||||
|
||||
it("renders selected branch suites in the public summary image without provider content", () => {
|
||||
const page = renderPublicCampaignSummary(campaign());
|
||||
expect(page).toContain("<strong>1/2</strong>");
|
||||
expect(page).toContain("<span>branch-only-chat</span>");
|
||||
expect(page).not.toContain("<strong>0/0</strong>");
|
||||
expect(page).not.toContain("onerror");
|
||||
expect(page).not.toContain("branch-model");
|
||||
});
|
||||
|
||||
it("retains ordinary catalog metadata and marks no-results branch selections failed", () => {
|
||||
const catalog = discoverReportCatalog({
|
||||
catalog: runnerMatrix,
|
||||
expected: [runnerMatrix[0]!.id],
|
||||
results: [],
|
||||
});
|
||||
expect(catalog).toEqual(runnerMatrix);
|
||||
expect(catalog[0]).toBe(runnerMatrix[0]);
|
||||
expect(campaign([])).toMatchObject({
|
||||
selected: 2,
|
||||
executed: 0,
|
||||
passed: 0,
|
||||
failed: 2,
|
||||
suites: [
|
||||
expect.objectContaining({ suiteId: "branch-only-chat", failed: 2 }),
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("uses known profile metadata in a new suite and rejects forged identities", () => {
|
||||
const known = runnerMatrix[0]!;
|
||||
const knownProfileId = `branch-only-chat.${known.profile.id}.${known.environment.id}.conversation`;
|
||||
expect(
|
||||
discoverReportCatalog({
|
||||
catalog: runnerMatrix,
|
||||
expected: [knownProfileId],
|
||||
results: [],
|
||||
}).at(-1)?.profile,
|
||||
).toBe(known.profile);
|
||||
expect(() =>
|
||||
discoverReportCatalog({
|
||||
catalog: [],
|
||||
expected: [id],
|
||||
results: [{ ...result(), suiteId: "forged-suite" }],
|
||||
}),
|
||||
).toThrow("Result identity does not match");
|
||||
expect(() =>
|
||||
discoverReportCatalog({
|
||||
catalog: [],
|
||||
expected: ["../../private"],
|
||||
results: [],
|
||||
}),
|
||||
).toThrow("Invalid report execution identity");
|
||||
expect(() =>
|
||||
discoverReportCatalog({ catalog: [], expected: [id, id], results: [] }),
|
||||
).toThrow("must be unique");
|
||||
});
|
||||
|
||||
it("regenerates branch-only cards and gallery from persisted JSON using trusted code", async () => {
|
||||
const bundle = await mkdtemp(
|
||||
path.join(os.tmpdir(), "branch-report-regenerate-"),
|
||||
);
|
||||
temporary.push(bundle);
|
||||
const evidence = path.join(bundle, "evidence", id, "attempt-1");
|
||||
await mkdir(evidence, { recursive: true });
|
||||
await writeFile(path.join(evidence, "final-state.png"), "fixture");
|
||||
await writeFile(
|
||||
path.join(bundle, "normalized-results.json"),
|
||||
JSON.stringify({ ...campaign(), suites: [] }),
|
||||
);
|
||||
await regenerateRunnerDashboard({ bundle, historyFile: null });
|
||||
const page = await readFile(path.join(bundle, "index.html"), "utf8");
|
||||
expect(page).toContain(`data-execution-id="${id}"`);
|
||||
expect(page).toContain(`data-execution-id="${missingId}"`);
|
||||
expect(page).toContain(
|
||||
`data-gallery-href="evidence/${id}/attempt-1/final-state.png"`,
|
||||
);
|
||||
const normalized = JSON.parse(
|
||||
await readFile(path.join(bundle, "normalized-results.json"), "utf8"),
|
||||
);
|
||||
expect(normalized.suites).toEqual([
|
||||
expect.objectContaining({
|
||||
suiteId: "branch-only-chat",
|
||||
selected: 2,
|
||||
passed: 1,
|
||||
failed: 1,
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("generates an explicit failed result with the correct unknown identity when no artifact exists", async () => {
|
||||
const root = await mkdtemp(
|
||||
path.join(os.tmpdir(), "branch-report-missing-"),
|
||||
);
|
||||
temporary.push(root);
|
||||
const out = path.join(root, "merged");
|
||||
const repo = path.resolve(import.meta.dirname, "../..");
|
||||
await promisify(execFile)(
|
||||
process.execPath,
|
||||
[
|
||||
path.join(repo, "cli/node_modules/tsx/dist/cli.mjs"),
|
||||
path.join(repo, "tests/runner-e2e/report.ts"),
|
||||
],
|
||||
{
|
||||
cwd: repo,
|
||||
env: {
|
||||
...process.env,
|
||||
PAPERCLIP_RUNNER_E2E_REPORT_ROOT: root,
|
||||
PAPERCLIP_RUNNER_E2E_REPORT_OUT: out,
|
||||
PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: JSON.stringify([missingId]),
|
||||
},
|
||||
},
|
||||
).catch((error: { code?: number }) => {
|
||||
if (error.code !== 1) throw error;
|
||||
});
|
||||
const normalized = JSON.parse(
|
||||
await readFile(path.join(out, "normalized-results.json"), "utf8"),
|
||||
);
|
||||
expect(normalized.results).toEqual([
|
||||
expect.objectContaining({
|
||||
executionId: missingId,
|
||||
suiteId: "branch-only-chat",
|
||||
profileId: "branch-provider",
|
||||
caseId: "reset",
|
||||
attempt: 0,
|
||||
status: "failed",
|
||||
}),
|
||||
]);
|
||||
expect(normalized.suites).toEqual([
|
||||
expect.objectContaining({
|
||||
suiteId: "branch-only-chat",
|
||||
failed: 1,
|
||||
executed: 0,
|
||||
}),
|
||||
]);
|
||||
expect(await readFile(path.join(out, "index.html"), "utf8")).toContain(
|
||||
`data-execution-id="${missingId}"`,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,130 @@
|
|||
import { createHash } from "node:crypto";
|
||||
import type { MatrixExecution, RunnerE2EResult } from "./types.js";
|
||||
|
||||
/** Display-only metadata: never load or reconstruct executable branch fixtures. */
|
||||
export interface ReportExecution {
|
||||
id: string;
|
||||
suiteDefinitionHash: string;
|
||||
suite: Pick<
|
||||
MatrixExecution["suite"],
|
||||
"id" | "label" | "description" | "expectedMatrixSize"
|
||||
>;
|
||||
profile: Pick<
|
||||
MatrixExecution["profile"],
|
||||
"id" | "label" | "generation" | "provider" | "model" | "expectedRuntimeMode"
|
||||
>;
|
||||
environment: Pick<
|
||||
MatrixExecution["environment"],
|
||||
"id" | "label" | "provider" | "expectedExecutionTarget"
|
||||
>;
|
||||
task: Pick<MatrixExecution["task"], "id" | "label">;
|
||||
}
|
||||
|
||||
export function parseReportExecutionId(id: string) {
|
||||
// These identities also become evidence paths. Reject path traversal and any
|
||||
// syntax other than the four catalog identity segments, even in imported JSON.
|
||||
if (typeof id !== "string")
|
||||
throw new Error("Report execution identity must be a string");
|
||||
const match =
|
||||
/^([a-z0-9][a-z0-9_-]*)\.([a-z0-9][a-z0-9_-]*)\.(local|daytona)\.([a-z0-9][a-z0-9_-]*)$/.exec(
|
||||
id,
|
||||
);
|
||||
if (!match || id.length > 512)
|
||||
throw new Error(
|
||||
`Invalid report execution identity: ${String(id).slice(0, 512)}`,
|
||||
);
|
||||
return {
|
||||
suiteId: match[1]!,
|
||||
profileId: match[2]!,
|
||||
environmentId: match[3]! as "local" | "daytona",
|
||||
caseId: match[4]!,
|
||||
};
|
||||
}
|
||||
|
||||
export function validateReportResultIdentity(result: RunnerE2EResult) {
|
||||
const identity = parseReportExecutionId(result.executionId);
|
||||
if (
|
||||
(result.suiteId !== undefined && result.suiteId !== identity.suiteId) ||
|
||||
result.profileId !== identity.profileId ||
|
||||
result.environmentId !== identity.environmentId ||
|
||||
result.caseId !== identity.caseId
|
||||
) {
|
||||
throw new Error(
|
||||
`Result identity does not match execution ID: ${result.executionId}`,
|
||||
);
|
||||
}
|
||||
return identity;
|
||||
}
|
||||
|
||||
export function discoverReportCatalog(input: {
|
||||
catalog: readonly ReportExecution[];
|
||||
expected: readonly string[];
|
||||
results: readonly RunnerE2EResult[];
|
||||
}): ReportExecution[] {
|
||||
const expected = new Set(input.expected);
|
||||
if (expected.size !== input.expected.length)
|
||||
throw new Error("Report expected identities must be unique");
|
||||
const identities = new Map(
|
||||
input.expected.map((id) => [id, parseReportExecutionId(id)]),
|
||||
);
|
||||
const results = new Map<string, RunnerE2EResult>();
|
||||
for (const result of input.results) {
|
||||
validateReportResultIdentity(result);
|
||||
if (expected.has(result.executionId))
|
||||
results.set(result.executionId, result);
|
||||
}
|
||||
const catalog = [...input.catalog];
|
||||
const knownIds = new Set(catalog.map((execution) => execution.id));
|
||||
const knownSuites = new Map(
|
||||
catalog.map((execution) => [execution.suite.id, execution.suite]),
|
||||
);
|
||||
for (const [id, identity] of identities) {
|
||||
if (knownIds.has(id)) continue;
|
||||
const result = results.get(id);
|
||||
const suiteIds = input.expected
|
||||
.filter(
|
||||
(candidate) => identities.get(candidate)?.suiteId === identity.suiteId,
|
||||
)
|
||||
.sort();
|
||||
const suite = knownSuites.get(identity.suiteId) ?? {
|
||||
id: identity.suiteId,
|
||||
label: identity.suiteId,
|
||||
description:
|
||||
"Suite discovered from retained campaign identities; full suite size is not known to this publisher.",
|
||||
expectedMatrixSize: suiteIds.length,
|
||||
};
|
||||
const knownProfile = input.catalog.find(
|
||||
(execution) => execution.profile.id === identity.profileId,
|
||||
)?.profile;
|
||||
const knownEnvironment = input.catalog.find(
|
||||
(execution) => execution.environment.id === identity.environmentId,
|
||||
)?.environment;
|
||||
const runtimeMode = result?.runtimeMode === "legacy" ? "legacy" : "native";
|
||||
catalog.push({
|
||||
id,
|
||||
suite,
|
||||
suiteDefinitionHash:
|
||||
result?.suiteDefinitionHash ??
|
||||
`selection-${createHash("sha256").update(JSON.stringify(suiteIds)).digest("hex")}`,
|
||||
profile: knownProfile ?? {
|
||||
id: identity.profileId,
|
||||
label: identity.profileId,
|
||||
generation: runtimeMode,
|
||||
expectedRuntimeMode: runtimeMode,
|
||||
provider:
|
||||
typeof result?.provider === "string" ? result.provider : "unknown",
|
||||
model: typeof result?.model === "string" ? result.model : "unknown",
|
||||
},
|
||||
environment: knownEnvironment ?? {
|
||||
id: identity.environmentId,
|
||||
label: identity.environmentId,
|
||||
provider: identity.environmentId,
|
||||
expectedExecutionTarget: {
|
||||
kind: identity.environmentId === "local" ? "local" : "remote",
|
||||
},
|
||||
},
|
||||
task: { id: identity.caseId, label: identity.caseId },
|
||||
});
|
||||
}
|
||||
return catalog;
|
||||
}
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
import { discoverReportCatalog } from "./report-catalog.js";
|
||||
import path from "node:path";
|
||||
import {
|
||||
copyFile,
|
||||
|
|
@ -188,6 +189,13 @@ async function main() {
|
|||
]);
|
||||
}
|
||||
|
||||
const reportCatalog = discoverReportCatalog({
|
||||
catalog: runnerMatrix,
|
||||
expected,
|
||||
results: [...candidates.values()].flatMap((entries) =>
|
||||
entries.map((entry) => entry.result),
|
||||
),
|
||||
});
|
||||
const selected: AggregatedResult[] = [];
|
||||
for (const executionId of expected) {
|
||||
const attempts = (candidates.get(executionId) ?? []).sort((left, right) => {
|
||||
|
|
@ -205,9 +213,9 @@ async function main() {
|
|||
});
|
||||
if (attempts.length === 0) {
|
||||
const now = new Date().toISOString();
|
||||
const execution = runnerMatrix.find(
|
||||
const execution = reportCatalog.find(
|
||||
(candidate) => candidate.id === executionId,
|
||||
);
|
||||
)!;
|
||||
const missing: RunnerE2EResult = {
|
||||
schema: "paperclip.runner-e2e.result/v2",
|
||||
executionId,
|
||||
|
|
|
|||
|
|
@ -1137,6 +1137,23 @@ describe("runner E2E evidence redaction", () => {
|
|||
).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("retains the two reviewed chat plan captures without admitting arbitrary chat PNGs", async () => {
|
||||
const root = await mkdtemp(path.join(os.tmpdir(), "runner-e2e-chat-captures-"));
|
||||
cleanupDirectories.push(root);
|
||||
const privateDir = path.join(root, "private");
|
||||
const uploadDir = path.join(root, "upload");
|
||||
await mkdir(privateDir, { recursive: true });
|
||||
for (const file of ["chat-plan-draft.png", "chat-plan-revised.png", "chat-secret.png", "chat-plan-extra.png"]) {
|
||||
await writeFile(path.join(privateDir, file), "fixture raster");
|
||||
}
|
||||
const packaged = await packageEvidence({ privateDir, uploadDir, secrets: [secret], expectPassScreenshot: false });
|
||||
expect(packaged.files.sort()).toEqual(["chat-plan-draft.png", "chat-plan-revised.png", "evidence-manifest.json"]);
|
||||
expect(packaged.leaks).toEqual([]);
|
||||
for (const file of packaged.files.filter((file) => file.endsWith(".png"))) {
|
||||
expect(await readFile(path.join(uploadDir, file), "utf8")).toBe("fixture raster");
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps raster evidence private to CI and rejects active SVG content", async () => {
|
||||
const root = await mkdtemp(
|
||||
path.join(os.tmpdir(), "runner-e2e-visual-evidence-test-"),
|
||||
|
|
|
|||
Loading…
Reference in New Issue