From e7136920959323756644d2f10a358b154fc7771f Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 11 Sep 2026 16:47:47 -0500 Subject: [PATCH] fix(runner-e2e): retain branch-only suites and chat screenshots in reports --- doc/plans/2026-09-10-agent-chat.md | 17 +- tests/runner-e2e/README.md | 10 + tests/runner-e2e/dashboard.ts | 35 ++-- tests/runner-e2e/evidence.ts | 2 + tests/runner-e2e/history-publish.ts | 10 +- tests/runner-e2e/history.test.ts | 16 ++ tests/runner-e2e/history.ts | 36 +++- tests/runner-e2e/public-summary-image.ts | 17 +- tests/runner-e2e/report-catalog.test.ts | 242 +++++++++++++++++++++++ tests/runner-e2e/report-catalog.ts | 130 ++++++++++++ tests/runner-e2e/report.ts | 12 +- tests/runner-e2e/support.test.ts | 17 ++ 12 files changed, 511 insertions(+), 33 deletions(-) create mode 100644 tests/runner-e2e/report-catalog.test.ts create mode 100644 tests/runner-e2e/report-catalog.ts diff --git a/doc/plans/2026-09-10-agent-chat.md b/doc/plans/2026-09-10-agent-chat.md index bb8bf2b1f9..0caac953b0 100644 --- a/doc/plans/2026-09-10-agent-chat.md +++ b/doc/plans/2026-09-10-agent-chat.md @@ -275,9 +275,20 @@ requested three-hour repair window. No acceptance assertions were disabled. - [Exact campaign results](https://d1p6rlowie26tp.cloudfront.net/runner-e2e/campaigns/gha-34648511170-1/summary.md) - [GitHub run and retained evidence](https://github.com/paperclipai/paperclip/actions/runs/34648511170) -Use the Markdown results for this branch-only suite: the trusted default-branch -HTML dashboard still uses its older catalog, while the normalized results and -Markdown report include all 24 chat cells. +The [HTML dashboard](https://d1p6rlowie26tp.cloudfront.net/runner-e2e/campaigns/gha-34648511170-1/index.html?report=agent-chat#suite-agent-chat) +was repaired from retained evidence after its older trusted catalog omitted the +branch-only suite. It now includes the chat suite and 32 screenshots, including +eight draft/revised plan captures recovered from their original Playwright +attachments. No paid cells were rerun; result records, tested SHA, timestamps, +usage, billing, attempts, and cleanup outcomes remain unchanged. + +Reporting now discovers validated display-only entries for unknown selected +execution IDs, and publication rejects missing declared screenshots. The exact +chat plan filenames are included in packaged evidence. All 165 runner unit tests +and runner TypeScript checks passed. Browser verification covered suite +filtering, restored plan images, and gallery navigation. This explicitly +authorized repair replaces only this campaign's report objects; normal +immutable-publication protections remain unchanged. The published summary and normalized results were verified after publication: exactly 24 unique expected cells, all passed on attempt 1, all cleanup checks diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 7e617cc6ee..c3a63a27eb 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -256,6 +256,12 @@ usage is labeled `unavailable` or `unpriced`; it is never presented as zero cost. The CI report job stages the same portable site at `normalized/index.html` inside the access-controlled merged report artifact. +The trusted publisher discovers display-only entries for selected execution IDs +absent from its local catalog, so branch-only suites remain visible in the +dashboard, filters, gallery, and summary image. It validates execution identity +and escapes display text without loading target-branch executable code. Unknown +suite cardinality is not treated as proof of full-suite coverage. + Permanent publication uses two explicit bundles. Both retain only normalized result PNG files with the explicit `public-runner-fixture` publication marker, including marked `failure.png` captures, so every campaign dashboard has its @@ -274,6 +280,10 @@ retains allowlisted inert per-attempt evidence (`.json`, `.log`, `.md`, and redaction. The GitHub Pages bundle is regenerated separately with the same declared-screenshot boundary. +Publication fails if any declared public screenshot is missing from the bundle. +The evidence packager explicitly retains `chat-plan-draft.png` and +`chat-plan-revised.png`; arbitrary chat-prefixed files remain excluded. + Both public bundles exclude video, archives, raw/unallowlisted logs, SVG or other active content, generated Playwright/blob/HTML report trees, and undeclared PNG files, and per-attempt XML. The root `junit.xml` remains public diff --git a/tests/runner-e2e/dashboard.ts b/tests/runner-e2e/dashboard.ts index 06a72bd9cb..a775909a06 100644 --- a/tests/runner-e2e/dashboard.ts +++ b/tests/runner-e2e/dashboard.ts @@ -1,5 +1,8 @@ +import { + discoverReportCatalog, + type ReportExecution, +} from "./report-catalog.js"; import type { - MatrixExecution, RunnerE2ECampaign, RunnerE2EHistoryIndex, RunnerE2EResult, @@ -22,7 +25,7 @@ export interface RunnerDashboardInput { title: string; generatedAt: string; expected: readonly string[]; - catalog: readonly MatrixExecution[]; + catalog: readonly ReportExecution[]; entries: readonly RunnerDashboardEntry[]; campaign?: RunnerE2ECampaign; history?: RunnerE2EHistoryIndex; @@ -135,7 +138,7 @@ function resolveScreenshots( } function renderCase( - execution: MatrixExecution, + execution: ReportExecution, expected: ReadonlySet, entryById: ReadonlyMap, ) { @@ -151,7 +154,11 @@ function renderCase( const label = state.replace("-", " "); const detail = entry?.errors.join("; ") || - (entry?.valid ? "All invariants passed" : "Not selected"); + (entry?.valid + ? "All invariants passed" + : selected + ? "No result artifact was uploaded" + : "Not selected"); const screenshots = resolveScreenshots(entry); const billing = entry ? summarizeExecutionBilling(entry.result) : null; const matcherResults = entry?.result.matcherResults ?? []; @@ -508,7 +515,7 @@ function renderHistory(history: RunnerE2EHistoryIndex | undefined) { } function renderSuiteMatrix(input: { - suiteCatalog: readonly MatrixExecution[]; + suiteCatalog: readonly ReportExecution[]; expected: ReadonlySet; entryById: ReadonlyMap; summary?: RunnerE2ESuiteSummary; @@ -580,6 +587,11 @@ function renderSuiteMatrix(input: { } export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { + const catalog = discoverReportCatalog({ + catalog: input.catalog, + expected: input.expected, + results: input.entries.map((entry) => entry.result), + }); const expected = new Set(input.expected); const entryById = new Map( input.entries.map((entry) => [entry.result.executionId, entry]), @@ -602,13 +614,13 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { ); const suites = [ ...new Map( - input.catalog.map((execution) => [execution.suite.id, execution.suite]), + catalog.map((execution) => [execution.suite.id, execution.suite]), ).values(), ]; const suiteSections = suites .map((suite) => renderSuiteMatrix({ - suiteCatalog: input.catalog.filter( + suiteCatalog: catalog.filter( (execution) => execution.suite.id === suite.id, ), expected, @@ -625,15 +637,12 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { ); const filterProfiles = [ ...new Map( - input.catalog.map((execution) => [ - execution.profile.id, - execution.profile, - ]), + catalog.map((execution) => [execution.profile.id, execution.profile]), ).values(), ]; const filterEnvironments = [ ...new Map( - input.catalog.map((execution) => [ + catalog.map((execution) => [ execution.environment.id, execution.environment, ]), @@ -1034,7 +1043,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { ${suiteSections} ${historySection} -
Generated ${html(input.generatedAt)}${input.catalog.length} catalog executions · Declared screenshots and sanitized structured evidence published
+
Generated ${html(input.generatedAt)}${catalog.length} catalog executions · Declared screenshots and sanitized structured evidence published
diff --git a/tests/runner-e2e/evidence.ts b/tests/runner-e2e/evidence.ts index c4a1cfe1a5..e09b027ece 100644 --- a/tests/runner-e2e/evidence.ts +++ b/tests/runner-e2e/evidence.ts @@ -38,6 +38,8 @@ const ALLOWED_ROOT_FILES = new Set([ "result.json", "final-state.png", "failure.png", + "chat-plan-draft.png", + "chat-plan-revised.png", "server.log", "playwright.log", "junit.xml", diff --git a/tests/runner-e2e/history-publish.ts b/tests/runner-e2e/history-publish.ts index f5dc913873..7fab9a53ac 100644 --- a/tests/runner-e2e/history-publish.ts +++ b/tests/runner-e2e/history-publish.ts @@ -313,10 +313,14 @@ export async function createBundleManifest( if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$/.test(campaignId)) { throw new Error("Campaign ID is unsafe for immutable object storage"); } + const retainedPaths = await relativeFiles(root, root, allowPublicSummary, publicScreenshots); + const retained = new Set(retainedPaths); + const missingScreenshots = [...publicScreenshots].filter((relative) => !retained.has(relative)); + if (missingScreenshots.length > 0) { + throw new Error(`Declared public screenshots are missing from the historical bundle: ${missingScreenshots.sort().join(", ")}`); + } const files = await Promise.all( - (await relativeFiles(root, root, allowPublicSummary, publicScreenshots)) - .sort() - .map(async (relative) => { + retainedPaths.sort().map(async (relative) => { const absolute = path.join(root, ...relative.split("/")); if ( relative === "public-images/campaign-summary.png" || diff --git a/tests/runner-e2e/history.test.ts b/tests/runner-e2e/history.test.ts index dcf20e24d7..c880bc134a 100644 --- a/tests/runner-e2e/history.test.ts +++ b/tests/runner-e2e/history.test.ts @@ -501,6 +501,22 @@ describe("historical publication security", () => { ).rejects.toThrow("does not match its raster file type"); }); + it("refuses a public bundle when any declared screenshot is absent", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "runner-missing-screenshot-")); + temporaryDirectories.push(root); + const directory = path.join(root, "evidence", "agent-chat.runner-codex.local.plan-handoff", "attempt-1"); + await mkdir(directory, { recursive: true }); + const base = "evidence/agent-chat.runner-codex.local.plan-handoff/attempt-1"; + const declared = new Set([`${base}/final-state.png`, `${base}/chat-plan-draft.png`]); + const png = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", "base64"); + await writeFile(path.join(directory, "final-state.png"), png); + await expect(createBundleManifest(root, "campaign-1", false, declared)) + .rejects.toThrow(`Declared public screenshots are missing from the historical bundle: ${base}/chat-plan-draft.png`); + await writeFile(path.join(directory, "chat-plan-draft.png"), png); + const manifest = await createBundleManifest(root, "campaign-1", false, declared); + expect(new Set(manifest.files.map((file) => file.path))).toEqual(declared); + }); + it("requires trusted-fixture opt-in and rejects unsafe screenshot paths", () => { const execution = runnerMatrix[0]!; const campaign = buildRunnerCampaign({ diff --git a/tests/runner-e2e/history.ts b/tests/runner-e2e/history.ts index bcb3871253..39077f9a33 100644 --- a/tests/runner-e2e/history.ts +++ b/tests/runner-e2e/history.ts @@ -1,3 +1,8 @@ +import { + discoverReportCatalog, + parseReportExecutionId, + type ReportExecution, +} from "./report-catalog.js"; import { runnerMatrix, runnerSuites } from "./catalog.js"; import { aggregateCampaignBilling, @@ -24,7 +29,12 @@ export function upgradeRunnerResult(result: RunnerE2EResult): RunnerE2EResult { const execution = runnerMatrix.find( (candidate) => candidate.id === executionId, ); - if (!execution) return result; + if (!execution) + return { + ...result, + executionId, + suiteId: result.suiteId ?? parseReportExecutionId(executionId).suiteId, + }; return { ...result, executionId, @@ -49,25 +59,39 @@ export function buildRunnerCampaign(input: { expected: readonly string[]; results: readonly RunnerE2EResult[]; eventName?: string | null; + catalog?: readonly ReportExecution[]; }): RunnerE2ECampaign { const expected = input.expected.map(canonicalExecutionId); const results = input.results.map((result) => ({ ...upgradeRunnerResult(result), billing: result.billing ?? summarizeExecutionBilling(result), })); + const knownCatalog = input.catalog ?? runnerMatrix; + const catalog = discoverReportCatalog({ + catalog: knownCatalog, + expected, + results, + }); + const discoveredSuites = [ + ...new Map( + catalog.map((execution) => [execution.suite.id, execution.suite]), + ).values(), + ]; const resultSource = results.find((result) => result.source)?.source; const source = { ...resolveRunnerE2ESource(resultSource), eventName: input.eventName ?? process.env.GITHUB_EVENT_NAME ?? null, }; - const suites = runnerSuites + const suites = discoveredSuites .map((suite) => { const suiteExpected = expected.filter((id) => id.startsWith(`${suite.id}.`), ); if (suiteExpected.length === 0) return null; const suiteResults = results.filter( - (result) => result.suiteId === suite.id, + (result) => + expected.includes(result.executionId) && + result.executionId.startsWith(`${suite.id}.`), ); const passed = suiteResults.filter( (result) => result.status === "passed" && result.cleanup === "passed", @@ -79,7 +103,7 @@ export function buildRunnerCampaign(input: { suiteId: suite.id, suiteDefinitionHash: suiteResults[0]?.suiteDefinitionHash ?? - runnerMatrix.find((execution) => execution.suite.id === suite.id)! + catalog.find((execution) => execution.suite.id === suite.id)! .suiteDefinitionHash, expected: suite.expectedMatrixSize, selected: suiteExpected.length, @@ -93,7 +117,9 @@ export function buildRunnerCampaign(input: { cleanupPassed: suiteResults.every( (result) => result.cleanup === "passed", ), - complete: suiteExpected.length === suite.expectedMatrixSize, + complete: + knownCatalog.some((execution) => execution.suite.id === suite.id) && + suiteExpected.length === suite.expectedMatrixSize, durationMs: suiteResults.reduce( (total, result) => total + result.durationMs, 0, diff --git a/tests/runner-e2e/public-summary-image.ts b/tests/runner-e2e/public-summary-image.ts index 13366da333..54dea1bd0f 100644 --- a/tests/runner-e2e/public-summary-image.ts +++ b/tests/runner-e2e/public-summary-image.ts @@ -1,3 +1,4 @@ +import { discoverReportCatalog } from "./report-catalog.js"; import path from "node:path"; import { mkdir } from "node:fs/promises"; import { chromium } from "@playwright/test"; @@ -24,8 +25,13 @@ function durationLabel(durationMs: number) { } export function renderPublicCampaignSummary(campaign: RunnerE2ECampaign) { + const catalog = discoverReportCatalog({ + catalog: runnerMatrix, + expected: campaign.expected, + results: campaign.results, + }); const catalogById = new Map( - runnerMatrix.map((execution) => [execution.id, execution]), + catalog.map((execution) => [execution.id, execution]), ); const expectedIds = [ ...new Set( @@ -53,10 +59,7 @@ export function renderPublicCampaignSummary(campaign: RunnerE2ECampaign) { ); const suites = [ ...new Map( - runnerMatrix.map((execution) => [ - execution.suite.id, - execution.suite.label, - ]), + catalog.map((execution) => [execution.suite.id, execution.suite.label]), ), ].map(([suiteId, label]) => { const selectedIds = expectedIds.filter( @@ -111,12 +114,12 @@ export function renderPublicCampaignSummary(campaign: RunnerE2ECampaign) {

Runner full-stack E2E

Campaign summary

-
${passed}/${expectedIds.length}Known executions passed
+
${passed}/${expectedIds.length}Selected executions passed
${expectedIds.length - passed}Failed or incomplete
${html(durationLabel(durationMs))}Total test time
${rows}
-
Generated from fixed catalog labels and sanitized numeric/status fields. Provider output is never rendered here.
+
Generated from trusted catalog labels, validated execution identities, and sanitized numeric/status fields. Provider output is never rendered here.
`; diff --git a/tests/runner-e2e/report-catalog.test.ts b/tests/runner-e2e/report-catalog.test.ts new file mode 100644 index 0000000000..ce54869bbd --- /dev/null +++ b/tests/runner-e2e/report-catalog.test.ts @@ -0,0 +1,242 @@ +import { renderPublicCampaignSummary } from "./public-summary-image.js"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { runnerMatrix } from "./catalog.js"; +import { discoverReportCatalog } from "./report-catalog.js"; +import { buildRunnerCampaign } from "./history.js"; +import { renderRunnerE2EDashboard } from "./dashboard.js"; +import { regenerateRunnerDashboard } from "./dashboard-regenerate.js"; +import type { RunnerE2EResult } from "./types.js"; + +const temporary: string[] = []; +afterEach(async () => { + await Promise.all( + temporary.splice(0).map((dir) => rm(dir, { recursive: true, force: true })), + ); +}); +const id = "branch-only-chat.branch-provider.local.conversation"; +const missingId = "branch-only-chat.branch-provider.local.reset"; +function result(): RunnerE2EResult { + return { + schema: "paperclip.runner-e2e.result/v2", + executionId: id, + suiteId: "branch-only-chat", + suiteDefinitionHash: "branch-definition", + profileId: "branch-provider", + environmentId: "local", + caseId: "conversation", + provider: '', + model: "branch-model", + runtimeMode: "native", + attempt: 1, + status: "passed", + cleanup: "passed", + durationMs: 1000, + startedAt: "2026-09-11T21:00:00.000Z", + finishedAt: "2026-09-11T21:00:01.000Z", + screenshots: [ + { + id: "final-state", + label: '', + file: "final-state.png", + publication: "public-runner-fixture", + }, + ], + }; +} +function campaign(results: RunnerE2EResult[] = [result()]) { + return buildRunnerCampaign({ + campaignId: "branch-campaign", + generatedAt: "2026-09-11T21:01:00.000Z", + expected: [id, missingId], + results, + }); +} + +describe("trusted report catalog discovery", () => { + it("aggregates unknown suites and renders their cases, missing results, and screenshot gallery", () => { + const summary = campaign(); + expect(summary.suites).toEqual([ + expect.objectContaining({ + suiteId: "branch-only-chat", + selected: 2, + executed: 1, + passed: 1, + failed: 1, + complete: false, + }), + ]); + const page = renderRunnerE2EDashboard({ + title: "Report", + generatedAt: summary.generatedAt, + expected: summary.expected, + catalog: [], + campaign: summary, + entries: [ + { + result: result(), + valid: true, + errors: [], + evidenceBaseHref: `evidence/${id}/attempt-1`, + evidenceFiles: ["final-state.png"], + }, + ], + }); + expect(page).toContain('id="suite-branch-only-chat"'); + expect(page).toContain('class="suite-summary"'); + expect(page).toContain(`data-execution-id="${id}"`); + expect(page).toContain(`data-execution-id="${missingId}"`); + expect(page).toContain('class="case case-missing"'); + expect(page).toContain("No result artifact was uploaded"); + expect(page).toContain( + `data-gallery-href="evidence/${id}/attempt-1/final-state.png"`, + ); + expect(page).toContain("View gallery · 1"); + expect(page).toContain("<img src=x onerror="alert(1)">"); + expect(page).not.toContain("alert("label")'); + }); + + it("renders selected branch suites in the public summary image without provider content", () => { + const page = renderPublicCampaignSummary(campaign()); + expect(page).toContain("1/2"); + expect(page).toContain("branch-only-chat"); + expect(page).not.toContain("0/0"); + expect(page).not.toContain("onerror"); + expect(page).not.toContain("branch-model"); + }); + + it("retains ordinary catalog metadata and marks no-results branch selections failed", () => { + const catalog = discoverReportCatalog({ + catalog: runnerMatrix, + expected: [runnerMatrix[0]!.id], + results: [], + }); + expect(catalog).toEqual(runnerMatrix); + expect(catalog[0]).toBe(runnerMatrix[0]); + expect(campaign([])).toMatchObject({ + selected: 2, + executed: 0, + passed: 0, + failed: 2, + suites: [ + expect.objectContaining({ suiteId: "branch-only-chat", failed: 2 }), + ], + }); + }); + + it("uses known profile metadata in a new suite and rejects forged identities", () => { + const known = runnerMatrix[0]!; + const knownProfileId = `branch-only-chat.${known.profile.id}.${known.environment.id}.conversation`; + expect( + discoverReportCatalog({ + catalog: runnerMatrix, + expected: [knownProfileId], + results: [], + }).at(-1)?.profile, + ).toBe(known.profile); + expect(() => + discoverReportCatalog({ + catalog: [], + expected: [id], + results: [{ ...result(), suiteId: "forged-suite" }], + }), + ).toThrow("Result identity does not match"); + expect(() => + discoverReportCatalog({ + catalog: [], + expected: ["../../private"], + results: [], + }), + ).toThrow("Invalid report execution identity"); + expect(() => + discoverReportCatalog({ catalog: [], expected: [id, id], results: [] }), + ).toThrow("must be unique"); + }); + + it("regenerates branch-only cards and gallery from persisted JSON using trusted code", async () => { + const bundle = await mkdtemp( + path.join(os.tmpdir(), "branch-report-regenerate-"), + ); + temporary.push(bundle); + const evidence = path.join(bundle, "evidence", id, "attempt-1"); + await mkdir(evidence, { recursive: true }); + await writeFile(path.join(evidence, "final-state.png"), "fixture"); + await writeFile( + path.join(bundle, "normalized-results.json"), + JSON.stringify({ ...campaign(), suites: [] }), + ); + await regenerateRunnerDashboard({ bundle, historyFile: null }); + const page = await readFile(path.join(bundle, "index.html"), "utf8"); + expect(page).toContain(`data-execution-id="${id}"`); + expect(page).toContain(`data-execution-id="${missingId}"`); + expect(page).toContain( + `data-gallery-href="evidence/${id}/attempt-1/final-state.png"`, + ); + const normalized = JSON.parse( + await readFile(path.join(bundle, "normalized-results.json"), "utf8"), + ); + expect(normalized.suites).toEqual([ + expect.objectContaining({ + suiteId: "branch-only-chat", + selected: 2, + passed: 1, + failed: 1, + }), + ]); + }); + + it("generates an explicit failed result with the correct unknown identity when no artifact exists", async () => { + const root = await mkdtemp( + path.join(os.tmpdir(), "branch-report-missing-"), + ); + temporary.push(root); + const out = path.join(root, "merged"); + const repo = path.resolve(import.meta.dirname, "../.."); + await promisify(execFile)( + process.execPath, + [ + path.join(repo, "cli/node_modules/tsx/dist/cli.mjs"), + path.join(repo, "tests/runner-e2e/report.ts"), + ], + { + cwd: repo, + env: { + ...process.env, + PAPERCLIP_RUNNER_E2E_REPORT_ROOT: root, + PAPERCLIP_RUNNER_E2E_REPORT_OUT: out, + PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: JSON.stringify([missingId]), + }, + }, + ).catch((error: { code?: number }) => { + if (error.code !== 1) throw error; + }); + const normalized = JSON.parse( + await readFile(path.join(out, "normalized-results.json"), "utf8"), + ); + expect(normalized.results).toEqual([ + expect.objectContaining({ + executionId: missingId, + suiteId: "branch-only-chat", + profileId: "branch-provider", + caseId: "reset", + attempt: 0, + status: "failed", + }), + ]); + expect(normalized.suites).toEqual([ + expect.objectContaining({ + suiteId: "branch-only-chat", + failed: 1, + executed: 0, + }), + ]); + expect(await readFile(path.join(out, "index.html"), "utf8")).toContain( + `data-execution-id="${missingId}"`, + ); + }); +}); diff --git a/tests/runner-e2e/report-catalog.ts b/tests/runner-e2e/report-catalog.ts new file mode 100644 index 0000000000..3400cb2c2d --- /dev/null +++ b/tests/runner-e2e/report-catalog.ts @@ -0,0 +1,130 @@ +import { createHash } from "node:crypto"; +import type { MatrixExecution, RunnerE2EResult } from "./types.js"; + +/** Display-only metadata: never load or reconstruct executable branch fixtures. */ +export interface ReportExecution { + id: string; + suiteDefinitionHash: string; + suite: Pick< + MatrixExecution["suite"], + "id" | "label" | "description" | "expectedMatrixSize" + >; + profile: Pick< + MatrixExecution["profile"], + "id" | "label" | "generation" | "provider" | "model" | "expectedRuntimeMode" + >; + environment: Pick< + MatrixExecution["environment"], + "id" | "label" | "provider" | "expectedExecutionTarget" + >; + task: Pick; +} + +export function parseReportExecutionId(id: string) { + // These identities also become evidence paths. Reject path traversal and any + // syntax other than the four catalog identity segments, even in imported JSON. + if (typeof id !== "string") + throw new Error("Report execution identity must be a string"); + const match = + /^([a-z0-9][a-z0-9_-]*)\.([a-z0-9][a-z0-9_-]*)\.(local|daytona)\.([a-z0-9][a-z0-9_-]*)$/.exec( + id, + ); + if (!match || id.length > 512) + throw new Error( + `Invalid report execution identity: ${String(id).slice(0, 512)}`, + ); + return { + suiteId: match[1]!, + profileId: match[2]!, + environmentId: match[3]! as "local" | "daytona", + caseId: match[4]!, + }; +} + +export function validateReportResultIdentity(result: RunnerE2EResult) { + const identity = parseReportExecutionId(result.executionId); + if ( + (result.suiteId !== undefined && result.suiteId !== identity.suiteId) || + result.profileId !== identity.profileId || + result.environmentId !== identity.environmentId || + result.caseId !== identity.caseId + ) { + throw new Error( + `Result identity does not match execution ID: ${result.executionId}`, + ); + } + return identity; +} + +export function discoverReportCatalog(input: { + catalog: readonly ReportExecution[]; + expected: readonly string[]; + results: readonly RunnerE2EResult[]; +}): ReportExecution[] { + const expected = new Set(input.expected); + if (expected.size !== input.expected.length) + throw new Error("Report expected identities must be unique"); + const identities = new Map( + input.expected.map((id) => [id, parseReportExecutionId(id)]), + ); + const results = new Map(); + for (const result of input.results) { + validateReportResultIdentity(result); + if (expected.has(result.executionId)) + results.set(result.executionId, result); + } + const catalog = [...input.catalog]; + const knownIds = new Set(catalog.map((execution) => execution.id)); + const knownSuites = new Map( + catalog.map((execution) => [execution.suite.id, execution.suite]), + ); + for (const [id, identity] of identities) { + if (knownIds.has(id)) continue; + const result = results.get(id); + const suiteIds = input.expected + .filter( + (candidate) => identities.get(candidate)?.suiteId === identity.suiteId, + ) + .sort(); + const suite = knownSuites.get(identity.suiteId) ?? { + id: identity.suiteId, + label: identity.suiteId, + description: + "Suite discovered from retained campaign identities; full suite size is not known to this publisher.", + expectedMatrixSize: suiteIds.length, + }; + const knownProfile = input.catalog.find( + (execution) => execution.profile.id === identity.profileId, + )?.profile; + const knownEnvironment = input.catalog.find( + (execution) => execution.environment.id === identity.environmentId, + )?.environment; + const runtimeMode = result?.runtimeMode === "legacy" ? "legacy" : "native"; + catalog.push({ + id, + suite, + suiteDefinitionHash: + result?.suiteDefinitionHash ?? + `selection-${createHash("sha256").update(JSON.stringify(suiteIds)).digest("hex")}`, + profile: knownProfile ?? { + id: identity.profileId, + label: identity.profileId, + generation: runtimeMode, + expectedRuntimeMode: runtimeMode, + provider: + typeof result?.provider === "string" ? result.provider : "unknown", + model: typeof result?.model === "string" ? result.model : "unknown", + }, + environment: knownEnvironment ?? { + id: identity.environmentId, + label: identity.environmentId, + provider: identity.environmentId, + expectedExecutionTarget: { + kind: identity.environmentId === "local" ? "local" : "remote", + }, + }, + task: { id: identity.caseId, label: identity.caseId }, + }); + } + return catalog; +} diff --git a/tests/runner-e2e/report.ts b/tests/runner-e2e/report.ts index 3b61eb5d14..8ddba5d834 100644 --- a/tests/runner-e2e/report.ts +++ b/tests/runner-e2e/report.ts @@ -1,3 +1,4 @@ +import { discoverReportCatalog } from "./report-catalog.js"; import path from "node:path"; import { copyFile, @@ -188,6 +189,13 @@ async function main() { ]); } + const reportCatalog = discoverReportCatalog({ + catalog: runnerMatrix, + expected, + results: [...candidates.values()].flatMap((entries) => + entries.map((entry) => entry.result), + ), + }); const selected: AggregatedResult[] = []; for (const executionId of expected) { const attempts = (candidates.get(executionId) ?? []).sort((left, right) => { @@ -205,9 +213,9 @@ async function main() { }); if (attempts.length === 0) { const now = new Date().toISOString(); - const execution = runnerMatrix.find( + const execution = reportCatalog.find( (candidate) => candidate.id === executionId, - ); + )!; const missing: RunnerE2EResult = { schema: "paperclip.runner-e2e.result/v2", executionId, diff --git a/tests/runner-e2e/support.test.ts b/tests/runner-e2e/support.test.ts index d4ad4bac35..d46d1ec1cf 100644 --- a/tests/runner-e2e/support.test.ts +++ b/tests/runner-e2e/support.test.ts @@ -1137,6 +1137,23 @@ describe("runner E2E evidence redaction", () => { ).rejects.toThrow(); }); + it("retains the two reviewed chat plan captures without admitting arbitrary chat PNGs", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "runner-e2e-chat-captures-")); + cleanupDirectories.push(root); + const privateDir = path.join(root, "private"); + const uploadDir = path.join(root, "upload"); + await mkdir(privateDir, { recursive: true }); + for (const file of ["chat-plan-draft.png", "chat-plan-revised.png", "chat-secret.png", "chat-plan-extra.png"]) { + await writeFile(path.join(privateDir, file), "fixture raster"); + } + const packaged = await packageEvidence({ privateDir, uploadDir, secrets: [secret], expectPassScreenshot: false }); + expect(packaged.files.sort()).toEqual(["chat-plan-draft.png", "chat-plan-revised.png", "evidence-manifest.json"]); + expect(packaged.leaks).toEqual([]); + for (const file of packaged.files.filter((file) => file.endsWith(".png"))) { + expect(await readFile(path.join(uploadDir, file), "utf8")).toBe("fixture raster"); + } + }); + it("keeps raster evidence private to CI and rejects active SVG content", async () => { const root = await mkdtemp( path.join(os.tmpdir(), "runner-e2e-visual-evidence-test-"),