diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.yml index 1150d96d54..7569708e7d 100644 --- a/.github/workflows/runner-full-stack-e2e.yml +++ b/.github/workflows/runner-full-stack-e2e.yml @@ -1104,14 +1104,44 @@ jobs: retention-days: 30 if-no-files-found: error - - name: Verify history source report and private screenshot evidence + - name: Qualify public preview raster sanitizer + if: always() + run: | + set -euo pipefail + if ! command -v convert >/dev/null 2>&1; then + sudo apt-get update -qq + sudo apt-get install --no-install-recommends -y imagemagick + fi + convert -version + + - name: Prepare public history bundle with redacted layout previews + id: prepare_public_history + if: always() + run: | + set -euo pipefail + pnpm test:e2e:runner:history:prepare -- \ + "$GITHUB_WORKSPACE/runner-e2e-merged-report/normalized" \ + "$GITHUB_WORKSPACE/runner-e2e-public-report/normalized" + + - name: Upload prepared public history source + if: always() && steps.prepare_public_history.outcome == 'success' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: runner-e2e-public-history-source-${{ github.run_id }}-${{ github.run_attempt }} + path: runner-e2e-public-report/ + retention-days: 1 + if-no-files-found: error + + - name: Verify prepared history source and public layout previews id: history_source_ready if: always() run: | set -euo pipefail - dashboard_root="runner-e2e-merged-report/normalized" - private_screenshot="$(find "$dashboard_root" -type f -name '*.png' -print -quit 2>/dev/null || true)" - if [ -f "$dashboard_root/index.html" ] && [ -n "$private_screenshot" ]; then + dashboard_root="runner-e2e-public-report/normalized" + public_preview="$(find "$dashboard_root/evidence" -type f -path '*/public-visuals/*.png' -print -quit 2>/dev/null || true)" + unexpected_png="$(find "$dashboard_root/evidence" -type f -name '*.png' ! -path '*/public-visuals/*.png' -print -quit 2>/dev/null || true)" + passed_count="$(jq '[.results[] | select(.status == "passed")] | length' "$dashboard_root/normalized-results.json" 2>/dev/null || echo invalid)" + if [ "${{ steps.prepare_public_history.outcome }}" = "success" ] && [ -f "$dashboard_root/index.html" ] && [ -z "$unexpected_png" ] && { [ "$passed_count" = "0" ] || [ -n "$public_preview" ]; }; then echo "ready=true" >> "$GITHUB_OUTPUT" else echo "ready=false" >> "$GITHUB_OUTPUT" @@ -1154,10 +1184,10 @@ jobs: - run: pnpm install --frozen-lockfile - - name: Download access-controlled normalized campaign + - name: Download prepared public campaign uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - name: runner-e2e-report-${{ github.run_id }}-${{ github.run_attempt }} + name: runner-e2e-public-history-source-${{ github.run_id }}-${{ github.run_attempt }} path: runner-e2e-merged-report - name: Exchange GitHub OIDC identity for scoped AWS credentials diff --git a/package.json b/package.json index 0a7fb82bab..e8aed3cb59 100644 --- a/package.json +++ b/package.json @@ -68,6 +68,7 @@ "test:e2e:runner:dashboard": "node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/dashboard-regenerate.ts", "test:e2e:runner:models:update": "node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/openrouter-models-update.ts", "test:e2e:runner:history:publish": "node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/history-publish.ts", + "test:e2e:runner:history:prepare": "node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/history-public-bundle.ts", "test:e2e:runner:unit": "vitest run --config tests/runner-e2e/vitest.config.ts", "test:e2e:runner:typecheck": "tsc -p tests/runner-e2e/tsconfig.json", "test:e2e:runner:report": "node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/report.ts", diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index ad62738f60..af3c18f381 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -204,16 +204,18 @@ usage is labeled `unavailable` or `unpriced`; it is never presented as zero cost. The CI report job stages the same portable site at `normalized/index.html` inside the access-controlled merged report artifact. -Permanent public history has a narrower boundary. Before uploading to S3 or -packaging the optional GitHub Pages artifact, the publisher removes raster and -video evidence, archives, and the generated Playwright/blob/HTML report trees. -It then regenerates the dashboard against only the remaining allowlisted, -inert structured per-attempt evidence (`.json`, `.log`, `.md`, and `.txt`). -Per-attempt XML is excluded because browsers can process XML/XSLT. The root -`junit.xml` remains public because the report aggregator builds it from fixed -markup and XML-escaped fields. Public dashboards therefore contain results and -accounting but no attempt screenshots, videos, traces, or generated Playwright -reports. +Permanent public history has a narrower boundary. A trusted job without +provider or AWS credentials copies successful declared PNG screenshots into a +separate publication tree. It validates the PNG container, reduces each image +to at most 160 pixels on either edge, applies a strong blur, limits the image to +24 colors, and removes metadata. These layout previews preserve coarse UI +state while making rendered task and provider text unreadable. The job then +removes the full-resolution raster files, failure screenshots, video, archives, +SVG, and generated Playwright/blob/HTML report trees. Only the derived +`public-visuals/*.png` previews and allowlisted inert evidence (`.json`, `.log`, +`.md`, and `.txt`) remain. Per-attempt XML is excluded because browsers can +process XML/XSLT. The root `junit.xml` remains public because the report +aggregator builds it from fixed markup and XML-escaped fields. ### Billing interpretation @@ -380,11 +382,12 @@ bundle digest fails closed. GitHub Pages remains the stable latest dashboard. Enable Pages with GitHub Actions as its source and set `RUNNER_FULL_STACK_E2E_PUBLISH_PAGES=true`. -The publisher prunes screenshots, video, archives, and generated report trees, -then regenerates the public dashboard before either the CloudFront-backed S3 -history or optional Pages artifact is created. Public per-attempt evidence is -limited to allowlisted inert structured text. Databases, Paperclip homes, -workspaces, raw/unredacted logs, credentials, and visual evidence are never +The trusted report job creates a separate public bundle before the AWS role is +available. It publishes only blurred, low-resolution, metadata-free layout +previews for successful declared screenshots and allowlisted inert structured +text. The S3/CloudFront history and optional Pages mirror use this same bundle. +Full-resolution and failure screenshots, video, archives, generated reports, +databases, Paperclip homes, workspaces, raw logs, and credentials are never published. Sanitized allowlisted `.log` copies may be public only after exact-value/key-shape scanning and redaction. diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md index 518e9f1a47..1e5e2a1e85 100644 --- a/tests/runner-e2e/SECURITY.md +++ b/tests/runner-e2e/SECURITY.md @@ -179,8 +179,9 @@ latest pointers are mutable, and S3 versioning makes those updates recoverable. ## Public evidence boundary CloudFront and GitHub Pages are public. Fixture identifiers, timing, token -usage, costs, normalized results, and allowlisted inert structured per-attempt -evidence are expected public data. Screenshots, video, archives, generated +usage, costs, normalized results, allowlisted inert structured per-attempt +evidence, and deliberately degraded layout previews are expected public data. +Full-resolution and failure screenshots, video, archives, generated Playwright/blob/HTML report trees, credentials, Paperclip homes, databases, workspaces, master keys, raw/unredacted logs, and unallowlisted files are not. Only allowlisted `.log` copies that passed exact-value/key-shape scanning and @@ -189,20 +190,29 @@ redaction may cross the public boundary. The packaged evidence uploaded as a 30-day GitHub Actions artifact has a different, access-controlled boundary. Text is exact-value and key-shape scanned and redacted. PNG and WebM are raw-byte scanned but cannot be inspected -for credentials rendered as pixels, so they remain only in local evidence and -the access-controlled artifact. SVG is rejected during packaging because it is -active content. +for credentials rendered as pixels, so full-resolution files remain only in +local evidence and the access-controlled artifact. SVG is rejected during +packaging because it is active content. -Before permanent publication, the campaign publisher prunes raster/video -files, archives, and generated report trees. It then regenerates the dashboard -from the remaining allowlisted `.json`, `.log`, `.md`, and `.txt` evidence and -accepts only that dashboard, normalized JSON/JUnit/summary, fixed -branding assets, and the inert structured evidence paths. Per-attempt XML is -excluded because browsers can process XML/XSLT; the only public XML is the -root `junit.xml`, which the report aggregator constructs from fixed markup and -XML-escaped fields. The same pruned tree feeds both S3/CloudFront history and -the optional GitHub Pages artifact. A leak fails the cell and withholds the -unsafe file. +Before permanent publication, the trusted report job creates a separate tree. +It accepts only declared screenshots from passing results. It validates a +bounded, non-interlaced PNG container before invoking ImageMagick with strict +memory, disk, thread, and time limits. It reduces each image to at most 160 +pixels on either edge, applies a strong blur, limits the palette, removes alpha +and metadata, and validates the new PNG again. This is a layout preview, not +diagnostic evidence. It then prunes full-resolution raster files, all failure +images, video, archives, active SVG, and generated reports. This transformation +runs before AWS credentials are available. The AWS job downloads only the +prepared public tree. + +The remaining allowlist contains `public-visuals/*.png`, `.json`, `.log`, `.md`, +and `.txt` evidence, plus the generated dashboard, normalized +JSON/JUnit/summary, and fixed branding assets. Per-attempt XML is excluded +because browsers can process XML/XSLT. The only public XML is the root +`junit.xml`, which the report aggregator constructs from fixed markup and +XML-escaped fields. The same prepared tree feeds S3/CloudFront history and the +optional GitHub Pages artifact. Any malformed image, transformation error, +unexpected file, or scan failure withholds publication. Rotate the affected credential immediately if a secret-scanning failure or unexpected public object is observed. Preserve the access-controlled Actions diff --git a/tests/runner-e2e/dashboard.ts b/tests/runner-e2e/dashboard.ts index 23f80352f7..5580888c80 100644 --- a/tests/runner-e2e/dashboard.ts +++ b/tests/runner-e2e/dashboard.ts @@ -889,7 +889,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {

Full-stack acceptance campaign

${html(input.title)}

-

A browser-verified matrix of runner profiles, execution environments, and deterministic task contracts. Visual evidence is retained in the access-controlled workflow artifact; public history contains inert structured evidence only.

+

A browser-verified matrix of runner profiles, execution environments, and deterministic task contracts. Full-resolution visual evidence stays in the access-controlled workflow artifact. Public history includes blurred low-resolution layout previews and inert structured evidence.

@@ -897,7 +897,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {
${failed}Failed
${html(durationLabel(totalDuration))}Test time
- +
@@ -918,7 +918,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { ${suiteSections} ${historySection} - +
diff --git a/tests/runner-e2e/history-index.ts b/tests/runner-e2e/history-index.ts index 3f93ed5cfd..c656a191c1 100644 --- a/tests/runner-e2e/history-index.ts +++ b/tests/runner-e2e/history-index.ts @@ -196,7 +196,7 @@ export function renderRunnerHistoryIndex(history: RunnerE2EHistoryIndex) {

Historical test reporting

Runner E2E campaigns

-

Each row is one workflow campaign against a Paperclip revision. Open a report for its configuration matrices, matchers, per-test billing, and sanitized structured evidence. Visual evidence remains in access-controlled workflow artifacts.

+

Each row is one workflow campaign against a Paperclip revision. Open a report for its configuration matrices, matchers, per-test billing, sanitized structured evidence, and blurred low-resolution layout previews. Full-resolution visual evidence remains in access-controlled workflow artifacts.

${campaigns.length}Campaigns
diff --git a/tests/runner-e2e/history-public-bundle.ts b/tests/runner-e2e/history-public-bundle.ts new file mode 100644 index 0000000000..0508f8062c --- /dev/null +++ b/tests/runner-e2e/history-public-bundle.ts @@ -0,0 +1,482 @@ +import { execFile } from "node:child_process"; +import { constants as fsConstants } from "node:fs"; +import path from "node:path"; +import { promisify } from "node:util"; +import { + cp, + copyFile, + lstat, + mkdir, + mkdtemp, + readFile, + readdir, + rm, + writeFile, +} from "node:fs/promises"; +import { regenerateRunnerDashboard } from "./dashboard-regenerate.js"; + +const execFileAsync = promisify(execFile); +const PUBLIC_EVIDENCE_EXTENSIONS = new Set([".json", ".log", ".md", ".txt"]); +const PRIVATE_EVIDENCE_DIRECTORIES = new Set([ + "blob-report", + "html-report", + "playwright-output", +]); +const PUBLIC_VISUAL_DIRECTORY = "public-visuals"; +const PUBLIC_SCREENSHOT_NAME = + /^(?:final-state|plan-[a-z0-9-]+|question-[a-z0-9-]+)\.png$/; +const PNG_SIGNATURE = Buffer.from([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, +]); +const MAX_PRIVATE_PNG_BYTES = 32 * 1024 * 1024; +const MAX_PRIVATE_PNG_PIXELS = 64 * 1024 * 1024; +const MAX_PRIVATE_PNG_EDGE = 32 * 1024; +const MAX_PUBLIC_PREVIEW_EDGE = 160; +const MAX_PUBLIC_PREVIEW_BYTES = 256 * 1024; +const PRIVATE_PNG_CHUNKS = new Set([ + "IHDR", + "IDAT", + "IEND", + "PLTE", + "tRNS", + "gAMA", + "cHRM", + "sRGB", + "pHYs", +]); +const PUBLIC_PNG_CHUNKS = new Set(["IHDR", "IDAT", "IEND", "PLTE"]); + +interface PublishedResult { + executionId?: unknown; + attempt?: unknown; + status?: unknown; + evidenceValid?: unknown; + screenshots?: unknown; + [key: string]: unknown; +} + +interface PublishedCampaign { + results?: unknown; + [key: string]: unknown; +} + +export type PublicPreviewTransformer = ( + source: string, + destination: string, +) => Promise; + +function publicVisualPath(evidencePath: string) { + const segments = evidencePath.split("/"); + return ( + segments.length === 2 && + segments[0] === PUBLIC_VISUAL_DIRECTORY && + PUBLIC_SCREENSHOT_NAME.test(segments[1] ?? "") + ); +} + +export function isPublicHistoryEvidencePath(relative: string) { + const match = relative.match( + /^evidence\/[A-Za-z0-9._-]+\/attempt-[1-9][0-9]*\/(.+)$/, + ); + if (!match) return false; + const evidencePath = match[1]!; + const segments = evidencePath.split("/"); + if ( + segments.some( + (segment) => + !segment || + segment === "." || + segment === ".." || + PRIVATE_EVIDENCE_DIRECTORIES.has(segment), + ) + ) { + return false; + } + if (publicVisualPath(evidencePath)) return true; + return PUBLIC_EVIDENCE_EXTENSIONS.has( + path.posix.extname(evidencePath).toLowerCase(), + ); +} + +async function pruneEvidenceDirectory(root: string, current: string) { + const entries = await readdir(current, { withFileTypes: true }); + for (const entry of entries) { + const absolute = path.join(current, entry.name); + if (entry.isDirectory()) { + await pruneEvidenceDirectory(root, absolute); + if ((await readdir(absolute)).length === 0) { + await rm(absolute, { recursive: true }); + } + continue; + } + const relative = path.relative(root, absolute).split(path.sep).join("/"); + if (!entry.isFile() || !isPublicHistoryEvidencePath(relative)) { + await rm(absolute, { force: true }); + } else if (path.posix.extname(relative).toLowerCase() === ".png") { + // Only derived public previews can pass the PNG path allowlist. Validate + // the bounded inert container again when the AWS publisher prunes its + // downloaded bundle. + await validatePublicPreview(absolute); + } + } +} + +export async function prunePrivateHistoryEvidence(root: string) { + const evidenceRoot = path.join(root, "evidence"); + const metadata = await lstat(evidenceRoot).catch(() => null); + if (!metadata) return; + if (!metadata.isDirectory()) { + throw new Error("Historical evidence root must be a directory"); + } + await pruneEvidenceDirectory(root, evidenceRoot); +} + +function crc32(buffer: Buffer) { + let crc = 0xffffffff; + for (const byte of buffer) { + crc ^= byte; + for (let bit = 0; bit < 8; bit += 1) { + crc = (crc >>> 1) ^ (crc & 1 ? 0xedb88320 : 0); + } + } + return (crc ^ 0xffffffff) >>> 0; +} + +function inspectPng( + value: Buffer, + options: { + label: string; + maxBytes: number; + maxEdge?: number; + maxPixels?: number; + chunks: ReadonlySet; + }, +) { + if (value.length > options.maxBytes) { + throw new Error(`${options.label} exceeds the PNG byte limit`); + } + if ( + value.length < PNG_SIGNATURE.length || + !value.subarray(0, PNG_SIGNATURE.length).equals(PNG_SIGNATURE) + ) { + throw new Error(`${options.label} is not a PNG`); + } + let offset = PNG_SIGNATURE.length; + let width = 0; + let height = 0; + let sawHeader = false; + let sawData = false; + let sawEnd = false; + while (offset < value.length) { + if (offset + 12 > value.length) { + throw new Error(`${options.label} has a truncated PNG chunk`); + } + const length = value.readUInt32BE(offset); + const dataStart = offset + 8; + const dataEnd = dataStart + length; + const chunkEnd = dataEnd + 4; + if (chunkEnd > value.length) { + throw new Error(`${options.label} has an invalid PNG chunk length`); + } + const type = value.toString("ascii", offset + 4, offset + 8); + if (!options.chunks.has(type)) { + throw new Error(`${options.label} contains forbidden PNG chunk ${type}`); + } + const expectedCrc = value.readUInt32BE(dataEnd); + const actualCrc = crc32(value.subarray(offset + 4, dataEnd)); + if (actualCrc !== expectedCrc) { + throw new Error(`${options.label} contains a corrupt PNG chunk`); + } + if (type === "IHDR") { + if (sawHeader || offset !== PNG_SIGNATURE.length || length !== 13) { + throw new Error(`${options.label} has an invalid PNG header`); + } + sawHeader = true; + width = value.readUInt32BE(dataStart); + height = value.readUInt32BE(dataStart + 4); + const bitDepth = value[dataStart + 8]; + const colorType = value[dataStart + 9]; + const compression = value[dataStart + 10]; + const filter = value[dataStart + 11]; + const interlace = value[dataStart + 12]; + if ( + width < 1 || + height < 1 || + bitDepth !== 8 || + ![2, 3, 6].includes(colorType ?? -1) || + compression !== 0 || + filter !== 0 || + interlace !== 0 + ) { + throw new Error(`${options.label} uses an unsupported PNG encoding`); + } + if ( + options.maxEdge && + (width > options.maxEdge || height > options.maxEdge) + ) { + throw new Error(`${options.label} exceeds the PNG dimensions`); + } + if (options.maxPixels && width * height > options.maxPixels) { + throw new Error(`${options.label} exceeds the PNG pixel limit`); + } + } else if (!sawHeader) { + throw new Error(`${options.label} has data before its PNG header`); + } else if (type === "IDAT") { + sawData = true; + } else if (type === "IEND") { + if (!sawData || length !== 0 || chunkEnd !== value.length) { + throw new Error(`${options.label} has an invalid PNG end marker`); + } + sawEnd = true; + } + offset = chunkEnd; + } + if (!sawHeader || !sawData || !sawEnd) { + throw new Error(`${options.label} is an incomplete PNG`); + } +} + +async function validatePrivateScreenshot(file: string) { + const metadata = await lstat(file); + if (!metadata.isFile() || metadata.isSymbolicLink()) { + throw new Error("Public preview source must be a regular file"); + } + inspectPng(await readFile(file), { + label: "Public preview source", + maxBytes: MAX_PRIVATE_PNG_BYTES, + maxEdge: MAX_PRIVATE_PNG_EDGE, + maxPixels: MAX_PRIVATE_PNG_PIXELS, + chunks: PRIVATE_PNG_CHUNKS, + }); +} + +async function validatePublicPreview(file: string) { + const metadata = await lstat(file); + if (!metadata.isFile() || metadata.isSymbolicLink()) { + throw new Error("Public preview output must be a regular file"); + } + inspectPng(await readFile(file), { + label: "Public preview output", + maxBytes: MAX_PUBLIC_PREVIEW_BYTES, + maxEdge: MAX_PUBLIC_PREVIEW_EDGE, + chunks: PUBLIC_PNG_CHUNKS, + }); +} + +export async function createPublicLayoutPreview( + source: string, + destination: string, +) { + await validatePrivateScreenshot(source); + await mkdir(path.dirname(destination), { recursive: true }); + const temporary = await mkdtemp( + path.join(path.dirname(destination), ".preview-"), + ); + const output = path.join(temporary, "preview.png"); + try { + await execFileAsync( + process.env.RUNNER_E2E_IMAGE_MAGICK_BINARY ?? "convert", + [ + "-limit", + "memory", + "128MiB", + "-limit", + "map", + "256MiB", + "-limit", + "disk", + "256MiB", + "-limit", + "thread", + "1", + "-limit", + "time", + "30", + source, + "-background", + "#f3f4f6", + "-alpha", + "remove", + "-alpha", + "off", + "-resize", + `${MAX_PUBLIC_PREVIEW_EDGE}x${MAX_PUBLIC_PREVIEW_EDGE}>`, + "-blur", + "0x2.5", + "-colors", + "24", + "-strip", + "-define", + "png:exclude-chunks=all", + `PNG8:${output}`, + ], + { timeout: 45_000, maxBuffer: 1024 * 1024 }, + ); + await validatePublicPreview(output); + await copyFile(output, destination, fsConstants.COPYFILE_EXCL); + } finally { + await rm(temporary, { recursive: true, force: true }); + } +} + +async function assertTreeContainsNoLinks(root: string, current = root) { + const entries = await readdir(current, { withFileTypes: true }); + for (const entry of entries) { + const absolute = path.join(current, entry.name); + const metadata = await lstat(absolute); + if (metadata.isSymbolicLink()) { + throw new Error("Public history source must not contain symbolic links"); + } + if (metadata.isDirectory()) await assertTreeContainsNoLinks(root, absolute); + } +} + +async function removeExistingPublicVisuals(current: string) { + const entries = await readdir(current, { withFileTypes: true }).catch( + () => [], + ); + for (const entry of entries) { + const absolute = path.join(current, entry.name); + if (!entry.isDirectory()) continue; + if (entry.name === PUBLIC_VISUAL_DIRECTORY) { + await rm(absolute, { recursive: true, force: true }); + continue; + } + await removeExistingPublicVisuals(absolute); + } +} + +function safePassedScreenshot(input: unknown) { + if (!input || typeof input !== "object") { + throw new Error("Passing result has invalid screenshot metadata"); + } + const screenshot = input as Record; + if ( + typeof screenshot.id !== "string" || + typeof screenshot.label !== "string" || + typeof screenshot.file !== "string" || + !PUBLIC_SCREENSHOT_NAME.test(screenshot.file) + ) { + throw new Error("Passing result has unsafe screenshot metadata"); + } + return { + ...screenshot, + label: `${screenshot.label} (redacted layout preview)`, + file: `${PUBLIC_VISUAL_DIRECTORY}/${screenshot.file}`, + privateFile: screenshot.file, + }; +} + +export async function preparePublicHistoryBundle(input: { + source: string; + destination: string; + transform?: PublicPreviewTransformer; +}) { + const source = path.resolve(input.source); + const destination = path.resolve(input.destination); + if ( + source === destination || + source.startsWith(`${destination}${path.sep}`) || + destination.startsWith(`${source}${path.sep}`) + ) { + throw new Error("Public history source and destination must not overlap"); + } + const sourceMetadata = await lstat(source); + if (!sourceMetadata.isDirectory() || sourceMetadata.isSymbolicLink()) { + throw new Error("Public history source must be a directory"); + } + await assertTreeContainsNoLinks(source); + await rm(destination, { recursive: true, force: true }); + await cp(source, destination, { recursive: true, force: false }); + + const normalizedFile = path.join(destination, "normalized-results.json"); + const campaign = JSON.parse( + await readFile(normalizedFile, "utf8"), + ) as PublishedCampaign; + if (!Array.isArray(campaign.results)) { + throw new Error("Public history source has invalid campaign results"); + } + await removeExistingPublicVisuals(path.join(destination, "evidence")); + + const transform = input.transform ?? createPublicLayoutPreview; + let previewCount = 0; + const results: PublishedResult[] = []; + for (const untrustedResult of campaign.results) { + if (!untrustedResult || typeof untrustedResult !== "object") { + throw new Error("Public history source has an invalid result"); + } + const result = untrustedResult as PublishedResult; + if (result.status !== "passed" || result.evidenceValid === false) { + results.push(result); + continue; + } + if ( + typeof result.executionId !== "string" || + !/^[A-Za-z0-9][A-Za-z0-9._-]{0,299}$/.test(result.executionId) || + !Number.isSafeInteger(result.attempt) || + Number(result.attempt) < 1 || + !Array.isArray(result.screenshots) || + result.screenshots.length === 0 + ) { + throw new Error("Passing result lacks safe screenshot provenance"); + } + const base = path.join( + destination, + "evidence", + result.executionId, + `attempt-${String(result.attempt)}`, + ); + const publicVisuals = path.join(base, PUBLIC_VISUAL_DIRECTORY); + await rm(publicVisuals, { recursive: true, force: true }); + await mkdir(publicVisuals, { recursive: true }); + const screenshots = []; + const seen = new Set(); + for (const entry of result.screenshots) { + const screenshot = safePassedScreenshot(entry); + if (seen.has(screenshot.privateFile)) { + throw new Error("Passing result has duplicate screenshot metadata"); + } + seen.add(screenshot.privateFile); + const sourceScreenshot = path.join(base, screenshot.privateFile); + const publicScreenshot = path.join(publicVisuals, screenshot.privateFile); + await validatePrivateScreenshot(sourceScreenshot); + await transform(sourceScreenshot, publicScreenshot); + await validatePublicPreview(publicScreenshot); + const { privateFile: _, ...publishedScreenshot } = screenshot; + screenshots.push(publishedScreenshot); + previewCount += 1; + } + results.push({ ...result, screenshots }); + } + await writeFile( + normalizedFile, + `${JSON.stringify({ ...campaign, results }, null, 2)}\n`, + "utf8", + ); + await prunePrivateHistoryEvidence(destination); + await regenerateRunnerDashboard({ bundle: destination, historyFile: null }); + console.log( + `Prepared ${previewCount} public low-resolution layout preview(s); full-resolution visual evidence remains private`, + ); + return { previewCount }; +} + +async function main() { + const [source, destination] = process.argv.slice(2); + if (!source || !destination) { + throw new Error( + "Usage: history-public-bundle.ts ", + ); + } + await preparePublicHistoryBundle({ source, destination }); +} + +if ( + process.argv[1] && + path.resolve(process.argv[1]) === path.resolve(import.meta.filename) +) { + await main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/tests/runner-e2e/history-publish.ts b/tests/runner-e2e/history-publish.ts index 2e404ab136..8f22b2726d 100644 --- a/tests/runner-e2e/history-publish.ts +++ b/tests/runner-e2e/history-publish.ts @@ -2,15 +2,7 @@ import { createHash } from "node:crypto"; import { execFile } from "node:child_process"; import path from "node:path"; import { promisify } from "node:util"; -import { - mkdtemp, - lstat, - readFile, - readdir, - rm, - stat, - writeFile, -} from "node:fs/promises"; +import { mkdtemp, readFile, readdir, stat, writeFile } from "node:fs/promises"; import os from "node:os"; import { regenerateRunnerDashboard } from "./dashboard-regenerate.js"; import { renderRunnerHistoryIndex } from "./history-index.js"; @@ -20,6 +12,12 @@ import { mergeRunnerHistory, } from "./history.js"; import type { RunnerE2ECampaign, RunnerE2EHistoryIndex } from "./types.js"; +import { + isPublicHistoryEvidencePath, + prunePrivateHistoryEvidence, +} from "./history-public-bundle.js"; + +export { prunePrivateHistoryEvidence } from "./history-public-bundle.js"; const execFileAsync = promisify(execFile); const MUTABLE_HISTORY_FILES = new Set([ @@ -34,36 +32,6 @@ const PUBLISH_ROOT_FILES = new Set([ "normalized-results.json", "summary.md", ]); -const PUBLIC_EVIDENCE_EXTENSIONS = new Set([".json", ".log", ".md", ".txt"]); -const PRIVATE_EVIDENCE_DIRECTORIES = new Set([ - "blob-report", - "html-report", - "playwright-output", -]); - -function publicEvidencePath(relative: string) { - const match = relative.match( - /^evidence\/[A-Za-z0-9._-]+\/attempt-[1-9][0-9]*\/(.+)$/, - ); - if (!match) return false; - const evidencePath = match[1]!; - const segments = evidencePath.split("/"); - if ( - segments.some( - (segment) => - !segment || - segment === "." || - segment === ".." || - PRIVATE_EVIDENCE_DIRECTORIES.has(segment), - ) - ) { - return false; - } - return PUBLIC_EVIDENCE_EXTENSIONS.has( - path.posix.extname(evidencePath).toLowerCase(), - ); -} - export function isHistoricalBundlePathAllowed(relative: string) { if ( relative.includes("\\") || @@ -79,35 +47,7 @@ export function isHistoricalBundlePathAllowed(relative: string) { ) { return true; } - return publicEvidencePath(relative); -} - -async function pruneEvidenceDirectory(root: string, current: string) { - const entries = await readdir(current, { withFileTypes: true }); - for (const entry of entries) { - const absolute = path.join(current, entry.name); - if (entry.isDirectory()) { - await pruneEvidenceDirectory(root, absolute); - if ((await readdir(absolute)).length === 0) { - await rm(absolute, { recursive: true }); - } - continue; - } - const relative = path.relative(root, absolute).split(path.sep).join("/"); - if (!entry.isFile() || !publicEvidencePath(relative)) { - await rm(absolute, { force: true }); - } - } -} - -export async function prunePrivateHistoryEvidence(root: string) { - const evidenceRoot = path.join(root, "evidence"); - const metadata = await lstat(evidenceRoot).catch(() => null); - if (!metadata) return; - if (!metadata.isDirectory()) { - throw new Error("Historical evidence root must be a directory"); - } - await pruneEvidenceDirectory(root, evidenceRoot); + return isPublicHistoryEvidencePath(relative); } interface BundleManifest { @@ -362,10 +302,10 @@ async function main() { // Campaign bundles are immutable and must not capture a mutable history // file left in a reused local directory. The root landing page below is the // only dashboard that embeds navigation across campaigns. - // Raster/video pixels are not OCR-scanned for secrets, and generated HTML, - // archives, and SVG may contain or execute active/private content. Preserve - // those in the access-controlled workflow artifact but remove them from the - // directory shared by public S3 and Pages publication. + // The trusted report job replaces successful screenshots with low-resolution + // blurred previews before this AWS-credentialed process sees the bundle. + // Remove every remaining private raster/video, generated HTML, archive, SVG, + // and other non-allowlisted path before S3 or Pages publication. await prunePrivateHistoryEvidence(reportRoot); await regenerateRunnerDashboard({ bundle: reportRoot, historyFile: null }); const manifest = await createBundleManifest(reportRoot, campaign.campaignId); diff --git a/tests/runner-e2e/history.test.ts b/tests/runner-e2e/history.test.ts index 556d09d83a..eabce7d269 100644 --- a/tests/runner-e2e/history.test.ts +++ b/tests/runner-e2e/history.test.ts @@ -1,4 +1,11 @@ -import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { + copyFile, + mkdtemp, + mkdir, + readFile, + rm, + writeFile, +} from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; @@ -9,7 +16,6 @@ import { buildHistoryPointers, createBundleManifest, isHistoricalBundlePathAllowed, - prunePrivateHistoryEvidence, validateHistoryDestination, } from "./history-publish.js"; import { @@ -20,9 +26,14 @@ import { mergeRunnerHistory, } from "./history.js"; import { renderRunnerHistoryIndex } from "./history-index.js"; +import { preparePublicHistoryBundle } from "./history-public-bundle.js"; import type { MatrixExecution, RunnerE2EResult } from "./types.js"; const temporaryDirectories: string[] = []; +const safePng = Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAQAAAADCAIAAAA7ljmRAAAAFElEQVQI12M0TpvJAANMDEgAhQMALHABOEZNdkwAAAAASUVORK5CYII=", + "base64", +); afterEach(async () => { vi.unstubAllEnvs(); await Promise.all( @@ -185,7 +196,7 @@ describe("runner E2E campaign history", () => { expect(index).toContain("65/66 passed"); expect(index).toContain("Open report →"); expect(index).toContain( - "Visual evidence remains in access-controlled workflow artifacts", + "Full-resolution visual evidence remains in access-controlled workflow artifacts", ); expect(index).toContain("Inert structured public evidence"); expect(index).not.toContain("data-gallery-dialog"); @@ -194,10 +205,12 @@ describe("runner E2E campaign history", () => { }); describe("historical publication security", () => { - it("keeps visual and active evidence private when building the public dashboard", async () => { + it("publishes blurred layout previews while keeping raw visuals private", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "runner-landing-test-")); - const output = path.join(root, "landing"); + const source = path.join(root, "private"); + const output = path.join(root, "public"); temporaryDirectories.push(root); + await mkdir(source); const execution = runnerMatrix[0]!; const campaignResult = { ...result(execution, "passed"), @@ -216,17 +229,23 @@ describe("historical publication security", () => { results: [campaignResult], }); const evidenceDirectory = path.join( - root, + source, + "evidence", + execution.id, + "attempt-1", + ); + const publicEvidenceDirectory = path.join( + output, "evidence", execution.id, "attempt-1", ); await mkdir(evidenceDirectory, { recursive: true }); await writeFile( - path.join(root, "normalized-results.json"), + path.join(source, "normalized-results.json"), JSON.stringify(campaign), ); - await writeFile(path.join(evidenceDirectory, "final-state.png"), "png"); + await writeFile(path.join(evidenceDirectory, "final-state.png"), safePng); await writeFile(path.join(evidenceDirectory, "failure.webm"), "webm"); await writeFile(path.join(evidenceDirectory, "unsafe.svg"), ""); await writeFile( @@ -234,6 +253,15 @@ describe("historical publication security", () => { "", ); await writeFile(path.join(evidenceDirectory, "result.json"), "{}\n"); + await mkdir(path.join(evidenceDirectory, "public-visuals")); + await writeFile( + path.join( + evidenceDirectory, + "public-visuals", + "plan-target-injected.png", + ), + safePng, + ); await mkdir(path.join(evidenceDirectory, "snapshots")); await writeFile( path.join(evidenceDirectory, "snapshots", "api-state.json"), @@ -250,45 +278,54 @@ describe("historical publication security", () => { "private archive", ); - await prunePrivateHistoryEvidence(root); - await regenerateRunnerDashboard({ - bundle: root, - outputDirectory: output, - evidenceHrefPrefix: "campaigns/campaign-1", + await preparePublicHistoryBundle({ + source, + destination: output, + transform: async (privateScreenshot, publicPreview) => { + await copyFile(privateScreenshot, publicPreview); + }, }); const dashboard = await readFile(path.join(output, "index.html"), "utf8"); - expect(dashboard).not.toContain( - `campaigns/campaign-1/evidence/${execution.id}/attempt-1/final-state.png`, - ); - expect(dashboard).toContain("Visual evidence · workflow artifact only"); expect(dashboard).toContain( - "public history contains inert structured evidence only", + `evidence/${execution.id}/attempt-1/public-visuals/final-state.png`, ); - expect(dashboard).toContain("Public history excludes visual evidence"); + await expect( + readFile(path.join(publicEvidenceDirectory, "final-state.png")), + ).rejects.toThrow(); + await expect( + readFile( + path.join(publicEvidenceDirectory, "public-visuals", "final-state.png"), + ), + ).resolves.toEqual(safePng); + await expect( + readFile( + path.join( + publicEvidenceDirectory, + "public-visuals", + "plan-target-injected.png", + ), + ), + ).rejects.toThrow(); await expect( readFile(path.join(evidenceDirectory, "final-state.png")), - ).rejects.toThrow(); + ).resolves.toEqual(safePng); + for (const relative of [ + "failure.webm", + "unsafe.svg", + "junit.xml", + "html-report/index.html", + "blob-report/report.zip", + ]) { + await expect( + readFile(path.join(publicEvidenceDirectory, ...relative.split("/"))), + ).rejects.toThrow(); + } await expect( - readFile(path.join(evidenceDirectory, "failure.webm")), - ).rejects.toThrow(); - await expect( - readFile(path.join(evidenceDirectory, "unsafe.svg")), - ).rejects.toThrow(); - await expect( - readFile(path.join(evidenceDirectory, "junit.xml")), - ).rejects.toThrow(); - await expect( - readFile(path.join(evidenceDirectory, "html-report", "index.html")), - ).rejects.toThrow(); - await expect( - readFile(path.join(evidenceDirectory, "blob-report", "report.zip")), - ).rejects.toThrow(); - await expect( - readFile(path.join(evidenceDirectory, "result.json"), "utf8"), + readFile(path.join(publicEvidenceDirectory, "result.json"), "utf8"), ).resolves.toBe("{}\n"); await expect( readFile( - path.join(evidenceDirectory, "snapshots", "api-state.json"), + path.join(publicEvidenceDirectory, "snapshots", "api-state.json"), "utf8", ), ).resolves.toBe("{}\n"); @@ -299,8 +336,8 @@ describe("historical publication security", () => { ).toBe("paperclip.runner-e2e.campaign/v2"); await expect( regenerateRunnerDashboard({ - bundle: root, - outputDirectory: output, + bundle: source, + outputDirectory: path.join(root, "invalid"), evidenceHrefPrefix: "../unsafe", }), ).rejects.toThrow("safe relative URL path"); @@ -341,6 +378,16 @@ describe("historical publication security", () => { "evidence/core-compatibility.profile.local.case/attempt-1/final-state.png", ), ).toBe(false); + expect( + isHistoricalBundlePathAllowed( + "evidence/core-compatibility.profile.local.case/attempt-1/public-visuals/final-state.png", + ), + ).toBe(true); + expect( + isHistoricalBundlePathAllowed( + "evidence/core-compatibility.profile.local.case/attempt-1/public-visuals/failure.png", + ), + ).toBe(false); expect( isHistoricalBundlePathAllowed( "evidence/core-compatibility.profile.local.case/attempt-1/failure.webm", diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts index 93623e2d24..3e39c0639f 100644 --- a/tests/runner-e2e/workflow-security.test.ts +++ b/tests/runner-e2e/workflow-security.test.ts @@ -642,6 +642,12 @@ describe("public repository paid workflow security", () => { expect( report.indexOf("Select latest workflow attempt per cell"), ).toBeLessThan(report.indexOf("Collect blob reports")); + expect(report).toContain( + "runner-e2e-public-history-source-${{ github.run_id }}-${{ github.run_attempt }}", + ); + expect(publisher).toContain( + "runner-e2e-public-history-source-${{ github.run_id }}-${{ github.run_attempt }}", + ); expect(publisher).toContain( 'echo "name=github-pages-${{ github.run_id }}-${{ github.run_attempt }}"', ); @@ -661,6 +667,10 @@ describe("public repository paid workflow security", () => { path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"), "utf8", ); + const report = workflow.slice( + workflow.indexOf(" report:"), + workflow.indexOf(" publish_history:"), + ); const publisher = workflow.slice(workflow.indexOf(" publish_history:")); expect(publisher).toContain("id-token: write"); expect(publisher).toContain("name: runner-e2e-history"); @@ -671,9 +681,21 @@ describe("public repository paid workflow security", () => { expect(publisher).not.toMatch(/aws s3 (?:rm|sync .*--delete)/); expect(workflow).toContain("history_source_ready"); expect(workflow).toContain( - "Verify history source report and private screenshot evidence", + "Prepare public history bundle with redacted layout previews", ); - expect(workflow).toContain("private_screenshot="); + expect(workflow).toContain( + "Verify prepared history source and public layout previews", + ); + expect(workflow).toContain("public_preview="); + expect(workflow).toContain("unexpected_png="); + expect(workflow).toContain("passed_count="); + expect(workflow).toContain("pnpm test:e2e:runner:history:prepare"); + expect(report).not.toContain("id-token: write"); + expect(report).not.toMatch( + /(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA)_API_KEY/, + ); + expect(publisher).not.toContain("Qualify public preview raster sanitizer"); + expect(publisher).not.toContain("runner-e2e-report-${{ github.run_id }}"); expect(workflow).toContain("Publish pruned immutable history"); expect(workflow).toContain("Publish latest structured dashboard"); expect(workflow).not.toContain("dashboard_ready");