feat(e2e): publish safe runner layout previews
This commit is contained in:
Dotta
parent
bf95a7eae2
commit
e7a073778e
|
|
@ -1104,14 +1104,44 @@ jobs:
|
|||
retention-days: 30
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Verify history source report and private screenshot evidence
|
||||
- name: Qualify public preview raster sanitizer
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if ! command -v convert >/dev/null 2>&1; then
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install --no-install-recommends -y imagemagick
|
||||
fi
|
||||
convert -version
|
||||
|
||||
- name: Prepare public history bundle with redacted layout previews
|
||||
id: prepare_public_history
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
pnpm test:e2e:runner:history:prepare -- \
|
||||
"$GITHUB_WORKSPACE/runner-e2e-merged-report/normalized" \
|
||||
"$GITHUB_WORKSPACE/runner-e2e-public-report/normalized"
|
||||
|
||||
- name: Upload prepared public history source
|
||||
if: always() && steps.prepare_public_history.outcome == 'success'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: runner-e2e-public-history-source-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: runner-e2e-public-report/
|
||||
retention-days: 1
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Verify prepared history source and public layout previews
|
||||
id: history_source_ready
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
dashboard_root="runner-e2e-merged-report/normalized"
|
||||
private_screenshot="$(find "$dashboard_root" -type f -name '*.png' -print -quit 2>/dev/null || true)"
|
||||
if [ -f "$dashboard_root/index.html" ] && [ -n "$private_screenshot" ]; then
|
||||
dashboard_root="runner-e2e-public-report/normalized"
|
||||
public_preview="$(find "$dashboard_root/evidence" -type f -path '*/public-visuals/*.png' -print -quit 2>/dev/null || true)"
|
||||
unexpected_png="$(find "$dashboard_root/evidence" -type f -name '*.png' ! -path '*/public-visuals/*.png' -print -quit 2>/dev/null || true)"
|
||||
passed_count="$(jq '[.results[] | select(.status == "passed")] | length' "$dashboard_root/normalized-results.json" 2>/dev/null || echo invalid)"
|
||||
if [ "${{ steps.prepare_public_history.outcome }}" = "success" ] && [ -f "$dashboard_root/index.html" ] && [ -z "$unexpected_png" ] && { [ "$passed_count" = "0" ] || [ -n "$public_preview" ]; }; then
|
||||
echo "ready=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "ready=false" >> "$GITHUB_OUTPUT"
|
||||
|
|
@ -1154,10 +1184,10 @@ jobs:
|
|||
|
||||
- run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Download access-controlled normalized campaign
|
||||
- name: Download prepared public campaign
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
||||
with:
|
||||
name: runner-e2e-report-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
name: runner-e2e-public-history-source-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: runner-e2e-merged-report
|
||||
|
||||
- name: Exchange GitHub OIDC identity for scoped AWS credentials
|
||||
|
|
|
|||
|
|
@ -68,6 +68,7 @@
|
|||
"test:e2e:runner:dashboard": "node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/dashboard-regenerate.ts",
|
||||
"test:e2e:runner:models:update": "node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/openrouter-models-update.ts",
|
||||
"test:e2e:runner:history:publish": "node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/history-publish.ts",
|
||||
"test:e2e:runner:history:prepare": "node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/history-public-bundle.ts",
|
||||
"test:e2e:runner:unit": "vitest run --config tests/runner-e2e/vitest.config.ts",
|
||||
"test:e2e:runner:typecheck": "tsc -p tests/runner-e2e/tsconfig.json",
|
||||
"test:e2e:runner:report": "node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/report.ts",
|
||||
|
|
|
|||
|
|
@ -204,16 +204,18 @@ usage is labeled `unavailable` or `unpriced`; it is never presented as zero
|
|||
cost. The CI report job stages the same portable site at
|
||||
`normalized/index.html` inside the access-controlled merged report artifact.
|
||||
|
||||
Permanent public history has a narrower boundary. Before uploading to S3 or
|
||||
packaging the optional GitHub Pages artifact, the publisher removes raster and
|
||||
video evidence, archives, and the generated Playwright/blob/HTML report trees.
|
||||
It then regenerates the dashboard against only the remaining allowlisted,
|
||||
inert structured per-attempt evidence (`.json`, `.log`, `.md`, and `.txt`).
|
||||
Per-attempt XML is excluded because browsers can process XML/XSLT. The root
|
||||
`junit.xml` remains public because the report aggregator builds it from fixed
|
||||
markup and XML-escaped fields. Public dashboards therefore contain results and
|
||||
accounting but no attempt screenshots, videos, traces, or generated Playwright
|
||||
reports.
|
||||
Permanent public history has a narrower boundary. A trusted job without
|
||||
provider or AWS credentials copies successful declared PNG screenshots into a
|
||||
separate publication tree. It validates the PNG container, reduces each image
|
||||
to at most 160 pixels on either edge, applies a strong blur, limits the image to
|
||||
24 colors, and removes metadata. These layout previews preserve coarse UI
|
||||
state while making rendered task and provider text unreadable. The job then
|
||||
removes the full-resolution raster files, failure screenshots, video, archives,
|
||||
SVG, and generated Playwright/blob/HTML report trees. Only the derived
|
||||
`public-visuals/*.png` previews and allowlisted inert evidence (`.json`, `.log`,
|
||||
`.md`, and `.txt`) remain. Per-attempt XML is excluded because browsers can
|
||||
process XML/XSLT. The root `junit.xml` remains public because the report
|
||||
aggregator builds it from fixed markup and XML-escaped fields.
|
||||
|
||||
### Billing interpretation
|
||||
|
||||
|
|
@ -380,11 +382,12 @@ bundle digest fails closed.
|
|||
|
||||
GitHub Pages remains the stable latest dashboard. Enable Pages with GitHub
|
||||
Actions as its source and set `RUNNER_FULL_STACK_E2E_PUBLISH_PAGES=true`.
|
||||
The publisher prunes screenshots, video, archives, and generated report trees,
|
||||
then regenerates the public dashboard before either the CloudFront-backed S3
|
||||
history or optional Pages artifact is created. Public per-attempt evidence is
|
||||
limited to allowlisted inert structured text. Databases, Paperclip homes,
|
||||
workspaces, raw/unredacted logs, credentials, and visual evidence are never
|
||||
The trusted report job creates a separate public bundle before the AWS role is
|
||||
available. It publishes only blurred, low-resolution, metadata-free layout
|
||||
previews for successful declared screenshots and allowlisted inert structured
|
||||
text. The S3/CloudFront history and optional Pages mirror use this same bundle.
|
||||
Full-resolution and failure screenshots, video, archives, generated reports,
|
||||
databases, Paperclip homes, workspaces, raw logs, and credentials are never
|
||||
published. Sanitized allowlisted `.log` copies may be public only after
|
||||
exact-value/key-shape scanning and redaction.
|
||||
|
||||
|
|
|
|||
|
|
@ -179,8 +179,9 @@ latest pointers are mutable, and S3 versioning makes those updates recoverable.
|
|||
## Public evidence boundary
|
||||
|
||||
CloudFront and GitHub Pages are public. Fixture identifiers, timing, token
|
||||
usage, costs, normalized results, and allowlisted inert structured per-attempt
|
||||
evidence are expected public data. Screenshots, video, archives, generated
|
||||
usage, costs, normalized results, allowlisted inert structured per-attempt
|
||||
evidence, and deliberately degraded layout previews are expected public data.
|
||||
Full-resolution and failure screenshots, video, archives, generated
|
||||
Playwright/blob/HTML report trees, credentials, Paperclip homes, databases,
|
||||
workspaces, master keys, raw/unredacted logs, and unallowlisted files are not.
|
||||
Only allowlisted `.log` copies that passed exact-value/key-shape scanning and
|
||||
|
|
@ -189,20 +190,29 @@ redaction may cross the public boundary.
|
|||
The packaged evidence uploaded as a 30-day GitHub Actions artifact has a
|
||||
different, access-controlled boundary. Text is exact-value and key-shape
|
||||
scanned and redacted. PNG and WebM are raw-byte scanned but cannot be inspected
|
||||
for credentials rendered as pixels, so they remain only in local evidence and
|
||||
the access-controlled artifact. SVG is rejected during packaging because it is
|
||||
active content.
|
||||
for credentials rendered as pixels, so full-resolution files remain only in
|
||||
local evidence and the access-controlled artifact. SVG is rejected during
|
||||
packaging because it is active content.
|
||||
|
||||
Before permanent publication, the campaign publisher prunes raster/video
|
||||
files, archives, and generated report trees. It then regenerates the dashboard
|
||||
from the remaining allowlisted `.json`, `.log`, `.md`, and `.txt` evidence and
|
||||
accepts only that dashboard, normalized JSON/JUnit/summary, fixed
|
||||
branding assets, and the inert structured evidence paths. Per-attempt XML is
|
||||
excluded because browsers can process XML/XSLT; the only public XML is the
|
||||
root `junit.xml`, which the report aggregator constructs from fixed markup and
|
||||
XML-escaped fields. The same pruned tree feeds both S3/CloudFront history and
|
||||
the optional GitHub Pages artifact. A leak fails the cell and withholds the
|
||||
unsafe file.
|
||||
Before permanent publication, the trusted report job creates a separate tree.
|
||||
It accepts only declared screenshots from passing results. It validates a
|
||||
bounded, non-interlaced PNG container before invoking ImageMagick with strict
|
||||
memory, disk, thread, and time limits. It reduces each image to at most 160
|
||||
pixels on either edge, applies a strong blur, limits the palette, removes alpha
|
||||
and metadata, and validates the new PNG again. This is a layout preview, not
|
||||
diagnostic evidence. It then prunes full-resolution raster files, all failure
|
||||
images, video, archives, active SVG, and generated reports. This transformation
|
||||
runs before AWS credentials are available. The AWS job downloads only the
|
||||
prepared public tree.
|
||||
|
||||
The remaining allowlist contains `public-visuals/*.png`, `.json`, `.log`, `.md`,
|
||||
and `.txt` evidence, plus the generated dashboard, normalized
|
||||
JSON/JUnit/summary, and fixed branding assets. Per-attempt XML is excluded
|
||||
because browsers can process XML/XSLT. The only public XML is the root
|
||||
`junit.xml`, which the report aggregator constructs from fixed markup and
|
||||
XML-escaped fields. The same prepared tree feeds S3/CloudFront history and the
|
||||
optional GitHub Pages artifact. Any malformed image, transformation error,
|
||||
unexpected file, or scan failure withholds publication.
|
||||
|
||||
Rotate the affected credential immediately if a secret-scanning failure or
|
||||
unexpected public object is observed. Preserve the access-controlled Actions
|
||||
|
|
|
|||
|
|
@ -889,7 +889,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {
|
|||
<div>
|
||||
<p class="eyebrow">Full-stack acceptance campaign</p>
|
||||
<h1>${html(input.title)}</h1>
|
||||
<p class="lede">A browser-verified matrix of runner profiles, execution environments, and deterministic task contracts. Visual evidence is retained in the access-controlled workflow artifact; public history contains inert structured evidence only.</p>
|
||||
<p class="lede">A browser-verified matrix of runner profiles, execution environments, and deterministic task contracts. Full-resolution visual evidence stays in the access-controlled workflow artifact. Public history includes blurred low-resolution layout previews and inert structured evidence.</p>
|
||||
</div>
|
||||
<div class="report-actions">
|
||||
<div class="summary" aria-label="Campaign summary">
|
||||
|
|
@ -897,7 +897,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {
|
|||
<div class="metric"><strong>${failed}</strong><span>Failed</span></div>
|
||||
<div class="metric"><strong>${html(durationLabel(totalDuration))}</strong><span>Test time</span></div>
|
||||
</div>
|
||||
<button class="gallery-launch" type="button" data-gallery-open ${screenshotCount === 0 ? "disabled" : ""}>${screenshotCount === 0 ? "Visual evidence · workflow artifact only" : `View gallery · ${screenshotCount}`}</button>
|
||||
<button class="gallery-launch" type="button" data-gallery-open ${screenshotCount === 0 ? "disabled" : ""}>${screenshotCount === 0 ? "Full-resolution visuals · workflow artifact" : `View gallery · ${screenshotCount}`}</button>
|
||||
</div>
|
||||
</header>
|
||||
<section class="billing-overview" aria-label="Campaign billing summary">
|
||||
|
|
@ -918,7 +918,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {
|
|||
</nav>
|
||||
${suiteSections}
|
||||
${historySection}
|
||||
<footer><span>Generated ${html(input.generatedAt)}</span><span>${input.catalog.length} catalog executions · Public history excludes visual evidence</span></footer>
|
||||
<footer><span>Generated ${html(input.generatedAt)}</span><span>${input.catalog.length} catalog executions · Public previews are blurred and low resolution</span></footer>
|
||||
</main>
|
||||
<dialog class="gallery-dialog" data-gallery-dialog aria-labelledby="gallery-title">
|
||||
<div class="gallery-shell">
|
||||
|
|
|
|||
|
|
@ -196,7 +196,7 @@ export function renderRunnerHistoryIndex(history: RunnerE2EHistoryIndex) {
|
|||
<div>
|
||||
<p class="eyebrow">Historical test reporting</p>
|
||||
<h1>Runner E2E campaigns</h1>
|
||||
<p class="lede">Each row is one workflow campaign against a Paperclip revision. Open a report for its configuration matrices, matchers, per-test billing, and sanitized structured evidence. Visual evidence remains in access-controlled workflow artifacts.</p>
|
||||
<p class="lede">Each row is one workflow campaign against a Paperclip revision. Open a report for its configuration matrices, matchers, per-test billing, sanitized structured evidence, and blurred low-resolution layout previews. Full-resolution visual evidence remains in access-controlled workflow artifacts.</p>
|
||||
</div>
|
||||
<div class="summary" aria-label="History summary">
|
||||
<div class="metric"><strong>${campaigns.length}</strong><span>Campaigns</span></div>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,482 @@
|
|||
import { execFile } from "node:child_process";
|
||||
import { constants as fsConstants } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import {
|
||||
cp,
|
||||
copyFile,
|
||||
lstat,
|
||||
mkdir,
|
||||
mkdtemp,
|
||||
readFile,
|
||||
readdir,
|
||||
rm,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { regenerateRunnerDashboard } from "./dashboard-regenerate.js";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const PUBLIC_EVIDENCE_EXTENSIONS = new Set([".json", ".log", ".md", ".txt"]);
|
||||
const PRIVATE_EVIDENCE_DIRECTORIES = new Set([
|
||||
"blob-report",
|
||||
"html-report",
|
||||
"playwright-output",
|
||||
]);
|
||||
const PUBLIC_VISUAL_DIRECTORY = "public-visuals";
|
||||
const PUBLIC_SCREENSHOT_NAME =
|
||||
/^(?:final-state|plan-[a-z0-9-]+|question-[a-z0-9-]+)\.png$/;
|
||||
const PNG_SIGNATURE = Buffer.from([
|
||||
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
|
||||
]);
|
||||
const MAX_PRIVATE_PNG_BYTES = 32 * 1024 * 1024;
|
||||
const MAX_PRIVATE_PNG_PIXELS = 64 * 1024 * 1024;
|
||||
const MAX_PRIVATE_PNG_EDGE = 32 * 1024;
|
||||
const MAX_PUBLIC_PREVIEW_EDGE = 160;
|
||||
const MAX_PUBLIC_PREVIEW_BYTES = 256 * 1024;
|
||||
const PRIVATE_PNG_CHUNKS = new Set([
|
||||
"IHDR",
|
||||
"IDAT",
|
||||
"IEND",
|
||||
"PLTE",
|
||||
"tRNS",
|
||||
"gAMA",
|
||||
"cHRM",
|
||||
"sRGB",
|
||||
"pHYs",
|
||||
]);
|
||||
const PUBLIC_PNG_CHUNKS = new Set(["IHDR", "IDAT", "IEND", "PLTE"]);
|
||||
|
||||
interface PublishedResult {
|
||||
executionId?: unknown;
|
||||
attempt?: unknown;
|
||||
status?: unknown;
|
||||
evidenceValid?: unknown;
|
||||
screenshots?: unknown;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
interface PublishedCampaign {
|
||||
results?: unknown;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export type PublicPreviewTransformer = (
|
||||
source: string,
|
||||
destination: string,
|
||||
) => Promise<void>;
|
||||
|
||||
function publicVisualPath(evidencePath: string) {
|
||||
const segments = evidencePath.split("/");
|
||||
return (
|
||||
segments.length === 2 &&
|
||||
segments[0] === PUBLIC_VISUAL_DIRECTORY &&
|
||||
PUBLIC_SCREENSHOT_NAME.test(segments[1] ?? "")
|
||||
);
|
||||
}
|
||||
|
||||
export function isPublicHistoryEvidencePath(relative: string) {
|
||||
const match = relative.match(
|
||||
/^evidence\/[A-Za-z0-9._-]+\/attempt-[1-9][0-9]*\/(.+)$/,
|
||||
);
|
||||
if (!match) return false;
|
||||
const evidencePath = match[1]!;
|
||||
const segments = evidencePath.split("/");
|
||||
if (
|
||||
segments.some(
|
||||
(segment) =>
|
||||
!segment ||
|
||||
segment === "." ||
|
||||
segment === ".." ||
|
||||
PRIVATE_EVIDENCE_DIRECTORIES.has(segment),
|
||||
)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
if (publicVisualPath(evidencePath)) return true;
|
||||
return PUBLIC_EVIDENCE_EXTENSIONS.has(
|
||||
path.posix.extname(evidencePath).toLowerCase(),
|
||||
);
|
||||
}
|
||||
|
||||
async function pruneEvidenceDirectory(root: string, current: string) {
|
||||
const entries = await readdir(current, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const absolute = path.join(current, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
await pruneEvidenceDirectory(root, absolute);
|
||||
if ((await readdir(absolute)).length === 0) {
|
||||
await rm(absolute, { recursive: true });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const relative = path.relative(root, absolute).split(path.sep).join("/");
|
||||
if (!entry.isFile() || !isPublicHistoryEvidencePath(relative)) {
|
||||
await rm(absolute, { force: true });
|
||||
} else if (path.posix.extname(relative).toLowerCase() === ".png") {
|
||||
// Only derived public previews can pass the PNG path allowlist. Validate
|
||||
// the bounded inert container again when the AWS publisher prunes its
|
||||
// downloaded bundle.
|
||||
await validatePublicPreview(absolute);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function prunePrivateHistoryEvidence(root: string) {
|
||||
const evidenceRoot = path.join(root, "evidence");
|
||||
const metadata = await lstat(evidenceRoot).catch(() => null);
|
||||
if (!metadata) return;
|
||||
if (!metadata.isDirectory()) {
|
||||
throw new Error("Historical evidence root must be a directory");
|
||||
}
|
||||
await pruneEvidenceDirectory(root, evidenceRoot);
|
||||
}
|
||||
|
||||
function crc32(buffer: Buffer) {
|
||||
let crc = 0xffffffff;
|
||||
for (const byte of buffer) {
|
||||
crc ^= byte;
|
||||
for (let bit = 0; bit < 8; bit += 1) {
|
||||
crc = (crc >>> 1) ^ (crc & 1 ? 0xedb88320 : 0);
|
||||
}
|
||||
}
|
||||
return (crc ^ 0xffffffff) >>> 0;
|
||||
}
|
||||
|
||||
function inspectPng(
|
||||
value: Buffer,
|
||||
options: {
|
||||
label: string;
|
||||
maxBytes: number;
|
||||
maxEdge?: number;
|
||||
maxPixels?: number;
|
||||
chunks: ReadonlySet<string>;
|
||||
},
|
||||
) {
|
||||
if (value.length > options.maxBytes) {
|
||||
throw new Error(`${options.label} exceeds the PNG byte limit`);
|
||||
}
|
||||
if (
|
||||
value.length < PNG_SIGNATURE.length ||
|
||||
!value.subarray(0, PNG_SIGNATURE.length).equals(PNG_SIGNATURE)
|
||||
) {
|
||||
throw new Error(`${options.label} is not a PNG`);
|
||||
}
|
||||
let offset = PNG_SIGNATURE.length;
|
||||
let width = 0;
|
||||
let height = 0;
|
||||
let sawHeader = false;
|
||||
let sawData = false;
|
||||
let sawEnd = false;
|
||||
while (offset < value.length) {
|
||||
if (offset + 12 > value.length) {
|
||||
throw new Error(`${options.label} has a truncated PNG chunk`);
|
||||
}
|
||||
const length = value.readUInt32BE(offset);
|
||||
const dataStart = offset + 8;
|
||||
const dataEnd = dataStart + length;
|
||||
const chunkEnd = dataEnd + 4;
|
||||
if (chunkEnd > value.length) {
|
||||
throw new Error(`${options.label} has an invalid PNG chunk length`);
|
||||
}
|
||||
const type = value.toString("ascii", offset + 4, offset + 8);
|
||||
if (!options.chunks.has(type)) {
|
||||
throw new Error(`${options.label} contains forbidden PNG chunk ${type}`);
|
||||
}
|
||||
const expectedCrc = value.readUInt32BE(dataEnd);
|
||||
const actualCrc = crc32(value.subarray(offset + 4, dataEnd));
|
||||
if (actualCrc !== expectedCrc) {
|
||||
throw new Error(`${options.label} contains a corrupt PNG chunk`);
|
||||
}
|
||||
if (type === "IHDR") {
|
||||
if (sawHeader || offset !== PNG_SIGNATURE.length || length !== 13) {
|
||||
throw new Error(`${options.label} has an invalid PNG header`);
|
||||
}
|
||||
sawHeader = true;
|
||||
width = value.readUInt32BE(dataStart);
|
||||
height = value.readUInt32BE(dataStart + 4);
|
||||
const bitDepth = value[dataStart + 8];
|
||||
const colorType = value[dataStart + 9];
|
||||
const compression = value[dataStart + 10];
|
||||
const filter = value[dataStart + 11];
|
||||
const interlace = value[dataStart + 12];
|
||||
if (
|
||||
width < 1 ||
|
||||
height < 1 ||
|
||||
bitDepth !== 8 ||
|
||||
![2, 3, 6].includes(colorType ?? -1) ||
|
||||
compression !== 0 ||
|
||||
filter !== 0 ||
|
||||
interlace !== 0
|
||||
) {
|
||||
throw new Error(`${options.label} uses an unsupported PNG encoding`);
|
||||
}
|
||||
if (
|
||||
options.maxEdge &&
|
||||
(width > options.maxEdge || height > options.maxEdge)
|
||||
) {
|
||||
throw new Error(`${options.label} exceeds the PNG dimensions`);
|
||||
}
|
||||
if (options.maxPixels && width * height > options.maxPixels) {
|
||||
throw new Error(`${options.label} exceeds the PNG pixel limit`);
|
||||
}
|
||||
} else if (!sawHeader) {
|
||||
throw new Error(`${options.label} has data before its PNG header`);
|
||||
} else if (type === "IDAT") {
|
||||
sawData = true;
|
||||
} else if (type === "IEND") {
|
||||
if (!sawData || length !== 0 || chunkEnd !== value.length) {
|
||||
throw new Error(`${options.label} has an invalid PNG end marker`);
|
||||
}
|
||||
sawEnd = true;
|
||||
}
|
||||
offset = chunkEnd;
|
||||
}
|
||||
if (!sawHeader || !sawData || !sawEnd) {
|
||||
throw new Error(`${options.label} is an incomplete PNG`);
|
||||
}
|
||||
}
|
||||
|
||||
async function validatePrivateScreenshot(file: string) {
|
||||
const metadata = await lstat(file);
|
||||
if (!metadata.isFile() || metadata.isSymbolicLink()) {
|
||||
throw new Error("Public preview source must be a regular file");
|
||||
}
|
||||
inspectPng(await readFile(file), {
|
||||
label: "Public preview source",
|
||||
maxBytes: MAX_PRIVATE_PNG_BYTES,
|
||||
maxEdge: MAX_PRIVATE_PNG_EDGE,
|
||||
maxPixels: MAX_PRIVATE_PNG_PIXELS,
|
||||
chunks: PRIVATE_PNG_CHUNKS,
|
||||
});
|
||||
}
|
||||
|
||||
async function validatePublicPreview(file: string) {
|
||||
const metadata = await lstat(file);
|
||||
if (!metadata.isFile() || metadata.isSymbolicLink()) {
|
||||
throw new Error("Public preview output must be a regular file");
|
||||
}
|
||||
inspectPng(await readFile(file), {
|
||||
label: "Public preview output",
|
||||
maxBytes: MAX_PUBLIC_PREVIEW_BYTES,
|
||||
maxEdge: MAX_PUBLIC_PREVIEW_EDGE,
|
||||
chunks: PUBLIC_PNG_CHUNKS,
|
||||
});
|
||||
}
|
||||
|
||||
export async function createPublicLayoutPreview(
|
||||
source: string,
|
||||
destination: string,
|
||||
) {
|
||||
await validatePrivateScreenshot(source);
|
||||
await mkdir(path.dirname(destination), { recursive: true });
|
||||
const temporary = await mkdtemp(
|
||||
path.join(path.dirname(destination), ".preview-"),
|
||||
);
|
||||
const output = path.join(temporary, "preview.png");
|
||||
try {
|
||||
await execFileAsync(
|
||||
process.env.RUNNER_E2E_IMAGE_MAGICK_BINARY ?? "convert",
|
||||
[
|
||||
"-limit",
|
||||
"memory",
|
||||
"128MiB",
|
||||
"-limit",
|
||||
"map",
|
||||
"256MiB",
|
||||
"-limit",
|
||||
"disk",
|
||||
"256MiB",
|
||||
"-limit",
|
||||
"thread",
|
||||
"1",
|
||||
"-limit",
|
||||
"time",
|
||||
"30",
|
||||
source,
|
||||
"-background",
|
||||
"#f3f4f6",
|
||||
"-alpha",
|
||||
"remove",
|
||||
"-alpha",
|
||||
"off",
|
||||
"-resize",
|
||||
`${MAX_PUBLIC_PREVIEW_EDGE}x${MAX_PUBLIC_PREVIEW_EDGE}>`,
|
||||
"-blur",
|
||||
"0x2.5",
|
||||
"-colors",
|
||||
"24",
|
||||
"-strip",
|
||||
"-define",
|
||||
"png:exclude-chunks=all",
|
||||
`PNG8:${output}`,
|
||||
],
|
||||
{ timeout: 45_000, maxBuffer: 1024 * 1024 },
|
||||
);
|
||||
await validatePublicPreview(output);
|
||||
await copyFile(output, destination, fsConstants.COPYFILE_EXCL);
|
||||
} finally {
|
||||
await rm(temporary, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function assertTreeContainsNoLinks(root: string, current = root) {
|
||||
const entries = await readdir(current, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const absolute = path.join(current, entry.name);
|
||||
const metadata = await lstat(absolute);
|
||||
if (metadata.isSymbolicLink()) {
|
||||
throw new Error("Public history source must not contain symbolic links");
|
||||
}
|
||||
if (metadata.isDirectory()) await assertTreeContainsNoLinks(root, absolute);
|
||||
}
|
||||
}
|
||||
|
||||
async function removeExistingPublicVisuals(current: string) {
|
||||
const entries = await readdir(current, { withFileTypes: true }).catch(
|
||||
() => [],
|
||||
);
|
||||
for (const entry of entries) {
|
||||
const absolute = path.join(current, entry.name);
|
||||
if (!entry.isDirectory()) continue;
|
||||
if (entry.name === PUBLIC_VISUAL_DIRECTORY) {
|
||||
await rm(absolute, { recursive: true, force: true });
|
||||
continue;
|
||||
}
|
||||
await removeExistingPublicVisuals(absolute);
|
||||
}
|
||||
}
|
||||
|
||||
function safePassedScreenshot(input: unknown) {
|
||||
if (!input || typeof input !== "object") {
|
||||
throw new Error("Passing result has invalid screenshot metadata");
|
||||
}
|
||||
const screenshot = input as Record<string, unknown>;
|
||||
if (
|
||||
typeof screenshot.id !== "string" ||
|
||||
typeof screenshot.label !== "string" ||
|
||||
typeof screenshot.file !== "string" ||
|
||||
!PUBLIC_SCREENSHOT_NAME.test(screenshot.file)
|
||||
) {
|
||||
throw new Error("Passing result has unsafe screenshot metadata");
|
||||
}
|
||||
return {
|
||||
...screenshot,
|
||||
label: `${screenshot.label} (redacted layout preview)`,
|
||||
file: `${PUBLIC_VISUAL_DIRECTORY}/${screenshot.file}`,
|
||||
privateFile: screenshot.file,
|
||||
};
|
||||
}
|
||||
|
||||
export async function preparePublicHistoryBundle(input: {
|
||||
source: string;
|
||||
destination: string;
|
||||
transform?: PublicPreviewTransformer;
|
||||
}) {
|
||||
const source = path.resolve(input.source);
|
||||
const destination = path.resolve(input.destination);
|
||||
if (
|
||||
source === destination ||
|
||||
source.startsWith(`${destination}${path.sep}`) ||
|
||||
destination.startsWith(`${source}${path.sep}`)
|
||||
) {
|
||||
throw new Error("Public history source and destination must not overlap");
|
||||
}
|
||||
const sourceMetadata = await lstat(source);
|
||||
if (!sourceMetadata.isDirectory() || sourceMetadata.isSymbolicLink()) {
|
||||
throw new Error("Public history source must be a directory");
|
||||
}
|
||||
await assertTreeContainsNoLinks(source);
|
||||
await rm(destination, { recursive: true, force: true });
|
||||
await cp(source, destination, { recursive: true, force: false });
|
||||
|
||||
const normalizedFile = path.join(destination, "normalized-results.json");
|
||||
const campaign = JSON.parse(
|
||||
await readFile(normalizedFile, "utf8"),
|
||||
) as PublishedCampaign;
|
||||
if (!Array.isArray(campaign.results)) {
|
||||
throw new Error("Public history source has invalid campaign results");
|
||||
}
|
||||
await removeExistingPublicVisuals(path.join(destination, "evidence"));
|
||||
|
||||
const transform = input.transform ?? createPublicLayoutPreview;
|
||||
let previewCount = 0;
|
||||
const results: PublishedResult[] = [];
|
||||
for (const untrustedResult of campaign.results) {
|
||||
if (!untrustedResult || typeof untrustedResult !== "object") {
|
||||
throw new Error("Public history source has an invalid result");
|
||||
}
|
||||
const result = untrustedResult as PublishedResult;
|
||||
if (result.status !== "passed" || result.evidenceValid === false) {
|
||||
results.push(result);
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
typeof result.executionId !== "string" ||
|
||||
!/^[A-Za-z0-9][A-Za-z0-9._-]{0,299}$/.test(result.executionId) ||
|
||||
!Number.isSafeInteger(result.attempt) ||
|
||||
Number(result.attempt) < 1 ||
|
||||
!Array.isArray(result.screenshots) ||
|
||||
result.screenshots.length === 0
|
||||
) {
|
||||
throw new Error("Passing result lacks safe screenshot provenance");
|
||||
}
|
||||
const base = path.join(
|
||||
destination,
|
||||
"evidence",
|
||||
result.executionId,
|
||||
`attempt-${String(result.attempt)}`,
|
||||
);
|
||||
const publicVisuals = path.join(base, PUBLIC_VISUAL_DIRECTORY);
|
||||
await rm(publicVisuals, { recursive: true, force: true });
|
||||
await mkdir(publicVisuals, { recursive: true });
|
||||
const screenshots = [];
|
||||
const seen = new Set<string>();
|
||||
for (const entry of result.screenshots) {
|
||||
const screenshot = safePassedScreenshot(entry);
|
||||
if (seen.has(screenshot.privateFile)) {
|
||||
throw new Error("Passing result has duplicate screenshot metadata");
|
||||
}
|
||||
seen.add(screenshot.privateFile);
|
||||
const sourceScreenshot = path.join(base, screenshot.privateFile);
|
||||
const publicScreenshot = path.join(publicVisuals, screenshot.privateFile);
|
||||
await validatePrivateScreenshot(sourceScreenshot);
|
||||
await transform(sourceScreenshot, publicScreenshot);
|
||||
await validatePublicPreview(publicScreenshot);
|
||||
const { privateFile: _, ...publishedScreenshot } = screenshot;
|
||||
screenshots.push(publishedScreenshot);
|
||||
previewCount += 1;
|
||||
}
|
||||
results.push({ ...result, screenshots });
|
||||
}
|
||||
await writeFile(
|
||||
normalizedFile,
|
||||
`${JSON.stringify({ ...campaign, results }, null, 2)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
await prunePrivateHistoryEvidence(destination);
|
||||
await regenerateRunnerDashboard({ bundle: destination, historyFile: null });
|
||||
console.log(
|
||||
`Prepared ${previewCount} public low-resolution layout preview(s); full-resolution visual evidence remains private`,
|
||||
);
|
||||
return { previewCount };
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const [source, destination] = process.argv.slice(2);
|
||||
if (!source || !destination) {
|
||||
throw new Error(
|
||||
"Usage: history-public-bundle.ts <private-report-directory> <public-report-directory>",
|
||||
);
|
||||
}
|
||||
await preparePublicHistoryBundle({ source, destination });
|
||||
}
|
||||
|
||||
if (
|
||||
process.argv[1] &&
|
||||
path.resolve(process.argv[1]) === path.resolve(import.meta.filename)
|
||||
) {
|
||||
await main().catch((error) => {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
|
|
@ -2,15 +2,7 @@ import { createHash } from "node:crypto";
|
|||
import { execFile } from "node:child_process";
|
||||
import path from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import {
|
||||
mkdtemp,
|
||||
lstat,
|
||||
readFile,
|
||||
readdir,
|
||||
rm,
|
||||
stat,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { mkdtemp, readFile, readdir, stat, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import { regenerateRunnerDashboard } from "./dashboard-regenerate.js";
|
||||
import { renderRunnerHistoryIndex } from "./history-index.js";
|
||||
|
|
@ -20,6 +12,12 @@ import {
|
|||
mergeRunnerHistory,
|
||||
} from "./history.js";
|
||||
import type { RunnerE2ECampaign, RunnerE2EHistoryIndex } from "./types.js";
|
||||
import {
|
||||
isPublicHistoryEvidencePath,
|
||||
prunePrivateHistoryEvidence,
|
||||
} from "./history-public-bundle.js";
|
||||
|
||||
export { prunePrivateHistoryEvidence } from "./history-public-bundle.js";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const MUTABLE_HISTORY_FILES = new Set([
|
||||
|
|
@ -34,36 +32,6 @@ const PUBLISH_ROOT_FILES = new Set([
|
|||
"normalized-results.json",
|
||||
"summary.md",
|
||||
]);
|
||||
const PUBLIC_EVIDENCE_EXTENSIONS = new Set([".json", ".log", ".md", ".txt"]);
|
||||
const PRIVATE_EVIDENCE_DIRECTORIES = new Set([
|
||||
"blob-report",
|
||||
"html-report",
|
||||
"playwright-output",
|
||||
]);
|
||||
|
||||
function publicEvidencePath(relative: string) {
|
||||
const match = relative.match(
|
||||
/^evidence\/[A-Za-z0-9._-]+\/attempt-[1-9][0-9]*\/(.+)$/,
|
||||
);
|
||||
if (!match) return false;
|
||||
const evidencePath = match[1]!;
|
||||
const segments = evidencePath.split("/");
|
||||
if (
|
||||
segments.some(
|
||||
(segment) =>
|
||||
!segment ||
|
||||
segment === "." ||
|
||||
segment === ".." ||
|
||||
PRIVATE_EVIDENCE_DIRECTORIES.has(segment),
|
||||
)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
return PUBLIC_EVIDENCE_EXTENSIONS.has(
|
||||
path.posix.extname(evidencePath).toLowerCase(),
|
||||
);
|
||||
}
|
||||
|
||||
export function isHistoricalBundlePathAllowed(relative: string) {
|
||||
if (
|
||||
relative.includes("\\") ||
|
||||
|
|
@ -79,35 +47,7 @@ export function isHistoricalBundlePathAllowed(relative: string) {
|
|||
) {
|
||||
return true;
|
||||
}
|
||||
return publicEvidencePath(relative);
|
||||
}
|
||||
|
||||
async function pruneEvidenceDirectory(root: string, current: string) {
|
||||
const entries = await readdir(current, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const absolute = path.join(current, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
await pruneEvidenceDirectory(root, absolute);
|
||||
if ((await readdir(absolute)).length === 0) {
|
||||
await rm(absolute, { recursive: true });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const relative = path.relative(root, absolute).split(path.sep).join("/");
|
||||
if (!entry.isFile() || !publicEvidencePath(relative)) {
|
||||
await rm(absolute, { force: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function prunePrivateHistoryEvidence(root: string) {
|
||||
const evidenceRoot = path.join(root, "evidence");
|
||||
const metadata = await lstat(evidenceRoot).catch(() => null);
|
||||
if (!metadata) return;
|
||||
if (!metadata.isDirectory()) {
|
||||
throw new Error("Historical evidence root must be a directory");
|
||||
}
|
||||
await pruneEvidenceDirectory(root, evidenceRoot);
|
||||
return isPublicHistoryEvidencePath(relative);
|
||||
}
|
||||
|
||||
interface BundleManifest {
|
||||
|
|
@ -362,10 +302,10 @@ async function main() {
|
|||
// Campaign bundles are immutable and must not capture a mutable history
|
||||
// file left in a reused local directory. The root landing page below is the
|
||||
// only dashboard that embeds navigation across campaigns.
|
||||
// Raster/video pixels are not OCR-scanned for secrets, and generated HTML,
|
||||
// archives, and SVG may contain or execute active/private content. Preserve
|
||||
// those in the access-controlled workflow artifact but remove them from the
|
||||
// directory shared by public S3 and Pages publication.
|
||||
// The trusted report job replaces successful screenshots with low-resolution
|
||||
// blurred previews before this AWS-credentialed process sees the bundle.
|
||||
// Remove every remaining private raster/video, generated HTML, archive, SVG,
|
||||
// and other non-allowlisted path before S3 or Pages publication.
|
||||
await prunePrivateHistoryEvidence(reportRoot);
|
||||
await regenerateRunnerDashboard({ bundle: reportRoot, historyFile: null });
|
||||
const manifest = await createBundleManifest(reportRoot, campaign.campaignId);
|
||||
|
|
|
|||
|
|
@ -1,4 +1,11 @@
|
|||
import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import {
|
||||
copyFile,
|
||||
mkdtemp,
|
||||
mkdir,
|
||||
readFile,
|
||||
rm,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
|
@ -9,7 +16,6 @@ import {
|
|||
buildHistoryPointers,
|
||||
createBundleManifest,
|
||||
isHistoricalBundlePathAllowed,
|
||||
prunePrivateHistoryEvidence,
|
||||
validateHistoryDestination,
|
||||
} from "./history-publish.js";
|
||||
import {
|
||||
|
|
@ -20,9 +26,14 @@ import {
|
|||
mergeRunnerHistory,
|
||||
} from "./history.js";
|
||||
import { renderRunnerHistoryIndex } from "./history-index.js";
|
||||
import { preparePublicHistoryBundle } from "./history-public-bundle.js";
|
||||
import type { MatrixExecution, RunnerE2EResult } from "./types.js";
|
||||
|
||||
const temporaryDirectories: string[] = [];
|
||||
const safePng = Buffer.from(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAQAAAADCAIAAAA7ljmRAAAAFElEQVQI12M0TpvJAANMDEgAhQMALHABOEZNdkwAAAAASUVORK5CYII=",
|
||||
"base64",
|
||||
);
|
||||
afterEach(async () => {
|
||||
vi.unstubAllEnvs();
|
||||
await Promise.all(
|
||||
|
|
@ -185,7 +196,7 @@ describe("runner E2E campaign history", () => {
|
|||
expect(index).toContain("65/66 passed");
|
||||
expect(index).toContain("Open report →");
|
||||
expect(index).toContain(
|
||||
"Visual evidence remains in access-controlled workflow artifacts",
|
||||
"Full-resolution visual evidence remains in access-controlled workflow artifacts",
|
||||
);
|
||||
expect(index).toContain("Inert structured public evidence");
|
||||
expect(index).not.toContain("data-gallery-dialog");
|
||||
|
|
@ -194,10 +205,12 @@ describe("runner E2E campaign history", () => {
|
|||
});
|
||||
|
||||
describe("historical publication security", () => {
|
||||
it("keeps visual and active evidence private when building the public dashboard", async () => {
|
||||
it("publishes blurred layout previews while keeping raw visuals private", async () => {
|
||||
const root = await mkdtemp(path.join(os.tmpdir(), "runner-landing-test-"));
|
||||
const output = path.join(root, "landing");
|
||||
const source = path.join(root, "private");
|
||||
const output = path.join(root, "public");
|
||||
temporaryDirectories.push(root);
|
||||
await mkdir(source);
|
||||
const execution = runnerMatrix[0]!;
|
||||
const campaignResult = {
|
||||
...result(execution, "passed"),
|
||||
|
|
@ -216,17 +229,23 @@ describe("historical publication security", () => {
|
|||
results: [campaignResult],
|
||||
});
|
||||
const evidenceDirectory = path.join(
|
||||
root,
|
||||
source,
|
||||
"evidence",
|
||||
execution.id,
|
||||
"attempt-1",
|
||||
);
|
||||
const publicEvidenceDirectory = path.join(
|
||||
output,
|
||||
"evidence",
|
||||
execution.id,
|
||||
"attempt-1",
|
||||
);
|
||||
await mkdir(evidenceDirectory, { recursive: true });
|
||||
await writeFile(
|
||||
path.join(root, "normalized-results.json"),
|
||||
path.join(source, "normalized-results.json"),
|
||||
JSON.stringify(campaign),
|
||||
);
|
||||
await writeFile(path.join(evidenceDirectory, "final-state.png"), "png");
|
||||
await writeFile(path.join(evidenceDirectory, "final-state.png"), safePng);
|
||||
await writeFile(path.join(evidenceDirectory, "failure.webm"), "webm");
|
||||
await writeFile(path.join(evidenceDirectory, "unsafe.svg"), "<svg />");
|
||||
await writeFile(
|
||||
|
|
@ -234,6 +253,15 @@ describe("historical publication security", () => {
|
|||
"<?xml-stylesheet href='https://example.test/private.xsl'?>",
|
||||
);
|
||||
await writeFile(path.join(evidenceDirectory, "result.json"), "{}\n");
|
||||
await mkdir(path.join(evidenceDirectory, "public-visuals"));
|
||||
await writeFile(
|
||||
path.join(
|
||||
evidenceDirectory,
|
||||
"public-visuals",
|
||||
"plan-target-injected.png",
|
||||
),
|
||||
safePng,
|
||||
);
|
||||
await mkdir(path.join(evidenceDirectory, "snapshots"));
|
||||
await writeFile(
|
||||
path.join(evidenceDirectory, "snapshots", "api-state.json"),
|
||||
|
|
@ -250,45 +278,54 @@ describe("historical publication security", () => {
|
|||
"private archive",
|
||||
);
|
||||
|
||||
await prunePrivateHistoryEvidence(root);
|
||||
await regenerateRunnerDashboard({
|
||||
bundle: root,
|
||||
outputDirectory: output,
|
||||
evidenceHrefPrefix: "campaigns/campaign-1",
|
||||
await preparePublicHistoryBundle({
|
||||
source,
|
||||
destination: output,
|
||||
transform: async (privateScreenshot, publicPreview) => {
|
||||
await copyFile(privateScreenshot, publicPreview);
|
||||
},
|
||||
});
|
||||
const dashboard = await readFile(path.join(output, "index.html"), "utf8");
|
||||
expect(dashboard).not.toContain(
|
||||
`campaigns/campaign-1/evidence/${execution.id}/attempt-1/final-state.png`,
|
||||
);
|
||||
expect(dashboard).toContain("Visual evidence · workflow artifact only");
|
||||
expect(dashboard).toContain(
|
||||
"public history contains inert structured evidence only",
|
||||
`evidence/${execution.id}/attempt-1/public-visuals/final-state.png`,
|
||||
);
|
||||
expect(dashboard).toContain("Public history excludes visual evidence");
|
||||
await expect(
|
||||
readFile(path.join(publicEvidenceDirectory, "final-state.png")),
|
||||
).rejects.toThrow();
|
||||
await expect(
|
||||
readFile(
|
||||
path.join(publicEvidenceDirectory, "public-visuals", "final-state.png"),
|
||||
),
|
||||
).resolves.toEqual(safePng);
|
||||
await expect(
|
||||
readFile(
|
||||
path.join(
|
||||
publicEvidenceDirectory,
|
||||
"public-visuals",
|
||||
"plan-target-injected.png",
|
||||
),
|
||||
),
|
||||
).rejects.toThrow();
|
||||
await expect(
|
||||
readFile(path.join(evidenceDirectory, "final-state.png")),
|
||||
).rejects.toThrow();
|
||||
).resolves.toEqual(safePng);
|
||||
for (const relative of [
|
||||
"failure.webm",
|
||||
"unsafe.svg",
|
||||
"junit.xml",
|
||||
"html-report/index.html",
|
||||
"blob-report/report.zip",
|
||||
]) {
|
||||
await expect(
|
||||
readFile(path.join(publicEvidenceDirectory, ...relative.split("/"))),
|
||||
).rejects.toThrow();
|
||||
}
|
||||
await expect(
|
||||
readFile(path.join(evidenceDirectory, "failure.webm")),
|
||||
).rejects.toThrow();
|
||||
await expect(
|
||||
readFile(path.join(evidenceDirectory, "unsafe.svg")),
|
||||
).rejects.toThrow();
|
||||
await expect(
|
||||
readFile(path.join(evidenceDirectory, "junit.xml")),
|
||||
).rejects.toThrow();
|
||||
await expect(
|
||||
readFile(path.join(evidenceDirectory, "html-report", "index.html")),
|
||||
).rejects.toThrow();
|
||||
await expect(
|
||||
readFile(path.join(evidenceDirectory, "blob-report", "report.zip")),
|
||||
).rejects.toThrow();
|
||||
await expect(
|
||||
readFile(path.join(evidenceDirectory, "result.json"), "utf8"),
|
||||
readFile(path.join(publicEvidenceDirectory, "result.json"), "utf8"),
|
||||
).resolves.toBe("{}\n");
|
||||
await expect(
|
||||
readFile(
|
||||
path.join(evidenceDirectory, "snapshots", "api-state.json"),
|
||||
path.join(publicEvidenceDirectory, "snapshots", "api-state.json"),
|
||||
"utf8",
|
||||
),
|
||||
).resolves.toBe("{}\n");
|
||||
|
|
@ -299,8 +336,8 @@ describe("historical publication security", () => {
|
|||
).toBe("paperclip.runner-e2e.campaign/v2");
|
||||
await expect(
|
||||
regenerateRunnerDashboard({
|
||||
bundle: root,
|
||||
outputDirectory: output,
|
||||
bundle: source,
|
||||
outputDirectory: path.join(root, "invalid"),
|
||||
evidenceHrefPrefix: "../unsafe",
|
||||
}),
|
||||
).rejects.toThrow("safe relative URL path");
|
||||
|
|
@ -341,6 +378,16 @@ describe("historical publication security", () => {
|
|||
"evidence/core-compatibility.profile.local.case/attempt-1/final-state.png",
|
||||
),
|
||||
).toBe(false);
|
||||
expect(
|
||||
isHistoricalBundlePathAllowed(
|
||||
"evidence/core-compatibility.profile.local.case/attempt-1/public-visuals/final-state.png",
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
isHistoricalBundlePathAllowed(
|
||||
"evidence/core-compatibility.profile.local.case/attempt-1/public-visuals/failure.png",
|
||||
),
|
||||
).toBe(false);
|
||||
expect(
|
||||
isHistoricalBundlePathAllowed(
|
||||
"evidence/core-compatibility.profile.local.case/attempt-1/failure.webm",
|
||||
|
|
|
|||
|
|
@ -642,6 +642,12 @@ describe("public repository paid workflow security", () => {
|
|||
expect(
|
||||
report.indexOf("Select latest workflow attempt per cell"),
|
||||
).toBeLessThan(report.indexOf("Collect blob reports"));
|
||||
expect(report).toContain(
|
||||
"runner-e2e-public-history-source-${{ github.run_id }}-${{ github.run_attempt }}",
|
||||
);
|
||||
expect(publisher).toContain(
|
||||
"runner-e2e-public-history-source-${{ github.run_id }}-${{ github.run_attempt }}",
|
||||
);
|
||||
expect(publisher).toContain(
|
||||
'echo "name=github-pages-${{ github.run_id }}-${{ github.run_attempt }}"',
|
||||
);
|
||||
|
|
@ -661,6 +667,10 @@ describe("public repository paid workflow security", () => {
|
|||
path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"),
|
||||
"utf8",
|
||||
);
|
||||
const report = workflow.slice(
|
||||
workflow.indexOf(" report:"),
|
||||
workflow.indexOf(" publish_history:"),
|
||||
);
|
||||
const publisher = workflow.slice(workflow.indexOf(" publish_history:"));
|
||||
expect(publisher).toContain("id-token: write");
|
||||
expect(publisher).toContain("name: runner-e2e-history");
|
||||
|
|
@ -671,9 +681,21 @@ describe("public repository paid workflow security", () => {
|
|||
expect(publisher).not.toMatch(/aws s3 (?:rm|sync .*--delete)/);
|
||||
expect(workflow).toContain("history_source_ready");
|
||||
expect(workflow).toContain(
|
||||
"Verify history source report and private screenshot evidence",
|
||||
"Prepare public history bundle with redacted layout previews",
|
||||
);
|
||||
expect(workflow).toContain("private_screenshot=");
|
||||
expect(workflow).toContain(
|
||||
"Verify prepared history source and public layout previews",
|
||||
);
|
||||
expect(workflow).toContain("public_preview=");
|
||||
expect(workflow).toContain("unexpected_png=");
|
||||
expect(workflow).toContain("passed_count=");
|
||||
expect(workflow).toContain("pnpm test:e2e:runner:history:prepare");
|
||||
expect(report).not.toContain("id-token: write");
|
||||
expect(report).not.toMatch(
|
||||
/(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA)_API_KEY/,
|
||||
);
|
||||
expect(publisher).not.toContain("Qualify public preview raster sanitizer");
|
||||
expect(publisher).not.toContain("runner-e2e-report-${{ github.run_id }}");
|
||||
expect(workflow).toContain("Publish pruned immutable history");
|
||||
expect(workflow).toContain("Publish latest structured dashboard");
|
||||
expect(workflow).not.toContain("dashboard_ready");
|
||||
|
|
|
|||
Loading…
Reference in New Issue