diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.yml index 3179f5c8e1..52b069de93 100644 --- a/.github/workflows/runner-full-stack-e2e.yml +++ b/.github/workflows/runner-full-stack-e2e.yml @@ -1102,6 +1102,8 @@ jobs: PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-${{ github.run_attempt }} PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }} + PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: ${{ vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL }} + PAPERCLIP_RUNNER_E2E_HISTORY_PREFIX: ${{ vars.RUNNER_E2E_HISTORY_PREFIX || 'runner-e2e' }} run: | set +e pnpm test:e2e:runner:report diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index b9ffb1dd6c..c2e29273fe 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -266,6 +266,11 @@ same metrics per campaign/suite/execution, source SHA/ref, definition fingerprints, completeness, retries, and cleanup. Trend charts compare only complete campaigns by default; partial/manual selections remain browsable. `summary.md` carries the current totals into the GitHub Actions job summary. +In CI, its **View results** section links to the exact immutable public campaign +report, the workflow and per-cell logs, and the access-controlled report +artifacts. Each cell name links to its exact section in the campaign report. +The public campaign links become available after the history publisher +finishes. The artifact links remain available for 30 days. ### Iterate on a published dashboard without rerunning paid tests diff --git a/tests/runner-e2e/dashboard.ts b/tests/runner-e2e/dashboard.ts index 6fe3f8d782..06a72bd9cb 100644 --- a/tests/runner-e2e/dashboard.ts +++ b/tests/runner-e2e/dashboard.ts @@ -247,6 +247,7 @@ function renderCase( ` : ""; return `
+ !segment || + segment === "." || + segment === ".." || + !/^[A-Za-z0-9._~-]+$/.test(segment), + ) + ) { + throw new Error( + "RUNNER_E2E_HISTORY_PREFIX must be a safe non-empty key prefix", + ); + } + const publicUrl = new URL(input.publicBaseUrl); + if ( + publicUrl.protocol !== "https:" || + publicUrl.username || + publicUrl.password || + publicUrl.search || + publicUrl.hash + ) { + throw new Error( + "RUNNER_E2E_HISTORY_PUBLIC_BASE_URL must be a credential-free HTTPS URL", + ); + } + return { + prefix, + publicBaseUrl: publicUrl.href.replace(/\/+$/, ""), + }; +} + +export function validateHistoryDestination(input: { + bucket: string; + prefix: string; + publicBaseUrl: string; +}) { + if (!/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(input.bucket)) { + throw new Error("RUNNER_E2E_HISTORY_S3_BUCKET is not a valid bucket name"); + } + return validateHistoryPublicDestination(input); +} + +export function runnerE2ECampaignPublicUrl( + destination: RunnerE2EHistoryPublicDestination, + campaignId: string, +) { + return `${destination.publicBaseUrl}/${destination.prefix}/campaigns/${encodeURIComponent(campaignId)}/index.html`; +} diff --git a/tests/runner-e2e/history-publish.ts b/tests/runner-e2e/history-publish.ts index db865998c2..f5dc913873 100644 --- a/tests/runner-e2e/history-publish.ts +++ b/tests/runner-e2e/history-publish.ts @@ -19,6 +19,7 @@ import os from "node:os"; import { writePublicCampaignSummaryImage } from "./public-summary-image.js"; import { regenerateRunnerDashboard } from "./dashboard-regenerate.js"; import { renderRunnerHistoryIndex } from "./history-index.js"; +import { validateHistoryDestination } from "./history-destination.js"; import { PUBLIC_RUNNER_SCREENSHOT_MARKER } from "./screenshot-policy.js"; import { campaignHistoryRecord, @@ -259,39 +260,7 @@ function json(value: unknown) { return `${JSON.stringify(value, null, 2)}\n`; } -export function validateHistoryDestination(input: { - bucket: string; - prefix: string; - publicBaseUrl: string; -}) { - if (!/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(input.bucket)) { - throw new Error("RUNNER_E2E_HISTORY_S3_BUCKET is not a valid bucket name"); - } - const prefix = input.prefix.replace(/^\/+|\/+$/g, ""); - if ( - !prefix || - prefix - .split("/") - .some((segment) => !segment || segment === "." || segment === "..") - ) { - throw new Error( - "RUNNER_E2E_HISTORY_PREFIX must be a safe non-empty key prefix", - ); - } - const publicUrl = new URL(input.publicBaseUrl); - if ( - publicUrl.protocol !== "https:" || - publicUrl.username || - publicUrl.password || - publicUrl.search || - publicUrl.hash - ) { - throw new Error( - "RUNNER_E2E_HISTORY_PUBLIC_BASE_URL must be a credential-free HTTPS URL", - ); - } - return { prefix, publicBaseUrl: publicUrl.href.replace(/\/$/, "") }; -} +export { validateHistoryDestination }; async function relativeFiles( root: string, diff --git a/tests/runner-e2e/history.test.ts b/tests/runner-e2e/history.test.ts index 32c6a2bc53..cec6d79128 100644 --- a/tests/runner-e2e/history.test.ts +++ b/tests/runner-e2e/history.test.ts @@ -634,6 +634,13 @@ describe("historical publication security", () => { publicBaseUrl: "https://history.paperclip.ai/", }), ).toThrow("safe non-empty key prefix"); + expect(() => + validateHistoryDestination({ + bucket: "paperclip-runner-e2e-history", + prefix: "runner-e2e?other", + publicBaseUrl: "https://history.paperclip.ai/", + }), + ).toThrow("safe non-empty key prefix"); expect(() => validateHistoryDestination({ bucket: "paperclip-runner-e2e-history", diff --git a/tests/runner-e2e/report.test.ts b/tests/runner-e2e/report.test.ts index 4ac6453cf0..f1e7b0e21f 100644 --- a/tests/runner-e2e/report.test.ts +++ b/tests/runner-e2e/report.test.ts @@ -150,6 +150,9 @@ describe("runner E2E report aggregation", () => { GITHUB_SERVER_URL: "https://github.com", GITHUB_REPOSITORY: "paperclipai/paperclip", GITHUB_RUN_ID: "123456", + PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: + "https://reports.example.test/", + PAPERCLIP_RUNNER_E2E_HISTORY_PREFIX: "/runner-e2e/", }, }, ); @@ -203,6 +206,9 @@ describe("runner E2E report aggregation", () => { expect(dashboard).toContain(" { expect(await readFile(path.join(output, "index.html"), "utf8")).toBe( dashboard, ); + const summary = await readFile(path.join(output, "summary.md"), "utf8"); + expect(summary).toContain("## View results"); + expect(summary).toContain( + "[Open the exact interactive campaign report](https://reports.example.test/runner-e2e/campaigns/gha-123456-1/index.html)", + ); + expect(summary).toContain( + "[Open the workflow run and per-cell job logs](https://github.com/paperclipai/paperclip/actions/runs/123456)", + ); + expect(summary).toContain( + "[Download the merged report and per-cell evidence](https://github.com/paperclipai/paperclip/actions/runs/123456#artifacts)", + ); + expect(summary).toContain( + `[core-compatibility.${executionId}](https://reports.example.test/runner-e2e/campaigns/gha-123456-1/index.html#execution-core-compatibility.${executionId})`, + ); }); it("prefers a valid rerun over a higher attempt number from an older campaign", async () => { diff --git a/tests/runner-e2e/report.ts b/tests/runner-e2e/report.ts index cc58fe73d9..3b61eb5d14 100644 --- a/tests/runner-e2e/report.ts +++ b/tests/runner-e2e/report.ts @@ -15,6 +15,7 @@ import { upgradeRunnerResult, } from "./history.js"; import { resolveRunnerE2ESource } from "./source.js"; +import { runnerE2ESummaryLinks } from "./summary-links.js"; import type { RunnerE2EResult } from "./types.js"; const repositoryRoot = path.resolve(import.meta.dirname, "../.."); @@ -296,11 +297,33 @@ async function main() { writeFile(path.join(output, "index.html"), dashboard, "utf8"), ]); + const summaryLinks = runnerE2ESummaryLinks({ + campaignId: normalized.campaignId, + workflowRunUrl: normalized.source.workflowRunUrl, + historyPublicBaseUrl: + process.env.PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL, + historyPrefix: process.env.PAPERCLIP_RUNNER_E2E_HISTORY_PREFIX, + }); + const publicCampaignUrl = summaryLinks.find( + (link) => link.kind === "campaign", + )?.url; + const summaryLines = [ "# Runner Full-Stack E2E", "", `Passed: ${normalized.passed}/${selected.length}`, "", + ...(summaryLinks.length > 0 + ? [ + "## View results", + "", + ...summaryLinks.map( + (link) => + `- [${link.label}](${link.url})${link.note ? ` — ${link.note}` : ""}`, + ), + "", + ] + : []), `Tokens: ${billing.llm.inputTokens} input / ${billing.llm.outputTokens} output / ${billing.llm.cachedInputTokens} cached`, "", `Provider-reported LLM cost: $${billing.reportedLlmCostUsd.toFixed(6)} (${billing.llm.runsWithReportedCost}/${billing.llm.runCount} runs priced)`, @@ -314,7 +337,10 @@ async function main() { const resolved = resolvedResults[index]!; const cellBilling = resolved.billing!; const runtimeCost = cellBilling.runtime.estimatedListCostUsd; - return `| ${resolved.executionId} | ${resolved.attempt} | ${entry.valid ? "pass" : "fail"} | ${resolved.runtimeMode} | ${Math.round(resolved.durationMs / 1000)}s | ${cellBilling.llm.inputTokens}/${cellBilling.llm.outputTokens} | $${cellBilling.reportedCostUsd.toFixed(6)} (${cellBilling.llm.costStatus}) | ${runtimeCost === undefined ? cellBilling.runtime.costStatus : `$${runtimeCost.toFixed(6)} est.`} | ${detail} |`; + const cell = publicCampaignUrl + ? `[${resolved.executionId}](${publicCampaignUrl}#execution-${encodeURIComponent(resolved.executionId)})` + : resolved.executionId; + return `| ${cell} | ${resolved.attempt} | ${entry.valid ? "pass" : "fail"} | ${resolved.runtimeMode} | ${Math.round(resolved.durationMs / 1000)}s | ${cellBilling.llm.inputTokens}/${cellBilling.llm.outputTokens} | $${cellBilling.reportedCostUsd.toFixed(6)} (${cellBilling.llm.costStatus}) | ${runtimeCost === undefined ? cellBilling.runtime.costStatus : `$${runtimeCost.toFixed(6)} est.`} | ${detail} |`; }), "", ]; diff --git a/tests/runner-e2e/summary-links.test.ts b/tests/runner-e2e/summary-links.test.ts new file mode 100644 index 0000000000..9ab4185137 --- /dev/null +++ b/tests/runner-e2e/summary-links.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from "vitest"; +import { runnerE2ESummaryLinks } from "./summary-links.js"; + +describe("runner E2E summary links", () => { + it("builds exact public, workflow, and artifact links", () => { + expect( + runnerE2ESummaryLinks({ + campaignId: "gha-34026735033-1", + workflowRunUrl: + "https://github.com/paperclipai/paperclip/actions/runs/34026735033", + historyPublicBaseUrl: "https://reports.example.test///", + historyPrefix: "/runner-e2e/", + }), + ).toEqual([ + { + kind: "campaign", + label: "Open the exact interactive campaign report", + url: "https://reports.example.test/runner-e2e/campaigns/gha-34026735033-1/index.html", + note: "available after the history publisher finishes", + }, + { + kind: "workflow", + label: "Open the workflow run and per-cell job logs", + url: "https://github.com/paperclipai/paperclip/actions/runs/34026735033", + }, + { + kind: "artifacts", + label: "Download the merged report and per-cell evidence", + url: "https://github.com/paperclipai/paperclip/actions/runs/34026735033#artifacts", + note: "GitHub access required; retained for 30 days", + }, + ]); + }); + + it("omits unsafe or incomplete destinations", () => { + expect( + runnerE2ESummaryLinks({ + campaignId: "../other-campaign", + workflowRunUrl: "https://token@example.test/actions/runs/1", + historyPublicBaseUrl: "http://reports.example.test", + historyPrefix: "runner-e2e", + }), + ).toEqual([]); + expect( + runnerE2ESummaryLinks({ + campaignId: "gha-1-1", + workflowRunUrl: null, + historyPublicBaseUrl: null, + historyPrefix: null, + }), + ).toEqual([]); + expect( + runnerE2ESummaryLinks({ + campaignId: "gha-1-1", + workflowRunUrl: null, + historyPublicBaseUrl: "https://reports.example.test", + historyPrefix: "runner-e2e#other", + }), + ).toEqual([]); + }); +}); diff --git a/tests/runner-e2e/summary-links.ts b/tests/runner-e2e/summary-links.ts new file mode 100644 index 0000000000..b69d5b8150 --- /dev/null +++ b/tests/runner-e2e/summary-links.ts @@ -0,0 +1,73 @@ +import { + runnerE2ECampaignPublicUrl, + validateHistoryPublicDestination, +} from "./history-destination.js"; + +export interface RunnerE2ESummaryLink { + kind: "campaign" | "workflow" | "artifacts"; + label: string; + url: string; + note?: string; +} + +function safeHttpsUrl(value: string | null | undefined) { + const input = value?.trim(); + if (!input) return null; + try { + const url = new URL(input); + if ( + url.protocol !== "https:" || + url.username || + url.password || + url.search || + url.hash + ) { + return null; + } + return url; + } catch { + return null; + } +} + +export function runnerE2ESummaryLinks(input: { + campaignId: string; + workflowRunUrl: string | null | undefined; + historyPublicBaseUrl: string | null | undefined; + historyPrefix: string | null | undefined; +}): RunnerE2ESummaryLink[] { + const links: RunnerE2ESummaryLink[] = []; + if (/^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$/.test(input.campaignId)) { + try { + const destination = validateHistoryPublicDestination({ + publicBaseUrl: input.historyPublicBaseUrl?.trim() ?? "", + prefix: input.historyPrefix?.trim() ?? "", + }); + links.push({ + kind: "campaign", + label: "Open the exact interactive campaign report", + url: runnerE2ECampaignPublicUrl(destination, input.campaignId), + note: "available after the history publisher finishes", + }); + } catch { + // Invalid or absent history configuration must not prevent report merging. + } + } + + const workflowRun = safeHttpsUrl(input.workflowRunUrl); + if (workflowRun) { + links.push({ + kind: "workflow", + label: "Open the workflow run and per-cell job logs", + url: workflowRun.href, + }); + workflowRun.hash = "artifacts"; + links.push({ + kind: "artifacts", + label: "Download the merged report and per-cell evidence", + url: workflowRun.href, + note: "GitHub access required; retained for 30 days", + }); + } + return links; +} diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts index 03a68f6af8..4dce7c374a 100644 --- a/tests/runner-e2e/workflow-security.test.ts +++ b/tests/runner-e2e/workflow-security.test.ts @@ -671,6 +671,12 @@ describe("public repository paid workflow security", () => { expect(report).toContain( "PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e", ); + expect(report).toContain( + "PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: ${{ vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL }}", + ); + expect(report).toContain( + "PAPERCLIP_RUNNER_E2E_HISTORY_PREFIX: ${{ vars.RUNNER_E2E_HISTORY_PREFIX || 'runner-e2e' }}", + ); expect( report.indexOf("Select latest workflow attempt per cell"), ).toBeLessThan(report.indexOf("Collect blob reports"));