From 9ecd93a54dadac1a835716eefa37a80b78d196f2 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sun, 6 Sep 2026 09:09:51 -0500 Subject: [PATCH] test(e2e): link runner campaign summaries (#12927) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Paperclip uses a paid full-stack campaign to verify runner behavior across providers and environments. > - The campaign already creates an interactive report, workflow logs, and retained evidence artifacts. > - The merge job summary shows result totals but does not link to those resources. > - Reviewers must search several workflow jobs and artifacts to find the executed cells. > - This pull request adds direct and safe links to the exact campaign, each cell, the workflow logs, and the artifacts. > - The benefit is that a reviewer can inspect a result from the Actions summary with one click. ## Linked Issues or Issue Description **What existing behavior does this improve?** This improves the `Merge and enforce campaign result` summary in the `Runner Full-Stack E2E` workflow. **Subsystem affected** The runner E2E report generator and its GitHub Actions workflow are affected. **Current behavior** The summary lists each selected cell and its result. It does not link to the published campaign report, the workflow logs, or the evidence artifacts. **Proposed behavior** The summary includes a `View results` section. It links to the exact immutable campaign report, the workflow logs, and the artifacts. Each cell name links to its stable section in the campaign report. **Reason and benefit** The current summary does not show reviewers where to inspect the run. Direct links make the result evidence discoverable without manual URL construction or artifact searches. **Breaking changes** None. This change only adds links and stable HTML anchors to existing report output. **Additional context** Related: #12904. The cited successful campaign is [run 34026735033](https://github.com/paperclipai/paperclip/actions/runs/34026735033). ## What Changed - Add a safe URL builder for public campaign, workflow, and artifact links. - Add a `View results` section to the GitHub Actions campaign summary. - Link each summary table cell to its exact section in the immutable campaign report. - Add stable execution anchors to the generated dashboard. - Reject non-HTTPS, credential-bearing, malformed, and ambiguous link destinations. - Document the new links and their retention or publication timing. ## Verification - `pnpm test:e2e:runner:unit` — 116 tests passed. - `pnpm test:e2e:runner:typecheck` — passed. - `pnpm typecheck` — passed, including migration safety. - `pnpm build` — passed. - `pnpm exec prettier --check ...` for all changed files — passed. - `git diff --check origin/master...HEAD` — passed. - The full local server suite also ran. One unrelated macOS workspace-runtime file passed 157 tests and failed 4 existing path and port assumptions. Two failures compare `/var` with `/private/var`. Two failures cannot reserve a port outside a hard-coded range. This PR does not change that file or its dependencies. ## Risks - The immutable campaign link becomes available after the history publisher completes. The workflow and artifact links remain available while publication runs. - The artifact link requires GitHub access and follows the existing 30-day retention period. - Invalid configured URLs are omitted instead of being rendered into the summary. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used OpenAI Codex desktop agent with GPT-5. The runtime does not expose the context-window size. The agent used repository inspection, agentic reasoning, code execution, and GitHub CLI tools. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/no-internal-issue-references`, `fix/sandbox-secret-resolution`, `feat/adapter-retry-backoff`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --- .github/workflows/runner-full-stack-e2e.yml | 2 + tests/runner-e2e/README.md | 5 ++ tests/runner-e2e/dashboard.ts | 1 + tests/runner-e2e/history-destination.ts | 60 +++++++++++++++++ tests/runner-e2e/history-publish.ts | 35 +--------- tests/runner-e2e/history.test.ts | 7 ++ tests/runner-e2e/report.test.ts | 20 ++++++ tests/runner-e2e/report.ts | 28 +++++++- tests/runner-e2e/summary-links.test.ts | 61 +++++++++++++++++ tests/runner-e2e/summary-links.ts | 73 +++++++++++++++++++++ tests/runner-e2e/workflow-security.test.ts | 6 ++ 11 files changed, 264 insertions(+), 34 deletions(-) create mode 100644 tests/runner-e2e/history-destination.ts create mode 100644 tests/runner-e2e/summary-links.test.ts create mode 100644 tests/runner-e2e/summary-links.ts diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.yml index 3179f5c8e1..52b069de93 100644 --- a/.github/workflows/runner-full-stack-e2e.yml +++ b/.github/workflows/runner-full-stack-e2e.yml @@ -1102,6 +1102,8 @@ jobs: PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-${{ github.run_attempt }} PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }} + PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: ${{ vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL }} + PAPERCLIP_RUNNER_E2E_HISTORY_PREFIX: ${{ vars.RUNNER_E2E_HISTORY_PREFIX || 'runner-e2e' }} run: | set +e pnpm test:e2e:runner:report diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index b9ffb1dd6c..c2e29273fe 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -266,6 +266,11 @@ same metrics per campaign/suite/execution, source SHA/ref, definition fingerprints, completeness, retries, and cleanup. Trend charts compare only complete campaigns by default; partial/manual selections remain browsable. `summary.md` carries the current totals into the GitHub Actions job summary. +In CI, its **View results** section links to the exact immutable public campaign +report, the workflow and per-cell logs, and the access-controlled report +artifacts. Each cell name links to its exact section in the campaign report. +The public campaign links become available after the history publisher +finishes. The artifact links remain available for 30 days. ### Iterate on a published dashboard without rerunning paid tests diff --git a/tests/runner-e2e/dashboard.ts b/tests/runner-e2e/dashboard.ts index 6fe3f8d782..06a72bd9cb 100644 --- a/tests/runner-e2e/dashboard.ts +++ b/tests/runner-e2e/dashboard.ts @@ -247,6 +247,7 @@ function renderCase( ` : ""; return `
+ !segment || + segment === "." || + segment === ".." || + !/^[A-Za-z0-9._~-]+$/.test(segment), + ) + ) { + throw new Error( + "RUNNER_E2E_HISTORY_PREFIX must be a safe non-empty key prefix", + ); + } + const publicUrl = new URL(input.publicBaseUrl); + if ( + publicUrl.protocol !== "https:" || + publicUrl.username || + publicUrl.password || + publicUrl.search || + publicUrl.hash + ) { + throw new Error( + "RUNNER_E2E_HISTORY_PUBLIC_BASE_URL must be a credential-free HTTPS URL", + ); + } + return { + prefix, + publicBaseUrl: publicUrl.href.replace(/\/+$/, ""), + }; +} + +export function validateHistoryDestination(input: { + bucket: string; + prefix: string; + publicBaseUrl: string; +}) { + if (!/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(input.bucket)) { + throw new Error("RUNNER_E2E_HISTORY_S3_BUCKET is not a valid bucket name"); + } + return validateHistoryPublicDestination(input); +} + +export function runnerE2ECampaignPublicUrl( + destination: RunnerE2EHistoryPublicDestination, + campaignId: string, +) { + return `${destination.publicBaseUrl}/${destination.prefix}/campaigns/${encodeURIComponent(campaignId)}/index.html`; +} diff --git a/tests/runner-e2e/history-publish.ts b/tests/runner-e2e/history-publish.ts index db865998c2..f5dc913873 100644 --- a/tests/runner-e2e/history-publish.ts +++ b/tests/runner-e2e/history-publish.ts @@ -19,6 +19,7 @@ import os from "node:os"; import { writePublicCampaignSummaryImage } from "./public-summary-image.js"; import { regenerateRunnerDashboard } from "./dashboard-regenerate.js"; import { renderRunnerHistoryIndex } from "./history-index.js"; +import { validateHistoryDestination } from "./history-destination.js"; import { PUBLIC_RUNNER_SCREENSHOT_MARKER } from "./screenshot-policy.js"; import { campaignHistoryRecord, @@ -259,39 +260,7 @@ function json(value: unknown) { return `${JSON.stringify(value, null, 2)}\n`; } -export function validateHistoryDestination(input: { - bucket: string; - prefix: string; - publicBaseUrl: string; -}) { - if (!/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(input.bucket)) { - throw new Error("RUNNER_E2E_HISTORY_S3_BUCKET is not a valid bucket name"); - } - const prefix = input.prefix.replace(/^\/+|\/+$/g, ""); - if ( - !prefix || - prefix - .split("/") - .some((segment) => !segment || segment === "." || segment === "..") - ) { - throw new Error( - "RUNNER_E2E_HISTORY_PREFIX must be a safe non-empty key prefix", - ); - } - const publicUrl = new URL(input.publicBaseUrl); - if ( - publicUrl.protocol !== "https:" || - publicUrl.username || - publicUrl.password || - publicUrl.search || - publicUrl.hash - ) { - throw new Error( - "RUNNER_E2E_HISTORY_PUBLIC_BASE_URL must be a credential-free HTTPS URL", - ); - } - return { prefix, publicBaseUrl: publicUrl.href.replace(/\/$/, "") }; -} +export { validateHistoryDestination }; async function relativeFiles( root: string, diff --git a/tests/runner-e2e/history.test.ts b/tests/runner-e2e/history.test.ts index 32c6a2bc53..cec6d79128 100644 --- a/tests/runner-e2e/history.test.ts +++ b/tests/runner-e2e/history.test.ts @@ -634,6 +634,13 @@ describe("historical publication security", () => { publicBaseUrl: "https://history.paperclip.ai/", }), ).toThrow("safe non-empty key prefix"); + expect(() => + validateHistoryDestination({ + bucket: "paperclip-runner-e2e-history", + prefix: "runner-e2e?other", + publicBaseUrl: "https://history.paperclip.ai/", + }), + ).toThrow("safe non-empty key prefix"); expect(() => validateHistoryDestination({ bucket: "paperclip-runner-e2e-history", diff --git a/tests/runner-e2e/report.test.ts b/tests/runner-e2e/report.test.ts index 4ac6453cf0..f1e7b0e21f 100644 --- a/tests/runner-e2e/report.test.ts +++ b/tests/runner-e2e/report.test.ts @@ -150,6 +150,9 @@ describe("runner E2E report aggregation", () => { GITHUB_SERVER_URL: "https://github.com", GITHUB_REPOSITORY: "paperclipai/paperclip", GITHUB_RUN_ID: "123456", + PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: + "https://reports.example.test/", + PAPERCLIP_RUNNER_E2E_HISTORY_PREFIX: "/runner-e2e/", }, }, ); @@ -203,6 +206,9 @@ describe("runner E2E report aggregation", () => { expect(dashboard).toContain(" { expect(await readFile(path.join(output, "index.html"), "utf8")).toBe( dashboard, ); + const summary = await readFile(path.join(output, "summary.md"), "utf8"); + expect(summary).toContain("## View results"); + expect(summary).toContain( + "[Open the exact interactive campaign report](https://reports.example.test/runner-e2e/campaigns/gha-123456-1/index.html)", + ); + expect(summary).toContain( + "[Open the workflow run and per-cell job logs](https://github.com/paperclipai/paperclip/actions/runs/123456)", + ); + expect(summary).toContain( + "[Download the merged report and per-cell evidence](https://github.com/paperclipai/paperclip/actions/runs/123456#artifacts)", + ); + expect(summary).toContain( + `[core-compatibility.${executionId}](https://reports.example.test/runner-e2e/campaigns/gha-123456-1/index.html#execution-core-compatibility.${executionId})`, + ); }); it("prefers a valid rerun over a higher attempt number from an older campaign", async () => { diff --git a/tests/runner-e2e/report.ts b/tests/runner-e2e/report.ts index cc58fe73d9..3b61eb5d14 100644 --- a/tests/runner-e2e/report.ts +++ b/tests/runner-e2e/report.ts @@ -15,6 +15,7 @@ import { upgradeRunnerResult, } from "./history.js"; import { resolveRunnerE2ESource } from "./source.js"; +import { runnerE2ESummaryLinks } from "./summary-links.js"; import type { RunnerE2EResult } from "./types.js"; const repositoryRoot = path.resolve(import.meta.dirname, "../.."); @@ -296,11 +297,33 @@ async function main() { writeFile(path.join(output, "index.html"), dashboard, "utf8"), ]); + const summaryLinks = runnerE2ESummaryLinks({ + campaignId: normalized.campaignId, + workflowRunUrl: normalized.source.workflowRunUrl, + historyPublicBaseUrl: + process.env.PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL, + historyPrefix: process.env.PAPERCLIP_RUNNER_E2E_HISTORY_PREFIX, + }); + const publicCampaignUrl = summaryLinks.find( + (link) => link.kind === "campaign", + )?.url; + const summaryLines = [ "# Runner Full-Stack E2E", "", `Passed: ${normalized.passed}/${selected.length}`, "", + ...(summaryLinks.length > 0 + ? [ + "## View results", + "", + ...summaryLinks.map( + (link) => + `- [${link.label}](${link.url})${link.note ? ` — ${link.note}` : ""}`, + ), + "", + ] + : []), `Tokens: ${billing.llm.inputTokens} input / ${billing.llm.outputTokens} output / ${billing.llm.cachedInputTokens} cached`, "", `Provider-reported LLM cost: $${billing.reportedLlmCostUsd.toFixed(6)} (${billing.llm.runsWithReportedCost}/${billing.llm.runCount} runs priced)`, @@ -314,7 +337,10 @@ async function main() { const resolved = resolvedResults[index]!; const cellBilling = resolved.billing!; const runtimeCost = cellBilling.runtime.estimatedListCostUsd; - return `| ${resolved.executionId} | ${resolved.attempt} | ${entry.valid ? "pass" : "fail"} | ${resolved.runtimeMode} | ${Math.round(resolved.durationMs / 1000)}s | ${cellBilling.llm.inputTokens}/${cellBilling.llm.outputTokens} | $${cellBilling.reportedCostUsd.toFixed(6)} (${cellBilling.llm.costStatus}) | ${runtimeCost === undefined ? cellBilling.runtime.costStatus : `$${runtimeCost.toFixed(6)} est.`} | ${detail} |`; + const cell = publicCampaignUrl + ? `[${resolved.executionId}](${publicCampaignUrl}#execution-${encodeURIComponent(resolved.executionId)})` + : resolved.executionId; + return `| ${cell} | ${resolved.attempt} | ${entry.valid ? "pass" : "fail"} | ${resolved.runtimeMode} | ${Math.round(resolved.durationMs / 1000)}s | ${cellBilling.llm.inputTokens}/${cellBilling.llm.outputTokens} | $${cellBilling.reportedCostUsd.toFixed(6)} (${cellBilling.llm.costStatus}) | ${runtimeCost === undefined ? cellBilling.runtime.costStatus : `$${runtimeCost.toFixed(6)} est.`} | ${detail} |`; }), "", ]; diff --git a/tests/runner-e2e/summary-links.test.ts b/tests/runner-e2e/summary-links.test.ts new file mode 100644 index 0000000000..9ab4185137 --- /dev/null +++ b/tests/runner-e2e/summary-links.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from "vitest"; +import { runnerE2ESummaryLinks } from "./summary-links.js"; + +describe("runner E2E summary links", () => { + it("builds exact public, workflow, and artifact links", () => { + expect( + runnerE2ESummaryLinks({ + campaignId: "gha-34026735033-1", + workflowRunUrl: + "https://github.com/paperclipai/paperclip/actions/runs/34026735033", + historyPublicBaseUrl: "https://reports.example.test///", + historyPrefix: "/runner-e2e/", + }), + ).toEqual([ + { + kind: "campaign", + label: "Open the exact interactive campaign report", + url: "https://reports.example.test/runner-e2e/campaigns/gha-34026735033-1/index.html", + note: "available after the history publisher finishes", + }, + { + kind: "workflow", + label: "Open the workflow run and per-cell job logs", + url: "https://github.com/paperclipai/paperclip/actions/runs/34026735033", + }, + { + kind: "artifacts", + label: "Download the merged report and per-cell evidence", + url: "https://github.com/paperclipai/paperclip/actions/runs/34026735033#artifacts", + note: "GitHub access required; retained for 30 days", + }, + ]); + }); + + it("omits unsafe or incomplete destinations", () => { + expect( + runnerE2ESummaryLinks({ + campaignId: "../other-campaign", + workflowRunUrl: "https://token@example.test/actions/runs/1", + historyPublicBaseUrl: "http://reports.example.test", + historyPrefix: "runner-e2e", + }), + ).toEqual([]); + expect( + runnerE2ESummaryLinks({ + campaignId: "gha-1-1", + workflowRunUrl: null, + historyPublicBaseUrl: null, + historyPrefix: null, + }), + ).toEqual([]); + expect( + runnerE2ESummaryLinks({ + campaignId: "gha-1-1", + workflowRunUrl: null, + historyPublicBaseUrl: "https://reports.example.test", + historyPrefix: "runner-e2e#other", + }), + ).toEqual([]); + }); +}); diff --git a/tests/runner-e2e/summary-links.ts b/tests/runner-e2e/summary-links.ts new file mode 100644 index 0000000000..b69d5b8150 --- /dev/null +++ b/tests/runner-e2e/summary-links.ts @@ -0,0 +1,73 @@ +import { + runnerE2ECampaignPublicUrl, + validateHistoryPublicDestination, +} from "./history-destination.js"; + +export interface RunnerE2ESummaryLink { + kind: "campaign" | "workflow" | "artifacts"; + label: string; + url: string; + note?: string; +} + +function safeHttpsUrl(value: string | null | undefined) { + const input = value?.trim(); + if (!input) return null; + try { + const url = new URL(input); + if ( + url.protocol !== "https:" || + url.username || + url.password || + url.search || + url.hash + ) { + return null; + } + return url; + } catch { + return null; + } +} + +export function runnerE2ESummaryLinks(input: { + campaignId: string; + workflowRunUrl: string | null | undefined; + historyPublicBaseUrl: string | null | undefined; + historyPrefix: string | null | undefined; +}): RunnerE2ESummaryLink[] { + const links: RunnerE2ESummaryLink[] = []; + if (/^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$/.test(input.campaignId)) { + try { + const destination = validateHistoryPublicDestination({ + publicBaseUrl: input.historyPublicBaseUrl?.trim() ?? "", + prefix: input.historyPrefix?.trim() ?? "", + }); + links.push({ + kind: "campaign", + label: "Open the exact interactive campaign report", + url: runnerE2ECampaignPublicUrl(destination, input.campaignId), + note: "available after the history publisher finishes", + }); + } catch { + // Invalid or absent history configuration must not prevent report merging. + } + } + + const workflowRun = safeHttpsUrl(input.workflowRunUrl); + if (workflowRun) { + links.push({ + kind: "workflow", + label: "Open the workflow run and per-cell job logs", + url: workflowRun.href, + }); + workflowRun.hash = "artifacts"; + links.push({ + kind: "artifacts", + label: "Download the merged report and per-cell evidence", + url: workflowRun.href, + note: "GitHub access required; retained for 30 days", + }); + } + return links; +} diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts index 03a68f6af8..4dce7c374a 100644 --- a/tests/runner-e2e/workflow-security.test.ts +++ b/tests/runner-e2e/workflow-security.test.ts @@ -671,6 +671,12 @@ describe("public repository paid workflow security", () => { expect(report).toContain( "PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e", ); + expect(report).toContain( + "PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: ${{ vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL }}", + ); + expect(report).toContain( + "PAPERCLIP_RUNNER_E2E_HISTORY_PREFIX: ${{ vars.RUNNER_E2E_HISTORY_PREFIX || 'runner-e2e' }}", + ); expect( report.indexOf("Select latest workflow attempt per cell"), ).toBeLessThan(report.indexOf("Collect blob reports"));