From 9b231a37a3668737665e62ed5a1a9e759630b271 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 4 Sep 2026 01:58:05 -0500 Subject: [PATCH] ci(runner): preserve evidence across failed-job reruns --- .github/workflows/runner-full-stack-e2e.yml | 71 +++- .../runner-e2e/select-rerun-artifacts.test.ts | 297 +++++++++++++++++ tests/runner-e2e/select-rerun-artifacts.ts | 304 ++++++++++++++++++ tests/runner-e2e/workflow-security.test.ts | 46 +++ 4 files changed, 706 insertions(+), 12 deletions(-) create mode 100644 tests/runner-e2e/select-rerun-artifacts.test.ts create mode 100644 tests/runner-e2e/select-rerun-artifacts.ts diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.yml index 1e49688578..4a599b0a51 100644 --- a/.github/workflows/runner-full-stack-e2e.yml +++ b/.github/workflows/runner-full-stack-e2e.yml @@ -410,9 +410,11 @@ jobs: run: | set -euo pipefail if [ "$NEEDS_DAYTONA" != true ]; then - echo "image=" >> "$GITHUB_OUTPUT" - echo "source_revision=" >> "$GITHUB_OUTPUT" - echo "content_id=" >> "$GITHUB_OUTPUT" + { + echo "image=" + echo "source_revision=" + echo "content_id=" + } >> "$GITHUB_OUTPUT" exit 0 fi [[ "$IMAGE_CONTENT_ID" =~ ^[0-9a-f]{64}$ ]] @@ -456,9 +458,11 @@ jobs: .config.User == "daytona" and (.config.Env | any(startswith("PAPERCLIP_RUNNER_PROVIDER_PACK_ROOT=")))' \ <<< "$image_config" >/dev/null - echo "image=$immutable" >> "$GITHUB_OUTPUT" - echo "source_revision=$source_revision" >> "$GITHUB_OUTPUT" - echo "content_id=$published_content_id" >> "$GITHUB_OUTPUT" + { + echo "image=$immutable" + echo "source_revision=$source_revision" + echo "content_id=$published_content_id" + } >> "$GITHUB_OUTPUT" build_runner_artifacts: name: Build reusable runner campaign artifacts @@ -951,6 +955,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 permissions: + actions: read contents: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -978,22 +983,52 @@ jobs: - run: pnpm install --frozen-lockfile + - name: Resolve workflow job attempts + env: + GH_TOKEN: ${{ github.token }} + REPOSITORY: ${{ github.repository }} + RUN_ID: ${{ github.run_id }} + run: | + set -euo pipefail + gh api --paginate --slurp \ + "repos/$REPOSITORY/actions/runs/$RUN_ID/jobs?filter=all&per_page=100" \ + > runner-e2e-job-pages.json + for attempt in $(seq 1 "${{ github.run_attempt }}"); do + gh api "repos/$REPOSITORY/actions/runs/$RUN_ID/attempts/$attempt" \ + --jq '{run_attempt, run_started_at}' + done > runner-e2e-attempts.jsonl + jq -s '.' runner-e2e-attempts.jsonl > runner-e2e-attempts.json + jq --slurpfile attempts runner-e2e-attempts.json \ + '{jobs: [.[].jobs[]], attempts: $attempts[0]}' \ + runner-e2e-job-pages.json > runner-e2e-jobs.json + - name: Download cell evidence id: download_evidence continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - pattern: runner-e2e-${{ github.run_id }}-${{ github.run_attempt }}-* + pattern: runner-e2e-${{ github.run_id }}-*-* path: downloaded-runner-e2e - merge-multiple: true + merge-multiple: false - name: Retry cell evidence download after transport failure if: steps.download_evidence.outcome == 'failure' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - pattern: runner-e2e-${{ github.run_id }}-${{ github.run_attempt }}-* + pattern: runner-e2e-${{ github.run_id }}-*-* path: downloaded-runner-e2e - merge-multiple: true + merge-multiple: false + + - name: Select latest workflow attempt per cell + if: always() + env: + PAPERCLIP_RUNNER_E2E_ARTIFACT_ROOT: ${{ github.workspace }}/downloaded-runner-e2e + PAPERCLIP_RUNNER_E2E_SELECTED_ROOT: ${{ github.workspace }}/selected-runner-e2e + PAPERCLIP_RUNNER_E2E_JOBS_JSON: ${{ github.workspace }}/runner-e2e-jobs.json + PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: ${{ needs.catalog.outputs.execution_ids }} + PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} + PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }} + run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/select-rerun-artifacts.ts - name: Collect blob reports run: | @@ -1005,7 +1040,7 @@ jobs: if [ ! -e "$target" ]; then cp "$report" "$target" fi - done < <(find downloaded-runner-e2e -path '*/blob-report/*.zip' -print0) + done < <(find selected-runner-e2e -path '*/blob-report/*.zip' -print0) - name: Merge Playwright HTML and JUnit if: always() @@ -1016,7 +1051,7 @@ jobs: - name: Aggregate normalized campaign results if: always() env: - PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/downloaded-runner-e2e + PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e PAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalized PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: ${{ needs.catalog.outputs.execution_ids }} PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-${{ github.run_attempt }} @@ -1058,6 +1093,8 @@ jobs: if: always() && needs.catalog.result == 'success' && needs.report.outputs.history_source_ready == 'true' runs-on: ubuntu-latest timeout-minutes: 15 + outputs: + pages_artifact_name: ${{ steps.pages_artifact_name.outputs.name }} concurrency: group: runner-e2e-history-publish cancel-in-progress: false @@ -1107,10 +1144,16 @@ jobs: RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: ${{ vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL }} run: pnpm test:e2e:runner:history:publish + - name: Resolve Pages artifact name + id: pages_artifact_name + if: vars.RUNNER_FULL_STACK_E2E_PUBLISH_PAGES == 'true' + run: echo "name=github-pages-${{ github.run_id }}-${{ github.run_attempt }}" >> "$GITHUB_OUTPUT" + - name: Package pruned structured dashboard for GitHub Pages if: vars.RUNNER_FULL_STACK_E2E_PUBLISH_PAGES == 'true' uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4 with: + name: ${{ steps.pages_artifact_name.outputs.name }} path: runner-e2e-merged-report/normalized pages: @@ -1128,3 +1171,7 @@ jobs: - name: Deploy to GitHub Pages id: deployment uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4 + with: + # If only this failed job is rerun, GitHub retains the successful + # publisher job's output from the earlier workflow attempt. + artifact_name: ${{ needs.publish_history.outputs.pages_artifact_name }} diff --git a/tests/runner-e2e/select-rerun-artifacts.test.ts b/tests/runner-e2e/select-rerun-artifacts.test.ts new file mode 100644 index 0000000000..baba8e0d82 --- /dev/null +++ b/tests/runner-e2e/select-rerun-artifacts.test.ts @@ -0,0 +1,297 @@ +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { selectRerunArtifacts } from "./select-rerun-artifacts.js"; +import type { RunnerE2EResult } from "./types.js"; + +const RUN_ID = "33843305626"; +const SOURCE_SHA = "0123456789abcdef0123456789abcdef01234567"; +const SOURCE_REF = "refs/heads/fix/runner-paid-matrix-integrity-v2"; +const WORKFLOW_RUN_URL = + "https://github.com/paperclipai/paperclip/actions/runs/33843305626"; +const RETAINED = "core-compatibility.legacy-codex.local.message-marker"; +const RERUN = "core-compatibility.runner-codex.local.plan-revise-accept"; +const cleanupDirectories: string[] = []; + +afterEach(async () => { + await Promise.all( + cleanupDirectories + .splice(0) + .map((directory) => rm(directory, { recursive: true, force: true })), + ); +}); + +function result(executionId: string, status: "passed" | "failed") { + const [suiteId, profileId, environmentId, caseId] = executionId.split("."); + return { + schema: "paperclip.runner-e2e.result/v2", + executionId, + suiteId, + source: { + sha: SOURCE_SHA, + ref: SOURCE_REF, + workflowRunUrl: WORKFLOW_RUN_URL, + }, + attempt: 1, + status, + ...(status === "failed" + ? { failureClass: "candidate_failure" as const, error: "failed" } + : {}), + profileId: profileId!, + environmentId: environmentId as RunnerE2EResult["environmentId"], + caseId: caseId!, + provider: "codex", + model: "fixture-model", + runtimeMode: "native", + startedAt: "2026-09-04T00:00:00.000Z", + finishedAt: "2026-09-04T00:00:01.000Z", + durationMs: 1_000, + cleanup: status === "passed" ? "passed" : "not_started", + } satisfies RunnerE2EResult; +} + +async function addArtifact(input: { + root: string; + executionId: string; + workflowAttempt: number; + status: "passed" | "failed"; + sourceSha?: string; + campaignName?: string; +}) { + const artifactName = `runner-e2e-${RUN_ID}-${input.workflowAttempt}-${input.executionId}`; + const campaignName = + input.campaignName ?? + `gha-${RUN_ID}-${input.workflowAttempt}-${input.executionId}`; + const directory = path.join( + input.root, + artifactName, + campaignName, + "results", + "attempt-1", + ); + await mkdir(directory, { recursive: true }); + const value = result(input.executionId, input.status); + await writeFile( + path.join(directory, "result.json"), + JSON.stringify({ + ...value, + source: { ...value.source, sha: input.sourceSha ?? value.source.sha }, + }), + ); + return { artifactName, campaignName }; +} + +async function fixture() { + const root = await mkdtemp( + path.join(os.tmpdir(), "runner-e2e-rerun-artifacts-"), + ); + cleanupDirectories.push(root); + return { + root, + artifactRoot: path.join(root, "downloaded"), + selectedRoot: path.join(root, "selected"), + }; +} + +function selectionInput(paths: Awaited>) { + return { + ...paths, + jobs: { + jobs: [ + { + name: RETAINED, + run_attempt: 1, + started_at: "2026-09-04T00:00:01.000Z", + }, + { + name: RERUN, + run_attempt: 1, + started_at: "2026-09-04T00:00:02.000Z", + }, + // filter=all synthesizes this attempt-2 row even though GitHub retained + // the successful attempt-1 job. Its original start time exposes it. + { + name: RETAINED, + run_attempt: 2, + started_at: "2026-09-04T00:00:01.000Z", + }, + { + name: RERUN, + run_attempt: 2, + started_at: "2026-09-04T01:00:01.000Z", + }, + ], + attempts: [ + { + run_attempt: 1, + run_started_at: "2026-09-04T00:00:00.000Z", + }, + { + run_attempt: 2, + run_started_at: "2026-09-04T01:00:00.000Z", + }, + ], + }, + expectedExecutionIds: [RETAINED, RERUN], + workflowRunId: RUN_ID, + workflowRunAttempt: 2, + sourceSha: SOURCE_SHA, + sourceRef: SOURCE_REF, + workflowRunUrl: WORKFLOW_RUN_URL, + }; +} + +describe("runner E2E workflow rerun artifact selection", () => { + it("combines retained successes with the latest rerun artifact", async () => { + const paths = await fixture(); + await addArtifact({ + root: paths.artifactRoot, + executionId: RETAINED, + workflowAttempt: 1, + status: "passed", + }); + await addArtifact({ + root: paths.artifactRoot, + executionId: RERUN, + workflowAttempt: 1, + status: "failed", + }); + const latest = await addArtifact({ + root: paths.artifactRoot, + executionId: RERUN, + workflowAttempt: 2, + status: "passed", + }); + + const selected = await selectRerunArtifacts(selectionInput(paths)); + + expect(selected).toEqual([ + { + executionId: RETAINED, + workflowAttempt: 1, + artifactName: `runner-e2e-${RUN_ID}-1-${RETAINED}`, + }, + { + executionId: RERUN, + workflowAttempt: 2, + artifactName: latest.artifactName, + }, + ]); + const selectedResult = JSON.parse( + await readFile( + path.join( + paths.selectedRoot, + latest.artifactName, + latest.campaignName, + "results", + "attempt-1", + "result.json", + ), + "utf8", + ), + ); + expect(selectedResult.status).toBe("passed"); + }); + + it("never falls back when the latest workflow attempt has no artifact", async () => { + const paths = await fixture(); + await addArtifact({ + root: paths.artifactRoot, + executionId: RETAINED, + workflowAttempt: 1, + status: "passed", + }); + await addArtifact({ + root: paths.artifactRoot, + executionId: RERUN, + workflowAttempt: 1, + status: "passed", + }); + + const selected = await selectRerunArtifacts(selectionInput(paths)); + + expect(selected.map((entry) => entry.executionId)).toEqual([RETAINED]); + }); + + it("does not let an older pass mask a latest failed artifact", async () => { + const paths = await fixture(); + await addArtifact({ + root: paths.artifactRoot, + executionId: RETAINED, + workflowAttempt: 1, + status: "passed", + }); + await addArtifact({ + root: paths.artifactRoot, + executionId: RERUN, + workflowAttempt: 1, + status: "passed", + }); + const latest = await addArtifact({ + root: paths.artifactRoot, + executionId: RERUN, + workflowAttempt: 2, + status: "failed", + }); + + await selectRerunArtifacts(selectionInput(paths)); + + const selectedResult = JSON.parse( + await readFile( + path.join( + paths.selectedRoot, + latest.artifactName, + latest.campaignName, + "results", + "attempt-1", + "result.json", + ), + "utf8", + ), + ); + expect(selectedResult.status).toBe("failed"); + }); + + it("rejects artifacts from another source", async () => { + const paths = await fixture(); + await addArtifact({ + root: paths.artifactRoot, + executionId: RETAINED, + workflowAttempt: 1, + status: "passed", + sourceSha: "ffffffffffffffffffffffffffffffffffffffff", + }); + await addArtifact({ + root: paths.artifactRoot, + executionId: RERUN, + workflowAttempt: 2, + status: "passed", + }); + + await expect(selectRerunArtifacts(selectionInput(paths))).rejects.toThrow( + "contains result from another source", + ); + }); + + it("rejects an artifact carrying another campaign", async () => { + const paths = await fixture(); + await addArtifact({ + root: paths.artifactRoot, + executionId: RETAINED, + workflowAttempt: 1, + status: "passed", + }); + await addArtifact({ + root: paths.artifactRoot, + executionId: RERUN, + workflowAttempt: 2, + status: "passed", + campaignName: `gha-another-run-2-${RERUN}`, + }); + + await expect(selectRerunArtifacts(selectionInput(paths))).rejects.toThrow( + /must contain only its exact campaign/u, + ); + }); +}); diff --git a/tests/runner-e2e/select-rerun-artifacts.ts b/tests/runner-e2e/select-rerun-artifacts.ts new file mode 100644 index 0000000000..af5294fd09 --- /dev/null +++ b/tests/runner-e2e/select-rerun-artifacts.ts @@ -0,0 +1,304 @@ +import { cp, mkdir, readFile, readdir } from "node:fs/promises"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import type { RunnerE2EResult } from "./types.js"; + +interface WorkflowJob { + name: string; + run_attempt: number; + started_at: string; +} + +interface WorkflowJobsResponse { + jobs: WorkflowJob[]; + attempts: Array<{ + run_attempt: number; + run_started_at: string; + }>; +} + +export interface SelectRerunArtifactsInput { + artifactRoot: string; + selectedRoot: string; + jobs: WorkflowJobsResponse; + expectedExecutionIds: readonly string[]; + workflowRunId: string; + workflowRunAttempt: number; + sourceSha: string; + sourceRef: string; + workflowRunUrl: string; +} + +function safeIdentifier(value: string, label: string) { + if (!value || !/^[A-Za-z0-9_.-]+$/u.test(value)) { + throw new Error( + `${label} contains unsafe characters: ${JSON.stringify(value)}`, + ); + } + return value; +} + +function positiveInteger(value: unknown, label: string) { + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 1) { + throw new Error(`${label} must be a positive integer`); + } + return value; +} + +function timestamp(value: unknown, label: string) { + if (typeof value !== "string" || !Number.isFinite(Date.parse(value))) { + throw new Error(`${label} must be an ISO timestamp`); + } + return Date.parse(value); +} + +async function walk(root: string): Promise { + const entries = await readdir(root, { withFileTypes: true }); + const files: string[] = []; + for (const entry of entries) { + const full = path.join(root, entry.name); + if (entry.isDirectory()) files.push(...(await walk(full))); + else if (entry.isFile()) files.push(full); + } + return files; +} + +/** + * Selects the artifact produced by the latest workflow job for each execution. + * GitHub reruns retain earlier-attempt artifacts, so validity or result + * timestamps must never be used to choose between workflow attempts. + */ +export async function selectRerunArtifacts(input: SelectRerunArtifactsInput) { + const runId = safeIdentifier(input.workflowRunId, "workflow run ID"); + const currentAttempt = positiveInteger( + input.workflowRunAttempt, + "workflow run attempt", + ); + const expected = input.expectedExecutionIds.map((executionId) => + safeIdentifier(executionId, "execution ID"), + ); + if (expected.length === 0 || new Set(expected).size !== expected.length) { + throw new Error("expected execution IDs must be non-empty and unique"); + } + const preexistingSelections = await readdir(input.selectedRoot).catch( + (error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return []; + throw error; + }, + ); + if (preexistingSelections.length > 0) { + throw new Error("selected artifact root must start empty"); + } + const expectedSet = new Set(expected); + const attemptStartedAt = new Map(); + for (const attemptMetadata of input.jobs.attempts) { + const attempt = positiveInteger( + attemptMetadata.run_attempt, + "workflow metadata attempt", + ); + if (attempt > currentAttempt || attemptStartedAt.has(attempt)) { + throw new Error(`invalid workflow metadata for attempt ${attempt}`); + } + attemptStartedAt.set( + attempt, + timestamp( + attemptMetadata.run_started_at, + `workflow attempt ${attempt} start`, + ), + ); + } + for (let attempt = 1; attempt <= currentAttempt; attempt += 1) { + if (!attemptStartedAt.has(attempt)) { + throw new Error(`workflow metadata omitted attempt ${attempt}`); + } + } + const attemptsByExecution = new Map>(); + for (const candidate of input.jobs.jobs) { + if (!expectedSet.has(candidate.name)) continue; + const attempt = positiveInteger(candidate.run_attempt, "job run attempt"); + if (attempt > currentAttempt) { + throw new Error( + `job ${candidate.name} claims future workflow attempt ${attempt}`, + ); + } + const jobStartedAt = timestamp( + candidate.started_at, + `job ${candidate.name} start`, + ); + // GitHub's filter=all response synthesizes current-attempt rows for jobs + // retained from an earlier attempt. Their started_at remains before the + // current attempt's trusted run_started_at, so they are not real reruns. + if (jobStartedAt < attemptStartedAt.get(attempt)!) continue; + const attempts = attemptsByExecution.get(candidate.name) ?? new Map(); + if (attempts.has(attempt)) { + throw new Error( + `workflow attempt ${attempt} contains duplicate job ${candidate.name}`, + ); + } + attempts.set(attempt, candidate); + attemptsByExecution.set(candidate.name, attempts); + } + + const latestAttemptByExecution = new Map(); + for (const executionId of expected) { + const attempts = [...(attemptsByExecution.get(executionId)?.keys() ?? [])]; + if (attempts.length === 0) { + throw new Error( + `workflow job history omitted expected execution ${executionId}`, + ); + } + latestAttemptByExecution.set(executionId, Math.max(...attempts)); + } + + const recognizedArtifactNames = new Map< + string, + { executionId: string; workflowAttempt: number } + >(); + for (const executionId of expected) { + for (const workflowAttempt of attemptsByExecution + .get(executionId)! + .keys()) { + recognizedArtifactNames.set( + `runner-e2e-${runId}-${workflowAttempt}-${executionId}`, + { executionId, workflowAttempt }, + ); + } + } + + const artifactEntries = await readdir(input.artifactRoot, { + withFileTypes: true, + }).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return []; + throw error; + }); + const artifactDirectories = new Map(); + for (const entry of artifactEntries) { + const identity = recognizedArtifactNames.get(entry.name); + if (!identity || !entry.isDirectory()) { + throw new Error(`downloaded unexpected runner artifact ${entry.name}`); + } + artifactDirectories.set( + entry.name, + path.join(input.artifactRoot, entry.name), + ); + } + + const selections: Array<{ + executionId: string; + workflowAttempt: number; + artifactName: string; + }> = []; + for (const executionId of expected) { + const workflowAttempt = latestAttemptByExecution.get(executionId)!; + const artifactName = `runner-e2e-${runId}-${workflowAttempt}-${executionId}`; + const artifactDirectory = artifactDirectories.get(artifactName); + // A latest job without an artifact must remain missing. Falling back to an + // older successful artifact would mask an infrastructure/upload failure. + if (!artifactDirectory) continue; + + const campaignName = `gha-${runId}-${workflowAttempt}-${executionId}`; + const topLevelEntries = await readdir(artifactDirectory, { + withFileTypes: true, + }); + if ( + topLevelEntries.length !== 1 || + topLevelEntries[0]?.name !== campaignName || + !topLevelEntries[0].isDirectory() + ) { + throw new Error( + `${artifactName} must contain only its exact campaign ${campaignName}`, + ); + } + const campaignDirectory = path.join(artifactDirectory, campaignName); + const resultFiles = (await walk(campaignDirectory)).filter( + (file) => path.basename(file) === "result.json", + ); + if (resultFiles.length === 0) { + throw new Error(`${artifactName} contains no normalized result`); + } + for (const resultFile of resultFiles) { + const result = JSON.parse( + await readFile(resultFile, "utf8"), + ) as RunnerE2EResult; + if (result.executionId !== executionId) { + throw new Error( + `${artifactName} contains result for ${String(result.executionId)}`, + ); + } + const source = result.source; + if ( + !source || + source.sha !== input.sourceSha || + source.ref !== input.sourceRef || + source.workflowRunUrl !== input.workflowRunUrl + ) { + throw new Error(`${artifactName} contains result from another source`); + } + } + const destination = path.join( + input.selectedRoot, + artifactName, + campaignName, + ); + await mkdir(path.dirname(destination), { recursive: true }); + await cp(campaignDirectory, destination, { + recursive: true, + force: false, + errorOnExist: true, + }); + selections.push({ executionId, workflowAttempt, artifactName }); + } + return selections; +} + +async function main() { + const required = (name: string) => { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`${name} is required`); + return value; + }; + const expected = JSON.parse( + required("PAPERCLIP_RUNNER_E2E_EXPECTED_IDS"), + ) as unknown; + if ( + !Array.isArray(expected) || + expected.some((value) => typeof value !== "string") + ) { + throw new Error( + "PAPERCLIP_RUNNER_E2E_EXPECTED_IDS must be a JSON string array", + ); + } + const jobs = JSON.parse( + await readFile(required("PAPERCLIP_RUNNER_E2E_JOBS_JSON"), "utf8"), + ) as WorkflowJobsResponse; + if (!jobs || !Array.isArray(jobs.jobs) || !Array.isArray(jobs.attempts)) { + throw new Error("workflow jobs JSON must contain jobs and attempts arrays"); + } + const runId = required("GITHUB_RUN_ID"); + const serverUrl = required("GITHUB_SERVER_URL"); + const repository = required("GITHUB_REPOSITORY"); + const selections = await selectRerunArtifacts({ + artifactRoot: required("PAPERCLIP_RUNNER_E2E_ARTIFACT_ROOT"), + selectedRoot: required("PAPERCLIP_RUNNER_E2E_SELECTED_ROOT"), + jobs, + expectedExecutionIds: expected, + workflowRunId: runId, + workflowRunAttempt: Number(required("GITHUB_RUN_ATTEMPT")), + sourceSha: required("PAPERCLIP_RUNNER_E2E_SOURCE_SHA"), + sourceRef: required("PAPERCLIP_RUNNER_E2E_SOURCE_REF"), + workflowRunUrl: `${serverUrl}/${repository}/actions/runs/${runId}`, + }); + console.log( + `Selected ${selections.length}/${expected.length} latest cell artifacts`, + ); +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href +) { + await main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts index 240a59cc5d..d73194e5f4 100644 --- a/tests/runner-e2e/workflow-security.test.ts +++ b/tests/runner-e2e/workflow-security.test.ts @@ -545,6 +545,52 @@ describe("public repository paid workflow security", () => { expect(testJob).not.toContain("build-provider-pack.mjs"); }); + it("binds rerun evidence and Pages artifacts to the exact workflow attempt", async () => { + const workflow = await readFile( + path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"), + "utf8", + ); + const reportStart = workflow.indexOf(" report:"); + const publisherStart = workflow.indexOf(" publish_history:", reportStart); + const report = workflow.slice(reportStart, publisherStart); + const publisher = workflow.slice(publisherStart); + + expect(report).toContain("actions: read"); + expect(report).toContain( + '"repos/$REPOSITORY/actions/runs/$RUN_ID/jobs?filter=all&per_page=100"', + ); + expect(report).toContain("gh api --paginate --slurp"); + expect(report).toContain( + '"repos/$REPOSITORY/actions/runs/$RUN_ID/attempts/$attempt"', + ); + expect(report).toContain("--jq '{run_attempt, run_started_at}'"); + expect(report).toContain("pattern: runner-e2e-${{ github.run_id }}-*-*"); + expect(report).not.toContain( + "pattern: runner-e2e-${{ github.run_id }}-${{ github.run_attempt }}-*", + ); + expect(report.match(/merge-multiple: false/g)).toHaveLength(2); + expect(report).toContain("Select latest workflow attempt per cell"); + expect(report).toContain("tests/runner-e2e/select-rerun-artifacts.ts"); + expect(report).toContain( + "PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e", + ); + expect( + report.indexOf("Select latest workflow attempt per cell"), + ).toBeLessThan(report.indexOf("Collect blob reports")); + expect(publisher).toContain( + 'echo "name=github-pages-${{ github.run_id }}-${{ github.run_attempt }}"', + ); + expect(publisher).toContain( + "pages_artifact_name: ${{ steps.pages_artifact_name.outputs.name }}", + ); + expect(publisher).toContain( + "name: ${{ steps.pages_artifact_name.outputs.name }}", + ); + expect(publisher).toContain( + "artifact_name: ${{ needs.publish_history.outputs.pages_artifact_name }}", + ); + }); + it("uses environment-scoped OIDC for a no-delete history publisher", async () => { const workflow = await readFile( path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"),