ci(runner): preserve evidence across failed-job reruns
This commit is contained in:
Dotta
parent
f90e475c97
commit
9b231a37a3
|
|
@ -410,9 +410,11 @@ jobs:
|
|||
run: |
|
||||
set -euo pipefail
|
||||
if [ "$NEEDS_DAYTONA" != true ]; then
|
||||
echo "image=" >> "$GITHUB_OUTPUT"
|
||||
echo "source_revision=" >> "$GITHUB_OUTPUT"
|
||||
echo "content_id=" >> "$GITHUB_OUTPUT"
|
||||
{
|
||||
echo "image="
|
||||
echo "source_revision="
|
||||
echo "content_id="
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
[[ "$IMAGE_CONTENT_ID" =~ ^[0-9a-f]{64}$ ]]
|
||||
|
|
@ -456,9 +458,11 @@ jobs:
|
|||
.config.User == "daytona" and
|
||||
(.config.Env | any(startswith("PAPERCLIP_RUNNER_PROVIDER_PACK_ROOT=")))' \
|
||||
<<< "$image_config" >/dev/null
|
||||
echo "image=$immutable" >> "$GITHUB_OUTPUT"
|
||||
echo "source_revision=$source_revision" >> "$GITHUB_OUTPUT"
|
||||
echo "content_id=$published_content_id" >> "$GITHUB_OUTPUT"
|
||||
{
|
||||
echo "image=$immutable"
|
||||
echo "source_revision=$source_revision"
|
||||
echo "content_id=$published_content_id"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
build_runner_artifacts:
|
||||
name: Build reusable runner campaign artifacts
|
||||
|
|
@ -951,6 +955,7 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
|
|
@ -978,22 +983,52 @@ jobs:
|
|||
|
||||
- run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Resolve workflow job attempts
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
REPOSITORY: ${{ github.repository }}
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
gh api --paginate --slurp \
|
||||
"repos/$REPOSITORY/actions/runs/$RUN_ID/jobs?filter=all&per_page=100" \
|
||||
> runner-e2e-job-pages.json
|
||||
for attempt in $(seq 1 "${{ github.run_attempt }}"); do
|
||||
gh api "repos/$REPOSITORY/actions/runs/$RUN_ID/attempts/$attempt" \
|
||||
--jq '{run_attempt, run_started_at}'
|
||||
done > runner-e2e-attempts.jsonl
|
||||
jq -s '.' runner-e2e-attempts.jsonl > runner-e2e-attempts.json
|
||||
jq --slurpfile attempts runner-e2e-attempts.json \
|
||||
'{jobs: [.[].jobs[]], attempts: $attempts[0]}' \
|
||||
runner-e2e-job-pages.json > runner-e2e-jobs.json
|
||||
|
||||
- name: Download cell evidence
|
||||
id: download_evidence
|
||||
continue-on-error: true
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
||||
with:
|
||||
pattern: runner-e2e-${{ github.run_id }}-${{ github.run_attempt }}-*
|
||||
pattern: runner-e2e-${{ github.run_id }}-*-*
|
||||
path: downloaded-runner-e2e
|
||||
merge-multiple: true
|
||||
merge-multiple: false
|
||||
|
||||
- name: Retry cell evidence download after transport failure
|
||||
if: steps.download_evidence.outcome == 'failure'
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
||||
with:
|
||||
pattern: runner-e2e-${{ github.run_id }}-${{ github.run_attempt }}-*
|
||||
pattern: runner-e2e-${{ github.run_id }}-*-*
|
||||
path: downloaded-runner-e2e
|
||||
merge-multiple: true
|
||||
merge-multiple: false
|
||||
|
||||
- name: Select latest workflow attempt per cell
|
||||
if: always()
|
||||
env:
|
||||
PAPERCLIP_RUNNER_E2E_ARTIFACT_ROOT: ${{ github.workspace }}/downloaded-runner-e2e
|
||||
PAPERCLIP_RUNNER_E2E_SELECTED_ROOT: ${{ github.workspace }}/selected-runner-e2e
|
||||
PAPERCLIP_RUNNER_E2E_JOBS_JSON: ${{ github.workspace }}/runner-e2e-jobs.json
|
||||
PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: ${{ needs.catalog.outputs.execution_ids }}
|
||||
PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}
|
||||
PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }}
|
||||
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/select-rerun-artifacts.ts
|
||||
|
||||
- name: Collect blob reports
|
||||
run: |
|
||||
|
|
@ -1005,7 +1040,7 @@ jobs:
|
|||
if [ ! -e "$target" ]; then
|
||||
cp "$report" "$target"
|
||||
fi
|
||||
done < <(find downloaded-runner-e2e -path '*/blob-report/*.zip' -print0)
|
||||
done < <(find selected-runner-e2e -path '*/blob-report/*.zip' -print0)
|
||||
|
||||
- name: Merge Playwright HTML and JUnit
|
||||
if: always()
|
||||
|
|
@ -1016,7 +1051,7 @@ jobs:
|
|||
- name: Aggregate normalized campaign results
|
||||
if: always()
|
||||
env:
|
||||
PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/downloaded-runner-e2e
|
||||
PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e
|
||||
PAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalized
|
||||
PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: ${{ needs.catalog.outputs.execution_ids }}
|
||||
PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
|
|
@ -1058,6 +1093,8 @@ jobs:
|
|||
if: always() && needs.catalog.result == 'success' && needs.report.outputs.history_source_ready == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
outputs:
|
||||
pages_artifact_name: ${{ steps.pages_artifact_name.outputs.name }}
|
||||
concurrency:
|
||||
group: runner-e2e-history-publish
|
||||
cancel-in-progress: false
|
||||
|
|
@ -1107,10 +1144,16 @@ jobs:
|
|||
RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: ${{ vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL }}
|
||||
run: pnpm test:e2e:runner:history:publish
|
||||
|
||||
- name: Resolve Pages artifact name
|
||||
id: pages_artifact_name
|
||||
if: vars.RUNNER_FULL_STACK_E2E_PUBLISH_PAGES == 'true'
|
||||
run: echo "name=github-pages-${{ github.run_id }}-${{ github.run_attempt }}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Package pruned structured dashboard for GitHub Pages
|
||||
if: vars.RUNNER_FULL_STACK_E2E_PUBLISH_PAGES == 'true'
|
||||
uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4
|
||||
with:
|
||||
name: ${{ steps.pages_artifact_name.outputs.name }}
|
||||
path: runner-e2e-merged-report/normalized
|
||||
|
||||
pages:
|
||||
|
|
@ -1128,3 +1171,7 @@ jobs:
|
|||
- name: Deploy to GitHub Pages
|
||||
id: deployment
|
||||
uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4
|
||||
with:
|
||||
# If only this failed job is rerun, GitHub retains the successful
|
||||
# publisher job's output from the earlier workflow attempt.
|
||||
artifact_name: ${{ needs.publish_history.outputs.pages_artifact_name }}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,297 @@
|
|||
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { selectRerunArtifacts } from "./select-rerun-artifacts.js";
|
||||
import type { RunnerE2EResult } from "./types.js";
|
||||
|
||||
const RUN_ID = "33843305626";
|
||||
const SOURCE_SHA = "0123456789abcdef0123456789abcdef01234567";
|
||||
const SOURCE_REF = "refs/heads/fix/runner-paid-matrix-integrity-v2";
|
||||
const WORKFLOW_RUN_URL =
|
||||
"https://github.com/paperclipai/paperclip/actions/runs/33843305626";
|
||||
const RETAINED = "core-compatibility.legacy-codex.local.message-marker";
|
||||
const RERUN = "core-compatibility.runner-codex.local.plan-revise-accept";
|
||||
const cleanupDirectories: string[] = [];
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
cleanupDirectories
|
||||
.splice(0)
|
||||
.map((directory) => rm(directory, { recursive: true, force: true })),
|
||||
);
|
||||
});
|
||||
|
||||
function result(executionId: string, status: "passed" | "failed") {
|
||||
const [suiteId, profileId, environmentId, caseId] = executionId.split(".");
|
||||
return {
|
||||
schema: "paperclip.runner-e2e.result/v2",
|
||||
executionId,
|
||||
suiteId,
|
||||
source: {
|
||||
sha: SOURCE_SHA,
|
||||
ref: SOURCE_REF,
|
||||
workflowRunUrl: WORKFLOW_RUN_URL,
|
||||
},
|
||||
attempt: 1,
|
||||
status,
|
||||
...(status === "failed"
|
||||
? { failureClass: "candidate_failure" as const, error: "failed" }
|
||||
: {}),
|
||||
profileId: profileId!,
|
||||
environmentId: environmentId as RunnerE2EResult["environmentId"],
|
||||
caseId: caseId!,
|
||||
provider: "codex",
|
||||
model: "fixture-model",
|
||||
runtimeMode: "native",
|
||||
startedAt: "2026-09-04T00:00:00.000Z",
|
||||
finishedAt: "2026-09-04T00:00:01.000Z",
|
||||
durationMs: 1_000,
|
||||
cleanup: status === "passed" ? "passed" : "not_started",
|
||||
} satisfies RunnerE2EResult;
|
||||
}
|
||||
|
||||
async function addArtifact(input: {
|
||||
root: string;
|
||||
executionId: string;
|
||||
workflowAttempt: number;
|
||||
status: "passed" | "failed";
|
||||
sourceSha?: string;
|
||||
campaignName?: string;
|
||||
}) {
|
||||
const artifactName = `runner-e2e-${RUN_ID}-${input.workflowAttempt}-${input.executionId}`;
|
||||
const campaignName =
|
||||
input.campaignName ??
|
||||
`gha-${RUN_ID}-${input.workflowAttempt}-${input.executionId}`;
|
||||
const directory = path.join(
|
||||
input.root,
|
||||
artifactName,
|
||||
campaignName,
|
||||
"results",
|
||||
"attempt-1",
|
||||
);
|
||||
await mkdir(directory, { recursive: true });
|
||||
const value = result(input.executionId, input.status);
|
||||
await writeFile(
|
||||
path.join(directory, "result.json"),
|
||||
JSON.stringify({
|
||||
...value,
|
||||
source: { ...value.source, sha: input.sourceSha ?? value.source.sha },
|
||||
}),
|
||||
);
|
||||
return { artifactName, campaignName };
|
||||
}
|
||||
|
||||
async function fixture() {
|
||||
const root = await mkdtemp(
|
||||
path.join(os.tmpdir(), "runner-e2e-rerun-artifacts-"),
|
||||
);
|
||||
cleanupDirectories.push(root);
|
||||
return {
|
||||
root,
|
||||
artifactRoot: path.join(root, "downloaded"),
|
||||
selectedRoot: path.join(root, "selected"),
|
||||
};
|
||||
}
|
||||
|
||||
function selectionInput(paths: Awaited<ReturnType<typeof fixture>>) {
|
||||
return {
|
||||
...paths,
|
||||
jobs: {
|
||||
jobs: [
|
||||
{
|
||||
name: RETAINED,
|
||||
run_attempt: 1,
|
||||
started_at: "2026-09-04T00:00:01.000Z",
|
||||
},
|
||||
{
|
||||
name: RERUN,
|
||||
run_attempt: 1,
|
||||
started_at: "2026-09-04T00:00:02.000Z",
|
||||
},
|
||||
// filter=all synthesizes this attempt-2 row even though GitHub retained
|
||||
// the successful attempt-1 job. Its original start time exposes it.
|
||||
{
|
||||
name: RETAINED,
|
||||
run_attempt: 2,
|
||||
started_at: "2026-09-04T00:00:01.000Z",
|
||||
},
|
||||
{
|
||||
name: RERUN,
|
||||
run_attempt: 2,
|
||||
started_at: "2026-09-04T01:00:01.000Z",
|
||||
},
|
||||
],
|
||||
attempts: [
|
||||
{
|
||||
run_attempt: 1,
|
||||
run_started_at: "2026-09-04T00:00:00.000Z",
|
||||
},
|
||||
{
|
||||
run_attempt: 2,
|
||||
run_started_at: "2026-09-04T01:00:00.000Z",
|
||||
},
|
||||
],
|
||||
},
|
||||
expectedExecutionIds: [RETAINED, RERUN],
|
||||
workflowRunId: RUN_ID,
|
||||
workflowRunAttempt: 2,
|
||||
sourceSha: SOURCE_SHA,
|
||||
sourceRef: SOURCE_REF,
|
||||
workflowRunUrl: WORKFLOW_RUN_URL,
|
||||
};
|
||||
}
|
||||
|
||||
describe("runner E2E workflow rerun artifact selection", () => {
|
||||
it("combines retained successes with the latest rerun artifact", async () => {
|
||||
const paths = await fixture();
|
||||
await addArtifact({
|
||||
root: paths.artifactRoot,
|
||||
executionId: RETAINED,
|
||||
workflowAttempt: 1,
|
||||
status: "passed",
|
||||
});
|
||||
await addArtifact({
|
||||
root: paths.artifactRoot,
|
||||
executionId: RERUN,
|
||||
workflowAttempt: 1,
|
||||
status: "failed",
|
||||
});
|
||||
const latest = await addArtifact({
|
||||
root: paths.artifactRoot,
|
||||
executionId: RERUN,
|
||||
workflowAttempt: 2,
|
||||
status: "passed",
|
||||
});
|
||||
|
||||
const selected = await selectRerunArtifacts(selectionInput(paths));
|
||||
|
||||
expect(selected).toEqual([
|
||||
{
|
||||
executionId: RETAINED,
|
||||
workflowAttempt: 1,
|
||||
artifactName: `runner-e2e-${RUN_ID}-1-${RETAINED}`,
|
||||
},
|
||||
{
|
||||
executionId: RERUN,
|
||||
workflowAttempt: 2,
|
||||
artifactName: latest.artifactName,
|
||||
},
|
||||
]);
|
||||
const selectedResult = JSON.parse(
|
||||
await readFile(
|
||||
path.join(
|
||||
paths.selectedRoot,
|
||||
latest.artifactName,
|
||||
latest.campaignName,
|
||||
"results",
|
||||
"attempt-1",
|
||||
"result.json",
|
||||
),
|
||||
"utf8",
|
||||
),
|
||||
);
|
||||
expect(selectedResult.status).toBe("passed");
|
||||
});
|
||||
|
||||
it("never falls back when the latest workflow attempt has no artifact", async () => {
|
||||
const paths = await fixture();
|
||||
await addArtifact({
|
||||
root: paths.artifactRoot,
|
||||
executionId: RETAINED,
|
||||
workflowAttempt: 1,
|
||||
status: "passed",
|
||||
});
|
||||
await addArtifact({
|
||||
root: paths.artifactRoot,
|
||||
executionId: RERUN,
|
||||
workflowAttempt: 1,
|
||||
status: "passed",
|
||||
});
|
||||
|
||||
const selected = await selectRerunArtifacts(selectionInput(paths));
|
||||
|
||||
expect(selected.map((entry) => entry.executionId)).toEqual([RETAINED]);
|
||||
});
|
||||
|
||||
it("does not let an older pass mask a latest failed artifact", async () => {
|
||||
const paths = await fixture();
|
||||
await addArtifact({
|
||||
root: paths.artifactRoot,
|
||||
executionId: RETAINED,
|
||||
workflowAttempt: 1,
|
||||
status: "passed",
|
||||
});
|
||||
await addArtifact({
|
||||
root: paths.artifactRoot,
|
||||
executionId: RERUN,
|
||||
workflowAttempt: 1,
|
||||
status: "passed",
|
||||
});
|
||||
const latest = await addArtifact({
|
||||
root: paths.artifactRoot,
|
||||
executionId: RERUN,
|
||||
workflowAttempt: 2,
|
||||
status: "failed",
|
||||
});
|
||||
|
||||
await selectRerunArtifacts(selectionInput(paths));
|
||||
|
||||
const selectedResult = JSON.parse(
|
||||
await readFile(
|
||||
path.join(
|
||||
paths.selectedRoot,
|
||||
latest.artifactName,
|
||||
latest.campaignName,
|
||||
"results",
|
||||
"attempt-1",
|
||||
"result.json",
|
||||
),
|
||||
"utf8",
|
||||
),
|
||||
);
|
||||
expect(selectedResult.status).toBe("failed");
|
||||
});
|
||||
|
||||
it("rejects artifacts from another source", async () => {
|
||||
const paths = await fixture();
|
||||
await addArtifact({
|
||||
root: paths.artifactRoot,
|
||||
executionId: RETAINED,
|
||||
workflowAttempt: 1,
|
||||
status: "passed",
|
||||
sourceSha: "ffffffffffffffffffffffffffffffffffffffff",
|
||||
});
|
||||
await addArtifact({
|
||||
root: paths.artifactRoot,
|
||||
executionId: RERUN,
|
||||
workflowAttempt: 2,
|
||||
status: "passed",
|
||||
});
|
||||
|
||||
await expect(selectRerunArtifacts(selectionInput(paths))).rejects.toThrow(
|
||||
"contains result from another source",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects an artifact carrying another campaign", async () => {
|
||||
const paths = await fixture();
|
||||
await addArtifact({
|
||||
root: paths.artifactRoot,
|
||||
executionId: RETAINED,
|
||||
workflowAttempt: 1,
|
||||
status: "passed",
|
||||
});
|
||||
await addArtifact({
|
||||
root: paths.artifactRoot,
|
||||
executionId: RERUN,
|
||||
workflowAttempt: 2,
|
||||
status: "passed",
|
||||
campaignName: `gha-another-run-2-${RERUN}`,
|
||||
});
|
||||
|
||||
await expect(selectRerunArtifacts(selectionInput(paths))).rejects.toThrow(
|
||||
/must contain only its exact campaign/u,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,304 @@
|
|||
import { cp, mkdir, readFile, readdir } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import type { RunnerE2EResult } from "./types.js";
|
||||
|
||||
interface WorkflowJob {
|
||||
name: string;
|
||||
run_attempt: number;
|
||||
started_at: string;
|
||||
}
|
||||
|
||||
interface WorkflowJobsResponse {
|
||||
jobs: WorkflowJob[];
|
||||
attempts: Array<{
|
||||
run_attempt: number;
|
||||
run_started_at: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
export interface SelectRerunArtifactsInput {
|
||||
artifactRoot: string;
|
||||
selectedRoot: string;
|
||||
jobs: WorkflowJobsResponse;
|
||||
expectedExecutionIds: readonly string[];
|
||||
workflowRunId: string;
|
||||
workflowRunAttempt: number;
|
||||
sourceSha: string;
|
||||
sourceRef: string;
|
||||
workflowRunUrl: string;
|
||||
}
|
||||
|
||||
function safeIdentifier(value: string, label: string) {
|
||||
if (!value || !/^[A-Za-z0-9_.-]+$/u.test(value)) {
|
||||
throw new Error(
|
||||
`${label} contains unsafe characters: ${JSON.stringify(value)}`,
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function positiveInteger(value: unknown, label: string) {
|
||||
if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 1) {
|
||||
throw new Error(`${label} must be a positive integer`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function timestamp(value: unknown, label: string) {
|
||||
if (typeof value !== "string" || !Number.isFinite(Date.parse(value))) {
|
||||
throw new Error(`${label} must be an ISO timestamp`);
|
||||
}
|
||||
return Date.parse(value);
|
||||
}
|
||||
|
||||
async function walk(root: string): Promise<string[]> {
|
||||
const entries = await readdir(root, { withFileTypes: true });
|
||||
const files: string[] = [];
|
||||
for (const entry of entries) {
|
||||
const full = path.join(root, entry.name);
|
||||
if (entry.isDirectory()) files.push(...(await walk(full)));
|
||||
else if (entry.isFile()) files.push(full);
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
/**
|
||||
* Selects the artifact produced by the latest workflow job for each execution.
|
||||
* GitHub reruns retain earlier-attempt artifacts, so validity or result
|
||||
* timestamps must never be used to choose between workflow attempts.
|
||||
*/
|
||||
export async function selectRerunArtifacts(input: SelectRerunArtifactsInput) {
|
||||
const runId = safeIdentifier(input.workflowRunId, "workflow run ID");
|
||||
const currentAttempt = positiveInteger(
|
||||
input.workflowRunAttempt,
|
||||
"workflow run attempt",
|
||||
);
|
||||
const expected = input.expectedExecutionIds.map((executionId) =>
|
||||
safeIdentifier(executionId, "execution ID"),
|
||||
);
|
||||
if (expected.length === 0 || new Set(expected).size !== expected.length) {
|
||||
throw new Error("expected execution IDs must be non-empty and unique");
|
||||
}
|
||||
const preexistingSelections = await readdir(input.selectedRoot).catch(
|
||||
(error: NodeJS.ErrnoException) => {
|
||||
if (error.code === "ENOENT") return [];
|
||||
throw error;
|
||||
},
|
||||
);
|
||||
if (preexistingSelections.length > 0) {
|
||||
throw new Error("selected artifact root must start empty");
|
||||
}
|
||||
const expectedSet = new Set(expected);
|
||||
const attemptStartedAt = new Map<number, number>();
|
||||
for (const attemptMetadata of input.jobs.attempts) {
|
||||
const attempt = positiveInteger(
|
||||
attemptMetadata.run_attempt,
|
||||
"workflow metadata attempt",
|
||||
);
|
||||
if (attempt > currentAttempt || attemptStartedAt.has(attempt)) {
|
||||
throw new Error(`invalid workflow metadata for attempt ${attempt}`);
|
||||
}
|
||||
attemptStartedAt.set(
|
||||
attempt,
|
||||
timestamp(
|
||||
attemptMetadata.run_started_at,
|
||||
`workflow attempt ${attempt} start`,
|
||||
),
|
||||
);
|
||||
}
|
||||
for (let attempt = 1; attempt <= currentAttempt; attempt += 1) {
|
||||
if (!attemptStartedAt.has(attempt)) {
|
||||
throw new Error(`workflow metadata omitted attempt ${attempt}`);
|
||||
}
|
||||
}
|
||||
const attemptsByExecution = new Map<string, Map<number, WorkflowJob>>();
|
||||
for (const candidate of input.jobs.jobs) {
|
||||
if (!expectedSet.has(candidate.name)) continue;
|
||||
const attempt = positiveInteger(candidate.run_attempt, "job run attempt");
|
||||
if (attempt > currentAttempt) {
|
||||
throw new Error(
|
||||
`job ${candidate.name} claims future workflow attempt ${attempt}`,
|
||||
);
|
||||
}
|
||||
const jobStartedAt = timestamp(
|
||||
candidate.started_at,
|
||||
`job ${candidate.name} start`,
|
||||
);
|
||||
// GitHub's filter=all response synthesizes current-attempt rows for jobs
|
||||
// retained from an earlier attempt. Their started_at remains before the
|
||||
// current attempt's trusted run_started_at, so they are not real reruns.
|
||||
if (jobStartedAt < attemptStartedAt.get(attempt)!) continue;
|
||||
const attempts = attemptsByExecution.get(candidate.name) ?? new Map();
|
||||
if (attempts.has(attempt)) {
|
||||
throw new Error(
|
||||
`workflow attempt ${attempt} contains duplicate job ${candidate.name}`,
|
||||
);
|
||||
}
|
||||
attempts.set(attempt, candidate);
|
||||
attemptsByExecution.set(candidate.name, attempts);
|
||||
}
|
||||
|
||||
const latestAttemptByExecution = new Map<string, number>();
|
||||
for (const executionId of expected) {
|
||||
const attempts = [...(attemptsByExecution.get(executionId)?.keys() ?? [])];
|
||||
if (attempts.length === 0) {
|
||||
throw new Error(
|
||||
`workflow job history omitted expected execution ${executionId}`,
|
||||
);
|
||||
}
|
||||
latestAttemptByExecution.set(executionId, Math.max(...attempts));
|
||||
}
|
||||
|
||||
const recognizedArtifactNames = new Map<
|
||||
string,
|
||||
{ executionId: string; workflowAttempt: number }
|
||||
>();
|
||||
for (const executionId of expected) {
|
||||
for (const workflowAttempt of attemptsByExecution
|
||||
.get(executionId)!
|
||||
.keys()) {
|
||||
recognizedArtifactNames.set(
|
||||
`runner-e2e-${runId}-${workflowAttempt}-${executionId}`,
|
||||
{ executionId, workflowAttempt },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const artifactEntries = await readdir(input.artifactRoot, {
|
||||
withFileTypes: true,
|
||||
}).catch((error: NodeJS.ErrnoException) => {
|
||||
if (error.code === "ENOENT") return [];
|
||||
throw error;
|
||||
});
|
||||
const artifactDirectories = new Map<string, string>();
|
||||
for (const entry of artifactEntries) {
|
||||
const identity = recognizedArtifactNames.get(entry.name);
|
||||
if (!identity || !entry.isDirectory()) {
|
||||
throw new Error(`downloaded unexpected runner artifact ${entry.name}`);
|
||||
}
|
||||
artifactDirectories.set(
|
||||
entry.name,
|
||||
path.join(input.artifactRoot, entry.name),
|
||||
);
|
||||
}
|
||||
|
||||
const selections: Array<{
|
||||
executionId: string;
|
||||
workflowAttempt: number;
|
||||
artifactName: string;
|
||||
}> = [];
|
||||
for (const executionId of expected) {
|
||||
const workflowAttempt = latestAttemptByExecution.get(executionId)!;
|
||||
const artifactName = `runner-e2e-${runId}-${workflowAttempt}-${executionId}`;
|
||||
const artifactDirectory = artifactDirectories.get(artifactName);
|
||||
// A latest job without an artifact must remain missing. Falling back to an
|
||||
// older successful artifact would mask an infrastructure/upload failure.
|
||||
if (!artifactDirectory) continue;
|
||||
|
||||
const campaignName = `gha-${runId}-${workflowAttempt}-${executionId}`;
|
||||
const topLevelEntries = await readdir(artifactDirectory, {
|
||||
withFileTypes: true,
|
||||
});
|
||||
if (
|
||||
topLevelEntries.length !== 1 ||
|
||||
topLevelEntries[0]?.name !== campaignName ||
|
||||
!topLevelEntries[0].isDirectory()
|
||||
) {
|
||||
throw new Error(
|
||||
`${artifactName} must contain only its exact campaign ${campaignName}`,
|
||||
);
|
||||
}
|
||||
const campaignDirectory = path.join(artifactDirectory, campaignName);
|
||||
const resultFiles = (await walk(campaignDirectory)).filter(
|
||||
(file) => path.basename(file) === "result.json",
|
||||
);
|
||||
if (resultFiles.length === 0) {
|
||||
throw new Error(`${artifactName} contains no normalized result`);
|
||||
}
|
||||
for (const resultFile of resultFiles) {
|
||||
const result = JSON.parse(
|
||||
await readFile(resultFile, "utf8"),
|
||||
) as RunnerE2EResult;
|
||||
if (result.executionId !== executionId) {
|
||||
throw new Error(
|
||||
`${artifactName} contains result for ${String(result.executionId)}`,
|
||||
);
|
||||
}
|
||||
const source = result.source;
|
||||
if (
|
||||
!source ||
|
||||
source.sha !== input.sourceSha ||
|
||||
source.ref !== input.sourceRef ||
|
||||
source.workflowRunUrl !== input.workflowRunUrl
|
||||
) {
|
||||
throw new Error(`${artifactName} contains result from another source`);
|
||||
}
|
||||
}
|
||||
const destination = path.join(
|
||||
input.selectedRoot,
|
||||
artifactName,
|
||||
campaignName,
|
||||
);
|
||||
await mkdir(path.dirname(destination), { recursive: true });
|
||||
await cp(campaignDirectory, destination, {
|
||||
recursive: true,
|
||||
force: false,
|
||||
errorOnExist: true,
|
||||
});
|
||||
selections.push({ executionId, workflowAttempt, artifactName });
|
||||
}
|
||||
return selections;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const required = (name: string) => {
|
||||
const value = process.env[name]?.trim();
|
||||
if (!value) throw new Error(`${name} is required`);
|
||||
return value;
|
||||
};
|
||||
const expected = JSON.parse(
|
||||
required("PAPERCLIP_RUNNER_E2E_EXPECTED_IDS"),
|
||||
) as unknown;
|
||||
if (
|
||||
!Array.isArray(expected) ||
|
||||
expected.some((value) => typeof value !== "string")
|
||||
) {
|
||||
throw new Error(
|
||||
"PAPERCLIP_RUNNER_E2E_EXPECTED_IDS must be a JSON string array",
|
||||
);
|
||||
}
|
||||
const jobs = JSON.parse(
|
||||
await readFile(required("PAPERCLIP_RUNNER_E2E_JOBS_JSON"), "utf8"),
|
||||
) as WorkflowJobsResponse;
|
||||
if (!jobs || !Array.isArray(jobs.jobs) || !Array.isArray(jobs.attempts)) {
|
||||
throw new Error("workflow jobs JSON must contain jobs and attempts arrays");
|
||||
}
|
||||
const runId = required("GITHUB_RUN_ID");
|
||||
const serverUrl = required("GITHUB_SERVER_URL");
|
||||
const repository = required("GITHUB_REPOSITORY");
|
||||
const selections = await selectRerunArtifacts({
|
||||
artifactRoot: required("PAPERCLIP_RUNNER_E2E_ARTIFACT_ROOT"),
|
||||
selectedRoot: required("PAPERCLIP_RUNNER_E2E_SELECTED_ROOT"),
|
||||
jobs,
|
||||
expectedExecutionIds: expected,
|
||||
workflowRunId: runId,
|
||||
workflowRunAttempt: Number(required("GITHUB_RUN_ATTEMPT")),
|
||||
sourceSha: required("PAPERCLIP_RUNNER_E2E_SOURCE_SHA"),
|
||||
sourceRef: required("PAPERCLIP_RUNNER_E2E_SOURCE_REF"),
|
||||
workflowRunUrl: `${serverUrl}/${repository}/actions/runs/${runId}`,
|
||||
});
|
||||
console.log(
|
||||
`Selected ${selections.length}/${expected.length} latest cell artifacts`,
|
||||
);
|
||||
}
|
||||
|
||||
if (
|
||||
process.argv[1] &&
|
||||
import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href
|
||||
) {
|
||||
await main().catch((error) => {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
|
|
@ -545,6 +545,52 @@ describe("public repository paid workflow security", () => {
|
|||
expect(testJob).not.toContain("build-provider-pack.mjs");
|
||||
});
|
||||
|
||||
it("binds rerun evidence and Pages artifacts to the exact workflow attempt", async () => {
|
||||
const workflow = await readFile(
|
||||
path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"),
|
||||
"utf8",
|
||||
);
|
||||
const reportStart = workflow.indexOf(" report:");
|
||||
const publisherStart = workflow.indexOf(" publish_history:", reportStart);
|
||||
const report = workflow.slice(reportStart, publisherStart);
|
||||
const publisher = workflow.slice(publisherStart);
|
||||
|
||||
expect(report).toContain("actions: read");
|
||||
expect(report).toContain(
|
||||
'"repos/$REPOSITORY/actions/runs/$RUN_ID/jobs?filter=all&per_page=100"',
|
||||
);
|
||||
expect(report).toContain("gh api --paginate --slurp");
|
||||
expect(report).toContain(
|
||||
'"repos/$REPOSITORY/actions/runs/$RUN_ID/attempts/$attempt"',
|
||||
);
|
||||
expect(report).toContain("--jq '{run_attempt, run_started_at}'");
|
||||
expect(report).toContain("pattern: runner-e2e-${{ github.run_id }}-*-*");
|
||||
expect(report).not.toContain(
|
||||
"pattern: runner-e2e-${{ github.run_id }}-${{ github.run_attempt }}-*",
|
||||
);
|
||||
expect(report.match(/merge-multiple: false/g)).toHaveLength(2);
|
||||
expect(report).toContain("Select latest workflow attempt per cell");
|
||||
expect(report).toContain("tests/runner-e2e/select-rerun-artifacts.ts");
|
||||
expect(report).toContain(
|
||||
"PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e",
|
||||
);
|
||||
expect(
|
||||
report.indexOf("Select latest workflow attempt per cell"),
|
||||
).toBeLessThan(report.indexOf("Collect blob reports"));
|
||||
expect(publisher).toContain(
|
||||
'echo "name=github-pages-${{ github.run_id }}-${{ github.run_attempt }}"',
|
||||
);
|
||||
expect(publisher).toContain(
|
||||
"pages_artifact_name: ${{ steps.pages_artifact_name.outputs.name }}",
|
||||
);
|
||||
expect(publisher).toContain(
|
||||
"name: ${{ steps.pages_artifact_name.outputs.name }}",
|
||||
);
|
||||
expect(publisher).toContain(
|
||||
"artifact_name: ${{ needs.publish_history.outputs.pages_artifact_name }}",
|
||||
);
|
||||
});
|
||||
|
||||
it("uses environment-scoped OIDC for a no-delete history publisher", async () => {
|
||||
const workflow = await readFile(
|
||||
path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"),
|
||||
|
|
|
|||
Loading…
Reference in New Issue