diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.yml index 577928ae7d..b5f82534c8 100644 --- a/.github/workflows/runner-full-stack-e2e.yml +++ b/.github/workflows/runner-full-stack-e2e.yml @@ -60,6 +60,7 @@ jobs: max_parallel_default: ${{ steps.runner.outputs.max_parallel_default }} max_parallel_limit: ${{ steps.runner.outputs.max_parallel_limit }} target_sha: ${{ steps.target.outputs.sha }} + target_ref: ${{ steps.target.outputs.ref }} steps: - name: Require default branch and allowlisted numeric actor IDs env: @@ -113,6 +114,7 @@ jobs: exit 1 fi echo "sha=$target_sha" >> "$GITHUB_OUTPUT" + echo "ref=refs/heads/$TARGET_BRANCH" >> "$GITHUB_OUTPUT" echo "Resolved the requested repository branch to $target_sha." - name: Select paid test runner @@ -548,7 +550,9 @@ jobs: - name: Name immutable shared campaign outputs id: build_artifact_name - run: echo "name=runner-e2e-build-${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT" + env: + TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} + run: echo "name=runner-e2e-build-${TARGET_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT" - name: Upload immutable shared campaign outputs uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -674,7 +678,9 @@ jobs: - name: Name immutable remote provider pack id: provider_pack_artifact_name if: needs.catalog.outputs.needs_remote_provider_pack == 'true' - run: echo "name=runner-e2e-provider-pack-${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT" + env: + TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} + run: echo "name=runner-e2e-provider-pack-${TARGET_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT" - name: Upload immutable remote provider pack if: needs.catalog.outputs.needs_remote_provider_pack == 'true' @@ -892,6 +898,8 @@ jobs: PAPERCLIP_E2E_DAYTONA_IMAGE: ${{ needs.daytona_image.outputs.image }} PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH: ${{ github.workspace }}/packages/paperclip-runner/provider-pack PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.executionId }} + PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} + PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }} run: pnpm test:e2e:runner -- --id "${{ matrix.executionId }}" - name: Upload access-controlled packaged cell evidence @@ -973,6 +981,8 @@ jobs: PAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalized PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: ${{ needs.catalog.outputs.execution_ids }} PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-${{ github.run_attempt }} + PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} + PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }} run: | set +e pnpm test:e2e:runner:report diff --git a/tests/runner-e2e/history.test.ts b/tests/runner-e2e/history.test.ts index 2a0c85c37a..556d09d83a 100644 --- a/tests/runner-e2e/history.test.ts +++ b/tests/runner-e2e/history.test.ts @@ -1,7 +1,7 @@ import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { runnerMatrix } from "./catalog.js"; import { regenerateRunnerDashboard } from "./dashboard-regenerate.js"; import { renderRunnerE2EDashboard } from "./dashboard.js"; @@ -24,6 +24,7 @@ import type { MatrixExecution, RunnerE2EResult } from "./types.js"; const temporaryDirectories: string[] = []; afterEach(async () => { + vi.unstubAllEnvs(); await Promise.all( temporaryDirectories .splice(0) @@ -55,6 +56,35 @@ function result(execution: MatrixExecution, status: "passed" | "failed") { } describe("runner E2E campaign history", () => { + it("records the resolved paid target instead of the trusted workflow checkout", () => { + vi.stubEnv("PAPERCLIP_RUNNER_E2E_SOURCE_SHA", "target-sha"); + vi.stubEnv("PAPERCLIP_RUNNER_E2E_SOURCE_REF", "refs/heads/target"); + vi.stubEnv("GITHUB_SHA", "trusted-master-sha"); + vi.stubEnv("GITHUB_REF", "refs/heads/master"); + const execution = runnerMatrix[0]!; + const campaign = buildRunnerCampaign({ + campaignId: "target-provenance", + generatedAt: "2026-08-28T00:01:00.000Z", + expected: [execution.id], + results: [ + { + ...result(execution, "passed"), + source: { + sha: "retained-result-sha", + ref: "refs/heads/retained-result", + workflowRunUrl: "https://example.test/actions/runs/1", + }, + }, + ], + }); + + expect(campaign.source).toMatchObject({ + sha: "target-sha", + ref: "refs/heads/target", + workflowRunUrl: "https://example.test/actions/runs/1", + }); + }); + it("migrates v1 execution IDs and keeps partial suite runs out of overall trends", () => { expect(canonicalExecutionId("legacy-codex.local.message-marker")).toBe( "core-compatibility.legacy-codex.local.message-marker", diff --git a/tests/runner-e2e/history.ts b/tests/runner-e2e/history.ts index 9101836ca7..bcb3871253 100644 --- a/tests/runner-e2e/history.ts +++ b/tests/runner-e2e/history.ts @@ -3,6 +3,7 @@ import { aggregateCampaignBilling, summarizeExecutionBilling, } from "./billing.js"; +import { resolveRunnerE2ESource } from "./source.js"; import type { RunnerE2ECampaign, RunnerE2EHistoryCampaign, @@ -56,15 +57,7 @@ export function buildRunnerCampaign(input: { })); const resultSource = results.find((result) => result.source)?.source; const source = { - sha: resultSource?.sha ?? process.env.GITHUB_SHA ?? null, - ref: resultSource?.ref ?? process.env.GITHUB_REF ?? null, - workflowRunUrl: - resultSource?.workflowRunUrl ?? - (process.env.GITHUB_SERVER_URL && - process.env.GITHUB_REPOSITORY && - process.env.GITHUB_RUN_ID - ? `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}` - : null), + ...resolveRunnerE2ESource(resultSource), eventName: input.eventName ?? process.env.GITHUB_EVENT_NAME ?? null, }; const suites = runnerSuites diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index d2575b43a7..d0fe4ef3d2 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -39,6 +39,7 @@ import { RunnerSelectorError, selectRunnerExecutions, } from "./selectors.js"; +import { resolveRunnerE2ESource } from "./source.js"; import { CREDENTIAL_NAMES, type MatrixExecution, @@ -280,16 +281,7 @@ function syntheticResult( executionId: execution.id, suiteId: execution.suite.id, suiteDefinitionHash: execution.suiteDefinitionHash, - source: { - sha: process.env.GITHUB_SHA ?? null, - ref: process.env.GITHUB_REF ?? null, - workflowRunUrl: - process.env.GITHUB_SERVER_URL && - process.env.GITHUB_REPOSITORY && - process.env.GITHUB_RUN_ID - ? `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}` - : null, - }, + source: resolveRunnerE2ESource(), ...(execution.profile.ranking ? { rankingSnapshot: execution.profile.ranking } : {}), diff --git a/tests/runner-e2e/report.test.ts b/tests/runner-e2e/report.test.ts index 0aef99e8e3..13cc9eba01 100644 --- a/tests/runner-e2e/report.test.ts +++ b/tests/runner-e2e/report.test.ts @@ -38,6 +38,11 @@ describe("runner E2E report aggregation", () => { startedAt: "2026-08-26T00:00:00.000Z", finishedAt: "2026-08-26T00:00:01.000Z", durationMs: 1_000, + source: { + sha: "forged-result-sha", + ref: "refs/heads/forged-result", + workflowRunUrl: "https://example.test/actions/runs/forged", + }, runIds: ["run-2"], usage: { inputTokens: 1_250, @@ -123,6 +128,15 @@ describe("runner E2E report aggregation", () => { PAPERCLIP_RUNNER_E2E_REPORT_ROOT: root, PAPERCLIP_RUNNER_E2E_REPORT_OUT: output, PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: JSON.stringify([executionId]), + PAPERCLIP_RUNNER_E2E_SOURCE_SHA: + "0123456789abcdef0123456789abcdef01234567", + PAPERCLIP_RUNNER_E2E_SOURCE_REF: + "refs/heads/fix/runner-paid-source-attribution", + GITHUB_SHA: "trusted-default-workflow-sha", + GITHUB_REF: "refs/heads/master", + GITHUB_SERVER_URL: "https://github.com", + GITHUB_REPOSITORY: "paperclipai/paperclip", + GITHUB_RUN_ID: "123456", }, }, ); @@ -137,6 +151,12 @@ describe("runner E2E report aggregation", () => { failed: 0, retries: 1, cleanupPassed: true, + source: { + sha: "0123456789abcdef0123456789abcdef01234567", + ref: "refs/heads/fix/runner-paid-source-attribution", + workflowRunUrl: + "https://github.com/paperclipai/paperclip/actions/runs/123456", + }, }); expect(normalized.billing).toMatchObject({ reportedLlmCostUsd: 0.0125, @@ -149,6 +169,12 @@ describe("runner E2E report aggregation", () => { expect(normalized.results[0]).toMatchObject({ attempt: 2, evidenceValid: true, + source: { + sha: "0123456789abcdef0123456789abcdef01234567", + ref: "refs/heads/fix/runner-paid-source-attribution", + workflowRunUrl: + "https://github.com/paperclipai/paperclip/actions/runs/123456", + }, }); const dashboard = await readFile( path.join(output, "dashboard.html"), diff --git a/tests/runner-e2e/report.ts b/tests/runner-e2e/report.ts index 4e5930440e..4857a7136f 100644 --- a/tests/runner-e2e/report.ts +++ b/tests/runner-e2e/report.ts @@ -14,6 +14,7 @@ import { canonicalExecutionId, upgradeRunnerResult, } from "./history.js"; +import { resolveRunnerE2ESource } from "./source.js"; import type { RunnerE2EResult } from "./types.js"; const repositoryRoot = path.resolve(import.meta.dirname, "../.."); @@ -245,6 +246,10 @@ async function main() { ]); const resolvedResults = selected.map((entry) => ({ ...entry.result, + // Cell evidence is produced by target-controlled code. The trusted report + // stamps the immutable target selected by the authorization job instead + // of allowing retained result metadata to claim another revision. + source: resolveRunnerE2ESource(entry.result.source), status: entry.valid ? entry.result.status : ("failed" as const), billing: summarizeExecutionBilling(entry.result), })); diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index 3bc394a32d..6386b864f7 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -16,6 +16,7 @@ import { numberedPlanStepCount, providerSessionContinuityFailures, } from "./run-observations.js"; +import { resolveRunnerE2ESource } from "./source.js"; import { assertSecretFree, findSecretLeakInJsonValues, @@ -1804,16 +1805,7 @@ for (const execution of executions) { executionId: execution.id, suiteId: execution.suite.id, suiteDefinitionHash: execution.suiteDefinitionHash, - source: { - sha: process.env.GITHUB_SHA ?? null, - ref: process.env.GITHUB_REF ?? null, - workflowRunUrl: - process.env.GITHUB_SERVER_URL && - process.env.GITHUB_REPOSITORY && - process.env.GITHUB_RUN_ID - ? `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}` - : null, - }, + source: resolveRunnerE2ESource(), ...(execution.profile.ranking ? { rankingSnapshot: execution.profile.ranking } : {}), diff --git a/tests/runner-e2e/source.test.ts b/tests/runner-e2e/source.test.ts new file mode 100644 index 0000000000..969a5e4bbc --- /dev/null +++ b/tests/runner-e2e/source.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from "vitest"; +import { resolveRunnerE2ESource } from "./source.js"; + +describe("runner E2E source provenance", () => { + it("prefers the resolved target over result and workflow revision contexts", () => { + expect( + resolveRunnerE2ESource( + { + sha: "result-sha", + ref: "refs/heads/result", + workflowRunUrl: "https://example.test/result-run", + }, + { + PAPERCLIP_RUNNER_E2E_SOURCE_SHA: "target-sha", + PAPERCLIP_RUNNER_E2E_SOURCE_REF: "refs/heads/target", + GITHUB_SHA: "workflow-sha", + GITHUB_REF: "refs/heads/master", + GITHUB_SERVER_URL: "https://github.com", + GITHUB_REPOSITORY: "paperclipai/paperclip", + GITHUB_RUN_ID: "123", + }, + ), + ).toEqual({ + sha: "target-sha", + ref: "refs/heads/target", + workflowRunUrl: + "https://github.com/paperclipai/paperclip/actions/runs/123", + }); + }); + + it("falls back through retained result provenance, workflow context, and null", () => { + expect( + resolveRunnerE2ESource( + { + sha: "result-sha", + ref: null, + workflowRunUrl: null, + }, + { + GITHUB_SHA: "workflow-sha", + GITHUB_REF: "refs/heads/master", + GITHUB_SERVER_URL: "https://github.com", + GITHUB_REPOSITORY: "paperclipai/paperclip", + GITHUB_RUN_ID: "456", + }, + ), + ).toEqual({ + sha: "result-sha", + ref: "refs/heads/master", + workflowRunUrl: + "https://github.com/paperclipai/paperclip/actions/runs/456", + }); + expect(resolveRunnerE2ESource(null, {})).toEqual({ + sha: null, + ref: null, + workflowRunUrl: null, + }); + }); +}); diff --git a/tests/runner-e2e/source.ts b/tests/runner-e2e/source.ts new file mode 100644 index 0000000000..f3d3d1401c --- /dev/null +++ b/tests/runner-e2e/source.ts @@ -0,0 +1,35 @@ +import type { RunnerE2EResult } from "./types.js"; + +type RunnerE2ESource = NonNullable; + +function nonEmpty(value: string | null | undefined) { + const normalized = value?.trim(); + return normalized ? normalized : null; +} + +function workflowRunUrl(environment: NodeJS.ProcessEnv) { + const serverUrl = nonEmpty(environment.GITHUB_SERVER_URL); + const repository = nonEmpty(environment.GITHUB_REPOSITORY); + const runId = nonEmpty(environment.GITHUB_RUN_ID); + return serverUrl && repository && runId + ? `${serverUrl}/${repository}/actions/runs/${runId}` + : null; +} + +export function resolveRunnerE2ESource( + existing?: RunnerE2ESource | null, + environment: NodeJS.ProcessEnv = process.env, +): RunnerE2ESource { + return { + sha: + nonEmpty(environment.PAPERCLIP_RUNNER_E2E_SOURCE_SHA) ?? + nonEmpty(existing?.sha) ?? + nonEmpty(environment.GITHUB_SHA), + ref: + nonEmpty(environment.PAPERCLIP_RUNNER_E2E_SOURCE_REF) ?? + nonEmpty(existing?.ref) ?? + nonEmpty(environment.GITHUB_REF), + workflowRunUrl: + workflowRunUrl(environment) ?? nonEmpty(existing?.workflowRunUrl), + }; +} diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts index cf554d4c34..9838f7586b 100644 --- a/tests/runner-e2e/workflow-security.test.ts +++ b/tests/runner-e2e/workflow-security.test.ts @@ -92,6 +92,10 @@ describe("public repository paid workflow security", () => { "repos/$REPOSITORY/branches/$encoded_branch", ); expect(authorizeJob).toContain('echo "sha=$target_sha"'); + expect(authorizeJob).toContain( + "target_ref: ${{ steps.target.outputs.ref }}", + ); + expect(authorizeJob).toContain('echo "ref=refs/heads/$TARGET_BRANCH"'); expect(authorizeJob).not.toContain("actions/checkout@"); expect(authorizeJob).not.toContain("pnpm install"); expect(targetLockJob).toContain("name: Resolve target pnpm lockfile"); @@ -255,6 +259,14 @@ describe("public repository paid workflow security", () => { expect(fullStack).toContain( "if: always() && !cancelled() && needs.catalog.result == 'success'", ); + for (const targetProvenanceJob of [paidJob, reportJob]) { + expect(targetProvenanceJob).toContain( + "PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}", + ); + expect(targetProvenanceJob).toContain( + "PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }}", + ); + } for (const [secret, condition] of Object.entries({ OPENAI_API_KEY: "matrix.credentialName == 'OPENAI_API_KEY'", ANTHROPIC_API_KEY: "matrix.credentialName == 'ANTHROPIC_API_KEY'", @@ -356,6 +368,14 @@ describe("public repository paid workflow security", () => { expect(buildJob).toContain( "provider_pack_artifact_name: ${{ steps.provider_pack_artifact_name.outputs.name }}", ); + expect(buildJob).toContain( + "runner-e2e-build-${TARGET_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}", + ); + expect(buildJob).toContain( + "runner-e2e-provider-pack-${TARGET_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}", + ); + expect(buildJob).not.toContain("runner-e2e-build-${GITHUB_SHA}"); + expect(buildJob).not.toContain("runner-e2e-provider-pack-${GITHUB_SHA}"); expect(workflow).toContain("needs_runner_typescript="); expect(workflow).toContain("needs_native_binaries="); expect(workflow).toContain("needs_remote_provider_pack=");