diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.yml index b3f9a7cadb..177d5d53b5 100644 --- a/.github/workflows/runner-full-stack-e2e.yml +++ b/.github/workflows/runner-full-stack-e2e.yml @@ -60,6 +60,7 @@ jobs: max_parallel_default: ${{ steps.runner.outputs.max_parallel_default }} max_parallel_limit: ${{ steps.runner.outputs.max_parallel_limit }} target_sha: ${{ steps.target.outputs.sha }} + target_ref: ${{ steps.target.outputs.ref }} steps: - name: Require default branch and allowlisted numeric actor IDs env: @@ -113,6 +114,7 @@ jobs: exit 1 fi echo "sha=$target_sha" >> "$GITHUB_OUTPUT" + echo "ref=refs/heads/$TARGET_BRANCH" >> "$GITHUB_OUTPUT" echo "Resolved the requested repository branch to $target_sha." - name: Select paid test runner @@ -554,7 +556,9 @@ jobs: - name: Name immutable shared campaign outputs id: build_artifact_name - run: echo "name=runner-e2e-build-${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT" + env: + TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} + run: echo "name=runner-e2e-build-${TARGET_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT" - name: Upload immutable shared campaign outputs uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -684,7 +688,9 @@ jobs: - name: Name immutable remote provider pack id: provider_pack_artifact_name if: needs.catalog.outputs.needs_remote_provider_pack == 'true' - run: echo "name=runner-e2e-provider-pack-${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT" + env: + TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} + run: echo "name=runner-e2e-provider-pack-${TARGET_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT" - name: Upload immutable remote provider pack if: needs.catalog.outputs.needs_remote_provider_pack == 'true' @@ -895,6 +901,8 @@ jobs: PAPERCLIP_E2E_DAYTONA_IMAGE: ${{ needs.daytona_image.outputs.image }} PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH: ${{ github.workspace }}/packages/paperclip-runner/provider-pack PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.executionId }} + PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} + PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }} run: pnpm test:e2e:runner -- --id "${{ matrix.executionId }}" - name: Upload access-controlled packaged cell evidence @@ -980,6 +988,8 @@ jobs: PAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalized PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: ${{ needs.catalog.outputs.execution_ids }} PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-${{ github.run_attempt }} + PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} + PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }} run: | set +e pnpm test:e2e:runner:report diff --git a/tests/runner-e2e/report.test.ts b/tests/runner-e2e/report.test.ts index 0aef99e8e3..13cc9eba01 100644 --- a/tests/runner-e2e/report.test.ts +++ b/tests/runner-e2e/report.test.ts @@ -38,6 +38,11 @@ describe("runner E2E report aggregation", () => { startedAt: "2026-08-26T00:00:00.000Z", finishedAt: "2026-08-26T00:00:01.000Z", durationMs: 1_000, + source: { + sha: "forged-result-sha", + ref: "refs/heads/forged-result", + workflowRunUrl: "https://example.test/actions/runs/forged", + }, runIds: ["run-2"], usage: { inputTokens: 1_250, @@ -123,6 +128,15 @@ describe("runner E2E report aggregation", () => { PAPERCLIP_RUNNER_E2E_REPORT_ROOT: root, PAPERCLIP_RUNNER_E2E_REPORT_OUT: output, PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: JSON.stringify([executionId]), + PAPERCLIP_RUNNER_E2E_SOURCE_SHA: + "0123456789abcdef0123456789abcdef01234567", + PAPERCLIP_RUNNER_E2E_SOURCE_REF: + "refs/heads/fix/runner-paid-source-attribution", + GITHUB_SHA: "trusted-default-workflow-sha", + GITHUB_REF: "refs/heads/master", + GITHUB_SERVER_URL: "https://github.com", + GITHUB_REPOSITORY: "paperclipai/paperclip", + GITHUB_RUN_ID: "123456", }, }, ); @@ -137,6 +151,12 @@ describe("runner E2E report aggregation", () => { failed: 0, retries: 1, cleanupPassed: true, + source: { + sha: "0123456789abcdef0123456789abcdef01234567", + ref: "refs/heads/fix/runner-paid-source-attribution", + workflowRunUrl: + "https://github.com/paperclipai/paperclip/actions/runs/123456", + }, }); expect(normalized.billing).toMatchObject({ reportedLlmCostUsd: 0.0125, @@ -149,6 +169,12 @@ describe("runner E2E report aggregation", () => { expect(normalized.results[0]).toMatchObject({ attempt: 2, evidenceValid: true, + source: { + sha: "0123456789abcdef0123456789abcdef01234567", + ref: "refs/heads/fix/runner-paid-source-attribution", + workflowRunUrl: + "https://github.com/paperclipai/paperclip/actions/runs/123456", + }, }); const dashboard = await readFile( path.join(output, "dashboard.html"), diff --git a/tests/runner-e2e/report.ts b/tests/runner-e2e/report.ts index 4e5930440e..4857a7136f 100644 --- a/tests/runner-e2e/report.ts +++ b/tests/runner-e2e/report.ts @@ -14,6 +14,7 @@ import { canonicalExecutionId, upgradeRunnerResult, } from "./history.js"; +import { resolveRunnerE2ESource } from "./source.js"; import type { RunnerE2EResult } from "./types.js"; const repositoryRoot = path.resolve(import.meta.dirname, "../.."); @@ -245,6 +246,10 @@ async function main() { ]); const resolvedResults = selected.map((entry) => ({ ...entry.result, + // Cell evidence is produced by target-controlled code. The trusted report + // stamps the immutable target selected by the authorization job instead + // of allowing retained result metadata to claim another revision. + source: resolveRunnerE2ESource(entry.result.source), status: entry.valid ? entry.result.status : ("failed" as const), billing: summarizeExecutionBilling(entry.result), })); diff --git a/tests/runner-e2e/source.ts b/tests/runner-e2e/source.ts new file mode 100644 index 0000000000..f3d3d1401c --- /dev/null +++ b/tests/runner-e2e/source.ts @@ -0,0 +1,35 @@ +import type { RunnerE2EResult } from "./types.js"; + +type RunnerE2ESource = NonNullable; + +function nonEmpty(value: string | null | undefined) { + const normalized = value?.trim(); + return normalized ? normalized : null; +} + +function workflowRunUrl(environment: NodeJS.ProcessEnv) { + const serverUrl = nonEmpty(environment.GITHUB_SERVER_URL); + const repository = nonEmpty(environment.GITHUB_REPOSITORY); + const runId = nonEmpty(environment.GITHUB_RUN_ID); + return serverUrl && repository && runId + ? `${serverUrl}/${repository}/actions/runs/${runId}` + : null; +} + +export function resolveRunnerE2ESource( + existing?: RunnerE2ESource | null, + environment: NodeJS.ProcessEnv = process.env, +): RunnerE2ESource { + return { + sha: + nonEmpty(environment.PAPERCLIP_RUNNER_E2E_SOURCE_SHA) ?? + nonEmpty(existing?.sha) ?? + nonEmpty(environment.GITHUB_SHA), + ref: + nonEmpty(environment.PAPERCLIP_RUNNER_E2E_SOURCE_REF) ?? + nonEmpty(existing?.ref) ?? + nonEmpty(environment.GITHUB_REF), + workflowRunUrl: + workflowRunUrl(environment) ?? nonEmpty(existing?.workflowRunUrl), + }; +} diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts index d902142926..f24646f5a8 100644 --- a/tests/runner-e2e/workflow-security.test.ts +++ b/tests/runner-e2e/workflow-security.test.ts @@ -126,6 +126,10 @@ describe("public repository paid workflow security", () => { "repos/$REPOSITORY/branches/$encoded_branch", ); expect(authorizeJob).toContain('echo "sha=$target_sha"'); + expect(authorizeJob).toContain( + "target_ref: ${{ steps.target.outputs.ref }}", + ); + expect(authorizeJob).toContain('echo "ref=refs/heads/$TARGET_BRANCH"'); expect(authorizeJob).not.toContain("actions/checkout@"); expect(authorizeJob).not.toContain("pnpm install"); expect(targetLockJob).toContain("name: Resolve target pnpm lockfile"); @@ -286,6 +290,14 @@ describe("public repository paid workflow security", () => { "ref: ${{ needs.authorize.outputs.target_sha }}", ); expect(historyJob).not.toContain("Download resolved target lockfile"); + for (const targetProvenanceJob of [paidJob, reportJob]) { + expect(targetProvenanceJob).toContain( + "PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}", + ); + expect(targetProvenanceJob).toContain( + "PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }}", + ); + } for (const [secret, condition] of Object.entries({ OPENAI_API_KEY: "matrix.credentialName == 'OPENAI_API_KEY'", ANTHROPIC_API_KEY: "matrix.credentialName == 'ANTHROPIC_API_KEY'", @@ -387,6 +399,14 @@ describe("public repository paid workflow security", () => { expect(buildJob).toContain( "provider_pack_artifact_name: ${{ steps.provider_pack_artifact_name.outputs.name }}", ); + expect(buildJob).toContain( + "runner-e2e-build-${TARGET_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}", + ); + expect(buildJob).toContain( + "runner-e2e-provider-pack-${TARGET_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}", + ); + expect(buildJob).not.toContain("runner-e2e-build-${GITHUB_SHA}"); + expect(buildJob).not.toContain("runner-e2e-provider-pack-${GITHUB_SHA}"); expect(workflow).toContain("needs_runner_typescript="); expect(workflow).toContain("needs_native_binaries="); expect(workflow).toContain("needs_remote_provider_pack=");