RomanNum3ral
/
paperclip
mirror of https://github.com/paperclipai/paperclip.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix(runner-e2e): record trusted target provenance

This commit is contained in:
Dotta 2026-09-03 15:15:58 -05:00
parent 97f771e2ec
commit 33c7646d3f
10 changed files with 194 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
.github/workflows/runner-full-stack-e2e.yml vendored
View File

@ -60,6 +60,7 @@ jobs:
max_parallel_default: ${{ steps.runner.outputs.max_parallel_default }}
max_parallel_limit: ${{ steps.runner.outputs.max_parallel_limit }}
target_sha: ${{ steps.target.outputs.sha }}
target_ref: ${{ steps.target.outputs.ref }}
steps:
- name: Require default branch and allowlisted numeric actor IDs
env:
@ -113,6 +114,7 @@ jobs:
exit 1
fi
echo "sha=$target_sha" >> "$GITHUB_OUTPUT"
echo "ref=refs/heads/$TARGET_BRANCH" >> "$GITHUB_OUTPUT"
echo "Resolved the requested repository branch to $target_sha."
- name: Select paid test runner
@ -548,7 +550,9 @@ jobs:
- name: Name immutable shared campaign outputs
id: build_artifact_name
run: echo "name=runner-e2e-build-${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT"
env:
TARGET_SHA: ${{ needs.authorize.outputs.target_sha }}
run: echo "name=runner-e2e-build-${TARGET_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT"
- name: Upload immutable shared campaign outputs
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
@ -674,7 +678,9 @@ jobs:
- name: Name immutable remote provider pack
id: provider_pack_artifact_name
if: needs.catalog.outputs.needs_remote_provider_pack == 'true'
run: echo "name=runner-e2e-provider-pack-${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT"
env:
TARGET_SHA: ${{ needs.authorize.outputs.target_sha }}
run: echo "name=runner-e2e-provider-pack-${TARGET_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT"
- name: Upload immutable remote provider pack
if: needs.catalog.outputs.needs_remote_provider_pack == 'true'
@ -892,6 +898,8 @@ jobs:
PAPERCLIP_E2E_DAYTONA_IMAGE: ${{ needs.daytona_image.outputs.image }}
PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH: ${{ github.workspace }}/packages/paperclip-runner/provider-pack
PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.executionId }}
PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}
PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }}
run: pnpm test:e2e:runner -- --id "${{ matrix.executionId }}"
- name: Upload access-controlled packaged cell evidence
@ -973,6 +981,8 @@ jobs:
PAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalized
PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: ${{ needs.catalog.outputs.execution_ids }}
PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-${{ github.run_attempt }}
PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}
PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }}
run: |
set +e
pnpm test:e2e:runner:report

32
tests/runner-e2e/history.test.ts
View File

@ -1,7 +1,7 @@
import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { afterEach, describe, expect, it, vi } from "vitest";
import { runnerMatrix } from "./catalog.js";
import { regenerateRunnerDashboard } from "./dashboard-regenerate.js";
import { renderRunnerE2EDashboard } from "./dashboard.js";
@ -24,6 +24,7 @@ import type { MatrixExecution, RunnerE2EResult } from "./types.js";
const temporaryDirectories: string[] = [];
afterEach(async () => {
vi.unstubAllEnvs();
await Promise.all(
temporaryDirectories
.splice(0)
@ -55,6 +56,35 @@ function result(execution: MatrixExecution, status: "passed" | "failed") {
}
describe("runner E2E campaign history", () => {
it("records the resolved paid target instead of the trusted workflow checkout", () => {
vi.stubEnv("PAPERCLIP_RUNNER_E2E_SOURCE_SHA", "target-sha");
vi.stubEnv("PAPERCLIP_RUNNER_E2E_SOURCE_REF", "refs/heads/target");
vi.stubEnv("GITHUB_SHA", "trusted-master-sha");
vi.stubEnv("GITHUB_REF", "refs/heads/master");
const execution = runnerMatrix[0]!;
const campaign = buildRunnerCampaign({
campaignId: "target-provenance",
generatedAt: "2026-08-28T00:01:00.000Z",
expected: [execution.id],
results: [
{
...result(execution, "passed"),
source: {
sha: "retained-result-sha",
ref: "refs/heads/retained-result",
workflowRunUrl: "https://example.test/actions/runs/1",
},
},
],
});
expect(campaign.source).toMatchObject({
sha: "target-sha",
ref: "refs/heads/target",
workflowRunUrl: "https://example.test/actions/runs/1",
});
});
it("migrates v1 execution IDs and keeps partial suite runs out of overall trends", () => {
expect(canonicalExecutionId("legacy-codex.local.message-marker")).toBe(
"core-compatibility.legacy-codex.local.message-marker",

11
tests/runner-e2e/history.ts
View File

@ -3,6 +3,7 @@ import {
aggregateCampaignBilling,
summarizeExecutionBilling,
} from "./billing.js";
import { resolveRunnerE2ESource } from "./source.js";
import type {
RunnerE2ECampaign,
RunnerE2EHistoryCampaign,
@ -56,15 +57,7 @@ export function buildRunnerCampaign(input: {
}));
const resultSource = results.find((result) => result.source)?.source;
const source = {
sha: resultSource?.sha ?? process.env.GITHUB_SHA ?? null,
ref: resultSource?.ref ?? process.env.GITHUB_REF ?? null,
workflowRunUrl:
resultSource?.workflowRunUrl ??
(process.env.GITHUB_SERVER_URL &&
process.env.GITHUB_REPOSITORY &&
process.env.GITHUB_RUN_ID
? `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`
: null),
...resolveRunnerE2ESource(resultSource),
eventName: input.eventName ?? process.env.GITHUB_EVENT_NAME ?? null,
};
const suites = runnerSuites

12
tests/runner-e2e/launch.ts
View File

@ -39,6 +39,7 @@ import {
RunnerSelectorError,
selectRunnerExecutions,
} from "./selectors.js";
import { resolveRunnerE2ESource } from "./source.js";
import {
CREDENTIAL_NAMES,
type MatrixExecution,
@ -280,16 +281,7 @@ function syntheticResult(
executionId: execution.id,
suiteId: execution.suite.id,
suiteDefinitionHash: execution.suiteDefinitionHash,
source: {
sha: process.env.GITHUB_SHA ?? null,
ref: process.env.GITHUB_REF ?? null,
workflowRunUrl:
process.env.GITHUB_SERVER_URL &&
process.env.GITHUB_REPOSITORY &&
process.env.GITHUB_RUN_ID
? `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`
: null,
},
source: resolveRunnerE2ESource(),
...(execution.profile.ranking
? { rankingSnapshot: execution.profile.ranking }
: {}),

26
tests/runner-e2e/report.test.ts
View File

@ -38,6 +38,11 @@ describe("runner E2E report aggregation", () => {
startedAt: "2026-08-26T00:00:00.000Z",
finishedAt: "2026-08-26T00:00:01.000Z",
durationMs: 1_000,
source: {
sha: "forged-result-sha",
ref: "refs/heads/forged-result",
workflowRunUrl: "https://example.test/actions/runs/forged",
},
runIds: ["run-2"],
usage: {
inputTokens: 1_250,
@ -123,6 +128,15 @@ describe("runner E2E report aggregation", () => {
PAPERCLIP_RUNNER_E2E_REPORT_ROOT: root,
PAPERCLIP_RUNNER_E2E_REPORT_OUT: output,
PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: JSON.stringify([executionId]),
PAPERCLIP_RUNNER_E2E_SOURCE_SHA:
"0123456789abcdef0123456789abcdef01234567",
PAPERCLIP_RUNNER_E2E_SOURCE_REF:
"refs/heads/fix/runner-paid-source-attribution",
GITHUB_SHA: "trusted-default-workflow-sha",
GITHUB_REF: "refs/heads/master",
GITHUB_SERVER_URL: "https://github.com",
GITHUB_REPOSITORY: "paperclipai/paperclip",
GITHUB_RUN_ID: "123456",
},
},
);
@ -137,6 +151,12 @@ describe("runner E2E report aggregation", () => {
failed: 0,
retries: 1,
cleanupPassed: true,
source: {
sha: "0123456789abcdef0123456789abcdef01234567",
ref: "refs/heads/fix/runner-paid-source-attribution",
workflowRunUrl:
"https://github.com/paperclipai/paperclip/actions/runs/123456",
},
});
expect(normalized.billing).toMatchObject({
reportedLlmCostUsd: 0.0125,
@ -149,6 +169,12 @@ describe("runner E2E report aggregation", () => {
expect(normalized.results[0]).toMatchObject({
attempt: 2,
evidenceValid: true,
source: {
sha: "0123456789abcdef0123456789abcdef01234567",
ref: "refs/heads/fix/runner-paid-source-attribution",
workflowRunUrl:
"https://github.com/paperclipai/paperclip/actions/runs/123456",
},
});
const dashboard = await readFile(
path.join(output, "dashboard.html"),

5
tests/runner-e2e/report.ts
View File

@ -14,6 +14,7 @@ import {
canonicalExecutionId,
upgradeRunnerResult,
} from "./history.js";
import { resolveRunnerE2ESource } from "./source.js";
import type { RunnerE2EResult } from "./types.js";
const repositoryRoot = path.resolve(import.meta.dirname, "../..");
@ -245,6 +246,10 @@ async function main() {
]);
const resolvedResults = selected.map((entry) => ({
...entry.result,
// Cell evidence is produced by target-controlled code. The trusted report
// stamps the immutable target selected by the authorization job instead
// of allowing retained result metadata to claim another revision.
source: resolveRunnerE2ESource(entry.result.source),
status: entry.valid ? entry.result.status : ("failed" as const),
billing: summarizeExecutionBilling(entry.result),
}));

12
tests/runner-e2e/runner.spec.ts
View File

@ -16,6 +16,7 @@ import {
numberedPlanStepCount,
providerSessionContinuityFailures,
} from "./run-observations.js";
import { resolveRunnerE2ESource } from "./source.js";
import {
assertSecretFree,
findSecretLeakInJsonValues,
@ -1804,16 +1805,7 @@ for (const execution of executions) {
executionId: execution.id,
suiteId: execution.suite.id,
suiteDefinitionHash: execution.suiteDefinitionHash,
source: {
sha: process.env.GITHUB_SHA ?? null,
ref: process.env.GITHUB_REF ?? null,
workflowRunUrl:
process.env.GITHUB_SERVER_URL &&
process.env.GITHUB_REPOSITORY &&
process.env.GITHUB_RUN_ID
? `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`
: null,
},
source: resolveRunnerE2ESource(),
...(execution.profile.ranking
? { rankingSnapshot: execution.profile.ranking }
: {}),

59
tests/runner-e2e/source.test.ts Normal file
View File

@ -0,0 +1,59 @@
import { describe, expect, it } from "vitest";
import { resolveRunnerE2ESource } from "./source.js";
describe("runner E2E source provenance", () => {
it("prefers the resolved target over result and workflow revision contexts", () => {
expect(
resolveRunnerE2ESource(
{
sha: "result-sha",
ref: "refs/heads/result",
workflowRunUrl: "https://example.test/result-run",
},
{
PAPERCLIP_RUNNER_E2E_SOURCE_SHA: "target-sha",
PAPERCLIP_RUNNER_E2E_SOURCE_REF: "refs/heads/target",
GITHUB_SHA: "workflow-sha",
GITHUB_REF: "refs/heads/master",
GITHUB_SERVER_URL: "https://github.com",
GITHUB_REPOSITORY: "paperclipai/paperclip",
GITHUB_RUN_ID: "123",
},
),
).toEqual({
sha: "target-sha",
ref: "refs/heads/target",
workflowRunUrl:
"https://github.com/paperclipai/paperclip/actions/runs/123",
});
});
it("falls back through retained result provenance, workflow context, and null", () => {
expect(
resolveRunnerE2ESource(
{
sha: "result-sha",
ref: null,
workflowRunUrl: null,
},
{
GITHUB_SHA: "workflow-sha",
GITHUB_REF: "refs/heads/master",
GITHUB_SERVER_URL: "https://github.com",
GITHUB_REPOSITORY: "paperclipai/paperclip",
GITHUB_RUN_ID: "456",
},
),
).toEqual({
sha: "result-sha",
ref: "refs/heads/master",
workflowRunUrl:
"https://github.com/paperclipai/paperclip/actions/runs/456",
});
expect(resolveRunnerE2ESource(null, {})).toEqual({
sha: null,
ref: null,
workflowRunUrl: null,
});
});
});

35
tests/runner-e2e/source.ts Normal file
View File

@ -0,0 +1,35 @@
import type { RunnerE2EResult } from "./types.js";
type RunnerE2ESource = NonNullable<RunnerE2EResult["source"]>;
function nonEmpty(value: string | null | undefined) {
const normalized = value?.trim();
return normalized ? normalized : null;
}
function workflowRunUrl(environment: NodeJS.ProcessEnv) {
const serverUrl = nonEmpty(environment.GITHUB_SERVER_URL);
const repository = nonEmpty(environment.GITHUB_REPOSITORY);
const runId = nonEmpty(environment.GITHUB_RUN_ID);
return serverUrl && repository && runId
? `${serverUrl}/${repository}/actions/runs/${runId}`
: null;
}
export function resolveRunnerE2ESource(
existing?: RunnerE2ESource | null,
environment: NodeJS.ProcessEnv = process.env,
): RunnerE2ESource {
return {
sha:
nonEmpty(environment.PAPERCLIP_RUNNER_E2E_SOURCE_SHA) ??
nonEmpty(existing?.sha) ??
nonEmpty(environment.GITHUB_SHA),
ref:
nonEmpty(environment.PAPERCLIP_RUNNER_E2E_SOURCE_REF) ??
nonEmpty(existing?.ref) ??
nonEmpty(environment.GITHUB_REF),
workflowRunUrl:
workflowRunUrl(environment) ?? nonEmpty(existing?.workflowRunUrl),
};
}

20
tests/runner-e2e/workflow-security.test.ts
View File

@ -92,6 +92,10 @@ describe("public repository paid workflow security", () => {
"repos/$REPOSITORY/branches/$encoded_branch",
);
expect(authorizeJob).toContain('echo "sha=$target_sha"');
expect(authorizeJob).toContain(
"target_ref: ${{ steps.target.outputs.ref }}",
);
expect(authorizeJob).toContain('echo "ref=refs/heads/$TARGET_BRANCH"');
expect(authorizeJob).not.toContain("actions/checkout@");
expect(authorizeJob).not.toContain("pnpm install");
expect(targetLockJob).toContain("name: Resolve target pnpm lockfile");
@ -255,6 +259,14 @@ describe("public repository paid workflow security", () => {
expect(fullStack).toContain(
"if: always() && !cancelled() && needs.catalog.result == 'success'",
);
for (const targetProvenanceJob of [paidJob, reportJob]) {
expect(targetProvenanceJob).toContain(
"PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}",
);
expect(targetProvenanceJob).toContain(
"PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }}",
);
}
for (const [secret, condition] of Object.entries({
OPENAI_API_KEY: "matrix.credentialName == 'OPENAI_API_KEY'",
ANTHROPIC_API_KEY: "matrix.credentialName == 'ANTHROPIC_API_KEY'",
@ -356,6 +368,14 @@ describe("public repository paid workflow security", () => {
expect(buildJob).toContain(
"provider_pack_artifact_name: ${{ steps.provider_pack_artifact_name.outputs.name }}",
);
expect(buildJob).toContain(
"runner-e2e-build-${TARGET_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}",
);
expect(buildJob).toContain(
"runner-e2e-provider-pack-${TARGET_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}",
);
expect(buildJob).not.toContain("runner-e2e-build-${GITHUB_SHA}");
expect(buildJob).not.toContain("runner-e2e-provider-pack-${GITHUB_SHA}");
expect(workflow).toContain("needs_runner_typescript=");
expect(workflow).toContain("needs_native_binaries=");
expect(workflow).toContain("needs_remote_provider_pack=");