421 lines
17 KiB
TypeScript
421 lines
17 KiB
TypeScript
import { readdir, readFile } from "node:fs/promises";
|
|
import path from "node:path";
|
|
import { describe, expect, it } from "vitest";
|
|
|
|
const repositoryRoot = path.resolve(import.meta.dirname, "../..");
|
|
const fullStackTestNeeds =
|
|
/needs:\s*\[\s*authorize,\s*target_lock,\s*catalog,\s*daytona_image,\s*build_runner_artifacts,\s*build_remote_provider_pack,?\s*\]/u;
|
|
const buildRunnerNeeds =
|
|
/needs:\s*\[\s*authorize,\s*target_lock,\s*catalog,?\s*\]/u;
|
|
const buildRemoteProviderPackNeeds =
|
|
/needs:\s*\[\s*authorize,\s*target_lock,\s*catalog,\s*daytona_image,\s*build_runner_artifacts,?\s*\]/u;
|
|
|
|
describe("public repository paid workflow security", () => {
|
|
it("gates every provider-secret job with stable actor IDs", async () => {
|
|
const workflows = await Promise.all(
|
|
["runner-full-stack-e2e.yml", "runner-live-evals.yml", "e2e.yml"].map(
|
|
async (name) => ({
|
|
name,
|
|
contents: await readFile(
|
|
path.join(repositoryRoot, ".github/workflows", name),
|
|
"utf8",
|
|
),
|
|
}),
|
|
),
|
|
);
|
|
|
|
for (const { name, contents } of workflows) {
|
|
const authorize = contents.indexOf(" authorize:");
|
|
const reauthorize = contents.indexOf("Reauthorize");
|
|
const paidCheckout = contents.indexOf("actions/checkout@", reauthorize);
|
|
const providerAccess = contents.search(
|
|
/(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA)_API_KEY:\s*\$\{\{\s*[^}]*secrets\./,
|
|
);
|
|
expect(
|
|
authorize,
|
|
`${name} must have an authorization job`,
|
|
).toBeGreaterThan(0);
|
|
expect(
|
|
reauthorize,
|
|
`${name} must reauthorize partial job reruns`,
|
|
).toBeGreaterThan(authorize);
|
|
expect(
|
|
paidCheckout,
|
|
`${name} must authorize before checkout`,
|
|
).toBeGreaterThan(reauthorize);
|
|
expect(
|
|
providerAccess,
|
|
`${name} must authorize before provider access`,
|
|
).toBeGreaterThan(reauthorize);
|
|
expect(contents).toContain("RUNNER_E2E_ALLOWED_ACTOR_IDS");
|
|
expect(contents).toContain("github.actor_id");
|
|
expect(contents).toContain("github.triggering_actor");
|
|
expect(contents).toContain("refs/heads/$DEFAULT_BRANCH");
|
|
expect(contents).toContain("needs: authorize");
|
|
expect(contents).toContain("name: runner-e2e-paid");
|
|
expect(contents).not.toMatch(
|
|
/^\s*(?:pull_request|pull_request_target|push|workflow_call|workflow_run):/m,
|
|
);
|
|
const actionReferences = [
|
|
...contents.matchAll(/^\s*(?:-\s*)?uses:\s*([^\s#]+)/gm),
|
|
].map((match) => match[1]!);
|
|
expect(actionReferences.length).toBeGreaterThan(0);
|
|
for (const reference of actionReferences) {
|
|
expect(reference).toMatch(/^[^@]+@[0-9a-f]{40}$/);
|
|
}
|
|
}
|
|
|
|
const fullStack = workflows[0]!.contents;
|
|
const paidJob = fullStack.slice(
|
|
fullStack.indexOf(" test:"),
|
|
fullStack.indexOf(" report:"),
|
|
);
|
|
const authorizeJob = fullStack.slice(
|
|
fullStack.indexOf(" authorize:"),
|
|
fullStack.indexOf(" target_lock:"),
|
|
);
|
|
const targetLockJob = fullStack.slice(
|
|
fullStack.indexOf(" target_lock:"),
|
|
fullStack.indexOf(" catalog:"),
|
|
);
|
|
expect(authorizeJob).toContain(
|
|
"aws_runner='runs-on/fleet=paperclip-public-pr-x64/env=public-ci'",
|
|
);
|
|
expect(authorizeJob).toContain("github_runner='ubuntu-latest'");
|
|
expect(authorizeJob).toContain(
|
|
"AWS_PAID_RUNNER_ENABLED: ${{ vars.RUNNER_E2E_AWS_ENABLED }}",
|
|
);
|
|
expect(authorizeJob).toContain(
|
|
"Resolve requested repository branch to an immutable commit",
|
|
);
|
|
expect(authorizeJob).toContain(
|
|
"repos/$REPOSITORY/branches/$encoded_branch",
|
|
);
|
|
expect(authorizeJob).toContain('echo "sha=$target_sha"');
|
|
expect(authorizeJob).not.toContain("actions/checkout@");
|
|
expect(authorizeJob).not.toContain("pnpm install");
|
|
expect(targetLockJob).toContain("name: Resolve target pnpm lockfile");
|
|
expect(targetLockJob).toContain("needs: authorize");
|
|
expect(targetLockJob).toContain(
|
|
"ref: ${{ needs.authorize.outputs.target_sha }}",
|
|
);
|
|
expect(targetLockJob).toContain("persist-credentials: false");
|
|
expect(targetLockJob).toContain(
|
|
"pnpm install --ignore-scripts --no-frozen-lockfile --lockfile-only",
|
|
);
|
|
expect(targetLockJob).toContain(
|
|
"artifact_id: ${{ steps.upload.outputs.artifact-id }}",
|
|
);
|
|
expect(targetLockJob).toContain("lock_sha256:");
|
|
expect(targetLockJob).toContain(
|
|
"runner-e2e-target-pnpm-lock-${{ github.run_id }}-${{ github.run_attempt }}",
|
|
);
|
|
expect(targetLockJob).not.toContain("name: runner-e2e-paid");
|
|
expect(targetLockJob).not.toMatch(
|
|
/(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA)_API_KEY/,
|
|
);
|
|
expect(paidJob).toContain(
|
|
"runs-on: ${{ needs.authorize.outputs.test_runner }}",
|
|
);
|
|
expect(paidJob).toMatch(fullStackTestNeeds);
|
|
expect(paidJob).toContain("name: runner-e2e-paid");
|
|
expect(paidJob).toMatch(
|
|
/Reauthorize paid execution before provider access[\s\S]*actions\/checkout@[0-9a-f]{40}[\s\S]*persist-credentials: false[\s\S]*Download resolved target lockfile/,
|
|
);
|
|
const paidInstall = paidJob.indexOf(
|
|
"pnpm install --frozen-lockfile --ignore-scripts",
|
|
);
|
|
const daytonaPluginPreparation = paidJob.indexOf(
|
|
"Prepare bundled Daytona plugin without dependency lifecycle scripts",
|
|
);
|
|
const paidExecution = paidJob.indexOf("- name: Run paid cell");
|
|
expect(paidInstall).toBeGreaterThan(0);
|
|
expect(daytonaPluginPreparation).toBeGreaterThan(paidInstall);
|
|
expect(paidExecution).toBeGreaterThan(daytonaPluginPreparation);
|
|
const preparedBeforeProviderAccess = paidJob.slice(
|
|
daytonaPluginPreparation,
|
|
paidExecution,
|
|
);
|
|
expect(preparedBeforeProviderAccess).toContain(
|
|
"if: matrix.environmentId == 'daytona'",
|
|
);
|
|
expect(preparedBeforeProviderAccess).toContain(
|
|
'test -f "$daytona_root/pnpm-lock.yaml"',
|
|
);
|
|
expect(preparedBeforeProviderAccess).toContain(
|
|
"pnpm install --ignore-workspace --frozen-lockfile --ignore-scripts",
|
|
);
|
|
expect(preparedBeforeProviderAccess).not.toContain("--no-lockfile");
|
|
expect(preparedBeforeProviderAccess).toContain(
|
|
"node scripts/link-plugin-dev-sdk.mjs",
|
|
);
|
|
expect(preparedBeforeProviderAccess).toContain(
|
|
'"@paperclipai/plugin-daytona"',
|
|
);
|
|
expect(preparedBeforeProviderAccess).toContain('"@paperclipai/plugin-sdk"');
|
|
expect(preparedBeforeProviderAccess).toContain(
|
|
'realpath "$daytona_root/node_modules/@paperclipai/plugin-sdk"',
|
|
);
|
|
expect(preparedBeforeProviderAccess).toContain(
|
|
'pnpm --dir "$daytona_root" build',
|
|
);
|
|
expect(preparedBeforeProviderAccess).not.toContain("secrets.");
|
|
expect(preparedBeforeProviderAccess).not.toContain("pnpm rebuild");
|
|
expect(paidJob.slice(0, paidExecution)).not.toMatch(
|
|
/secrets\.(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA)_API_KEY/,
|
|
);
|
|
expect(authorizeJob).toContain('echo "max_parallel_limit=100"');
|
|
expect(fullStack).toContain('[ "$MAX_PARALLEL_LIMIT" -gt 100 ]');
|
|
expect(fullStack).toContain(
|
|
'[ "$MAX_PARALLEL" -gt "$MAX_PARALLEL_LIMIT" ]',
|
|
);
|
|
expect(fullStack).toContain(
|
|
"group: runner-full-stack-e2e-${{ inputs.target_branch || github.event.repository.default_branch }}",
|
|
);
|
|
expect(fullStack).toContain(
|
|
"cancel-in-progress: ${{ github.event_name == 'workflow_dispatch' && inputs.target_branch != '' && inputs.target_branch != github.event.repository.default_branch }}",
|
|
);
|
|
const targetCodeJobs = [
|
|
fullStack.slice(
|
|
fullStack.indexOf(" catalog:"),
|
|
fullStack.indexOf(" daytona_image:"),
|
|
),
|
|
fullStack.slice(
|
|
fullStack.indexOf(" daytona_image:"),
|
|
fullStack.indexOf(" build_runner_artifacts:"),
|
|
),
|
|
fullStack.slice(
|
|
fullStack.indexOf(" build_runner_artifacts:"),
|
|
fullStack.indexOf(" build_remote_provider_pack:"),
|
|
),
|
|
fullStack.slice(
|
|
fullStack.indexOf(" build_remote_provider_pack:"),
|
|
fullStack.indexOf(" test:"),
|
|
),
|
|
paidJob,
|
|
];
|
|
for (const targetCodeJob of targetCodeJobs) {
|
|
const checkout = targetCodeJob.indexOf("actions/checkout@");
|
|
const downloadLock = targetCodeJob.indexOf(
|
|
"Download resolved target lockfile",
|
|
);
|
|
const restoreLock = targetCodeJob.indexOf(
|
|
"Restore resolved target lockfile",
|
|
);
|
|
const setupNode = targetCodeJob.indexOf("actions/setup-node@");
|
|
const install = targetCodeJob.indexOf("pnpm install --frozen-lockfile");
|
|
expect(checkout).toBeGreaterThan(0);
|
|
expect(downloadLock).toBeGreaterThan(checkout);
|
|
expect(restoreLock).toBeGreaterThan(downloadLock);
|
|
if (setupNode >= 0) {
|
|
expect(setupNode).toBeGreaterThan(restoreLock);
|
|
}
|
|
if (install >= 0) {
|
|
expect(install).toBeGreaterThan(restoreLock);
|
|
}
|
|
expect(targetCodeJob).toContain(
|
|
"artifact-ids: ${{ needs.target_lock.outputs.artifact_id }}",
|
|
);
|
|
expect(targetCodeJob).toContain(
|
|
"EXPECTED_LOCK_SHA256: ${{ needs.target_lock.outputs.lock_sha256 }}",
|
|
);
|
|
}
|
|
expect(fullStack.match(/Download resolved target lockfile/g)).toHaveLength(
|
|
5,
|
|
);
|
|
expect(fullStack.match(/Restore resolved target lockfile/g)).toHaveLength(
|
|
5,
|
|
);
|
|
expect(
|
|
fullStack.match(
|
|
/ref: \$\{\{ needs\.authorize\.outputs\.target_sha \}\}/g,
|
|
),
|
|
).toHaveLength(6);
|
|
expect(fullStack.match(/ref: \$\{\{ github\.sha \}\}/g)).toHaveLength(2);
|
|
expect(fullStack.match(/persist-credentials: false/g)).toHaveLength(8);
|
|
expect(fullStack).not.toContain("ref: ${{ inputs.target_branch }}");
|
|
expect(fullStack).toContain(
|
|
"PAPERCLIP_RUNNER_SOURCE_REVISION=${TARGET_SHA}",
|
|
);
|
|
const reportJob = fullStack.slice(
|
|
fullStack.indexOf(" report:"),
|
|
fullStack.indexOf(" publish_history:"),
|
|
);
|
|
const historyJob = fullStack.slice(fullStack.indexOf(" publish_history:"));
|
|
expect(reportJob).toContain("ref: ${{ github.sha }}");
|
|
expect(reportJob).not.toContain(
|
|
"ref: ${{ needs.authorize.outputs.target_sha }}",
|
|
);
|
|
expect(reportJob).not.toContain("Download resolved target lockfile");
|
|
expect(historyJob).toContain("ref: ${{ github.sha }}");
|
|
expect(historyJob).not.toContain(
|
|
"ref: ${{ needs.authorize.outputs.target_sha }}",
|
|
);
|
|
expect(historyJob).not.toContain("Download resolved target lockfile");
|
|
for (const [secret, condition] of Object.entries({
|
|
OPENAI_API_KEY: "matrix.credentialName == 'OPENAI_API_KEY'",
|
|
ANTHROPIC_API_KEY: "matrix.credentialName == 'ANTHROPIC_API_KEY'",
|
|
OPENROUTER_API_KEY: "matrix.credentialName == 'OPENROUTER_API_KEY'",
|
|
DAYTONA_API_KEY: "matrix.environmentId == 'daytona'",
|
|
})) {
|
|
expect(fullStack).toContain(
|
|
`${secret}: \${{ ${condition} && secrets.${secret} || '' }}`,
|
|
);
|
|
}
|
|
});
|
|
|
|
it("keeps provider credentials inside explicitly gated paid workflows", async () => {
|
|
const workflowDirectory = path.join(repositoryRoot, ".github/workflows");
|
|
const allowedProviderWorkflows = new Set([
|
|
"e2e.yml",
|
|
"runner-full-stack-e2e.yml",
|
|
"runner-live-evals.yml",
|
|
]);
|
|
const names = (await readdir(workflowDirectory)).filter((name) =>
|
|
/\.ya?ml$/.test(name),
|
|
);
|
|
|
|
for (const name of names) {
|
|
const contents = await readFile(
|
|
path.join(workflowDirectory, name),
|
|
"utf8",
|
|
);
|
|
const providerSecretReferences = [
|
|
...contents.matchAll(
|
|
/secrets(?:\.(?:OPENAI_API_KEY|ANTHROPIC_API_KEY|OPENROUTER_API_KEY|DAYTONA_API_KEY)\b|\[['"](?:OPENAI_API_KEY|ANTHROPIC_API_KEY|OPENROUTER_API_KEY|DAYTONA_API_KEY)['"]\])/g,
|
|
),
|
|
];
|
|
if (providerSecretReferences.length > 0) {
|
|
expect(
|
|
allowedProviderWorkflows.has(name),
|
|
`${name} must not receive provider credentials`,
|
|
).toBe(true);
|
|
}
|
|
}
|
|
});
|
|
|
|
it("runs paid scheduled campaigns only on Sundays", async () => {
|
|
const workflows = await Promise.all(
|
|
["runner-full-stack-e2e.yml", "runner-live-evals.yml"].map((name) =>
|
|
readFile(path.join(repositoryRoot, ".github/workflows", name), "utf8"),
|
|
),
|
|
);
|
|
for (const workflow of workflows) {
|
|
const crons = [...workflow.matchAll(/cron:\s*"([^"]+)"/g)].map(
|
|
(match) => match[1]!,
|
|
);
|
|
expect(crons).toHaveLength(1);
|
|
expect(crons[0]).toMatch(/^\d{1,2} \d{1,2} \* \* 0$/);
|
|
expect(workflow).toContain("workflow_dispatch:");
|
|
}
|
|
});
|
|
|
|
it("builds runner outputs once without provider credentials and verifies them in every paid cell", async () => {
|
|
const workflow = await readFile(
|
|
path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"),
|
|
"utf8",
|
|
);
|
|
const buildJobStart = workflow.indexOf(" build_runner_artifacts:");
|
|
const testJobStart = workflow.indexOf(" test:", buildJobStart);
|
|
const reportJobStart = workflow.indexOf(" report:", testJobStart);
|
|
const buildJob = workflow.slice(buildJobStart, testJobStart);
|
|
const testJob = workflow.slice(testJobStart, reportJobStart);
|
|
|
|
expect(buildJobStart).toBeGreaterThan(0);
|
|
expect(testJobStart).toBeGreaterThan(buildJobStart);
|
|
expect(buildJob).toMatch(buildRunnerNeeds);
|
|
expect(buildJob).toMatch(buildRemoteProviderPackNeeds);
|
|
expect(buildJob).not.toContain("environment:");
|
|
expect(buildJob).not.toContain("secrets.");
|
|
expect(
|
|
buildJob.match(/pnpm install --frozen-lockfile --ignore-scripts/g),
|
|
).toHaveLength(2);
|
|
expect(buildJob).toContain(
|
|
"pnpm --filter @paperclipai/paperclip-runner build:typescript",
|
|
);
|
|
expect(buildJob).toContain(
|
|
"pnpm --filter @paperclipai/paperclip-runner build:runner-binaries",
|
|
);
|
|
expect(buildJob).toContain(
|
|
"node packages/paperclip-runner/scripts/build-provider-pack.mjs",
|
|
);
|
|
expect(buildJob).toContain(
|
|
"node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs",
|
|
);
|
|
expect(buildJob).toContain("runner-e2e-build-bundle.tar.gz.sha256");
|
|
expect(buildJob).toContain("runner-e2e-provider-pack.tar.gz.sha256");
|
|
expect(buildJob).toContain(
|
|
"build_artifact_name: ${{ steps.build_artifact_name.outputs.name }}",
|
|
);
|
|
expect(buildJob).toContain(
|
|
"needs.build_runner_artifacts.outputs.build_artifact_name",
|
|
);
|
|
expect(buildJob).toContain(
|
|
"provider_pack_artifact_name: ${{ steps.provider_pack_artifact_name.outputs.name }}",
|
|
);
|
|
expect(workflow).toContain("needs_runner_typescript=");
|
|
expect(workflow).toContain("needs_native_binaries=");
|
|
expect(workflow).toContain("needs_remote_provider_pack=");
|
|
|
|
expect(testJob).toMatch(fullStackTestNeeds);
|
|
expect(testJob).toContain("Download immutable campaign outputs");
|
|
expect(testJob).toContain("Download immutable remote provider pack");
|
|
expect(testJob).toContain(
|
|
"needs.build_runner_artifacts.outputs.build_artifact_name",
|
|
);
|
|
expect(testJob).toContain(
|
|
"needs.build_remote_provider_pack.outputs.provider_pack_artifact_name",
|
|
);
|
|
expect(testJob).toContain("sha256sum --check");
|
|
expect(testJob.indexOf("sha256sum --check")).toBeLessThan(
|
|
testJob.indexOf("tar --extract"),
|
|
);
|
|
expect(testJob).toContain(
|
|
"test -x packages/paperclip-runner/runner/target/debug/paperclip-runnerd",
|
|
);
|
|
expect(testJob).toContain(".payload.runnerSourceRevision == $revision");
|
|
expect(workflow).toContain("Qualify local provider Node interpreter");
|
|
expect(testJob).toContain(
|
|
"Materialize verified pinned OpenCode executable",
|
|
);
|
|
expect(testJob).toContain(
|
|
"matrix.profileId == 'legacy-opencode' || matrix.profileId == 'runner-opencode' || matrix.suiteId == 'openrouter-model-breadth'",
|
|
);
|
|
expect(testJob).toContain(
|
|
"node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs",
|
|
);
|
|
expect(testJob).not.toContain("postinstall.mjs");
|
|
expect(testJob).not.toContain("pnpm rebuild");
|
|
expect(testJob).not.toContain("build:typescript");
|
|
expect(testJob).not.toContain("build:runner-binaries");
|
|
expect(testJob).not.toContain("build-provider-pack.mjs");
|
|
});
|
|
|
|
it("uses environment-scoped OIDC for a no-delete history publisher", async () => {
|
|
const workflow = await readFile(
|
|
path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"),
|
|
"utf8",
|
|
);
|
|
const publisher = workflow.slice(workflow.indexOf(" publish_history:"));
|
|
expect(publisher).toContain("id-token: write");
|
|
expect(publisher).toContain("name: runner-e2e-history");
|
|
expect(publisher).toContain("aws-actions/configure-aws-credentials@");
|
|
expect(publisher).toContain("RUNNER_E2E_HISTORY_AWS_ROLE_ARN");
|
|
expect(publisher).not.toContain("cache: pnpm");
|
|
expect(publisher).not.toMatch(/AWS_(?:ACCESS|SECRET)_KEY/);
|
|
expect(publisher).not.toMatch(/aws s3 (?:rm|sync .*--delete)/);
|
|
expect(workflow).toContain("history_source_ready");
|
|
expect(workflow).toContain(
|
|
"Verify history source report and private screenshot evidence",
|
|
);
|
|
expect(workflow).toContain("private_screenshot=");
|
|
expect(workflow).toContain("Publish pruned immutable history");
|
|
expect(workflow).toContain("Publish latest structured dashboard");
|
|
expect(workflow).not.toContain("dashboard_ready");
|
|
expect(workflow).not.toContain("Publish latest screenshot dashboard");
|
|
expect(
|
|
workflow.indexOf("pnpm test:e2e:runner:history:publish"),
|
|
).toBeLessThan(workflow.indexOf("actions/upload-pages-artifact@"));
|
|
});
|
|
});
|