fix(ci): reuse cloud source verification for npm canaries (#13233)
Reuse the exact master source-verification result before npm canary publication, removing a duplicate verification matrix while preserving fail-closed release checks. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Devin Foley
GitHub
parent
52811c6ce6
commit
4fde92107e
|
|
@ -48,6 +48,33 @@ jobs:
|
|||
SOURCE_SHA: ${{ github.sha }}
|
||||
run: node scripts/cloud-readiness.mjs "$SOURCE_SHA"
|
||||
|
||||
source_verified:
|
||||
# npm canary publication reuses this exact-source verification proof.
|
||||
# Keep it independent of image/migrator availability, and fail closed when
|
||||
# any source check fails, is cancelled, or is skipped.
|
||||
name: Cloud source verified v1
|
||||
needs: [verify]
|
||||
if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
ref: ${{ github.sha }}
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: 24
|
||||
- name: Check the source verification consumer
|
||||
run: node --test scripts/cloud-source-verification.test.mjs
|
||||
- name: Record source verification
|
||||
env:
|
||||
SOURCE_SHA: ${{ github.sha }}
|
||||
run: |
|
||||
echo "Cloud source verified v1: $SOURCE_SHA" >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
ready:
|
||||
# Versioned consumer contract. Never add always() or continue-on-error:
|
||||
# failed, cancelled, or skipped prerequisites must not report readiness.
|
||||
|
|
|
|||
|
|
@ -335,7 +335,7 @@ jobs:
|
|||
run: node --test ./scripts/__tests__/e2e-shard.test.mjs
|
||||
|
||||
- name: Test release verify workflow wiring
|
||||
run: node --test ./scripts/__tests__/release-verify-workflow.test.mjs
|
||||
run: node --test ./scripts/__tests__/release-verify-workflow.test.mjs ./scripts/cloud-source-verification.test.mjs
|
||||
|
||||
- name: Test standalone package build concurrency
|
||||
run: node --test ./scripts/__tests__/build-standalone-concurrency.test.mjs
|
||||
|
|
|
|||
|
|
@ -296,10 +296,26 @@ jobs:
|
|||
retention-days: 30
|
||||
|
||||
verify_canary:
|
||||
if: github.event_name == 'push'
|
||||
uses: ./.github/workflows/release-verify.yml
|
||||
with:
|
||||
ref: ${{ github.sha }}
|
||||
name: Reuse exact-source verification
|
||||
if: github.repository == 'paperclipai/paperclip' && github.event_name == 'push' && github.ref == 'refs/heads/master'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 50
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
ref: ${{ github.sha }}
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: 24
|
||||
- name: Require successful source checks for this exact master push
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
SOURCE_SHA: ${{ github.sha }}
|
||||
run: node scripts/cloud-source-verification.mjs "$SOURCE_SHA"
|
||||
|
||||
publish_canary:
|
||||
if: github.event_name == 'push'
|
||||
|
|
|
|||
|
|
@ -26,11 +26,22 @@ build with the new identity must rebuild layers that depend on the base image;
|
|||
later builds can reuse those layers.
|
||||
|
||||
Verification and image building run concurrently, outside the full npm release's
|
||||
concurrency group. Different commits have independent groups. Source verification
|
||||
is initially duplicated with the normal npm release: this spends existing hosted
|
||||
runner capacity to avoid waiting behind an older release. No verification gate is
|
||||
removed from npm publication. Watch organization-wide runner queues when measuring
|
||||
the result.
|
||||
concurrency group. Different commits have independent groups. The npm canary
|
||||
release reuses `Cloud source verified v1` for the exact master push instead of
|
||||
starting a second copy of `Release Verify`. This source-only job depends on every
|
||||
source check but does not wait for Docker or migrator publication. npm canary
|
||||
publication remains possible when source verification passes and an image build
|
||||
fails. Stable releases and candidate-branch betas still run full verification.
|
||||
|
||||
The canary consumer requires the expected workflow ID and path, upstream source
|
||||
repository, master push event, full SHA, and a successful job in the latest run
|
||||
attempt. It checks the run again after reading the jobs to reject a concurrent
|
||||
rerun. Missing proof waits for up to 45 minutes; failed, skipped, cancelled,
|
||||
ambiguous, or mismatched proof cannot authorize publication. API failures fail
|
||||
closed. If a source check fails, fix it and rerun Cloud readiness before retrying
|
||||
the release. Use **Re-run all jobs** when a later attempt did not rerun the source
|
||||
proof; an earlier attempt's successful job is not accepted. This avoids duplicate
|
||||
test jobs on standard runners. Measure queue time to assess the timing gain.
|
||||
|
||||
Release verification spreads the general server suites across ten standard hosted
|
||||
runners, with the long chat suite split separately across three jobs. Each server
|
||||
|
|
@ -67,8 +78,9 @@ successfully verified a real master commit.
|
|||
## Timing and rollout
|
||||
|
||||
The reusable Runner chaos workflow scopes concurrency to the caller workflow
|
||||
and source ref. Cloud readiness and the npm release can verify the same commit
|
||||
at the same time. They must not cancel each other's required test job.
|
||||
and source ref. Cloud readiness, stable verification, and standalone evals can
|
||||
verify the same commit at the same time. They must not cancel each other's
|
||||
required test job.
|
||||
|
||||
Measure the complete path from a master merge to a healthy target running that
|
||||
exact commit. Keep readiness and deployment as separate milestones:
|
||||
|
|
|
|||
|
|
@ -35,13 +35,16 @@ test("chaos verification isolates callers that verify the same source commit", (
|
|||
assert.match(chaosWorkflow, /cancel-in-progress: true/);
|
||||
});
|
||||
|
||||
test("release workflow delegates stable and canary verification to the reusable workflow", () => {
|
||||
test("canary reuses exact-source proof while stable keeps full verification", () => {
|
||||
const releaseWorkflow = readWorkflow("release.yml");
|
||||
|
||||
assert.match(
|
||||
releaseWorkflow,
|
||||
/verify_canary:\n\s+if: github\.event_name == 'push'\n\s+uses: \.\/\.github\/workflows\/release-verify\.yml\n\s+with:\n\s+ref: \$\{\{ github\.sha \}\}/,
|
||||
);
|
||||
const canary = releaseWorkflow.split(" verify_canary:\n")[1].split("\n publish_canary:")[0];
|
||||
assert.match(canary, /github\.repository == 'paperclipai\/paperclip' && github\.event_name == 'push' && github\.ref == 'refs\/heads\/master'/);
|
||||
assert.match(canary, /actions: read/);
|
||||
assert.match(canary, /ref: \$\{\{ github\.sha \}\}/);
|
||||
assert.match(canary, /SOURCE_SHA: \$\{\{ github\.sha \}\}/);
|
||||
assert.match(canary, /run: node scripts\/cloud-source-verification\.mjs "\$SOURCE_SHA"/);
|
||||
assert.doesNotMatch(canary, /release-verify\.yml|continue-on-error|always\(\)/);
|
||||
assert.match(releaseWorkflow, /publish_canary:\n\s+if: github\.event_name == 'push'\n\s+needs: verify_canary/);
|
||||
// The stable lane is gated on the stable channel since the nightly lane
|
||||
// was added; a `needs:` line (for example a preflight job) may sit between
|
||||
// the gate and the delegation.
|
||||
|
|
@ -57,6 +60,17 @@ test("release workflow delegates stable and canary verification to the reusable
|
|||
);
|
||||
});
|
||||
|
||||
test("source proof requires every source check and does not wait on image publication", () => {
|
||||
const readiness = readWorkflow("cloud-readiness.yml");
|
||||
const proof = readiness.split(" source_verified:\n")[1].split("\n ready:")[0];
|
||||
assert.match(proof, /name: Cloud source verified v1/);
|
||||
assert.match(proof, /needs: \[verify\]/);
|
||||
assert.match(proof, /node --test scripts\/cloud-source-verification.test.mjs/);
|
||||
assert.match(proof, /SOURCE_SHA: \$\{\{ github\.sha \}\}/);
|
||||
assert.doesNotMatch(proof, /always\(\)|continue-on-error|needs:.*(?:image|artifacts)/);
|
||||
assert.match(readiness.split(" ready:\n")[1], /needs: \[verify, image, artifacts\]/);
|
||||
});
|
||||
|
||||
test("onboard smoke container binds beyond loopback so the mapped port is reachable", () => {
|
||||
const dockerfile = readFileSync(
|
||||
path.join(repoRoot, "docker/Dockerfile.onboard-smoke"),
|
||||
|
|
|
|||
|
|
@ -0,0 +1,99 @@
|
|||
import { appendFile } from "node:fs/promises";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
const repository = "paperclipai/paperclip";
|
||||
const workflowPath = ".github/workflows/cloud-readiness.yml";
|
||||
export const sourceVerificationJob = "Cloud source verified v1";
|
||||
|
||||
function assertSha(sha) {
|
||||
if (!/^[a-f0-9]{40}$/.test(sha ?? "")) throw new Error("A full lowercase source SHA is required.");
|
||||
}
|
||||
|
||||
function trustedRun(run, sha, workflowId) {
|
||||
return run.workflow_id === workflowId && run.path === workflowPath &&
|
||||
run.repository?.full_name === repository && run.head_repository?.full_name === repository &&
|
||||
run.head_sha === sha && run.head_branch === "master" && run.event === "push" &&
|
||||
Number.isSafeInteger(run.id) && run.id > 0 &&
|
||||
Number.isSafeInteger(run.run_attempt) && run.run_attempt > 0;
|
||||
}
|
||||
|
||||
// Consume one versioned job, independent of image/migrator availability. A
|
||||
// failed image build must not invalidate source checks that already passed.
|
||||
export async function readSourceVerification(sha, api) {
|
||||
assertSha(sha);
|
||||
const workflow = await api(`/repos/${repository}/actions/workflows/cloud-readiness.yml`);
|
||||
if (workflow.path !== workflowPath || !Number.isSafeInteger(workflow.id) || workflow.id < 1) {
|
||||
throw new Error("Cloud readiness workflow identity does not match.");
|
||||
}
|
||||
const listing = await api(`/repos/${repository}/actions/workflows/${workflow.id}/runs?head_sha=${sha}&event=push&branch=master&per_page=100`);
|
||||
if (!Array.isArray(listing.workflow_runs) || !Number.isSafeInteger(listing.total_count) ||
|
||||
listing.total_count < 0 || listing.total_count > 100 || listing.workflow_runs.length !== listing.total_count) {
|
||||
throw new Error("Cloud readiness run listing is incomplete.");
|
||||
}
|
||||
const run = listing.workflow_runs.filter((candidate) => trustedRun(candidate, sha, workflow.id))
|
||||
.sort((a, b) => b.id - a.id)[0];
|
||||
if (!run) return undefined;
|
||||
|
||||
// Attempt-specific jobs prevent an earlier successful attempt from blessing
|
||||
// a later rerun. Keep pagination even though today's matrix fits one page.
|
||||
const jobs = [];
|
||||
for (let page = 1; page <= 10; page += 1) {
|
||||
const batch = await api(`/repos/${repository}/actions/runs/${run.id}/attempts/${run.run_attempt}/jobs?per_page=100&page=${page}`);
|
||||
if (!Array.isArray(batch.jobs)) throw new Error("Cloud readiness job listing is malformed.");
|
||||
jobs.push(...batch.jobs);
|
||||
if (batch.jobs.length < 100) break;
|
||||
if (page === 10) throw new Error("Cloud readiness job listing is incomplete.");
|
||||
}
|
||||
const matches = jobs.filter((job) => job.name === sourceVerificationJob);
|
||||
if (matches.length > 1) throw new Error("Cloud source verification job is ambiguous.");
|
||||
const job = matches[0];
|
||||
if (job?.status === "completed" && job.conclusion === "success" &&
|
||||
job.head_sha === sha && job.run_id === run.id && job.run_attempt === run.run_attempt) {
|
||||
// Re-read the run after its jobs: a rerun that started during polling must
|
||||
// not let the previous attempt through. Changes are retried next poll.
|
||||
const current = await api(`/repos/${repository}/actions/runs/${run.id}`);
|
||||
if (!trustedRun(current, sha, workflow.id) || current.run_attempt !== run.run_attempt) return undefined;
|
||||
return { sha, runId: run.id, attempt: run.run_attempt, jobId: job.id };
|
||||
}
|
||||
if (job?.status === "completed" || run.status === "completed") {
|
||||
throw new Error(`Cloud source verification did not pass for ${sha} (run ${run.id}, attempt ${run.run_attempt}). Rerun Cloud readiness before retrying the release.`);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export async function waitForSourceVerification(sha, {
|
||||
api, now = Date.now, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
||||
timeoutMs = 45 * 60_000, intervalMs = 30_000, log = console.log,
|
||||
} = {}) {
|
||||
assertSha(sha);
|
||||
const deadline = now() + timeoutMs;
|
||||
log(`Waiting for ${sourceVerificationJob} for ${sha}.`);
|
||||
while (now() < deadline) {
|
||||
const result = await readSourceVerification(sha, api);
|
||||
if (result) return result;
|
||||
const remaining = deadline - now();
|
||||
if (remaining > 0) await sleep(Math.min(intervalMs, remaining));
|
||||
}
|
||||
throw new Error(`Cloud source verification timed out for ${sha}. Rerun Cloud readiness before retrying the release.`);
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
try {
|
||||
if (!process.env.GITHUB_TOKEN) throw new Error("GITHUB_TOKEN with Actions read access is required.");
|
||||
const api = async (path) => {
|
||||
const response = await fetch(`https://api.github.com${path}`, {
|
||||
headers: { Authorization: `Bearer ${process.env.GITHUB_TOKEN}`, Accept: "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28" },
|
||||
signal: AbortSignal.timeout(30_000), redirect: "error",
|
||||
});
|
||||
if (!response.ok) throw new Error(`GitHub Actions read failed (HTTP ${response.status}).`);
|
||||
return response.json();
|
||||
};
|
||||
const proof = await waitForSourceVerification(process.argv[2], { api });
|
||||
const message = `Source verification passed for ${proof.sha}: https://github.com/${repository}/actions/runs/${proof.runId}/attempts/${proof.attempt} (job ${proof.jobId}).`;
|
||||
console.log(message);
|
||||
if (process.env.GITHUB_STEP_SUMMARY) await appendFile(process.env.GITHUB_STEP_SUMMARY, `${message}\n`);
|
||||
} catch (error) {
|
||||
console.error(error.message);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,125 @@
|
|||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { readSourceVerification, sourceVerificationJob, waitForSourceVerification } from "./cloud-source-verification.mjs";
|
||||
|
||||
const sha = "a".repeat(40);
|
||||
const workflow = { id: 123, path: ".github/workflows/cloud-readiness.yml" };
|
||||
const baseRun = {
|
||||
id: 456, workflow_id: workflow.id, path: workflow.path, run_attempt: 2,
|
||||
repository: { full_name: "paperclipai/paperclip" }, head_repository: { full_name: "paperclipai/paperclip" },
|
||||
head_sha: sha, head_branch: "master", event: "push", status: "in_progress", conclusion: null,
|
||||
};
|
||||
const baseJob = { id: 789, name: sourceVerificationJob, run_id: 456, run_attempt: 2, head_sha: sha, status: "completed", conclusion: "success" };
|
||||
|
||||
function fixture({ runs = [baseRun], jobs = [baseJob], current = baseRun, total, workflowRecord = workflow } = {}) {
|
||||
const calls = [];
|
||||
const api = async (path) => {
|
||||
calls.push(path);
|
||||
if (path.endsWith("/workflows/cloud-readiness.yml")) return workflowRecord;
|
||||
if (path.includes("/workflows/123/runs?")) return { total_count: total ?? runs.length, workflow_runs: runs };
|
||||
if (path.includes("/attempts/")) {
|
||||
assert.ok(path.includes(`/attempts/${runs.at(-1).run_attempt}/jobs?`));
|
||||
const page = Number(new URL("https://api.github.com" + path).searchParams.get("page"));
|
||||
return { jobs: jobs.slice((page - 1) * 100, page * 100) };
|
||||
}
|
||||
if (path.endsWith("/runs/456")) return current;
|
||||
throw new Error(`Unexpected API path: ${path}`);
|
||||
};
|
||||
return { api, calls };
|
||||
}
|
||||
|
||||
test("source proof passes while image work is still running, or has failed", async () => {
|
||||
for (const overrides of [{}, { status: "completed", conclusion: "failure" }]) {
|
||||
const run = { ...baseRun, ...overrides };
|
||||
const { api, calls } = fixture({ runs: [run], current: run });
|
||||
assert.deepEqual(await readSourceVerification(sha, api), { sha, runId: 456, attempt: 2, jobId: 789 });
|
||||
assert.ok(calls.some((path) => path.includes(`head_sha=${sha}&event=push&branch=master`)));
|
||||
assert.ok(calls.some((path) => path.includes("/attempts/2/jobs?")));
|
||||
}
|
||||
});
|
||||
|
||||
test("only the expected workflow, repository, master push, and full SHA can supply proof", async () => {
|
||||
for (const overrides of [
|
||||
{ workflow_id: 999 }, { path: ".github/workflows/pr.yml" },
|
||||
{ repository: { full_name: "other/paperclip" } }, { head_repository: { full_name: "fork/paperclip" } },
|
||||
{ head_sha: "b".repeat(40) }, { head_branch: "feature" }, { event: "workflow_dispatch" },
|
||||
{ run_attempt: undefined }, { run_attempt: 0 },
|
||||
]) {
|
||||
const { api, calls } = fixture({ runs: [{ ...baseRun, ...overrides }] });
|
||||
assert.equal(await readSourceVerification(sha, api), undefined, JSON.stringify(overrides));
|
||||
assert.equal(calls.length, 2);
|
||||
}
|
||||
});
|
||||
|
||||
test("a newer pending run cannot fall back to an older successful run", async () => {
|
||||
const newer = { ...baseRun, id: 457, run_attempt: 1 };
|
||||
const { api } = fixture({ runs: [baseRun, newer], jobs: [] });
|
||||
assert.equal(await readSourceVerification(sha, api), undefined);
|
||||
});
|
||||
|
||||
test("job identities and terminal failures fail closed", async () => {
|
||||
for (const overrides of [
|
||||
{ head_sha: "b".repeat(40) }, { run_id: 999 }, { run_attempt: 1 },
|
||||
{ conclusion: "failure" }, { conclusion: "cancelled" }, { conclusion: "skipped" },
|
||||
]) {
|
||||
const { api } = fixture({ jobs: [{ ...baseJob, ...overrides }] });
|
||||
await assert.rejects(readSourceVerification(sha, api), /did not pass/);
|
||||
}
|
||||
});
|
||||
|
||||
test("a rerun racing the jobs read cannot reuse the previous attempt", async () => {
|
||||
const { api } = fixture({ current: { ...baseRun, run_attempt: 3 } });
|
||||
assert.equal(await readSourceVerification(sha, api), undefined);
|
||||
});
|
||||
|
||||
test("the versioned proof must exist and be unique", async () => {
|
||||
for (const jobs of [[], [{ ...baseJob, name: "Cloud deployable v1" }]]) {
|
||||
await assert.rejects(readSourceVerification(sha, fixture({ runs: [{ ...baseRun, status: "completed" }], jobs }).api), /did not pass/);
|
||||
}
|
||||
await assert.rejects(readSourceVerification(sha, fixture({ jobs: [baseJob, baseJob] }).api), /ambiguous/);
|
||||
});
|
||||
|
||||
test("proof may appear after the first page of jobs", async () => {
|
||||
const jobs = [...Array.from({ length: 100 }, (_, index) => ({ ...baseJob, name: `test ${index}` })), baseJob];
|
||||
const { api, calls } = fixture({ jobs });
|
||||
assert.equal((await readSourceVerification(sha, api)).jobId, 789);
|
||||
assert.ok(calls.some((path) => path.endsWith("page=2")));
|
||||
});
|
||||
|
||||
test("incomplete discovery and API failures cannot bless a release", async () => {
|
||||
for (const total of [2, 101, -1]) {
|
||||
await assert.rejects(readSourceVerification(sha, fixture({ total }).api), /incomplete/);
|
||||
}
|
||||
await assert.rejects(readSourceVerification(sha, fixture({ workflowRecord: { ...workflow, path: ".github/workflows/pr.yml" } }).api), /identity/);
|
||||
await assert.rejects(readSourceVerification(sha, async () => { throw new Error("API unavailable"); }), /API unavailable/);
|
||||
});
|
||||
|
||||
test("a missing or pending source proof waits and has a bounded deadline", async () => {
|
||||
let time = 0;
|
||||
let polls = 0;
|
||||
const { api } = fixture({ runs: [] });
|
||||
await assert.rejects(waitForSourceVerification(sha, {
|
||||
api: async (path) => { if (path.endsWith(".yml")) polls += 1; return api(path); },
|
||||
now: () => time, sleep: async (ms) => { time += ms; }, timeoutMs: 50, intervalMs: 30, log: () => {},
|
||||
}), /timed out/);
|
||||
assert.equal(time, 50);
|
||||
assert.equal(polls, 2);
|
||||
});
|
||||
|
||||
test("pending verification succeeds when its exact job completes", async () => {
|
||||
let time = 0;
|
||||
const pending = fixture({ jobs: [{ ...baseJob, status: "in_progress", conclusion: null }] });
|
||||
const passed = fixture();
|
||||
const proof = await waitForSourceVerification(sha, {
|
||||
api: (path) => (time ? passed.api : pending.api)(path), now: () => time,
|
||||
sleep: async (ms) => { time += ms; }, timeoutMs: 100, intervalMs: 30, log: () => {},
|
||||
});
|
||||
assert.equal(proof.sha, sha);
|
||||
assert.equal(time, 30);
|
||||
});
|
||||
|
||||
test("malformed source refs are rejected before any request", async () => {
|
||||
for (const ref of ["master", "a".repeat(7), "A".repeat(40), undefined]) {
|
||||
await assert.rejects(readSourceVerification(ref, () => assert.fail("must not request")), /full lowercase/);
|
||||
}
|
||||
});
|
||||
Loading…
Reference in New Issue