diff --git a/.github/scripts/tests/cloud-readiness.test.mjs b/.github/scripts/tests/cloud-readiness.test.mjs new file mode 100644 index 0000000000..25233a335b --- /dev/null +++ b/.github/scripts/tests/cloud-readiness.test.mjs @@ -0,0 +1,91 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { waitForCloudArtifacts } from "../../../scripts/cloud-readiness.mjs"; +import { previewManifest } from "../../../scripts/preview-artifacts.mjs"; + +const sha = "a".repeat(40); +const digest = `sha256:${"b".repeat(64)}`; +const json = (body, status = 200) => new Response(JSON.stringify(body), { status }); +function registry({ missing = new Set(), failure, wrongImage = false, wrongPackage = false } = {}) { + return async (url) => { + if (failure) return json({}, failure); + if (url.startsWith("https://registry.npmjs.org/")) { + const name = decodeURIComponent(new URL(url).pathname.split("/")[1]); + if (missing.has(name.split("/")[1])) return json({}, 404); + const pkg = previewManifest({ name, version: "0.0.0" }, sha); + return json({ ...pkg, ...(wrongPackage ? { gitHead: "c".repeat(40) } : {}), dist: { integrity: "sha512-fixture", tarball: "https://registry.npmjs.org/fixture.tgz" } }); + } + if (url.includes("/token?")) return json({ token: "fixture" }); + if (url.includes("/manifests/")) return missing.has("image") ? json({}, 404) : json({ config: { digest } }); + if (url.includes("/blobs/")) return json({ config: { Labels: { "org.opencontainers.image.revision": wrongImage ? "c".repeat(40) : sha } } }); + throw new Error(`Unexpected request: ${url}`); + }; +} + +test("readiness requires the image and both exact-source packages on the successful poll", async () => { + const missing = new Set(["image", "shared", "db"]); + let clock = 0; + const states = []; + const result = await waitForCloudArtifacts(sha, { + fetchImpl: registry({ missing }), now: () => clock, intervalMs: 10, timeoutMs: 100, log: (message) => states.push(message), + sleep: async (ms) => { + clock += ms; + if (clock === 10) missing.delete("image"); + if (clock === 20) missing.delete("shared"); + if (clock === 30) { missing.delete("db"); missing.add("image"); } + if (clock === 40) missing.delete("image"); + }, + }); + assert.equal(clock, 40, "an artifact disappearing before the final poll must prevent readiness"); + assert.deepEqual(result, { version: 1, sha, packageVersion: `0.0.0-preview.g${sha}` }); + assert.match(states.at(-1), /Cloud artifacts available/); +}); + +test("missing artifacts time out with a precise inventory and bounded sleep", async () => { + let clock = 0; + const sleeps = []; + await assert.rejects(waitForCloudArtifacts(sha, { + fetchImpl: registry({ missing: new Set(["db"]) }), now: () => clock, timeoutMs: 25, intervalMs: 20, log: () => {}, + sleep: async (ms) => { sleeps.push(ms); clock += ms; }, + }), /timed out.*missing: db/); + assert.deepEqual(sleeps, [20, 5]); +}); + +for (const fixture of [{ failure: 403 }, { failure: 503 }, { wrongImage: true }, { wrongPackage: true }]) { + test(`registry errors and identity mismatches fail without waiting: ${JSON.stringify(fixture)}`, async () => { + await assert.rejects(waitForCloudArtifacts(sha, { + fetchImpl: registry(fixture), sleep: async () => assert.fail("must not retry an invalid artifact or upstream error"), log: () => {}, + })); + }); +} + +test("invalid source and timing configuration are rejected before registry access", async () => { + const fetchImpl = async () => assert.fail("invalid inputs must not reach a registry"); + await assert.rejects(waitForCloudArtifacts("master", { fetchImpl }), /full immutable commit SHA/); + for (const options of [{ timeoutMs: 0 }, { intervalMs: -1 }, { timeoutMs: Infinity }]) { + await assert.rejects(waitForCloudArtifacts(sha, { ...options, fetchImpl }), /positive finite/); + } +}); + +test("the versioned readiness job requires successful source, image and artifact jobs", () => { + const workflow = readFileSync(new URL("../../workflows/cloud-readiness.yml", import.meta.url), "utf8"); + assert.match(workflow, /push:\s*\n\s*branches: \[master\]/); + assert.match(workflow, /group: cloud-readiness-\$\{\{ github.sha \}\}/); + assert.match(workflow, /uses: \.\/\.github\/workflows\/release-verify.yml\s+with:\s+ref: \$\{\{ github.sha \}\}/); + assert.match(workflow, /uses: \.\/\.github\/workflows\/docker-cloud.yml/); + const ready = workflow.split(" ready:")[1]; + assert.match(ready, /name: Cloud deployable v1/); + assert.match(ready, /needs: \[verify, image, artifacts\]/); + assert.match(ready, /if: github.repository == 'paperclipai\/paperclip' && github.ref == 'refs\/heads\/master'/); + assert.doesNotMatch(ready, /^\s*(?:if:.*always\(|continue-on-error:)/m); + assert.doesNotMatch(workflow, /secrets: inherit|id-token: write|actions: write|checks: write|uses: .*@v\d\b/); + const cloud = readFileSync(new URL("../../workflows/docker-cloud.yml", import.meta.url), "utf8"); + assert.doesNotMatch(cloud, /^ push:/m, "the master image must build only once"); + const migrator = readFileSync(new URL("../../workflows/cloud-artifacts.yml", import.meta.url), "utf8"); + assert.match(migrator, /push:\s*\n\s*branches: \[master\]/); + assert.match(migrator, /SOURCE_SHA: \$\{\{ github.sha \}\}/); + assert.match(migrator, /gh workflow run release.yml .*--ref master/); + assert.match(migrator, /--field channel=cloud-migrator/); + assert.match(migrator, /--field source_ref="\$SOURCE_SHA"/); +}); diff --git a/.github/workflows/cloud-readiness.yml b/.github/workflows/cloud-readiness.yml new file mode 100644 index 0000000000..df6c1dac61 --- /dev/null +++ b/.github/workflows/cloud-readiness.yml @@ -0,0 +1,66 @@ +name: Cloud readiness +run-name: Cloud readiness ${{ github.sha }} + +on: + push: + branches: [master] + workflow_dispatch: + +permissions: {} + +# Source verification must start outside the full npm release's queue. +concurrency: + group: cloud-readiness-${{ github.sha }} + cancel-in-progress: false + +jobs: + image: + if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master' + permissions: + contents: read + packages: write + uses: ./.github/workflows/docker-cloud.yml + + verify: + if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master' + permissions: + contents: read + uses: ./.github/workflows/release-verify.yml + with: + ref: ${{ github.sha }} + + artifacts: + if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master' + name: Wait for exact-source cloud artifacts + runs-on: ubuntu-latest + timeout-minutes: 35 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + - name: Wait for verified image and exact-source migrator + env: + SOURCE_SHA: ${{ github.sha }} + run: node scripts/cloud-readiness.mjs "$SOURCE_SHA" + + ready: + # Versioned consumer contract. Never add always() or continue-on-error: + # failed, cancelled, or skipped prerequisites must not report readiness. + name: Cloud deployable v1 + needs: [verify, image, artifacts] + if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master' + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Record cloud readiness + env: + SOURCE_SHA: ${{ github.sha }} + run: | + echo "Cloud deployable v1: $SOURCE_SHA" >> "$GITHUB_STEP_SUMMARY" + echo "Source verification passed; the full-SHA image and exact-source migrator are available." >> "$GITHUB_STEP_SUMMARY" + echo "Deployment tooling must still resolve and pin the image and migrator and validate migration compatibility." >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/docker-cloud.yml b/.github/workflows/docker-cloud.yml index 8fb2517bc0..e920dbfa71 100644 --- a/.github/workflows/docker-cloud.yml +++ b/.github/workflows/docker-cloud.yml @@ -1,8 +1,6 @@ name: Docker cloud on: - push: - branches: [master] workflow_dispatch: workflow_call: diff --git a/doc/DOCKER.md b/doc/DOCKER.md index ad8be8d9dc..757a2c3094 100644 --- a/doc/DOCKER.md +++ b/doc/DOCKER.md @@ -35,8 +35,8 @@ docker build -t paperclip-local \ ## Cloud image addresses The Docker workflow publishes the managed deployment image for Linux AMD64. -`Docker cloud` starts on each master push independently of the multi-platform -self-hosted build. Different commits use separate concurrency groups and existing +`Cloud readiness` starts `Docker cloud` on each master push independently of the +multi-platform self-hosted build. Different commits use separate concurrency groups and existing GitHub-hosted runners, so an older production or cloud build does not hold the new commit in a workflow queue. Available GitHub runner capacity still applies. Release tags and manual `Docker` dispatches call the same cloud build workflow. @@ -63,6 +63,10 @@ tests passed or that a compatible database migrator is available. Deployment tooling must still check those prerequisites and pin the resolved image digest; a rebuild of the same source can update the tag's digest. +The separate [cloud readiness check](cloud-build-readiness.md) combines source +verification, successful cloud image checks, and exact-source migrator +availability. It runs outside the full npm release's concurrency queue. + ## One-liner (build + run) ```sh diff --git a/doc/cloud-build-readiness.md b/doc/cloud-build-readiness.md new file mode 100644 index 0000000000..cd83ae5b79 --- /dev/null +++ b/doc/cloud-build-readiness.md @@ -0,0 +1,65 @@ +# Cloud build readiness + +The `Cloud readiness` workflow starts for every master push. Its versioned +`Cloud deployable v1` job succeeds only after all three prerequisites succeed: + +- The existing `Release Verify` workflow checks that exact commit, including + typecheck, builds, general and serialized tests, and Runner verification. +- The reusable `Docker cloud` workflow builds and verifies its Linux AMD64 + image, including Sentry resolution and orphan reaping, then publishes the + full-SHA cloud tag. Cloud readiness owns the master trigger so there is one + cloud build per push. Release tags and manual Docker runs retain their callers. +- The full-SHA image and both exact-source npm packages are visible. The + packages are `@paperclipai/shared` and `@paperclipai/db` at + `0.0.0-preview.g`, published through the migrator-only release lane. + Registry metadata must match the full commit, and the database package must + pin the matching shared package. + +Verification and image building run concurrently, outside the full npm release's +concurrency group. Different commits have independent groups. Source verification +is initially duplicated with the normal npm release: this spends existing hosted +runner capacity to avoid waiting behind an older release. No verification gate is +removed from npm publication. Watch organization-wide runner queues when measuring +the result. + +The artifact wait runs for up to 30 minutes and reports what is missing. Only +an HTTP 404 means publication is pending; authorization errors, upstream outages, +and identity mismatches fail the job. A failed, cancelled, or skipped prerequisite +cannot produce a successful readiness job. Retry the failed publication or build, +then rerun the failed readiness workflow jobs to check the same commit again. + +## Consumer contract + +`Cloud deployable v1` is a source-and-artifact readiness signal. A deployment +consumer must still resolve and pin the image digest and npm integrity/lockfile, +validate migration contents and compatibility, and apply its target health gates. +The check creates no release record and deploys no instance. A full-SHA tag by +itself, or a successful migrator dispatch, is not this readiness signal. + +For automatic selection, accept only a successful job named exactly +`Cloud deployable v1` in the latest attempt of a successful +`.github/workflows/cloud-readiness.yml` run in `paperclipai/paperclip`, with +event `push`, head branch `master`, and the expected full head SHA and repository. +Do not trust a similarly named check from another workflow or a manual branch run. +Order candidates by master ancestry, not job completion time: an older commit +finishing late must not roll a fleet backward. Fail closed on API errors. + +Existing npm canary discovery is unchanged by this producer workflow. Consumers +can adopt the versioned signal separately after the workflow has landed and +successfully verified a real master commit. + +## Timing and rollout + +Measure from the master push to completion of `Cloud deployable v1`. Record +queue time and the image, source-verification, and artifact-wait durations +separately. The slowest prerequisite determines readiness; shortening an already +faster prerequisite may have no effect on the total. + +Land full-SHA image publication, independent cloud builds, and migrator-only +publication before enabling this workflow. Until those producers are present, +the artifact wait cannot succeed. A manual dispatch on master can verify the +wiring, but automatic consumers should use push runs. Source verification and +registry checks can be rerun without deploying or changing mutable npm channels. + +When reverting this workflow, restore the master push trigger in +`docker-cloud.yml` in the same change so master images continue to build. diff --git a/scripts/cloud-readiness.mjs b/scripts/cloud-readiness.mjs new file mode 100644 index 0000000000..09ad83a938 --- /dev/null +++ b/scripts/cloud-readiness.mjs @@ -0,0 +1,46 @@ +#!/usr/bin/env node +import { pathToFileURL } from "node:url"; +import { imageExists, packageExists, versionFor } from "./preview-artifacts.mjs"; + +/** Read-only availability gate. Deployment still resolves and pins artifacts. */ +export async function waitForCloudArtifacts(sha, { + fetchImpl = fetch, + now = () => performance.now(), + sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), + timeoutMs = 30 * 60_000, + intervalMs = 20_000, + log = console.log, +} = {}) { + const version = versionFor(sha); + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0 || !Number.isFinite(intervalMs) || intervalMs <= 0) { + throw new Error("Cloud readiness requires positive finite timeout and poll interval."); + } + const deadline = now() + timeoutMs; + let previous; + let missing = ["image", "shared", "db"]; + while (now() < deadline) { + // Recheck every artifact on the successful poll. Only an explicit 404 + // means publication is pending; identity errors and upstream outages fail. + const results = await Promise.all([ + imageExists(sha, fetchImpl), + packageExists("@paperclipai/shared", sha, fetchImpl), + packageExists("@paperclipai/db", sha, fetchImpl), + ]); + missing = ["image", "shared", "db"].filter((_, index) => !results[index]); + if (missing.length === 0) { + log(`Cloud artifacts available for ${sha}: verified image and exact-source migrator ${version}.`); + return { version: 1, sha, packageVersion: version }; + } + const state = missing.join(", "); + if (state !== previous) log(`Waiting for cloud artifacts for ${sha}: ${state}.`); + previous = state; + const remaining = deadline - now(); + if (remaining > 0) await sleep(Math.min(intervalMs, remaining)); + } + throw new Error(`Cloud artifacts timed out for ${sha}; missing: ${missing.join(", ")}.`); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + try { await waitForCloudArtifacts(process.argv[2]); } + catch (error) { console.error(error.message); process.exitCode = 1; } +} diff --git a/scripts/preview-artifacts.test.mjs b/scripts/preview-artifacts.test.mjs index 10ce16ccb3..0417365b1b 100644 --- a/scripts/preview-artifacts.test.mjs +++ b/scripts/preview-artifacts.test.mjs @@ -179,7 +179,10 @@ test("commits sharing a short prefix use separate full-SHA image addresses", asy test("cloud builds start per commit and preserve tag promotion dependencies", () => { const docker = readFileSync(new URL("../.github/workflows/docker.yml", import.meta.url), "utf8"); const cloud = readFileSync(new URL("../.github/workflows/docker-cloud.yml", import.meta.url), "utf8"); - assert.match(cloud, /branches: \[master\]/); + const readiness = readFileSync(new URL("../.github/workflows/cloud-readiness.yml", import.meta.url), "utf8"); + assert.match(readiness, /branches: \[master\]/); + assert.match(readiness, /uses: \.\/\.github\/workflows\/docker-cloud.yml/); + assert.doesNotMatch(cloud, /^ push:/m); assert.match(cloud, /workflow_call:/); assert.match(cloud, /group: docker-cloud-\$\{\{ github.sha \}\}/); assert.match(cloud, /cancel-in-progress: false/);