ci: bind readiness to successful cloud image verification

Co-Authored-By: Paperclip <noreply@paperclip.ing>

.github/scripts/tests/cloud-readiness.test.mjs

@@ -0,0 +1,85 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import { waitForCloudArtifacts } from "../../../scripts/cloud-readiness.mjs";
+import { previewManifest } from "../../../scripts/preview-artifacts.mjs";
+
+const sha = "a".repeat(40);
+const digest = `sha256:${"b".repeat(64)}`;
+const json = (body, status = 200) => new Response(JSON.stringify(body), { status });
+function registry({ missing = new Set(), failure, wrongImage = false, wrongPackage = false } = {}) {
+  return async (url) => {
+    if (failure) return json({}, failure);
+    if (url.startsWith("https://registry.npmjs.org/")) {
+      const name = decodeURIComponent(new URL(url).pathname.split("/")[1]);
+      if (missing.has(name.split("/")[1])) return json({}, 404);
+      const pkg = previewManifest({ name, version: "0.0.0" }, sha);
+      return json({ ...pkg, ...(wrongPackage ? { gitHead: "c".repeat(40) } : {}), dist: { integrity: "sha512-fixture", tarball: "https://registry.npmjs.org/fixture.tgz" } });
+    }
+    if (url.includes("/token?")) return json({ token: "fixture" });
+    if (url.includes("/manifests/")) return missing.has("image") ? json({}, 404) : json({ config: { digest } });
+    if (url.includes("/blobs/")) return json({ config: { Labels: { "org.opencontainers.image.revision": wrongImage ? "c".repeat(40) : sha } } });
+    throw new Error(`Unexpected request: ${url}`);
+  };
+}
+
+test("readiness requires the image and both exact-source packages on the successful poll", async () => {
+  const missing = new Set(["image", "shared", "db"]);
+  let clock = 0;
+  const states = [];
+  const result = await waitForCloudArtifacts(sha, {
+    fetchImpl: registry({ missing }), now: () => clock, intervalMs: 10, timeoutMs: 100, log: (message) => states.push(message),
+    sleep: async (ms) => {
+      clock += ms;
+      if (clock === 10) missing.delete("image");
+      if (clock === 20) missing.delete("shared");
+      if (clock === 30) { missing.delete("db"); missing.add("image"); }
+      if (clock === 40) missing.delete("image");
+    },
+  });
+  assert.equal(clock, 40, "an artifact disappearing before the final poll must prevent readiness");
+  assert.deepEqual(result, { version: 1, sha, packageVersion: `0.0.0-preview.g${sha}` });
+  assert.match(states.at(-1), /Cloud artifacts available/);
+});
+
+test("missing artifacts time out with a precise inventory and bounded sleep", async () => {
+  let clock = 0;
+  const sleeps = [];
+  await assert.rejects(waitForCloudArtifacts(sha, {
+    fetchImpl: registry({ missing: new Set(["db"]) }), now: () => clock, timeoutMs: 25, intervalMs: 20, log: () => {},
+    sleep: async (ms) => { sleeps.push(ms); clock += ms; },
+  }), /timed out.*missing: db/);
+  assert.deepEqual(sleeps, [20, 5]);
+});
+
+for (const fixture of [{ failure: 403 }, { failure: 503 }, { wrongImage: true }, { wrongPackage: true }]) {
+  test(`registry errors and identity mismatches fail without waiting: ${JSON.stringify(fixture)}`, async () => {
+    await assert.rejects(waitForCloudArtifacts(sha, {
+      fetchImpl: registry(fixture), sleep: async () => assert.fail("must not retry an invalid artifact or upstream error"), log: () => {},
+    }));
+  });
+}
+
+test("invalid source and timing configuration are rejected before registry access", async () => {
+  const fetchImpl = async () => assert.fail("invalid inputs must not reach a registry");
+  await assert.rejects(waitForCloudArtifacts("master", { fetchImpl }), /full immutable commit SHA/);
+  for (const options of [{ timeoutMs: 0 }, { intervalMs: -1 }, { timeoutMs: Infinity }]) {
+    await assert.rejects(waitForCloudArtifacts(sha, { ...options, fetchImpl }), /positive finite/);
+  }
+});
+
+test("the versioned readiness job requires successful source, image and artifact jobs", () => {
+  const workflow = readFileSync(new URL("../../workflows/cloud-readiness.yml", import.meta.url), "utf8");
+  assert.match(workflow, /push:\s*\n\s*branches: \[master\]/);
+  assert.match(workflow, /group: cloud-readiness-\$\{\{ github.sha \}\}/);
+  assert.match(workflow, /uses: \.\/\.github\/workflows\/release-verify.yml\s+with:\s+ref: \$\{\{ github.sha \}\}/);
+  assert.match(workflow, /uses: \.\/\.github\/workflows\/docker-cloud.yml/);
+  const ready = workflow.split("  ready:")[1];
+  assert.match(ready, /name: Cloud deployable v1/);
+  assert.match(ready, /needs: \[verify, image, artifacts\]/);
+  assert.match(ready, /if: github.repository == 'paperclipai\/paperclip' && github.ref == 'refs\/heads\/master'/);
+  assert.doesNotMatch(ready, /^\s*(?:if:.*always\(|continue-on-error:)/m);
+  assert.doesNotMatch(workflow, /secrets: inherit|id-token: write|actions: write|checks: write|uses: .*@v\d\b/);
+  const cloud = readFileSync(new URL("../../workflows/docker-cloud.yml", import.meta.url), "utf8");
+  assert.doesNotMatch(cloud, /^  push:/m, "the master image must build only once");
+});

.github/workflows/cloud-readiness.yml

@@ -14,6 +14,13 @@ concurrency:
   cancel-in-progress: false
 
 jobs:
+  image:
+    if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master'
+    permissions:
+      contents: read
+      packages: write
+    uses: ./.github/workflows/docker-cloud.yml
+
   verify:
     if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master'
     permissions:
@@ -45,7 +52,7 @@ jobs:
     # Versioned consumer contract. Never add always() or continue-on-error:
     # failed, cancelled, or skipped prerequisites must not report readiness.
     name: Cloud deployable v1
-    needs: [verify, artifacts]
+    needs: [verify, image, artifacts]
     if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master'
     runs-on: ubuntu-latest
     timeout-minutes: 5

.github/workflows/docker-cloud.yml

@@ -1,8 +1,6 @@
 name: Docker cloud
 
 on:
-  push:
-    branches: [master]
   workflow_dispatch:
   workflow_call:
 

doc/DOCKER.md

@@ -27,8 +27,8 @@ docker build -t paperclip-local \
 ## Cloud image addresses
 
 The Docker workflow publishes the managed deployment image for Linux AMD64.
-`Docker cloud` starts on each master push independently of the multi-platform
-self-hosted build. Different commits use separate concurrency groups and existing
+`Cloud readiness` starts `Docker cloud` on each master push independently of the
+multi-platform self-hosted build. Different commits use separate concurrency groups and existing
 GitHub-hosted runners, so an older production or cloud build does not hold the
 new commit in a workflow queue. Available GitHub runner capacity still applies.
 Release tags and manual `Docker` dispatches call the same cloud build workflow.
@@ -49,6 +49,10 @@ tests passed or that a compatible database migrator is available. Deployment
 tooling must still check those prerequisites and pin the resolved image digest;
 a rebuild of the same source can update the tag's digest.
 
+The separate [cloud readiness check](cloud-build-readiness.md) combines source
+verification, successful cloud image checks, and exact-source migrator
+availability. It runs outside the full npm release's concurrency queue.
+
 ## One-liner (build + run)
 
 ```sh

doc/cloud-build-readiness.md

@@ -0,0 +1,65 @@
+# Cloud build readiness
+
+The `Cloud readiness` workflow starts for every master push. Its versioned
+`Cloud deployable v1` job succeeds only after all three prerequisites succeed:
+
+- The existing `Release Verify` workflow checks that exact commit, including
+  typecheck, builds, general and serialized tests, and Runner verification.
+- The reusable `Docker cloud` workflow builds and verifies its Linux AMD64
+  image, including Sentry resolution and orphan reaping, then publishes the
+  full-SHA cloud tag. Cloud readiness owns the master trigger so there is one
+  cloud build per push. Release tags and manual Docker runs retain their callers.
+- The full-SHA image and both exact-source npm packages are visible. The
+  packages are `@paperclipai/shared` and `@paperclipai/db` at
+  `0.0.0-preview.g<FULL_SHA>`, published through the migrator-only release lane.
+  Registry metadata must match the full commit, and the database package must
+  pin the matching shared package.
+
+Verification and image building run concurrently, outside the full npm release's
+concurrency group. Different commits have independent groups. Source verification
+is initially duplicated with the normal npm release: this spends existing hosted
+runner capacity to avoid waiting behind an older release. No verification gate is
+removed from npm publication. Watch organization-wide runner queues when measuring
+the result.
+
+The artifact wait runs for up to 30 minutes and reports what is missing. Only
+an HTTP 404 means publication is pending; authorization errors, upstream outages,
+and identity mismatches fail the job. A failed, cancelled, or skipped prerequisite
+cannot produce a successful readiness job. Retry the failed publication or build,
+then rerun the failed readiness workflow jobs to check the same commit again.
+
+## Consumer contract
+
+`Cloud deployable v1` is a source-and-artifact readiness signal. A deployment
+consumer must still resolve and pin the image digest and npm integrity/lockfile,
+validate migration contents and compatibility, and apply its target health gates.
+The check creates no release record and deploys no instance. A full-SHA tag by
+itself, or a successful migrator dispatch, is not this readiness signal.
+
+For automatic selection, accept only a successful job named exactly
+`Cloud deployable v1` in the latest attempt of a successful
+`.github/workflows/cloud-readiness.yml` run in `paperclipai/paperclip`, with
+event `push`, head branch `master`, and the expected full head SHA and repository.
+Do not trust a similarly named check from another workflow or a manual branch run.
+Order candidates by master ancestry, not job completion time: an older commit
+finishing late must not roll a fleet backward. Fail closed on API errors.
+
+Existing npm canary discovery is unchanged by this producer workflow. Consumers
+can adopt the versioned signal separately after the workflow has landed and
+successfully verified a real master commit.
+
+## Timing and rollout
+
+Measure from the master push to completion of `Cloud deployable v1`. Record
+queue time and the image, source-verification, and artifact-wait durations
+separately. The slowest prerequisite determines readiness; shortening an already
+faster prerequisite may have no effect on the total.
+
+Land full-SHA image publication, independent cloud builds, and migrator-only
+publication before enabling this workflow. Until those producers are present,
+the artifact wait cannot succeed. A manual dispatch on master can verify the
+wiring, but automatic consumers should use push runs. Source verification and
+registry checks can be rerun without deploying or changing mutable npm channels.
+
+When reverting this workflow, restore the master push trigger in
+`docker-cloud.yml` in the same change so master images continue to build.

scripts/preview-artifacts.test.mjs

@@ -178,7 +178,10 @@ test("commits sharing a short prefix use separate full-SHA image addresses", asy
 test("cloud builds start per commit and preserve tag promotion dependencies", () => {
   const docker = readFileSync(new URL("../.github/workflows/docker.yml", import.meta.url), "utf8");
   const cloud = readFileSync(new URL("../.github/workflows/docker-cloud.yml", import.meta.url), "utf8");
-  assert.match(cloud, /branches: \[master\]/);
+  const readiness = readFileSync(new URL("../.github/workflows/cloud-readiness.yml", import.meta.url), "utf8");
+  assert.match(readiness, /branches: \[master\]/);
+  assert.match(readiness, /uses: \.\/\.github\/workflows\/docker-cloud.yml/);
+  assert.doesNotMatch(cloud, /^  push:/m);
   assert.match(cloud, /workflow_call:/);
   assert.match(cloud, /group: docker-cloud-\$\{\{ github.sha \}\}/);
   assert.match(cloud, /cancel-in-progress: false/);