fix(ci): verify deployable cloud artifacts independently (#13192)

Verify source, build the cloud image, and wait for exact-source migrator packages concurrently. Emit Cloud deployable v1 only when every prerequisite succeeds for the merged full SHA.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

.github/scripts/tests/cloud-readiness.test.mjs

@@ -0,0 +1,91 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import { waitForCloudArtifacts } from "../../../scripts/cloud-readiness.mjs";
+import { previewManifest } from "../../../scripts/preview-artifacts.mjs";
+
+const sha = "a".repeat(40);
+const digest = `sha256:${"b".repeat(64)}`;
+const json = (body, status = 200) => new Response(JSON.stringify(body), { status });
+function registry({ missing = new Set(), failure, wrongImage = false, wrongPackage = false } = {}) {
+  return async (url) => {
+    if (failure) return json({}, failure);
+    if (url.startsWith("https://registry.npmjs.org/")) {
+      const name = decodeURIComponent(new URL(url).pathname.split("/")[1]);
+      if (missing.has(name.split("/")[1])) return json({}, 404);
+      const pkg = previewManifest({ name, version: "0.0.0" }, sha);
+      return json({ ...pkg, ...(wrongPackage ? { gitHead: "c".repeat(40) } : {}), dist: { integrity: "sha512-fixture", tarball: "https://registry.npmjs.org/fixture.tgz" } });
+    }
+    if (url.includes("/token?")) return json({ token: "fixture" });
+    if (url.includes("/manifests/")) return missing.has("image") ? json({}, 404) : json({ config: { digest } });
+    if (url.includes("/blobs/")) return json({ config: { Labels: { "org.opencontainers.image.revision": wrongImage ? "c".repeat(40) : sha } } });
+    throw new Error(`Unexpected request: ${url}`);
+  };
+}
+
+test("readiness requires the image and both exact-source packages on the successful poll", async () => {
+  const missing = new Set(["image", "shared", "db"]);
+  let clock = 0;
+  const states = [];
+  const result = await waitForCloudArtifacts(sha, {
+    fetchImpl: registry({ missing }), now: () => clock, intervalMs: 10, timeoutMs: 100, log: (message) => states.push(message),
+    sleep: async (ms) => {
+      clock += ms;
+      if (clock === 10) missing.delete("image");
+      if (clock === 20) missing.delete("shared");
+      if (clock === 30) { missing.delete("db"); missing.add("image"); }
+      if (clock === 40) missing.delete("image");
+    },
+  });
+  assert.equal(clock, 40, "an artifact disappearing before the final poll must prevent readiness");
+  assert.deepEqual(result, { version: 1, sha, packageVersion: `0.0.0-preview.g${sha}` });
+  assert.match(states.at(-1), /Cloud artifacts available/);
+});
+
+test("missing artifacts time out with a precise inventory and bounded sleep", async () => {
+  let clock = 0;
+  const sleeps = [];
+  await assert.rejects(waitForCloudArtifacts(sha, {
+    fetchImpl: registry({ missing: new Set(["db"]) }), now: () => clock, timeoutMs: 25, intervalMs: 20, log: () => {},
+    sleep: async (ms) => { sleeps.push(ms); clock += ms; },
+  }), /timed out.*missing: db/);
+  assert.deepEqual(sleeps, [20, 5]);
+});
+
+for (const fixture of [{ failure: 403 }, { failure: 503 }, { wrongImage: true }, { wrongPackage: true }]) {
+  test(`registry errors and identity mismatches fail without waiting: ${JSON.stringify(fixture)}`, async () => {
+    await assert.rejects(waitForCloudArtifacts(sha, {
+      fetchImpl: registry(fixture), sleep: async () => assert.fail("must not retry an invalid artifact or upstream error"), log: () => {},
+    }));
+  });
+}
+
+test("invalid source and timing configuration are rejected before registry access", async () => {
+  const fetchImpl = async () => assert.fail("invalid inputs must not reach a registry");
+  await assert.rejects(waitForCloudArtifacts("master", { fetchImpl }), /full immutable commit SHA/);
+  for (const options of [{ timeoutMs: 0 }, { intervalMs: -1 }, { timeoutMs: Infinity }]) {
+    await assert.rejects(waitForCloudArtifacts(sha, { ...options, fetchImpl }), /positive finite/);
+  }
+});
+
+test("the versioned readiness job requires successful source, image and artifact jobs", () => {
+  const workflow = readFileSync(new URL("../../workflows/cloud-readiness.yml", import.meta.url), "utf8");
+  assert.match(workflow, /push:\s*\n\s*branches: \[master\]/);
+  assert.match(workflow, /group: cloud-readiness-\$\{\{ github.sha \}\}/);
+  assert.match(workflow, /uses: \.\/\.github\/workflows\/release-verify.yml\s+with:\s+ref: \$\{\{ github.sha \}\}/);
+  assert.match(workflow, /uses: \.\/\.github\/workflows\/docker-cloud.yml/);
+  const ready = workflow.split("  ready:")[1];
+  assert.match(ready, /name: Cloud deployable v1/);
+  assert.match(ready, /needs: \[verify, image, artifacts\]/);
+  assert.match(ready, /if: github.repository == 'paperclipai\/paperclip' && github.ref == 'refs\/heads\/master'/);
+  assert.doesNotMatch(ready, /^\s*(?:if:.*always\(|continue-on-error:)/m);
+  assert.doesNotMatch(workflow, /secrets: inherit|id-token: write|actions: write|checks: write|uses: .*@v\d\b/);
+  const cloud = readFileSync(new URL("../../workflows/docker-cloud.yml", import.meta.url), "utf8");
+  assert.doesNotMatch(cloud, /^  push:/m, "the master image must build only once");
+  const migrator = readFileSync(new URL("../../workflows/cloud-artifacts.yml", import.meta.url), "utf8");
+  assert.match(migrator, /push:\s*\n\s*branches: \[master\]/);
+  assert.match(migrator, /SOURCE_SHA: \$\{\{ github.sha \}\}/);
+  assert.match(migrator, /gh workflow run release.yml .*--ref master/);
+  assert.match(migrator, /--field channel=cloud-migrator/);
+  assert.match(migrator, /--field source_ref="\$SOURCE_SHA"/);
+});

.github/workflows/cloud-readiness.yml

@@ -0,0 +1,66 @@
+name: Cloud readiness
+run-name: Cloud readiness ${{ github.sha }}
+
+on:
+  push:
+    branches: [master]
+  workflow_dispatch:
+
+permissions: {}
+
+# Source verification must start outside the full npm release's queue.
+concurrency:
+  group: cloud-readiness-${{ github.sha }}
+  cancel-in-progress: false
+
+jobs:
+  image:
+    if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master'
+    permissions:
+      contents: read
+      packages: write
+    uses: ./.github/workflows/docker-cloud.yml
+
+  verify:
+    if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master'
+    permissions:
+      contents: read
+    uses: ./.github/workflows/release-verify.yml
+    with:
+      ref: ${{ github.sha }}
+
+  artifacts:
+    if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master'
+    name: Wait for exact-source cloud artifacts
+    runs-on: ubuntu-latest
+    timeout-minutes: 35
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          persist-credentials: false
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
+        with:
+          node-version: 24
+      - name: Wait for verified image and exact-source migrator
+        env:
+          SOURCE_SHA: ${{ github.sha }}
+        run: node scripts/cloud-readiness.mjs "$SOURCE_SHA"
+
+  ready:
+    # Versioned consumer contract. Never add always() or continue-on-error:
+    # failed, cancelled, or skipped prerequisites must not report readiness.
+    name: Cloud deployable v1
+    needs: [verify, image, artifacts]
+    if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master'
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Record cloud readiness
+        env:
+          SOURCE_SHA: ${{ github.sha }}
+        run: |
+          echo "Cloud deployable v1: $SOURCE_SHA" >> "$GITHUB_STEP_SUMMARY"
+          echo "Source verification passed; the full-SHA image and exact-source migrator are available." >> "$GITHUB_STEP_SUMMARY"
+          echo "Deployment tooling must still resolve and pin the image and migrator and validate migration compatibility." >> "$GITHUB_STEP_SUMMARY"

.github/workflows/docker-cloud.yml

@@ -1,8 +1,6 @@
 name: Docker cloud
 
 on:
-  push:
-    branches: [master]
   workflow_dispatch:
   workflow_call:
 

doc/DOCKER.md

@@ -35,8 +35,8 @@ docker build -t paperclip-local \
 ## Cloud image addresses
 
 The Docker workflow publishes the managed deployment image for Linux AMD64.
-`Docker cloud` starts on each master push independently of the multi-platform
-self-hosted build. Different commits use separate concurrency groups and existing
+`Cloud readiness` starts `Docker cloud` on each master push independently of the
+multi-platform self-hosted build. Different commits use separate concurrency groups and existing
 GitHub-hosted runners, so an older production or cloud build does not hold the
 new commit in a workflow queue. Available GitHub runner capacity still applies.
 Release tags and manual `Docker` dispatches call the same cloud build workflow.
@@ -63,6 +63,10 @@ tests passed or that a compatible database migrator is available. Deployment
 tooling must still check those prerequisites and pin the resolved image digest;
 a rebuild of the same source can update the tag's digest.
 
+The separate [cloud readiness check](cloud-build-readiness.md) combines source
+verification, successful cloud image checks, and exact-source migrator
+availability. It runs outside the full npm release's concurrency queue.
+
 ## One-liner (build + run)
 
 ```sh

doc/cloud-build-readiness.md

@@ -0,0 +1,65 @@
+# Cloud build readiness
+
+The `Cloud readiness` workflow starts for every master push. Its versioned
+`Cloud deployable v1` job succeeds only after all three prerequisites succeed:
+
+- The existing `Release Verify` workflow checks that exact commit, including
+  typecheck, builds, general and serialized tests, and Runner verification.
+- The reusable `Docker cloud` workflow builds and verifies its Linux AMD64
+  image, including Sentry resolution and orphan reaping, then publishes the
+  full-SHA cloud tag. Cloud readiness owns the master trigger so there is one
+  cloud build per push. Release tags and manual Docker runs retain their callers.
+- The full-SHA image and both exact-source npm packages are visible. The
+  packages are `@paperclipai/shared` and `@paperclipai/db` at
+  `0.0.0-preview.g<FULL_SHA>`, published through the migrator-only release lane.
+  Registry metadata must match the full commit, and the database package must
+  pin the matching shared package.
+
+Verification and image building run concurrently, outside the full npm release's
+concurrency group. Different commits have independent groups. Source verification
+is initially duplicated with the normal npm release: this spends existing hosted
+runner capacity to avoid waiting behind an older release. No verification gate is
+removed from npm publication. Watch organization-wide runner queues when measuring
+the result.
+
+The artifact wait runs for up to 30 minutes and reports what is missing. Only
+an HTTP 404 means publication is pending; authorization errors, upstream outages,
+and identity mismatches fail the job. A failed, cancelled, or skipped prerequisite
+cannot produce a successful readiness job. Retry the failed publication or build,
+then rerun the failed readiness workflow jobs to check the same commit again.
+
+## Consumer contract
+
+`Cloud deployable v1` is a source-and-artifact readiness signal. A deployment
+consumer must still resolve and pin the image digest and npm integrity/lockfile,
+validate migration contents and compatibility, and apply its target health gates.
+The check creates no release record and deploys no instance. A full-SHA tag by
+itself, or a successful migrator dispatch, is not this readiness signal.
+
+For automatic selection, accept only a successful job named exactly
+`Cloud deployable v1` in the latest attempt of a successful
+`.github/workflows/cloud-readiness.yml` run in `paperclipai/paperclip`, with
+event `push`, head branch `master`, and the expected full head SHA and repository.
+Do not trust a similarly named check from another workflow or a manual branch run.
+Order candidates by master ancestry, not job completion time: an older commit
+finishing late must not roll a fleet backward. Fail closed on API errors.
+
+Existing npm canary discovery is unchanged by this producer workflow. Consumers
+can adopt the versioned signal separately after the workflow has landed and
+successfully verified a real master commit.
+
+## Timing and rollout
+
+Measure from the master push to completion of `Cloud deployable v1`. Record
+queue time and the image, source-verification, and artifact-wait durations
+separately. The slowest prerequisite determines readiness; shortening an already
+faster prerequisite may have no effect on the total.
+
+Land full-SHA image publication, independent cloud builds, and migrator-only
+publication before enabling this workflow. Until those producers are present,
+the artifact wait cannot succeed. A manual dispatch on master can verify the
+wiring, but automatic consumers should use push runs. Source verification and
+registry checks can be rerun without deploying or changing mutable npm channels.
+
+When reverting this workflow, restore the master push trigger in
+`docker-cloud.yml` in the same change so master images continue to build.

scripts/cloud-readiness.mjs

@@ -0,0 +1,46 @@
+#!/usr/bin/env node
+import { pathToFileURL } from "node:url";
+import { imageExists, packageExists, versionFor } from "./preview-artifacts.mjs";
+
+/** Read-only availability gate. Deployment still resolves and pins artifacts. */
+export async function waitForCloudArtifacts(sha, {
+  fetchImpl = fetch,
+  now = () => performance.now(),
+  sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
+  timeoutMs = 30 * 60_000,
+  intervalMs = 20_000,
+  log = console.log,
+} = {}) {
+  const version = versionFor(sha);
+  if (!Number.isFinite(timeoutMs) || timeoutMs <= 0 || !Number.isFinite(intervalMs) || intervalMs <= 0) {
+    throw new Error("Cloud readiness requires positive finite timeout and poll interval.");
+  }
+  const deadline = now() + timeoutMs;
+  let previous;
+  let missing = ["image", "shared", "db"];
+  while (now() < deadline) {
+    // Recheck every artifact on the successful poll. Only an explicit 404
+    // means publication is pending; identity errors and upstream outages fail.
+    const results = await Promise.all([
+      imageExists(sha, fetchImpl),
+      packageExists("@paperclipai/shared", sha, fetchImpl),
+      packageExists("@paperclipai/db", sha, fetchImpl),
+    ]);
+    missing = ["image", "shared", "db"].filter((_, index) => !results[index]);
+    if (missing.length === 0) {
+      log(`Cloud artifacts available for ${sha}: verified image and exact-source migrator ${version}.`);
+      return { version: 1, sha, packageVersion: version };
+    }
+    const state = missing.join(", ");
+    if (state !== previous) log(`Waiting for cloud artifacts for ${sha}: ${state}.`);
+    previous = state;
+    const remaining = deadline - now();
+    if (remaining > 0) await sleep(Math.min(intervalMs, remaining));
+  }
+  throw new Error(`Cloud artifacts timed out for ${sha}; missing: ${missing.join(", ")}.`);
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  try { await waitForCloudArtifacts(process.argv[2]); }
+  catch (error) { console.error(error.message); process.exitCode = 1; }
+}

scripts/preview-artifacts.test.mjs

@@ -179,7 +179,10 @@ test("commits sharing a short prefix use separate full-SHA image addresses", asy
 test("cloud builds start per commit and preserve tag promotion dependencies", () => {
   const docker = readFileSync(new URL("../.github/workflows/docker.yml", import.meta.url), "utf8");
   const cloud = readFileSync(new URL("../.github/workflows/docker-cloud.yml", import.meta.url), "utf8");
-  assert.match(cloud, /branches: \[master\]/);
+  const readiness = readFileSync(new URL("../.github/workflows/cloud-readiness.yml", import.meta.url), "utf8");
+  assert.match(readiness, /branches: \[master\]/);
+  assert.match(readiness, /uses: \.\/\.github\/workflows\/docker-cloud.yml/);
+  assert.doesNotMatch(cloud, /^  push:/m);
   assert.match(cloud, /workflow_call:/);
   assert.match(cloud, /group: docker-cloud-\$\{\{ github.sha \}\}/);
   assert.match(cloud, /cancel-in-progress: false/);