fix(release): publish exact-source cloud migrators on merge (#13188)

Publish exact-source shared and database migrator packages for each master merge through the existing trusted Release workflow, independently of the full release and image build.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

.github/workflows/cloud-artifacts.yml

@@ -0,0 +1,34 @@
+name: Cloud artifacts
+
+on:
+  push:
+    branches: [master]
+  workflow_dispatch:
+
+permissions: {}
+
+jobs:
+  dispatch_migrator:
+    name: Start exact-source cloud migrator publication
+    if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master'
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      actions: write
+    steps:
+      # This separate workflow starts at merge, outside the full npm release's
+      # concurrency group. Publication stays in release.yml so npm recognizes
+      # the established trusted-publisher identity and npm-canary environment.
+      # No source checkout or package code runs with the dispatch credential.
+      - name: Dispatch the migrator-only release
+        env:
+          GH_TOKEN: ${{ github.token }}
+          SOURCE_SHA: ${{ github.sha }}
+        run: |
+          set -euo pipefail
+          request_id="$(cat /proc/sys/kernel/random/uuid)"
+          gh workflow run release.yml --repo "$GITHUB_REPOSITORY" --ref master \
+            --field channel=cloud-migrator \
+            --field source_ref="$SOURCE_SHA" \
+            --field request_id="$request_id"
+          echo "Started Cloud migrator $SOURCE_SHA in release.yml (request $request_id)." >> "$GITHUB_STEP_SUMMARY"

.github/workflows/release.yml

@@ -1,5 +1,5 @@
 name: Release
-run-name: ${{ inputs.channel == 'preview' && format('Stack deploy {0} build', inputs.request_id) || 'Release' }}
+run-name: ${{ inputs.channel == 'preview' && format('Stack deploy {0} build', inputs.request_id) || inputs.channel == 'cloud-migrator' && format('Cloud migrator {0}', inputs.source_ref) || 'Release' }}
 
 on:
   push:
@@ -19,14 +19,15 @@ on:
           - beta
           - nightly
           - preview
+          - cloud-migrator
         default: stable
       source_ref:
-        description: Stable source ref, or full immutable SHA for a preview build
+        description: Stable source ref, or full immutable SHA for a preview or cloud migrator build
         required: true
         type: string
         default: master
       request_id:
-        description: (preview) CLI correlation UUID
+        description: (preview/cloud-migrator) Correlation UUID
         type: string
         default: ""
       preview_migrator:
@@ -56,7 +57,7 @@ on:
         default: false
 
 concurrency:
-  group: ${{ inputs.channel == 'preview' && format('preview-{0}', inputs.source_ref) || format('release-{0}-{1}', github.event_name, github.ref) }}
+  group: ${{ (inputs.channel == 'preview' || inputs.channel == 'cloud-migrator') && format('{0}-{1}', inputs.channel, inputs.source_ref) || format('release-{0}-{1}', github.event_name, github.ref) }}
   cancel-in-progress: false
 
 env:
@@ -76,7 +77,7 @@ env:
 jobs:
   plan_preview:
     name: Check preview artifacts
-    if: github.ref == 'refs/heads/master' && github.event_name == 'workflow_dispatch' && inputs.channel == 'preview' && !inputs.dry_run
+    if: github.ref == 'refs/heads/master' && github.event_name == 'workflow_dispatch' && (inputs.channel == 'preview' || inputs.channel == 'cloud-migrator') && !inputs.dry_run
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -96,7 +97,8 @@ jobs:
           SOURCE_SHA: ${{ inputs.source_ref }}
           REQUEST_ID: ${{ inputs.request_id }}
           PREVIEW_MIGRATOR: ${{ inputs.preview_migrator }}
-        run: node scripts/preview-artifacts.mjs plan "$SOURCE_SHA" "$REQUEST_ID" "$PREVIEW_MIGRATOR"
+          PLAN_COMMAND: ${{ inputs.channel == 'cloud-migrator' && 'plan-migrator' || 'plan' }}
+        run: node scripts/preview-artifacts.mjs "$PLAN_COMMAND" "$SOURCE_SHA" "$REQUEST_ID" "$PREVIEW_MIGRATOR"
 
   package_preview:
     name: Build preview migrator
@@ -144,6 +146,12 @@ jobs:
     if: github.ref == 'refs/heads/master' && needs.plan_preview.outputs.packages == 'true' && needs.package_preview.result == 'success'
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    # A manual preview and a merge-triggered migrator may compile in parallel.
+    # Serialize only publication so they cannot race an immutable npm version,
+    # without making the migrator wait for a preview's separate image build.
+    concurrency:
+      group: preview-package-publish-${{ inputs.source_ref }}
+      cancel-in-progress: false
     # Reuse release.yml's established npm trusted-publisher identity. This job
     # publishes only isolated preview versions; it cannot advance lane tags.
     environment: npm-canary
@@ -260,7 +268,7 @@ jobs:
     name: Verify preview artifacts
     needs: [plan_preview, image_preview, publish_image_preview, package_preview, publish_preview]
     if: >-
-      always() && needs.plan_preview.result == 'success' &&
+      always() && inputs.channel == 'preview' && needs.plan_preview.result == 'success' &&
       (needs.publish_image_preview.result == 'success' || needs.plan_preview.outputs.image == 'false') &&
       (needs.publish_preview.result == 'success' || needs.plan_preview.outputs.packages == 'false')
     runs-on: ubuntu-latest

doc/preview-release-artifacts.md

@@ -43,6 +43,31 @@ version 1, request ID, SHA, stage `build`, and status `ready`. It expires after
 
 ## Publishing configuration and isolation
 
+### Migrator publication on merge
+
+The `Cloud artifacts` workflow starts a `cloud-migrator` dispatch of `release.yml`
+for every push to `master`. This dispatch builds and publishes only the exact-source
+`@paperclipai/shared` and `@paperclipai/db` preview packages. It starts independently
+of the full npm release and does not wait for the Docker image. The normal Docker
+workflow supplies the image separately.
+
+The run title is `Cloud migrator <FULL_SHA>`. A successful `Cloud artifacts`
+dispatch job only confirms that GitHub accepted the request. Inspect the matching
+`release.yml` run to confirm publication completed. This path does not produce a
+`stack-deploy-result` or certify source-test success or deployment readiness.
+Cloud must still verify all deployment prerequisites.
+
+To retry one commit, dispatch `release.yml` on `master` with `channel=cloud-migrator`,
+the full SHA as `source_ref`, a new UUID v4 as `request_id`, and `dry_run=false`.
+`preview_migrator` is not required for this channel. Existing packages are verified
+and reused. Preview and migrator-only runs use separate workflow concurrency
+groups. Only their package publication jobs share a group for the same SHA, so
+they cannot publish the same version concurrently and the migrator does not wait
+for a preview's image build. Different SHAs publish in separate groups; the full
+release keeps its existing group.
+
+### Publisher identity
+
 Configure npm trusted publishing for **both packages** with repository
 `paperclipai/paperclip`, workflow `release.yml`, and environment `npm-canary`.
 The image publisher uses the same environment, whose deployment branch policy

scripts/preview-artifacts.mjs

@@ -74,6 +74,14 @@ export async function packageExists(name, sha, fetchImpl = fetch) {
   return true;
 }
 
+export async function planArtifacts(sha, { migrator = false, image = true, fetchImpl = fetch } = {}) {
+  versionFor(sha);
+  return {
+    image: image && !await imageExists(sha, fetchImpl),
+    packages: migrator && !(await packageExists("@paperclipai/shared", sha, fetchImpl) && await packageExists("@paperclipai/db", sha, fetchImpl)),
+  };
+}
+
 export async function imageExists(sha, fetchImpl = fetch) {
   versionFor(sha);
   const tokenRes = await fetchImpl("https://ghcr.io/token?service=ghcr.io&scope=repository:paperclipai/paperclip:pull", { signal: AbortSignal.timeout(30_000) });
@@ -178,12 +186,13 @@ export async function publishPreview(dir, sha, { fetchImpl = fetch, exec = execF
 if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
   const [command, ...args] = process.argv.slice(2);
   try {
-    if (command === "plan") {
+    if (command === "plan" || command === "plan-migrator") {
       const [sha, requestId, migrator] = args;
       validateRequest(sha, requestId);
       if (process.env.GITHUB_REF !== "refs/heads/master") throw new Error("Preview workflow definitions must run from master.");
-      const image = !await imageExists(sha);
-      const packages = migrator === "true" && !(await packageExists("@paperclipai/shared", sha) && await packageExists("@paperclipai/db", sha));
+      const { image, packages } = await planArtifacts(sha, {
+        image: command === "plan", migrator: command === "plan-migrator" || migrator === "true",
+      });
       appendFileSync(process.env.GITHUB_OUTPUT, `image=${image}\npackages=${packages}\n`);
     } else if (command === "pack") packPreview(...args);
     else if (command === "publish") await publishPreview(...args);
@@ -195,6 +204,6 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href)
       if (process.env.PREVIEW_MIGRATOR === "true" && !(await packageExists("@paperclipai/shared", sha) && await packageExists("@paperclipai/db", sha))) throw new Error("Preview packages are still missing.");
       mkdirSync("stack-deploy-result", { recursive: true });
       writeFileSync("stack-deploy-result/result.json", JSON.stringify({ version: 1, stage: "build", requestId, sha, status: "ready" }) + "\n");
-    } else throw new Error("Expected plan, pack, publish, publish-image, or result.");
+    } else throw new Error("Expected plan, plan-migrator, pack, publish, publish-image, or result.");
   } catch (error) { console.error(error.message); process.exitCode = 1; }
 }

scripts/preview-artifacts.test.mjs

@@ -1,5 +1,6 @@
 import test from "node:test";
 import assert from "node:assert/strict";
+import { planArtifacts } from "./preview-artifacts.mjs";
 import { readFileSync, mkdtempSync, writeFileSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
 import path from "node:path";
@@ -24,6 +25,35 @@ test("preview request requires immutable SHA and correlation UUID", () => {
   assert.throws(() => validateRequest(sha, "not-a-request"));
 });
 
+test("migrator-only planning never waits for GHCR and reuses complete exact-source packages", async () => {
+  for (const available of [[], ["@paperclipai/shared"], ["@paperclipai/shared", "@paperclipai/db"]]) {
+    const calls = [];
+    const result = await planArtifacts(sha, { image: false, migrator: true, fetchImpl: async (url) => {
+      assert.equal(new URL(url).hostname, "registry.npmjs.org");
+      const name = decodeURIComponent(new URL(url).pathname.split("/")[1]);
+      calls.push(name);
+      return available.includes(name) ? json({ ...manifest(name), dist: { integrity: "test-integrity", tarball: "https://registry.npmjs.org/package.tgz" } }) : json({}, 404);
+    } });
+    assert.deepEqual(result, { image: false, packages: available.length !== 2 });
+    assert.ok(calls.includes("@paperclipai/shared"));
+    if (available.length) assert.ok(calls.includes("@paperclipai/db"));
+  }
+});
+
+test("migrator-only planning rejects registry outages and mismatched source identity", async () => {
+  for (const response of [json({}, 403), json({}, 503), json({ ...manifest("@paperclipai/shared"), gitHead: "b".repeat(40) })]) {
+    await assert.rejects(planArtifacts(sha, { image: false, migrator: true, fetchImpl: async () => response }));
+  }
+});
+
+test("ordinary preview planning still requests a missing image without publishing unsolicited packages", async () => {
+  const result = await planArtifacts(sha, { fetchImpl: async (url) => {
+    assert.equal(new URL(url).hostname, "ghcr.io");
+    return url.includes("/token?") ? json({ token: "test-pull-token" }) : json({}, 404);
+  } });
+  assert.deepEqual(result, { image: true, packages: false });
+});
+
 test("preview manifests carry exact source, isolated versions and shared dependency", () => {
   const pkg = manifest("@paperclipai/db");
   assert.equal(pkg.version, `0.0.0-preview.g${sha}`);
@@ -86,6 +116,24 @@ test("preview workflow separates branch compilation from trusted publishing", ()
   assert.match(workflow, /Stack deploy \{0\} build/);
 });
 
+test("merge dispatch uses the existing publisher outside full-release concurrency without claiming image readiness", () => {
+  const dispatcher = readFileSync(new URL("../.github/workflows/cloud-artifacts.yml", import.meta.url), "utf8");
+  const release = readFileSync(new URL("../.github/workflows/release.yml", import.meta.url), "utf8");
+  assert.match(dispatcher, /branches: \[master\]/);
+  assert.match(dispatcher, /github.ref == 'refs\/heads\/master'/);
+  assert.match(dispatcher, /SOURCE_SHA: \$\{\{ github.sha \}\}/);
+  assert.match(dispatcher, /gh workflow run release.yml .*--ref master/);
+  assert.match(dispatcher, /--field channel=cloud-migrator/);
+  assert.doesNotMatch(dispatcher, /actions\/checkout|id-token: write|packages: write|secrets\./);
+  assert.match(release, /\(inputs.channel == 'preview' \|\| inputs.channel == 'cloud-migrator'\) && format\('\{0\}-\{1\}', inputs.channel, inputs.source_ref\)/);
+  const publisher = release.split("  publish_preview:")[1].split("  image_preview:")[0];
+  assert.match(publisher, /group: preview-package-publish-\$\{\{ inputs.source_ref \}\}/);
+  assert.match(publisher, /cancel-in-progress: false/);
+  assert.match(release, /PLAN_COMMAND: \$\{\{ inputs.channel == 'cloud-migrator' && 'plan-migrator' \|\| 'plan' \}\}/);
+  const result = release.split("  result_preview:")[1].split("  verify_canary:")[0];
+  assert.match(result, /always\(\) && inputs.channel == 'preview'/);
+});
+
 
 test("existing image reuse verifies the full revision behind the immutable tag", async () => {
   const digest = "sha256:" + "b".repeat(64);