From 5cc51fad06d9762da70801b70ca3d93cef068c7e Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Thu, 10 Sep 2026 21:07:38 -0700 Subject: [PATCH] fix(release): publish exact-source cloud migrators on merge (#13188) Publish exact-source shared and database migrator packages for each master merge through the existing trusted Release workflow, independently of the full release and image build. Co-Authored-By: Paperclip --- .github/workflows/cloud-artifacts.yml | 34 +++++++++++++++++++ .github/workflows/release.yml | 22 ++++++++---- doc/preview-release-artifacts.md | 25 ++++++++++++++ scripts/preview-artifacts.mjs | 17 +++++++--- scripts/preview-artifacts.test.mjs | 48 +++++++++++++++++++++++++++ 5 files changed, 135 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/cloud-artifacts.yml diff --git a/.github/workflows/cloud-artifacts.yml b/.github/workflows/cloud-artifacts.yml new file mode 100644 index 0000000000..53d8ab710b --- /dev/null +++ b/.github/workflows/cloud-artifacts.yml @@ -0,0 +1,34 @@ +name: Cloud artifacts + +on: + push: + branches: [master] + workflow_dispatch: + +permissions: {} + +jobs: + dispatch_migrator: + name: Start exact-source cloud migrator publication + if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + actions: write + steps: + # This separate workflow starts at merge, outside the full npm release's + # concurrency group. Publication stays in release.yml so npm recognizes + # the established trusted-publisher identity and npm-canary environment. + # No source checkout or package code runs with the dispatch credential. + - name: Dispatch the migrator-only release + env: + GH_TOKEN: ${{ github.token }} + SOURCE_SHA: ${{ github.sha }} + run: | + set -euo pipefail + request_id="$(cat /proc/sys/kernel/random/uuid)" + gh workflow run release.yml --repo "$GITHUB_REPOSITORY" --ref master \ + --field channel=cloud-migrator \ + --field source_ref="$SOURCE_SHA" \ + --field request_id="$request_id" + echo "Started Cloud migrator $SOURCE_SHA in release.yml (request $request_id)." >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4edc35b836..2f510813fe 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,5 +1,5 @@ name: Release -run-name: ${{ inputs.channel == 'preview' && format('Stack deploy {0} build', inputs.request_id) || 'Release' }} +run-name: ${{ inputs.channel == 'preview' && format('Stack deploy {0} build', inputs.request_id) || inputs.channel == 'cloud-migrator' && format('Cloud migrator {0}', inputs.source_ref) || 'Release' }} on: push: @@ -19,14 +19,15 @@ on: - beta - nightly - preview + - cloud-migrator default: stable source_ref: - description: Stable source ref, or full immutable SHA for a preview build + description: Stable source ref, or full immutable SHA for a preview or cloud migrator build required: true type: string default: master request_id: - description: (preview) CLI correlation UUID + description: (preview/cloud-migrator) Correlation UUID type: string default: "" preview_migrator: @@ -56,7 +57,7 @@ on: default: false concurrency: - group: ${{ inputs.channel == 'preview' && format('preview-{0}', inputs.source_ref) || format('release-{0}-{1}', github.event_name, github.ref) }} + group: ${{ (inputs.channel == 'preview' || inputs.channel == 'cloud-migrator') && format('{0}-{1}', inputs.channel, inputs.source_ref) || format('release-{0}-{1}', github.event_name, github.ref) }} cancel-in-progress: false env: @@ -76,7 +77,7 @@ env: jobs: plan_preview: name: Check preview artifacts - if: github.ref == 'refs/heads/master' && github.event_name == 'workflow_dispatch' && inputs.channel == 'preview' && !inputs.dry_run + if: github.ref == 'refs/heads/master' && github.event_name == 'workflow_dispatch' && (inputs.channel == 'preview' || inputs.channel == 'cloud-migrator') && !inputs.dry_run runs-on: ubuntu-latest permissions: contents: read @@ -96,7 +97,8 @@ jobs: SOURCE_SHA: ${{ inputs.source_ref }} REQUEST_ID: ${{ inputs.request_id }} PREVIEW_MIGRATOR: ${{ inputs.preview_migrator }} - run: node scripts/preview-artifacts.mjs plan "$SOURCE_SHA" "$REQUEST_ID" "$PREVIEW_MIGRATOR" + PLAN_COMMAND: ${{ inputs.channel == 'cloud-migrator' && 'plan-migrator' || 'plan' }} + run: node scripts/preview-artifacts.mjs "$PLAN_COMMAND" "$SOURCE_SHA" "$REQUEST_ID" "$PREVIEW_MIGRATOR" package_preview: name: Build preview migrator @@ -144,6 +146,12 @@ jobs: if: github.ref == 'refs/heads/master' && needs.plan_preview.outputs.packages == 'true' && needs.package_preview.result == 'success' runs-on: ubuntu-latest timeout-minutes: 30 + # A manual preview and a merge-triggered migrator may compile in parallel. + # Serialize only publication so they cannot race an immutable npm version, + # without making the migrator wait for a preview's separate image build. + concurrency: + group: preview-package-publish-${{ inputs.source_ref }} + cancel-in-progress: false # Reuse release.yml's established npm trusted-publisher identity. This job # publishes only isolated preview versions; it cannot advance lane tags. environment: npm-canary @@ -260,7 +268,7 @@ jobs: name: Verify preview artifacts needs: [plan_preview, image_preview, publish_image_preview, package_preview, publish_preview] if: >- - always() && needs.plan_preview.result == 'success' && + always() && inputs.channel == 'preview' && needs.plan_preview.result == 'success' && (needs.publish_image_preview.result == 'success' || needs.plan_preview.outputs.image == 'false') && (needs.publish_preview.result == 'success' || needs.plan_preview.outputs.packages == 'false') runs-on: ubuntu-latest diff --git a/doc/preview-release-artifacts.md b/doc/preview-release-artifacts.md index 11f0c2808d..d7a5fe5279 100644 --- a/doc/preview-release-artifacts.md +++ b/doc/preview-release-artifacts.md @@ -43,6 +43,31 @@ version 1, request ID, SHA, stage `build`, and status `ready`. It expires after ## Publishing configuration and isolation +### Migrator publication on merge + +The `Cloud artifacts` workflow starts a `cloud-migrator` dispatch of `release.yml` +for every push to `master`. This dispatch builds and publishes only the exact-source +`@paperclipai/shared` and `@paperclipai/db` preview packages. It starts independently +of the full npm release and does not wait for the Docker image. The normal Docker +workflow supplies the image separately. + +The run title is `Cloud migrator `. A successful `Cloud artifacts` +dispatch job only confirms that GitHub accepted the request. Inspect the matching +`release.yml` run to confirm publication completed. This path does not produce a +`stack-deploy-result` or certify source-test success or deployment readiness. +Cloud must still verify all deployment prerequisites. + +To retry one commit, dispatch `release.yml` on `master` with `channel=cloud-migrator`, +the full SHA as `source_ref`, a new UUID v4 as `request_id`, and `dry_run=false`. +`preview_migrator` is not required for this channel. Existing packages are verified +and reused. Preview and migrator-only runs use separate workflow concurrency +groups. Only their package publication jobs share a group for the same SHA, so +they cannot publish the same version concurrently and the migrator does not wait +for a preview's image build. Different SHAs publish in separate groups; the full +release keeps its existing group. + +### Publisher identity + Configure npm trusted publishing for **both packages** with repository `paperclipai/paperclip`, workflow `release.yml`, and environment `npm-canary`. The image publisher uses the same environment, whose deployment branch policy diff --git a/scripts/preview-artifacts.mjs b/scripts/preview-artifacts.mjs index 6a68f83492..6a6eff6fb2 100644 --- a/scripts/preview-artifacts.mjs +++ b/scripts/preview-artifacts.mjs @@ -74,6 +74,14 @@ export async function packageExists(name, sha, fetchImpl = fetch) { return true; } +export async function planArtifacts(sha, { migrator = false, image = true, fetchImpl = fetch } = {}) { + versionFor(sha); + return { + image: image && !await imageExists(sha, fetchImpl), + packages: migrator && !(await packageExists("@paperclipai/shared", sha, fetchImpl) && await packageExists("@paperclipai/db", sha, fetchImpl)), + }; +} + export async function imageExists(sha, fetchImpl = fetch) { versionFor(sha); const tokenRes = await fetchImpl("https://ghcr.io/token?service=ghcr.io&scope=repository:paperclipai/paperclip:pull", { signal: AbortSignal.timeout(30_000) }); @@ -178,12 +186,13 @@ export async function publishPreview(dir, sha, { fetchImpl = fetch, exec = execF if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { const [command, ...args] = process.argv.slice(2); try { - if (command === "plan") { + if (command === "plan" || command === "plan-migrator") { const [sha, requestId, migrator] = args; validateRequest(sha, requestId); if (process.env.GITHUB_REF !== "refs/heads/master") throw new Error("Preview workflow definitions must run from master."); - const image = !await imageExists(sha); - const packages = migrator === "true" && !(await packageExists("@paperclipai/shared", sha) && await packageExists("@paperclipai/db", sha)); + const { image, packages } = await planArtifacts(sha, { + image: command === "plan", migrator: command === "plan-migrator" || migrator === "true", + }); appendFileSync(process.env.GITHUB_OUTPUT, `image=${image}\npackages=${packages}\n`); } else if (command === "pack") packPreview(...args); else if (command === "publish") await publishPreview(...args); @@ -195,6 +204,6 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) if (process.env.PREVIEW_MIGRATOR === "true" && !(await packageExists("@paperclipai/shared", sha) && await packageExists("@paperclipai/db", sha))) throw new Error("Preview packages are still missing."); mkdirSync("stack-deploy-result", { recursive: true }); writeFileSync("stack-deploy-result/result.json", JSON.stringify({ version: 1, stage: "build", requestId, sha, status: "ready" }) + "\n"); - } else throw new Error("Expected plan, pack, publish, publish-image, or result."); + } else throw new Error("Expected plan, plan-migrator, pack, publish, publish-image, or result."); } catch (error) { console.error(error.message); process.exitCode = 1; } } diff --git a/scripts/preview-artifacts.test.mjs b/scripts/preview-artifacts.test.mjs index 33041e3882..10ce16ccb3 100644 --- a/scripts/preview-artifacts.test.mjs +++ b/scripts/preview-artifacts.test.mjs @@ -1,5 +1,6 @@ import test from "node:test"; import assert from "node:assert/strict"; +import { planArtifacts } from "./preview-artifacts.mjs"; import { readFileSync, mkdtempSync, writeFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; @@ -24,6 +25,35 @@ test("preview request requires immutable SHA and correlation UUID", () => { assert.throws(() => validateRequest(sha, "not-a-request")); }); +test("migrator-only planning never waits for GHCR and reuses complete exact-source packages", async () => { + for (const available of [[], ["@paperclipai/shared"], ["@paperclipai/shared", "@paperclipai/db"]]) { + const calls = []; + const result = await planArtifacts(sha, { image: false, migrator: true, fetchImpl: async (url) => { + assert.equal(new URL(url).hostname, "registry.npmjs.org"); + const name = decodeURIComponent(new URL(url).pathname.split("/")[1]); + calls.push(name); + return available.includes(name) ? json({ ...manifest(name), dist: { integrity: "test-integrity", tarball: "https://registry.npmjs.org/package.tgz" } }) : json({}, 404); + } }); + assert.deepEqual(result, { image: false, packages: available.length !== 2 }); + assert.ok(calls.includes("@paperclipai/shared")); + if (available.length) assert.ok(calls.includes("@paperclipai/db")); + } +}); + +test("migrator-only planning rejects registry outages and mismatched source identity", async () => { + for (const response of [json({}, 403), json({}, 503), json({ ...manifest("@paperclipai/shared"), gitHead: "b".repeat(40) })]) { + await assert.rejects(planArtifacts(sha, { image: false, migrator: true, fetchImpl: async () => response })); + } +}); + +test("ordinary preview planning still requests a missing image without publishing unsolicited packages", async () => { + const result = await planArtifacts(sha, { fetchImpl: async (url) => { + assert.equal(new URL(url).hostname, "ghcr.io"); + return url.includes("/token?") ? json({ token: "test-pull-token" }) : json({}, 404); + } }); + assert.deepEqual(result, { image: true, packages: false }); +}); + test("preview manifests carry exact source, isolated versions and shared dependency", () => { const pkg = manifest("@paperclipai/db"); assert.equal(pkg.version, `0.0.0-preview.g${sha}`); @@ -86,6 +116,24 @@ test("preview workflow separates branch compilation from trusted publishing", () assert.match(workflow, /Stack deploy \{0\} build/); }); +test("merge dispatch uses the existing publisher outside full-release concurrency without claiming image readiness", () => { + const dispatcher = readFileSync(new URL("../.github/workflows/cloud-artifacts.yml", import.meta.url), "utf8"); + const release = readFileSync(new URL("../.github/workflows/release.yml", import.meta.url), "utf8"); + assert.match(dispatcher, /branches: \[master\]/); + assert.match(dispatcher, /github.ref == 'refs\/heads\/master'/); + assert.match(dispatcher, /SOURCE_SHA: \$\{\{ github.sha \}\}/); + assert.match(dispatcher, /gh workflow run release.yml .*--ref master/); + assert.match(dispatcher, /--field channel=cloud-migrator/); + assert.doesNotMatch(dispatcher, /actions\/checkout|id-token: write|packages: write|secrets\./); + assert.match(release, /\(inputs.channel == 'preview' \|\| inputs.channel == 'cloud-migrator'\) && format\('\{0\}-\{1\}', inputs.channel, inputs.source_ref\)/); + const publisher = release.split(" publish_preview:")[1].split(" image_preview:")[0]; + assert.match(publisher, /group: preview-package-publish-\$\{\{ inputs.source_ref \}\}/); + assert.match(publisher, /cancel-in-progress: false/); + assert.match(release, /PLAN_COMMAND: \$\{\{ inputs.channel == 'cloud-migrator' && 'plan-migrator' \|\| 'plan' \}\}/); + const result = release.split(" result_preview:")[1].split(" verify_canary:")[0]; + assert.match(result, /always\(\) && inputs.channel == 'preview'/); +}); + test("existing image reuse verifies the full revision behind the immutable tag", async () => { const digest = "sha256:" + "b".repeat(64);