diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3621bc98c9..49c00e59fd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,5 @@ name: Release +run-name: ${{ inputs.channel == 'preview' && format('Stack deploy {0} build', inputs.request_id) || 'Release' }} on: push: @@ -17,12 +18,21 @@ on: - stable - beta - nightly + - preview default: stable source_ref: - description: (stable) Commit SHA, branch, or tag to publish as stable + description: Stable source ref, or full immutable SHA for a preview build required: true type: string default: master + request_id: + description: (preview) CLI correlation UUID + type: string + default: "" + preview_migrator: + description: (preview) Publish isolated shared and database packages if missing + type: boolean + default: false stable_date: description: Enter a UTC date in YYYY-MM-DD format, for example 2026-03-18. Do not enter a version string. The workflow will resolve that date to a stable version such as 2026.318.0, then 2026.318.1 for the next same-day stable. required: false @@ -46,7 +56,7 @@ on: default: false concurrency: - group: release-${{ github.event_name }}-${{ github.ref }} + group: ${{ inputs.channel == 'preview' && format('preview-{0}', inputs.source_ref) || format('release-{0}-{1}', github.event_name, github.ref) }} cancel-in-progress: false env: @@ -64,6 +74,219 @@ env: NPM_PUBLISH_VERIFY_DELAY_SECONDS: "10" jobs: + plan_preview: + name: Check preview artifacts + if: github.ref == 'refs/heads/master' && github.event_name == 'workflow_dispatch' && inputs.channel == 'preview' && !inputs.dry_run + runs-on: ubuntu-latest + permissions: + contents: read + outputs: + image: ${{ steps.plan.outputs.image }} + packages: ${{ steps.plan.outputs.packages }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + - name: Validate immutable source and inspect existing artifacts + id: plan + env: + SOURCE_SHA: ${{ inputs.source_ref }} + REQUEST_ID: ${{ inputs.request_id }} + PREVIEW_MIGRATOR: ${{ inputs.preview_migrator }} + run: node scripts/preview-artifacts.mjs plan "$SOURCE_SHA" "$REQUEST_ID" "$PREVIEW_MIGRATOR" + + package_preview: + name: Build preview migrator + needs: plan_preview + if: needs.plan_preview.outputs.packages == 'true' + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + path: trusted + persist-credentials: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ inputs.source_ref }} + path: source + persist-credentials: false + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 + with: + version: 9.15.4 + run_install: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + - name: Install build dependencies without lifecycle scripts + working-directory: source + run: pnpm install --ignore-scripts --no-frozen-lockfile + - name: Build and pack exact-source preview packages + env: + SOURCE_SHA: ${{ inputs.source_ref }} + run: node trusted/scripts/preview-artifacts.mjs pack source packages "$SOURCE_SHA" + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: preview-packages + overwrite: true + path: packages/*.tgz + if-no-files-found: error + retention-days: 7 + + publish_preview: + name: Publish preview migrator + needs: [plan_preview, package_preview] + if: github.ref == 'refs/heads/master' && needs.plan_preview.outputs.packages == 'true' && needs.package_preview.result == 'success' + runs-on: ubuntu-latest + timeout-minutes: 30 + # Reuse release.yml's established npm trusted-publisher identity. This job + # publishes only isolated preview versions; it cannot advance lane tags. + environment: npm-canary + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + - name: Install npm with trusted publishing support + run: npm install --global npm@11.18.0 --ignore-scripts + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: preview-packages + path: preview-packages + - name: Publish immutable preview packages without running package code + env: + SOURCE_SHA: ${{ inputs.source_ref }} + run: node scripts/preview-artifacts.mjs publish preview-packages "$SOURCE_SHA" + + image_preview: + name: Build preview cloud image + needs: plan_preview + if: needs.plan_preview.outputs.image == 'true' + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ inputs.source_ref }} + persist-credentials: false + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 + with: + version: 9.15.4 + run_install: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + - name: Prepare locked image context + env: + SOURCE_SHA: ${{ inputs.source_ref }} + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$SOURCE_SHA" + pnpm install --resolution-only --ignore-scripts --ignore-pnpmfile --no-frozen-lockfile + echo "TOOLS_EPOCH=$(date -u +%G-W%V)" >> "$GITHUB_ENV" + - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4 + - name: Build the immutable cloud image without registry credentials + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 + with: + context: . + target: cloud + platforms: linux/amd64 + push: false + provenance: false # Docker archives cannot carry registry attestations. + outputs: type=docker,dest=${{ runner.temp }}/preview-image.tar + tags: ghcr.io/paperclipai/paperclip:sha-${{ inputs.source_ref }}-cloud + build-args: | + CLOUD_BUNDLED_PLUGINS=daytona + CLOUD_BUNDLED_SERVER_DEPS=@sentry/node + PAPERCLIP_BUILD_COMMIT=${{ inputs.source_ref }} + PAPERCLIP_BUILD_VERSION=0.0.0-preview.g${{ inputs.source_ref }} + CLI_TOOLS_CACHE_EPOCH=${{ env.TOOLS_EPOCH }} + labels: | + org.opencontainers.image.revision=${{ inputs.source_ref }} + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: preview-image + overwrite: true + path: ${{ runner.temp }}/preview-image.tar + compression-level: 0 + if-no-files-found: error + retention-days: 1 + + publish_image_preview: + name: Publish preview cloud image + needs: [plan_preview, image_preview] + if: github.ref == 'refs/heads/master' && needs.plan_preview.outputs.image == 'true' && needs.image_preview.result == 'success' + runs-on: ubuntu-latest + timeout-minutes: 30 + # This existing environment has an external master-only branch policy. + environment: npm-canary + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: preview-image + path: preview-image + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Verify image identity and publish without executing image code + env: + SOURCE_SHA: ${{ inputs.source_ref }} + run: node scripts/preview-artifacts.mjs publish-image preview-image/preview-image.tar "$SOURCE_SHA" + + result_preview: + name: Verify preview artifacts + needs: [plan_preview, image_preview, publish_image_preview, package_preview, publish_preview] + if: >- + always() && needs.plan_preview.result == 'success' && + (needs.publish_image_preview.result == 'success' || needs.plan_preview.outputs.image == 'false') && + (needs.publish_preview.result == 'success' || needs.plan_preview.outputs.packages == 'false') + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + - name: Confirm exact artifacts are visible + env: + SOURCE_SHA: ${{ inputs.source_ref }} + REQUEST_ID: ${{ inputs.request_id }} + PREVIEW_MIGRATOR: ${{ inputs.preview_migrator }} + run: node scripts/preview-artifacts.mjs result "$SOURCE_SHA" "$REQUEST_ID" + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: stack-deploy-result + overwrite: true + path: stack-deploy-result/result.json + if-no-files-found: error + retention-days: 30 + verify_canary: if: github.event_name == 'push' uses: ./.github/workflows/release-verify.yml diff --git a/doc/preview-release-artifacts.md b/doc/preview-release-artifacts.md new file mode 100644 index 0000000000..11f0c2808d --- /dev/null +++ b/doc/preview-release-artifacts.md @@ -0,0 +1,93 @@ +# Preview deployment artifacts + +The `preview` channel in `.github/workflows/release.yml` builds deployment +artifacts for one immutable source commit. It does not create a GitHub release, +move a source branch, or advance any stable, beta, nightly, or canary alias. + +Dispatch `release.yml` on `master` with these inputs: + +| Input | Value | +| --- | --- | +| `channel` | `preview` | +| `source_ref` | Full lowercase 40-character commit SHA in this repository | +| `request_id` | UUID v4 identifying the operator's deployment request | +| `preview_migrator` | `true` when exact-source DB/shared packages are needed | +| `dry_run` | `false` | + +The workflow title is `Stack deploy build`. Consumers must find a +run by this identity, not by the latest run. Preview builds reject workflow +definitions that do not run from `master`. + +## Outputs and reuse + +The image uses `ghcr.io/paperclipai/paperclip:sha--cloud`. +Full-SHA tags keep separate commits with the same short prefix isolated. Normal +release images retain their existing short-tag convention. Build arguments carry the full commit SHA. +Preview builds do not import or overwrite the shared release cache or release +aliases. Missing images are built for Linux amd64, matching managed deployments. + +When requested, both `@paperclipai/shared` and `@paperclipai/db` use +`0.0.0-preview.g`. Workspace dependencies are pinned to exact versions. +Packages carry `gitHead` and `paperclipPreviewCommit` source identity. npm publishes +them under the `preview` dist-tag only. Normal consumers of `latest` or `canary` +continue to select normal releases. + +Registry 404 responses mean missing artifacts. Authentication errors, outages, +or existing package identity mismatches fail the workflow. Retries reuse matching +published artifacts, including a shared package published before a DB publish +failure. Allow npm's visibility polling to finish before retrying. + +The final `stack-deploy-result` artifact contains `result.json` with contract +version 1, request ID, SHA, stage `build`, and status `ready`. It expires after +30 days. This confirms artifact availability; it does not certify a tenant deploy. + +## Publishing configuration and isolation + +Configure npm trusted publishing for **both packages** with repository +`paperclipai/paperclip`, workflow `release.yml`, and environment `npm-canary`. +The image publisher uses the same environment, whose deployment branch policy +permits only master. Both publishers also check the workflow ref before running. +This uses the existing publisher identity rather than requiring another workflow +registration. The job uses npm with OIDC trusted publishing support and provenance. +The environment's existing protections still apply. + +Package compilation runs in a separate job with read-only repository access and +no npm, cloud-admin, or provider credentials. Trusted tooling from `master` packs +the requested source checkout. The existing bundled-package helper takes patch +configuration from that source checkout. Build artifacts contain only the two +fixed package tarballs. + +Publishing runs on a fresh runner with trusted tooling, without a checkout of +the requested branch. It checks package identity and exact dependencies, rejects +archive path aliases, and publishes with lifecycle scripts disabled and an explicit +registry and dist-tag. Image builds also run without registry write access and export a Docker archive. +A separate trusted publisher loads that archive as data, verifies its full revision, +platform, and image ID, then pushes only the SHA tag. It never runs the image. +Dependency resolution disables scripts and pnpmfile hooks. Preview actions are +pinned to full commit SHAs. + +The deploying control plane must independently verify package integrity, source +identity, SQL and migration journal contents, and schema compatibility. Publish +this workflow support before enabling an operator CLI that depends on it. Test +normal release selection and a new migration-bearing branch in staging before +allowing production use. + +## Local checks + +```sh +node --test scripts/preview-artifacts.test.mjs +pnpm test:release-registry +pnpm -r typecheck +pnpm test:run +pnpm build +``` + +For a package build without publishing, install dependencies in a disposable +checkout at an exact commit, then use the trusted helper: + +```sh +node scripts/preview-artifacts.mjs pack /path/to/source /path/to/output FULL_SHA +``` + +This executes source build scripts. Keep output outside the repository and use an +environment without publishing or cloud-admin credentials. diff --git a/package.json b/package.json index 309517bd71..9d7f21654b 100644 --- a/package.json +++ b/package.json @@ -59,7 +59,7 @@ "smoke:posthog-live": "node scripts/smoke/posthog-live.mjs", "smoke:pipelines-tutorial": "./scripts/smoke/pipelines-tutorial-smoke.sh", "smoke:terminal-bench-loop-skill": "node scripts/smoke/terminal-bench-loop-skill-smoke.mjs", - "test:release-registry": "node --test scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs", + "test:release-registry": "node --test scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs", "storybook-visual:baseline": "node scripts/storybook-visual-baseline.mjs", "test:storybook-visual": "node scripts/storybook-visual-baseline.mjs download && node scripts/storybook-visual-baseline.mjs verify && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts", "test:storybook-visual:update": "node scripts/storybook-visual-baseline.mjs download && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts --update-snapshots && node scripts/storybook-visual-baseline.mjs pack", diff --git a/scripts/prepare-bundled-package.mjs b/scripts/prepare-bundled-package.mjs index 891da00b71..aeb2bef0c5 100644 --- a/scripts/prepare-bundled-package.mjs +++ b/scripts/prepare-bundled-package.mjs @@ -116,8 +116,8 @@ export function selectBundledDependencyPatches( return selectedPatches; } -export function applyBundledDependencyPatches(destinationDir, bundledDependencies) { - const rootPackage = JSON.parse(readFileSync(resolve(repoRoot, "package.json"), "utf8")); +export function applyBundledDependencyPatches(destinationDir, bundledDependencies, sourceRoot = repoRoot) { + const rootPackage = JSON.parse(readFileSync(resolve(sourceRoot, "package.json"), "utf8")); const patchedDependencies = rootPackage.pnpm?.patchedDependencies ?? {}; for (const { packageName, patchPath } of selectBundledDependencyPatches( @@ -129,14 +129,14 @@ export function applyBundledDependencyPatches(destinationDir, bundledDependencie "patch", ["-p1", "--forward", "-d", resolve(destinationDir, "node_modules", packageName)], { - input: readFileSync(resolve(repoRoot, patchPath)), + input: readFileSync(resolve(sourceRoot, patchPath)), stdio: ["pipe", "inherit", "inherit"], }, ); } } -export function prepareBundledPackage(sourceDir, destinationDir) { +export function prepareBundledPackage(sourceDir, destinationDir, { sourceRoot = repoRoot } = {}) { const sourcePackagePath = resolve(sourceDir, "package.json"); const sourcePackage = JSON.parse(readFileSync(sourcePackagePath, "utf8")); const bundledDependencies = sourcePackage.bundleDependencies ?? sourcePackage.bundledDependencies ?? []; @@ -166,7 +166,7 @@ export function prepareBundledPackage(sourceDir, destinationDir) { { cwd: destinationDir, stdio: "inherit" }, ); writeFileSync(deployedPackagePath, `${JSON.stringify(publishManifest, null, 2)}\n`); - applyBundledDependencyPatches(destinationDir, bundledDependencies); + applyBundledDependencyPatches(destinationDir, bundledDependencies, sourceRoot); if (bundledDependencies.includes("acpx")) { const acpxPackage = JSON.parse( diff --git a/scripts/preview-artifacts.mjs b/scripts/preview-artifacts.mjs new file mode 100644 index 0000000000..6a68f83492 --- /dev/null +++ b/scripts/preview-artifacts.mjs @@ -0,0 +1,200 @@ +#!/usr/bin/env node +// Trusted release tooling. Packaging runs without publish credentials; publishing +// accepts only the two fixed package artifacts and never executes their scripts. +import { execFileSync } from "node:child_process"; +import { readFileSync, writeFileSync, mkdirSync, cpSync, renameSync, appendFileSync } from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { gunzipSync } from "node:zlib"; +import { createHash } from "node:crypto"; +import { materializePublishManifest, prepareBundledPackage } from "./prepare-bundled-package.mjs"; + +export const versionFor = (sha) => { + if (!/^[0-9a-f]{40}$/.test(sha ?? "")) throw new Error("Preview builds require a full immutable commit SHA."); + return `0.0.0-preview.g${sha}`; +}; +export function validateRequest(sha, requestId) { + versionFor(sha); + if (!/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(requestId ?? "")) throw new Error("A correlation UUID is required."); +} + +export function previewManifest(pkg, sha) { + if (!["@paperclipai/shared", "@paperclipai/db"].includes(pkg.name)) throw new Error("Unexpected preview package."); + const version = versionFor(sha); + const exact = structuredClone(pkg); + for (const section of ["dependencies", "optionalDependencies", "peerDependencies"]) { + for (const [name, specifier] of Object.entries(exact[section] ?? {})) { + if (typeof specifier === "string" && specifier.startsWith("workspace:")) exact[section][name] = version; + } + } + const result = materializePublishManifest({ ...exact, version }); + result.gitHead = sha; + result.paperclipPreviewCommit = sha; + if (pkg.name === "@paperclipai/db") result.dependencies = { ...result.dependencies, "@paperclipai/shared": version }; + return result; +} + +export function assertMetadata(pkg, name, sha) { + if (pkg?.publishConfig !== undefined || pkg?.name !== name || pkg.version !== versionFor(sha) || pkg.gitHead !== sha || pkg.paperclipPreviewCommit !== sha || + (name === "@paperclipai/db" && pkg.dependencies?.["@paperclipai/shared"] !== versionFor(sha))) { + throw new Error("Preview package identity or dependency pin mismatch."); + } +} + +export function tarManifest(bytes) { + const tar = gunzipSync(bytes, { maxOutputLength: 128 * 1024 * 1024 }); + let manifest; + for (let offset = 0; offset + 512 <= tar.length;) { + const h = tar.subarray(offset, offset + 512); + if (h.every((v) => v === 0)) break; + const field = (start, size) => h.subarray(start, start + size).toString("utf8").split("\0")[0].trim(); + const sizeText = field(124, 12); + if (!/^[0-7]+$/.test(sizeText)) throw new Error("Invalid package archive."); + const size = Number.parseInt(sizeText, 8); + if (offset + 512 + size > tar.length) throw new Error("Truncated package archive."); + const name = `${field(345, 155) ? field(345, 155) + "/" : ""}${field(0, 100)}`; + if (!name.startsWith("package/") || name.split("/").some((part) => part === "." || part === "..") || ![0, 48, 53].includes(h[156])) throw new Error("Unsupported package archive entry."); + if (name === "package/package.json") { + if (manifest || ![0, 48].includes(h[156])) throw new Error("Invalid package manifest entry."); + manifest = JSON.parse(tar.subarray(offset + 512, offset + 512 + size).toString("utf8")); + } + offset += 512 + Math.ceil(size / 512) * 512; + } + if (!manifest) throw new Error("Missing package manifest."); + return manifest; +} + +export async function packageExists(name, sha, fetchImpl = fetch) { + const response = await fetchImpl(`https://registry.npmjs.org/${encodeURIComponent(name)}/${versionFor(sha)}`, { signal: AbortSignal.timeout(30_000) }); + if (response.status === 404) return false; + if (!response.ok) throw new Error(`npm lookup failed: HTTP ${response.status}`); + const pkg = await response.json(); + assertMetadata(pkg, name, sha); + if (!pkg.dist?.integrity || !pkg.dist?.tarball) throw new Error("Published preview has no immutable distribution pin."); + return true; +} + +export async function imageExists(sha, fetchImpl = fetch) { + versionFor(sha); + const tokenRes = await fetchImpl("https://ghcr.io/token?service=ghcr.io&scope=repository:paperclipai/paperclip:pull", { signal: AbortSignal.timeout(30_000) }); + if (!tokenRes.ok) throw new Error(`GHCR lookup failed: HTTP ${tokenRes.status}`); + const { token } = await tokenRes.json(); + if (typeof token !== "string") throw new Error("GHCR did not return a pull token."); + const base = "https://ghcr.io/v2/paperclipai/paperclip"; + const headers = { Authorization: `Bearer ${token}`, Accept: "application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json" }; + const get = (url) => fetchImpl(url, { headers, redirect: "error", signal: AbortSignal.timeout(30_000) }); + let res = await get(`${base}/manifests/sha-${sha}-cloud`); + if (res.status === 404) return false; + if (!res.ok) throw new Error(`GHCR lookup failed: HTTP ${res.status}`); + let manifest = await res.json(); + const digest = (value) => { + if (typeof value !== "string" || !/^sha256:[0-9a-f]{64}$/.test(value)) throw new Error("Invalid image digest."); + return value; + }; + if (Array.isArray(manifest.manifests)) { + const amd64 = manifest.manifests.find((entry) => entry.platform?.os === "linux" && entry.platform?.architecture === "amd64"); + if (!amd64) throw new Error("Cloud image has no Linux amd64 manifest."); + res = await get(`${base}/manifests/${digest(amd64.digest)}`); + if (!res.ok) throw new Error(`GHCR manifest lookup failed: HTTP ${res.status}`); + manifest = await res.json(); + } + res = await fetchImpl(`${base}/blobs/${digest(manifest.config?.digest)}`, { headers, redirect: "manual", signal: AbortSignal.timeout(30_000) }); + // Registry blob storage may redirect to its signed storage URL. Follow only + // with no Authorization header, so the GHCR token cannot leave the registry. + if ([301, 302, 307, 308].includes(res.status)) { + const location = new URL(res.headers.get("location")); + if (location.protocol !== "https:" || location.username || location.password) throw new Error("Invalid registry blob redirect."); + res = await fetchImpl(location.href, { redirect: "error", signal: AbortSignal.timeout(30_000) }); + } + if (!res.ok) throw new Error(`GHCR config lookup failed: HTTP ${res.status}`); + const config = await res.json(); + if (config.config?.Labels?.["org.opencontainers.image.revision"] !== sha) throw new Error("Existing SHA image tag does not match the requested full commit."); + return true; +} + +/** Publication loads image data, but never runs a container or source scripts. */ +export async function publishImage(file, sha, { exec = execFileSync, fetchImpl = fetch } = {}) { + versionFor(sha); + const image = `ghcr.io/paperclipai/paperclip:sha-${sha}-cloud`; + if (await imageExists(sha, fetchImpl)) { console.log("Reusing the verified SHA cloud image."); return; } + exec("docker", ["load", "--input", path.resolve(file)], { encoding: "utf8", maxBuffer: 8 * 1024 * 1024 }); + const [metadata] = JSON.parse(exec("docker", ["image", "inspect", image], { encoding: "utf8", maxBuffer: 8 * 1024 * 1024 })); + if (metadata?.Config?.Labels?.["org.opencontainers.image.revision"] !== sha || metadata.Os !== "linux" || metadata.Architecture !== "amd64" || + !/^sha256:[0-9a-f]{64}$/.test(metadata.Id ?? "")) throw new Error("Built image identity or platform does not match the request."); + // Push only this verified image ID under the one permitted tag, regardless + // of any additional tag names present in the untrusted Docker archive. + exec("docker", ["tag", metadata.Id, image], { stdio: "inherit" }); + exec("docker", ["push", image], { stdio: "inherit" }); +} + +export function packPreview(source, output, sha, { exec = execFileSync } = {}) { + versionFor(sha); + source = path.resolve(source); output = path.resolve(output); + if (exec("git", ["rev-parse", "HEAD"], { cwd: source, encoding: "utf8" }).trim() !== sha) throw new Error("Source checkout differs from the requested commit."); + mkdirSync(output, { recursive: true }); + for (const short of ["shared", "db"]) { + exec("pnpm", ["--filter", `@paperclipai/${short}`, "build"], { cwd: source, stdio: "inherit" }); + const packageDir = path.join(source, "packages", short); + const originalText = readFileSync(path.join(packageDir, "package.json"), "utf8"); + const original = JSON.parse(originalText); + const pkg = previewManifest(original, sha); + const staging = path.join(output, `package-${short}`); + if ((pkg.bundleDependencies ?? []).length) { + // The established helper materializes patched embedded-postgres instead + // of publishing pnpm's dependency symlinks. + writeFileSync(path.join(packageDir, "package.json"), JSON.stringify(pkg)); + try { prepareBundledPackage(packageDir, staging, { sourceRoot: source }); } + finally { writeFileSync(path.join(packageDir, "package.json"), originalText); } + } else { + mkdirSync(staging, { recursive: true }); + cpSync(path.join(packageDir, "dist"), path.join(staging, "dist"), { recursive: true }); + writeFileSync(path.join(staging, "package.json"), JSON.stringify(pkg)); + } + const packed = JSON.parse(exec("npx", ["--yes", "npm@10.9.7", "pack", "--ignore-scripts", "--json", "--pack-destination", output], { cwd: staging, encoding: "utf8", maxBuffer: 8 * 1024 * 1024 })); + renameSync(path.join(output, path.basename(packed[0].filename)), path.join(output, `${short}.tgz`)); + assertMetadata(tarManifest(readFileSync(path.join(output, `${short}.tgz`))), `@paperclipai/${short}`, sha); + } +} + +export async function publishPreview(dir, sha, { fetchImpl = fetch, exec = execFileSync, sleep = (ms) => new Promise((r) => setTimeout(r, ms)) } = {}) { + for (const short of ["shared", "db"]) { + const name = `@paperclipai/${short}`; + const file = path.resolve(dir, `${short}.tgz`); + const bytes = readFileSync(file); + assertMetadata(tarManifest(bytes), name, sha); + if (await packageExists(name, sha, fetchImpl)) { console.log(`Reusing ${name}@${versionFor(sha)}`); continue; } + console.log(`Publishing ${name}@${versionFor(sha)} (${createHash("sha256").update(bytes).digest("hex").slice(0, 12)})`); + // No package checkout, lifecycle scripts, npmrc, or branch code runs here. + exec("npm", ["publish", file, "--tag", "preview", "--access", "public", "--ignore-scripts", "--provenance", "--registry", "https://registry.npmjs.org"], { stdio: "inherit" }); + let published = false; + for (let attempt = 0; attempt < 60; attempt++) { + if (await packageExists(name, sha, fetchImpl)) { published = true; break; } + await sleep(10_000); + } + if (!published) throw new Error("npm accepted the preview but it is not yet visible. Retry reuses published packages."); + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const [command, ...args] = process.argv.slice(2); + try { + if (command === "plan") { + const [sha, requestId, migrator] = args; + validateRequest(sha, requestId); + if (process.env.GITHUB_REF !== "refs/heads/master") throw new Error("Preview workflow definitions must run from master."); + const image = !await imageExists(sha); + const packages = migrator === "true" && !(await packageExists("@paperclipai/shared", sha) && await packageExists("@paperclipai/db", sha)); + appendFileSync(process.env.GITHUB_OUTPUT, `image=${image}\npackages=${packages}\n`); + } else if (command === "pack") packPreview(...args); + else if (command === "publish") await publishPreview(...args); + else if (command === "publish-image") await publishImage(...args); + else if (command === "result") { + const [sha, requestId] = args; + validateRequest(sha, requestId); + if (!await imageExists(sha)) throw new Error("Cloud image is still missing."); + if (process.env.PREVIEW_MIGRATOR === "true" && !(await packageExists("@paperclipai/shared", sha) && await packageExists("@paperclipai/db", sha))) throw new Error("Preview packages are still missing."); + mkdirSync("stack-deploy-result", { recursive: true }); + writeFileSync("stack-deploy-result/result.json", JSON.stringify({ version: 1, stage: "build", requestId, sha, status: "ready" }) + "\n"); + } else throw new Error("Expected plan, pack, publish, publish-image, or result."); + } catch (error) { console.error(error.message); process.exitCode = 1; } +} diff --git a/scripts/preview-artifacts.test.mjs b/scripts/preview-artifacts.test.mjs new file mode 100644 index 0000000000..746f246cf9 --- /dev/null +++ b/scripts/preview-artifacts.test.mjs @@ -0,0 +1,128 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync, mkdtempSync, writeFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { gzipSync } from "node:zlib"; +import { previewManifest, assertMetadata, validateRequest, versionFor, tarManifest, packageExists, imageExists, publishPreview, publishImage } from "./preview-artifacts.mjs"; + +const sha = "a".repeat(40); +const id = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; +const manifest = (name) => previewManifest({ name, version: "0.0.0", dependencies: name.endsWith("/db") ? { "@paperclipai/shared": "workspace:*" } : {}, publishConfig: { exports: { ".": "./dist/index.js" } } }, sha); +function pack(pkg) { + const b = Buffer.from(JSON.stringify(pkg)); const h = Buffer.alloc(512); + h.write("package/package.json"); h.write(b.length.toString(8).padStart(11, "0"), 124, 11); h[156] = 48; + const padded = Buffer.alloc(Math.ceil(b.length / 512) * 512); b.copy(padded); + return gzipSync(Buffer.concat([h, padded, Buffer.alloc(1024)])); +} +const json = (body, status = 200) => new Response(JSON.stringify(body), { status }); + +test("preview request requires immutable SHA and correlation UUID", () => { + validateRequest(sha, id); + for (const ref of ["master", "origin/master", "a".repeat(7), "$(unsafe)", "A".repeat(40)]) assert.throws(() => versionFor(ref)); + assert.throws(() => validateRequest(sha, "not-a-request")); +}); + +test("preview manifests carry exact source, isolated versions and shared dependency", () => { + const pkg = manifest("@paperclipai/db"); + assert.equal(pkg.version, `0.0.0-preview.g${sha}`); + assert.equal(pkg.dependencies["@paperclipai/shared"], pkg.version); + assert.deepEqual(pkg.exports, { ".": "./dist/index.js" }); + assertMetadata(pkg, "@paperclipai/db", sha); + assert.throws(() => assertMetadata({ ...pkg, gitHead: "b".repeat(40) }, pkg.name, sha)); + assert.throws(() => assertMetadata({ ...pkg, dependencies: { "@paperclipai/shared": "latest" } }, pkg.name, sha)); + assert.deepEqual(tarManifest(pack(pkg)), pkg); +}); + +test("only 404 means an artifact is missing; auth and outages are fatal", async () => { + assert.equal(await packageExists("@paperclipai/db", sha, async () => json({}, 404)), false); + await assert.rejects(packageExists("@paperclipai/db", sha, async () => json({}, 403))); + await assert.rejects(packageExists("@paperclipai/db", sha, async () => json({}, 503))); + await assert.rejects(imageExists(sha, async () => json({}, 503))); + assert.equal(await imageExists(sha, async (url) => url.includes("/token?") ? json({ token: "test-pull-token" }) : json({}, 404)), false); + await assert.rejects(packageExists("@paperclipai/db", sha, async () => json({ ...manifest("@paperclipai/db"), gitHead: "b".repeat(40) }))); +}); + +test("publishing reuses existing previews and never executes package lifecycle hooks", async () => { + const dir = mkdtempSync(path.join(tmpdir(), "preview-publish-test-")); + const published = new Set(["@paperclipai/shared"]); + const calls = []; + try { + for (const short of ["shared", "db"]) writeFileSync(path.join(dir, `${short}.tgz`), pack({ ...manifest(`@paperclipai/${short}`), scripts: { prepublishOnly: "do-not-run" } })); + await publishPreview(dir, sha, { + fetchImpl: async (url) => { + const name = decodeURIComponent(new URL(url).pathname.split("/")[1]); + return published.has(name) ? json({ ...manifest(name), dist: { integrity: "test-integrity", tarball: "https://registry.npmjs.org/package.tgz" } }) : json({}, 404); + }, + exec: (command, args) => { calls.push({ command, args }); published.add("@paperclipai/db"); }, + sleep: async () => {}, + }); + assert.equal(calls.length, 1); + assert.equal(calls[0].command, "npm"); + assert.ok(calls[0].args.includes("--ignore-scripts")); + assert.equal(calls[0].args[calls[0].args.indexOf("--tag") + 1], "preview"); + assert.ok(!calls[0].args.includes("canary")); + } finally { rmSync(dir, { recursive: true, force: true }); } +}); + +test("preview workflow separates branch compilation from trusted publishing", () => { + const workflow = readFileSync(new URL("../.github/workflows/release.yml", import.meta.url), "utf8"); + const builder = workflow.split(" package_preview:")[1].split(" publish_preview:")[0]; + const publisher = workflow.split(" publish_preview:")[1].split(" image_preview:")[0]; + const image = workflow.split(" image_preview:")[1].split(" publish_image_preview:")[0]; + const imagePublisher = workflow.split(" publish_image_preview:")[1].split(" result_preview:")[0]; + assert.doesNotMatch(builder, /id-token: write|packages: write|secrets\./); + assert.doesNotMatch(publisher, /ref: \$\{\{ inputs.source_ref|working-directory: source|pnpm install/); + assert.match(publisher, /environment: npm-canary/); + assert.match(image, /PAPERCLIP_BUILD_COMMIT=\$\{\{ inputs.source_ref \}\}/); + assert.doesNotMatch(image, /cache-(?:to|from):|canary-cloud|latest-cloud|packages: write|secrets\./); + assert.doesNotMatch(imagePublisher, /ref: \$\{\{ inputs.source_ref|docker\/build-push-action|pnpm install/); + assert.match(imagePublisher, /publish-image/); + assert.match(imagePublisher, /environment: npm-canary/); + assert.match(imagePublisher, /github.ref == 'refs\/heads\/master'/); + assert.match(publisher, /github.ref == 'refs\/heads\/master'/); + assert.doesNotMatch(workflow.split(" verify_canary:")[0], /uses: [^\n]+@v\d/); + assert.match(workflow, /Stack deploy \{0\} build/); +}); + + +test("existing image reuse verifies the full revision behind the immutable tag", async () => { + const digest = "sha256:" + "b".repeat(64); + for (const revision of [sha, "c".repeat(40)]) { + const fetchImpl = async (url) => url.includes("/token?") ? json({ token: "test-pull-token" }) : + url.includes("/blobs/") ? json({ config: { Labels: { "org.opencontainers.image.revision": revision } } }) : + url.endsWith(digest) ? json({ config: { digest } }) : json({ manifests: [{ digest, platform: { os: "linux", architecture: "amd64" } }] }); + if (revision === sha) assert.equal(await imageExists(sha, fetchImpl), true); + else await assert.rejects(imageExists(sha, fetchImpl), /full commit/); + } +}); + + +test("image publisher verifies source and platform before pushing exactly one immutable tag", async () => { + for (const revision of [sha, "c".repeat(40)]) { + const calls = []; + const operation = publishImage("preview-image.tar", sha, { + fetchImpl: async (url) => url.includes("/token?") ? json({ token: "test-pull-token" }) : json({}, 404), + exec: (command, args) => { + calls.push({ command, args }); + if (args[0] === "image") return JSON.stringify([{ Id: "sha256:" + "b".repeat(64), Os: "linux", Architecture: "amd64", Config: { Labels: { "org.opencontainers.image.revision": revision } } }]); + return ""; + }, + }); + if (revision === sha) { + await operation; + assert.deepEqual(calls.filter((call) => call.args[0] === "push").map((call) => call.args), [["push", `ghcr.io/paperclipai/paperclip:sha-${sha}-cloud`]]); + } else { await assert.rejects(operation, /identity/); assert.ok(!calls.some((call) => call.args[0] === "push")); } + assert.ok(!calls.some((call) => ["run", "build"].includes(call.args[0]))); + } +}); + + +test("commits sharing a short prefix use separate full-SHA image addresses", async () => { + const urls = []; + const fetchImpl = async (url) => { urls.push(url); return url.includes("/token?") ? json({ token: "test-pull-token" }) : json({}, 404); }; + const other = sha.slice(0, 7) + "b".repeat(33); + await imageExists(sha, fetchImpl); + await imageExists(other, fetchImpl); + assert.deepEqual(urls.filter((url) => url.includes("/manifests/")), [sha, other].map((commit) => `https://ghcr.io/v2/paperclipai/paperclip/manifests/sha-${commit}-cloud`)); +});