diff --git a/.github/scripts/tests/lockfile-refresh-workflows.test.mjs b/.github/scripts/tests/lockfile-refresh-workflows.test.mjs index 7c19cf2cb9..e0e9775c04 100644 --- a/.github/scripts/tests/lockfile-refresh-workflows.test.mjs +++ b/.github/scripts/tests/lockfile-refresh-workflows.test.mjs @@ -6,6 +6,7 @@ const workflows = [ '.github/workflows/refresh-lockfile.yml', '.github/workflows/pr-trusted.yml', '.github/workflows/docker.yml', + '.github/workflows/docker-cloud.yml', ]; test('lockfile repair workflows resolve dependencies instead of updating metadata only', async () => { diff --git a/.github/workflows/docker-cloud.yml b/.github/workflows/docker-cloud.yml new file mode 100644 index 0000000000..13e750e004 --- /dev/null +++ b/.github/workflows/docker-cloud.yml @@ -0,0 +1,264 @@ +name: Docker cloud + +on: + push: + branches: [master] + workflow_dispatch: + workflow_call: + +permissions: {} + +# Independent SHAs can build immediately on separate existing hosted runners. +# Repeated requests for the same source serialize without cancelling a build. +# No mutable canary channel is promoted here; docker.yml owns that operation. +concurrency: + group: docker-cloud-${{ github.sha }} + cancel-in-progress: false + +jobs: + build-and-push-cloud: + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + contents: read + packages: write + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + # Full history and tags so `git describe` below can compute the + # release version to stamp into the image. + fetch-depth: 0 + + # `.git` is dockerignored, so a running image cannot derive its own + # version and otherwise reports the source package.json placeholder in + # analytics and the debug panel. Compute it here from the pristine + # checkout (real CalVer drift from the nearest release tag) and pass it + # into the build. Empty when no release tag is reachable — the server + # then keeps its existing fallbacks. + - name: Compute build version + id: build-version + run: | + set -euo pipefail + case "${GITHUB_REF}" in + refs/tags/nightly/v*) + # Lane tags carry the exact published version; stamp it verbatim + # instead of describing drift from the nearest stable tag. + version="${GITHUB_REF#refs/tags/nightly/v}" + ;; + refs/tags/beta/v*) + version="${GITHUB_REF#refs/tags/beta/v}" + ;; + *) + version="$(git describe --tags --match 'v*' --long --dirty 2>/dev/null || true)" + ;; + esac + echo "version=${version}" >> "$GITHUB_OUTPUT" + echo "Stamping build version: ${version:-}" + + # ISO week stamp for the Dockerfile's tool layer: the layer caches + # across commits and re-pulls the @latest CLI tools when the week rolls + # over, instead of on every build. + - name: Compute tool cache epoch + id: tools-epoch + run: echo "epoch=$(date -u +%G-W%V)" >> "$GITHUB_OUTPUT" + + # Each SHA exports its own cache. Import recent first-parent caches so + # a late older build cannot overwrite a newer build's cache manifest. + # The legacy ref keeps the first builds warm during the transition. + - name: Select cloud cache ancestry + id: cloud-cache + env: + CACHE_IMAGE: ghcr.io/${{ github.repository }} + run: | + set -euo pipefail + { + echo 'sources<> "$GITHUB_OUTPUT" + + - name: Setup pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 + with: + version: 9.15.4 + run_install: false + + # No dependency cache here: this workflow publishes release images, and + # restoring a shared Actions cache into the build inputs would let a + # poisoned cache entry reach the published artifact. + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + + - name: Refresh lockfile for Docker build context + run: | + set -euo pipefail + pnpm install --resolution-only --ignore-scripts --no-frozen-lockfile + + changed="$(git status --porcelain)" + if [ -z "$changed" ]; then + echo "Lockfile already matches package metadata." + exit 0 + fi + + if printf '%s\n' "$changed" | grep -Fvq ' pnpm-lock.yaml'; then + echo "Unexpected files changed during lockfile refresh:" + echo "$changed" + exit 1 + fi + + echo "Using refreshed pnpm-lock.yaml in the Docker build context." + + - name: Free runner disk + run: | + set -euo pipefail + echo "Disk before cleanup:" + df -h + + pnpm store prune || true + sudo apt-get clean || true + sudo rm -rf \ + /usr/share/dotnet \ + /usr/share/swift \ + /usr/local/lib/android \ + /usr/local/share/boost \ + /usr/local/share/powershell \ + /opt/ghc \ + /opt/hostedtoolcache/CodeQL \ + /opt/hostedtoolcache/PyPy \ + /opt/hostedtoolcache/Ruby || true + docker system prune -af || true + + echo "Disk after cleanup:" + df -h + + - name: Login to GitHub Container Registry + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4 + + # Deployment tooling reads these labels from the registry to verify an + # image's schema expectations against a migrator before deploying it, + # without pulling the image. The server refuses to start when the + # database is missing bundled migrations, so orchestrators need a cheap + # way to check image/migrator compatibility up front. + - name: Compute schema migration labels + id: schema + run: | + set -euo pipefail + last=$(ls packages/db/src/migrations/*.sql | sed 's|.*/||' | LC_ALL=C sort | tail -1) + count=$(ls packages/db/src/migrations/*.sql | wc -l | tr -d ' ') + echo "last=${last}" >> "$GITHUB_OUTPUT" + echo "count=${count}" >> "$GITHUB_OUTPUT" + + # Published under the same lane tag set as the self-hosted image, with a + # `-cloud` suffix (nightly-cloud, latest-cloud, -cloud, + # sha--cloud). `:canary-cloud` follows the same retag-step + # ownership rule as `:canary` above. + - name: Docker meta (cloud) + id: meta-cloud + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6 + with: + images: ghcr.io/${{ github.repository }} + flavor: | + suffix=-cloud,onlatest=true + tags: | + type=raw,value=nightly,enable=${{ startsWith(github.ref, 'refs/tags/nightly/v') }} + type=raw,value=beta,enable=${{ startsWith(github.ref, 'refs/tags/beta/v') }} + type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }} + type=semver,pattern={{version}},enable=${{ startsWith(github.ref, 'refs/tags/v') }} + type=semver,pattern={{major}}.{{minor}},enable=${{ startsWith(github.ref, 'refs/tags/v') }} + type=sha + labels: | + io.github.paperclipai.schema.last-migration=${{ steps.schema.outputs.last }} + io.github.paperclipai.schema.migration-count=${{ steps.schema.outputs.count }} + + - name: Build and push (cloud) + id: build-cloud + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 + with: + context: . + target: cloud + # Space-separated sandbox-provider directory names to build into + # the variant; add here when managed deployments need another. + # CLOUD_BUNDLED_SERVER_DEPS names the optional peer packages the + # variant installs from server/package.json's declared version; + # add another name there when a managed tenant needs it. + build-args: | + CLOUD_BUNDLED_PLUGINS=daytona + CLOUD_BUNDLED_SERVER_DEPS=@sentry/node + PAPERCLIP_BUILD_VERSION=${{ steps.build-version.outputs.version }} + PAPERCLIP_BUILD_COMMIT=${{ github.sha }} + CLI_TOOLS_CACHE_EPOCH=${{ steps.tools-epoch.outputs.epoch }} + # amd64 only, unlike the self-hosted image above: the cloud variant + # is consumed exclusively by managed-deployment hosts, which run + # amd64. The QEMU-emulated arm64 half dominated this job's wall + # clock, and dropping it roughly halves time-to-deployable-image. + platforms: linux/amd64 + push: true + # Same-SHA builds serialize above; different SHAs never share a + # writable cache ref. Registry layers are content-addressed and + # shared even when cache manifests have separate tags. + cache-from: ${{ steps.cloud-cache.outputs.sources }} + cache-to: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-cloud-${{ github.sha }},mode=max + tags: ${{ steps.meta-cloud.outputs.tags }} + labels: ${{ steps.meta-cloud.outputs.labels }} + + # The cloud target installs @sentry/node at the version + # server/package.json declares, into a directory the server's own + # module resolution walks. Verify the image this job just pushed, not + # a local build, so a build-cache or layer-ordering regression is + # caught before any tenant runs the image. + + - name: Verify the pushed image resolves the declared Sentry version + env: + IMAGE: ghcr.io/${{ github.repository }}@${{ steps.build-cloud.outputs.digest }} + run: | + set -euo pipefail + + expected="$(node -e "process.stdout.write(require('./server/package.json').peerDependencies['@sentry/node'])")" + test -n "$expected" + + installed="$(docker run --rm --pull always \ + -v "$PWD/scripts/assert-cloud-image-sentry.mjs:/app/server/.ci-sentry-probe.mjs:ro" \ + --entrypoint node "$IMAGE" /app/server/.ci-sentry-probe.mjs)" + + echo "Declared optional peer version: $expected" + echo "Installed in the pushed image: $installed" + if [ "$installed" != "$expected" ]; then + echo "ERROR: the pushed image resolves @sentry/node@$installed, expected @sentry/node@$expected" >&2 + exit 1 + fi + echo "The pushed image resolves the declared @sentry/node version." + + # Verify the independently published cloud image without waiting for + # the self-hosted manifest job. The Sentry check already pulled it. + - name: Verify cloud PID 1 reaps orphaned processes + env: + IMAGE: ghcr.io/${{ github.repository }}@${{ steps.build-cloud.outputs.digest }} + run: docker run --rm -i "$IMAGE" sh -s < scripts/assert-orphan-reaping.sh + + # Cloud's commit resolver and preview-artifact planner use the full SHA. + # Publish that address only after checking this build's exact digest. + # Retagging reuses the registry manifest and does not rebuild the image. + - name: Publish verified full-SHA cloud tag + env: + IMAGE: ghcr.io/${{ github.repository }}@${{ steps.build-cloud.outputs.digest }} + FULL_SHA_TAG: ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-cloud + run: | + set -euo pipefail + revision="$(docker image inspect "$IMAGE" --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}')" + platform="$(docker image inspect "$IMAGE" --format '{{ .Os }}/{{ .Architecture }}')" + test "$revision" = "$GITHUB_SHA" + test "$platform" = linux/amd64 + docker buildx imagetools create --prefer-index=false --tag "$FULL_SHA_TAG" "$IMAGE" diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 3ac3871887..7f76a3894d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -349,8 +349,7 @@ jobs: # until the cgroup pid limit is exhausted and every fork() in the # container fails. Run against the pushed manifest rather than a local # build: the legs push by digest, so nothing is loaded into this - # runner's daemon. The cloud variant is FROM production and inherits the - # same ENTRYPOINT, so checking this image covers both. + # runner's daemon. The independent cloud workflow checks its own image. - name: Verify PID 1 reaps orphaned processes env: # Through the environment, not interpolated into the script body, so @@ -363,234 +362,15 @@ jobs: echo "Verifying orphan reaping in $image" docker run --rm -i --pull always "$image" sh -s < scripts/assert-orphan-reaping.sh - # The cloud variant carries built bundled plugins for managed deployments - # (see the `cloud` stage in the Dockerfile). It runs as its own job with no - # `needs:` on the stock publish above, so the two builds run in parallel and - # a failure or slow build in one never gates, delays, or skips the other. - # Both jobs share only the single top-level concurrency slot. Each job is a - # separate runner, so this one carries its own copy of the prep steps - # (checkout through schema labels) — the accepted cost of that isolation. + # Master cloud builds start independently in docker-cloud.yml. Tag builds + # and manual Docker dispatches call the same implementation, preserving the + # release tags and the canary promotion dependency below. build-and-push-cloud: - runs-on: ubuntu-latest - timeout-minutes: 60 + if: github.event_name != 'push' || github.ref != 'refs/heads/master' + uses: ./.github/workflows/docker-cloud.yml permissions: contents: read packages: write - steps: - - name: Checkout - uses: actions/checkout@v7 - with: - # Full history and tags so `git describe` below can compute the - # release version to stamp into the image. - fetch-depth: 0 - - # `.git` is dockerignored, so a running image cannot derive its own - # version and otherwise reports the source package.json placeholder in - # analytics and the debug panel. Compute it here from the pristine - # checkout (real CalVer drift from the nearest release tag) and pass it - # into the build. Empty when no release tag is reachable — the server - # then keeps its existing fallbacks. - - name: Compute build version - id: build-version - run: | - set -euo pipefail - case "${GITHUB_REF}" in - refs/tags/nightly/v*) - # Lane tags carry the exact published version; stamp it verbatim - # instead of describing drift from the nearest stable tag. - version="${GITHUB_REF#refs/tags/nightly/v}" - ;; - refs/tags/beta/v*) - version="${GITHUB_REF#refs/tags/beta/v}" - ;; - *) - version="$(git describe --tags --match 'v*' --long --dirty 2>/dev/null || true)" - ;; - esac - echo "version=${version}" >> "$GITHUB_OUTPUT" - echo "Stamping build version: ${version:-}" - - # ISO week stamp for the Dockerfile's tool layer: the layer caches - # across commits and re-pulls the @latest CLI tools when the week rolls - # over, instead of on every build. - - name: Compute tool cache epoch - id: tools-epoch - run: echo "epoch=$(date -u +%G-W%V)" >> "$GITHUB_OUTPUT" - - - name: Setup pnpm - uses: pnpm/action-setup@v6 - with: - version: 9.15.4 - run_install: false - - # No dependency cache here: this workflow publishes release images, and - # restoring a shared Actions cache into the build inputs would let a - # poisoned cache entry reach the published artifact. - - name: Setup Node.js - uses: actions/setup-node@v7 - with: - node-version: 24 - - - name: Refresh lockfile for Docker build context - run: | - set -euo pipefail - pnpm install --resolution-only --ignore-scripts --no-frozen-lockfile - - changed="$(git status --porcelain)" - if [ -z "$changed" ]; then - echo "Lockfile already matches package metadata." - exit 0 - fi - - if printf '%s\n' "$changed" | grep -Fvq ' pnpm-lock.yaml'; then - echo "Unexpected files changed during lockfile refresh:" - echo "$changed" - exit 1 - fi - - echo "Using refreshed pnpm-lock.yaml in the Docker build context." - - - name: Free runner disk - run: | - set -euo pipefail - echo "Disk before cleanup:" - df -h - - pnpm store prune || true - sudo apt-get clean || true - sudo rm -rf \ - /usr/share/dotnet \ - /usr/share/swift \ - /usr/local/lib/android \ - /usr/local/share/boost \ - /usr/local/share/powershell \ - /opt/ghc \ - /opt/hostedtoolcache/CodeQL \ - /opt/hostedtoolcache/PyPy \ - /opt/hostedtoolcache/Ruby || true - docker system prune -af || true - - echo "Disk after cleanup:" - df -h - - - name: Login to GitHub Container Registry - uses: docker/login-action@v4 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 - - # Deployment tooling reads these labels from the registry to verify an - # image's schema expectations against a migrator before deploying it, - # without pulling the image. The server refuses to start when the - # database is missing bundled migrations, so orchestrators need a cheap - # way to check image/migrator compatibility up front. - - name: Compute schema migration labels - id: schema - run: | - set -euo pipefail - last=$(ls packages/db/src/migrations/*.sql | sed 's|.*/||' | LC_ALL=C sort | tail -1) - count=$(ls packages/db/src/migrations/*.sql | wc -l | tr -d ' ') - echo "last=${last}" >> "$GITHUB_OUTPUT" - echo "count=${count}" >> "$GITHUB_OUTPUT" - - # Published under the same lane tag set as the self-hosted image, with a - # `-cloud` suffix (nightly-cloud, latest-cloud, -cloud, - # sha--cloud). `:canary-cloud` follows the same retag-step - # ownership rule as `:canary` above. - - name: Docker meta (cloud) - id: meta-cloud - uses: docker/metadata-action@v6 - with: - images: ghcr.io/${{ github.repository }} - flavor: | - suffix=-cloud,onlatest=true - tags: | - type=raw,value=nightly,enable=${{ startsWith(github.ref, 'refs/tags/nightly/v') }} - type=raw,value=beta,enable=${{ startsWith(github.ref, 'refs/tags/beta/v') }} - type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }} - type=semver,pattern={{version}},enable=${{ startsWith(github.ref, 'refs/tags/v') }} - type=semver,pattern={{major}}.{{minor}},enable=${{ startsWith(github.ref, 'refs/tags/v') }} - type=sha - labels: | - io.github.paperclipai.schema.last-migration=${{ steps.schema.outputs.last }} - io.github.paperclipai.schema.migration-count=${{ steps.schema.outputs.count }} - - - name: Build and push (cloud) - id: build-cloud - uses: docker/build-push-action@v7 - with: - context: . - target: cloud - # Space-separated sandbox-provider directory names to build into - # the variant; add here when managed deployments need another. - # CLOUD_BUNDLED_SERVER_DEPS names the optional peer packages the - # variant installs from server/package.json's declared version; - # add another name there when a managed tenant needs it. - build-args: | - CLOUD_BUNDLED_PLUGINS=daytona - CLOUD_BUNDLED_SERVER_DEPS=@sentry/node - PAPERCLIP_BUILD_VERSION=${{ steps.build-version.outputs.version }} - PAPERCLIP_BUILD_COMMIT=${{ github.sha }} - CLI_TOOLS_CACHE_EPOCH=${{ steps.tools-epoch.outputs.epoch }} - # amd64 only, unlike the self-hosted image above: the cloud variant - # is consumed exclusively by managed-deployment hosts, which run - # amd64. The QEMU-emulated arm64 half dominated this job's wall - # clock, and dropping it roughly halves time-to-deployable-image. - platforms: linux/amd64 - push: true - # Registry-backed BuildKit cache, separate ref from the self-hosted - # job so the two parallel builds never clobber each other's cache - # manifest (see the rationale on the job above). - cache-from: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-cloud - cache-to: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-cloud,mode=max - tags: ${{ steps.meta-cloud.outputs.tags }} - labels: ${{ steps.meta-cloud.outputs.labels }} - - # The cloud target installs @sentry/node at the version - # server/package.json declares, into a directory the server's own - # module resolution walks. Verify the image this job just pushed, not - # a local build, so a build-cache or layer-ordering regression is - # caught before any tenant runs the image. - - - name: Verify the pushed image resolves the declared Sentry version - env: - IMAGE: ghcr.io/${{ github.repository }}@${{ steps.build-cloud.outputs.digest }} - run: | - set -euo pipefail - - expected="$(node -e "process.stdout.write(require('./server/package.json').peerDependencies['@sentry/node'])")" - test -n "$expected" - - installed="$(docker run --rm --pull always \ - -v "$PWD/scripts/assert-cloud-image-sentry.mjs:/app/server/.ci-sentry-probe.mjs:ro" \ - --entrypoint node "$IMAGE" /app/server/.ci-sentry-probe.mjs)" - - echo "Declared optional peer version: $expected" - echo "Installed in the pushed image: $installed" - if [ "$installed" != "$expected" ]; then - echo "ERROR: the pushed image resolves @sentry/node@$installed, expected @sentry/node@$expected" >&2 - exit 1 - fi - echo "The pushed image resolves the declared @sentry/node version." - - # Cloud's commit resolver and preview-artifact planner use the full SHA. - # Publish that address only after checking this build's exact digest. - # Retagging reuses the registry manifest and does not rebuild the image. - - name: Publish verified full-SHA cloud tag - env: - IMAGE: ghcr.io/${{ github.repository }}@${{ steps.build-cloud.outputs.digest }} - FULL_SHA_TAG: ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-cloud - run: | - set -euo pipefail - revision="$(docker image inspect "$IMAGE" --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}')" - platform="$(docker image inspect "$IMAGE" --format '{{ .Os }}/{{ .Architecture }}')" - test "$revision" = "$GITHUB_SHA" - test "$platform" = linux/amd64 - docker buildx imagetools create --prefer-index=false --tag "$FULL_SHA_TAG" "$IMAGE" # Moves the mutable `:canary` / `:canary-cloud` channel tags. Kept OUT # of the build jobs and serialized in its own lane, and — the load- diff --git a/doc/DOCKER.md b/doc/DOCKER.md index e22aea05ed..f43d9a24b1 100644 --- a/doc/DOCKER.md +++ b/doc/DOCKER.md @@ -35,7 +35,19 @@ docker build -t paperclip-local \ ## Cloud image addresses The Docker workflow publishes the managed deployment image for Linux AMD64. -After the pushed image passes its Sentry check, the workflow verifies its +`Docker cloud` starts on each master push independently of the multi-platform +self-hosted build. Different commits use separate concurrency groups and existing +GitHub-hosted runners, so an older production or cloud build does not hold the +new commit in a workflow queue. Available GitHub runner capacity still applies. +Release tags and manual `Docker` dispatches call the same cloud build workflow. + +Each commit exports to its own `buildcache-cloud-` registry tag. +Builds import the current commit and nine first-parent ancestors, plus the +legacy `buildcache-cloud` fallback. This preserves reusable layers without +letting concurrent builds overwrite one shared cache manifest. Retain recent +cache tags if registry cleanup is configured; deleting them makes builds colder. + +After the pushed image passes its Sentry and orphan-reaping checks, the workflow verifies its commit label and platform and adds `ghcr.io/paperclipai/paperclip:sha--cloud`. This address lets commit-based deployment tooling reuse the normal build. Existing short-SHA and release tags remain available. diff --git a/scripts/preview-artifacts.test.mjs b/scripts/preview-artifacts.test.mjs index ad42cbd6bc..33041e3882 100644 --- a/scripts/preview-artifacts.test.mjs +++ b/scripts/preview-artifacts.test.mjs @@ -4,7 +4,7 @@ import { readFileSync, mkdtempSync, writeFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { gzipSync } from "node:zlib"; -import { spawnSync } from "node:child_process"; +import { execFileSync, spawnSync } from "node:child_process"; import { previewManifest, assertMetadata, validateRequest, versionFor, tarManifest, packageExists, imageExists, publishPreview, publishImage } from "./preview-artifacts.mjs"; const sha = "a".repeat(40); @@ -128,9 +128,58 @@ test("commits sharing a short prefix use separate full-SHA image addresses", asy assert.deepEqual(urls.filter((url) => url.includes("/manifests/")), [sha, other].map((commit) => `https://ghcr.io/v2/paperclipai/paperclip/manifests/sha-${commit}-cloud`)); }); +test("cloud builds start per commit and preserve tag promotion dependencies", () => { + const docker = readFileSync(new URL("../.github/workflows/docker.yml", import.meta.url), "utf8"); + const cloud = readFileSync(new URL("../.github/workflows/docker-cloud.yml", import.meta.url), "utf8"); + assert.match(cloud, /branches: \[master\]/); + assert.match(cloud, /workflow_call:/); + assert.match(cloud, /group: docker-cloud-\$\{\{ github.sha \}\}/); + assert.match(cloud, /cancel-in-progress: false/); + assert.doesNotMatch(cloud, /uses: .*@v\d\b/); + assert.match(cloud, /cache-to: type=registry,ref=ghcr.io\/\$\{\{ github.repository \}\}:buildcache-cloud-\$\{\{ github.sha \}\},mode=max/); + const caller = docker.split(" build-and-push-cloud:")[1].split(" promote_canary_channel:")[0]; + assert.match(caller, /if: github.event_name != 'push' \|\| github.ref != 'refs\/heads\/master'/); + assert.match(caller, /uses: .\/.github\/workflows\/docker-cloud.yml/); + assert.match(docker.split(" promote_canary_channel:")[1], /needs: \[merge-and-push, build-and-push-cloud\]/); + const reaping = cloud.indexOf(" - name: Verify cloud PID 1 reaps orphaned processes"); + assert.ok(reaping > cloud.indexOf(" - name: Verify the pushed image resolves the declared Sentry version")); + assert.ok(reaping < cloud.indexOf(" - name: Publish verified full-SHA cloud tag")); +}); + +test("cloud cache imports are bounded, follow master ancestry, and retain the legacy fallback", () => { + const workflow = readFileSync(new URL("../.github/workflows/docker-cloud.yml", import.meta.url), "utf8"); + const step = workflow.split(" - name: Select cloud cache ancestry")[1].split(" - name: Setup pnpm")[0]; + const script = step.split(" run: |\n")[1].split("\n").map((line) => line.replace(/^ {10}/, "")).join("\n"); + const dir = mkdtempSync(path.join(tmpdir(), "cloud-cache-test-")); + const output = path.join(dir, "output"); + const env = { ...process.env, GIT_AUTHOR_NAME: "Test", GIT_AUTHOR_EMAIL: "test@example.test", GIT_COMMITTER_NAME: "Test", GIT_COMMITTER_EMAIL: "test@example.test" }; + const git = (...args) => execFileSync("git", ["-c", "core.hooksPath=/dev/null", "-c", "commit.gpgsign=false", ...args], { cwd: dir, env, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim(); + try { + git("init", "--initial-branch=master"); + const commits = []; + for (let i = 0; i < 12; i++) { + git("commit", "--allow-empty", "-m", `main ${i}`); + commits.unshift(git("rev-parse", "HEAD")); + } + git("checkout", "-b", "topic", "HEAD~1"); + git("commit", "--allow-empty", "-m", "topic"); + git("checkout", "master"); + git("merge", "--no-ff", "topic", "-m", "merge topic"); + commits.unshift(git("rev-parse", "HEAD")); + const result = spawnSync("bash", ["-c", script], { cwd: dir, encoding: "utf8", env: { ...env, CACHE_IMAGE: "ghcr.io/paperclipai/paperclip", GITHUB_OUTPUT: output } }); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(readFileSync(output, "utf8").trim().split("\n"), [ + "sources< `type=registry,ref=ghcr.io/paperclipai/paperclip:buildcache-cloud-${commit}`), + "type=registry,ref=ghcr.io/paperclipai/paperclip:buildcache-cloud", + "CACHE_SOURCES", + ]); + } finally { rmSync(dir, { recursive: true, force: true }); } +}); + test("normal cloud builds publish the checked digest only when source and platform match", () => { - const workflow = readFileSync(new URL("../.github/workflows/docker.yml", import.meta.url), "utf8"); - const cloud = workflow.split(" build-and-push-cloud:")[1].split(" promote_canary_channel:")[0]; + const workflow = readFileSync(new URL("../.github/workflows/docker-cloud.yml", import.meta.url), "utf8"); + const cloud = workflow.split(" build-and-push-cloud:")[1]; const verify = cloud.indexOf(" - name: Verify the pushed image resolves the declared Sentry version"); const publish = cloud.indexOf(" - name: Publish verified full-SHA cloud tag"); assert.ok(verify >= 0 && publish > verify); diff --git a/server/src/__tests__/cloud-image-bundled-plugins.test.ts b/server/src/__tests__/cloud-image-bundled-plugins.test.ts index 464fd73308..0358db8f8b 100644 --- a/server/src/__tests__/cloud-image-bundled-plugins.test.ts +++ b/server/src/__tests__/cloud-image-bundled-plugins.test.ts @@ -20,6 +20,7 @@ import { BUNDLED_PLUGIN_CATALOG } from "../services/bundled-plugins.js"; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..", ".."); const dockerfile = readFileSync(path.join(repoRoot, "Dockerfile"), "utf8"); const workflow = readFileSync(path.join(repoRoot, ".github", "workflows", "docker.yml"), "utf8"); +const cloudWorkflow = readFileSync(path.join(repoRoot, ".github", "workflows", "docker-cloud.yml"), "utf8"); function parseList(source: string, pattern: RegExp, label: string): string[] { const match = source.match(pattern); @@ -35,7 +36,7 @@ const dockerfileDefault = parseList( "Dockerfile", ); const workflowArg = parseList( - workflow, + cloudWorkflow, /^\s*CLOUD_BUNDLED_PLUGINS=(.*)$/m, "docker workflow", ); @@ -77,16 +78,17 @@ describe("cloud image bundled plugins", () => { }); it("publishes the cloud image in its own job with no needs coupling", () => { - // The cloud publish runs as its own top-level job so the stock/production - // publish can never gate, delay, or skip it. Both jobs share only the - // single top-level concurrency slot; there is deliberately no `needs:` - // between them, so a failure in one is never coupled to the other. - const jobsSection = workflow.slice(workflow.indexOf("\njobs:\n")); + const caller = workflow.split(" build-and-push-cloud:")[1]?.split(" promote_canary_channel:")[0]; + expect(caller, "tag and manual builds must call the cloud workflow").toContain("uses: ./.github/workflows/docker-cloud.yml"); + expect(caller, "the reusable caller must also remain independent of production").not.toMatch(/^\s*needs:/m); + // The reusable cloud workflow owns its job and SHA concurrency group. + // Production publication must not gate, delay, or skip the cloud build. + const jobsSection = cloudWorkflow.slice(cloudWorkflow.indexOf("\njobs:\n")); const headers = [...jobsSection.matchAll(/^ {2}([\w-]+):[^\n]*$/gm)]; expect( headers.length, - "docker.yml must declare at least two jobs under jobs:", - ).toBeGreaterThanOrEqual(2); + "docker-cloud.yml must declare a cloud build job under jobs:", + ).toBeGreaterThanOrEqual(1); // Locate the job block that carries the cloud build (target: cloud) and // assert it declares no `needs:` — coupling it to another job would diff --git a/server/src/__tests__/cloud-image-sentry.test.ts b/server/src/__tests__/cloud-image-sentry.test.ts index 6eb4d14777..25a55e4baf 100644 --- a/server/src/__tests__/cloud-image-sentry.test.ts +++ b/server/src/__tests__/cloud-image-sentry.test.ts @@ -37,7 +37,7 @@ import { describe, expect, it } from "vitest"; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..", ".."); const dockerfile = readFileSync(path.join(repoRoot, "Dockerfile"), "utf8"); -const workflow = readFileSync(path.join(repoRoot, ".github", "workflows", "docker.yml"), "utf8"); +const workflow = readFileSync(path.join(repoRoot, ".github", "workflows", "docker-cloud.yml"), "utf8"); const serverPackageJson = JSON.parse( readFileSync(path.join(repoRoot, "server", "package.json"), "utf8"), ) as { peerDependencies?: Record }; diff --git a/server/src/__tests__/docker-build-stamp.test.ts b/server/src/__tests__/docker-build-stamp.test.ts index 454b8b912e..4794989feb 100644 --- a/server/src/__tests__/docker-build-stamp.test.ts +++ b/server/src/__tests__/docker-build-stamp.test.ts @@ -21,6 +21,7 @@ import { describe, expect, it } from "vitest"; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..", ".."); const dockerfile = readFileSync(path.join(repoRoot, "Dockerfile"), "utf8"); const workflow = readFileSync(path.join(repoRoot, ".github", "workflows", "docker.yml"), "utf8"); +const cloudWorkflow = readFileSync(path.join(repoRoot, ".github", "workflows", "docker-cloud.yml"), "utf8"); /** * Return the text of the Dockerfile stage that starts at the named target. @@ -68,7 +69,7 @@ describe("docker build-stamp wiring", () => { }); it("passes PAPERCLIP_BUILD_COMMIT as a build-arg for both image targets", () => { - const argLines = [...workflow.matchAll(/^\s*PAPERCLIP_BUILD_COMMIT=.*$/gm)]; + const argLines = [...`${workflow}\n${cloudWorkflow}`.matchAll(/^\s*PAPERCLIP_BUILD_COMMIT=.*$/gm)]; expect( argLines.length, "the docker workflow must pass PAPERCLIP_BUILD_COMMIT for the production and cloud builds",