ci: build cloud images independently for each merge (#13189)
Build cloud images independently for each master commit through a reusable workflow. Preserve production release dependencies and image runtime checks, and write cloud registry caches per commit with bounded ancestor imports to prevent overlapping builds from replacing each other's cache. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Devin Foley
GitHub
parent
59d74b68b2
commit
5c660a32f3
|
|
@ -6,6 +6,7 @@ const workflows = [
|
|||
'.github/workflows/refresh-lockfile.yml',
|
||||
'.github/workflows/pr-trusted.yml',
|
||||
'.github/workflows/docker.yml',
|
||||
'.github/workflows/docker-cloud.yml',
|
||||
];
|
||||
|
||||
test('lockfile repair workflows resolve dependencies instead of updating metadata only', async () => {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,264 @@
|
|||
name: Docker cloud
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [master]
|
||||
workflow_dispatch:
|
||||
workflow_call:
|
||||
|
||||
permissions: {}
|
||||
|
||||
# Independent SHAs can build immediately on separate existing hosted runners.
|
||||
# Repeated requests for the same source serialize without cancelling a build.
|
||||
# No mutable canary channel is promoted here; docker.yml owns that operation.
|
||||
concurrency:
|
||||
group: docker-cloud-${{ github.sha }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
build-and-push-cloud:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
# Full history and tags so `git describe` below can compute the
|
||||
# release version to stamp into the image.
|
||||
fetch-depth: 0
|
||||
|
||||
# `.git` is dockerignored, so a running image cannot derive its own
|
||||
# version and otherwise reports the source package.json placeholder in
|
||||
# analytics and the debug panel. Compute it here from the pristine
|
||||
# checkout (real CalVer drift from the nearest release tag) and pass it
|
||||
# into the build. Empty when no release tag is reachable — the server
|
||||
# then keeps its existing fallbacks.
|
||||
- name: Compute build version
|
||||
id: build-version
|
||||
run: |
|
||||
set -euo pipefail
|
||||
case "${GITHUB_REF}" in
|
||||
refs/tags/nightly/v*)
|
||||
# Lane tags carry the exact published version; stamp it verbatim
|
||||
# instead of describing drift from the nearest stable tag.
|
||||
version="${GITHUB_REF#refs/tags/nightly/v}"
|
||||
;;
|
||||
refs/tags/beta/v*)
|
||||
version="${GITHUB_REF#refs/tags/beta/v}"
|
||||
;;
|
||||
*)
|
||||
version="$(git describe --tags --match 'v*' --long --dirty 2>/dev/null || true)"
|
||||
;;
|
||||
esac
|
||||
echo "version=${version}" >> "$GITHUB_OUTPUT"
|
||||
echo "Stamping build version: ${version:-<none>}"
|
||||
|
||||
# ISO week stamp for the Dockerfile's tool layer: the layer caches
|
||||
# across commits and re-pulls the @latest CLI tools when the week rolls
|
||||
# over, instead of on every build.
|
||||
- name: Compute tool cache epoch
|
||||
id: tools-epoch
|
||||
run: echo "epoch=$(date -u +%G-W%V)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Each SHA exports its own cache. Import recent first-parent caches so
|
||||
# a late older build cannot overwrite a newer build's cache manifest.
|
||||
# The legacy ref keeps the first builds warm during the transition.
|
||||
- name: Select cloud cache ancestry
|
||||
id: cloud-cache
|
||||
env:
|
||||
CACHE_IMAGE: ghcr.io/${{ github.repository }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
{
|
||||
echo 'sources<<CACHE_SOURCES'
|
||||
for commit in $(git rev-list --first-parent --max-count=10 HEAD); do
|
||||
echo "type=registry,ref=$CACHE_IMAGE:buildcache-cloud-$commit"
|
||||
done
|
||||
echo "type=registry,ref=$CACHE_IMAGE:buildcache-cloud"
|
||||
echo 'CACHE_SOURCES'
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
|
||||
with:
|
||||
version: 9.15.4
|
||||
run_install: false
|
||||
|
||||
# No dependency cache here: this workflow publishes release images, and
|
||||
# restoring a shared Actions cache into the build inputs would let a
|
||||
# poisoned cache entry reach the published artifact.
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: 24
|
||||
|
||||
- name: Refresh lockfile for Docker build context
|
||||
run: |
|
||||
set -euo pipefail
|
||||
pnpm install --resolution-only --ignore-scripts --no-frozen-lockfile
|
||||
|
||||
changed="$(git status --porcelain)"
|
||||
if [ -z "$changed" ]; then
|
||||
echo "Lockfile already matches package metadata."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if printf '%s\n' "$changed" | grep -Fvq ' pnpm-lock.yaml'; then
|
||||
echo "Unexpected files changed during lockfile refresh:"
|
||||
echo "$changed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Using refreshed pnpm-lock.yaml in the Docker build context."
|
||||
|
||||
- name: Free runner disk
|
||||
run: |
|
||||
set -euo pipefail
|
||||
echo "Disk before cleanup:"
|
||||
df -h
|
||||
|
||||
pnpm store prune || true
|
||||
sudo apt-get clean || true
|
||||
sudo rm -rf \
|
||||
/usr/share/dotnet \
|
||||
/usr/share/swift \
|
||||
/usr/local/lib/android \
|
||||
/usr/local/share/boost \
|
||||
/usr/local/share/powershell \
|
||||
/opt/ghc \
|
||||
/opt/hostedtoolcache/CodeQL \
|
||||
/opt/hostedtoolcache/PyPy \
|
||||
/opt/hostedtoolcache/Ruby || true
|
||||
docker system prune -af || true
|
||||
|
||||
echo "Disk after cleanup:"
|
||||
df -h
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
|
||||
|
||||
# Deployment tooling reads these labels from the registry to verify an
|
||||
# image's schema expectations against a migrator before deploying it,
|
||||
# without pulling the image. The server refuses to start when the
|
||||
# database is missing bundled migrations, so orchestrators need a cheap
|
||||
# way to check image/migrator compatibility up front.
|
||||
- name: Compute schema migration labels
|
||||
id: schema
|
||||
run: |
|
||||
set -euo pipefail
|
||||
last=$(ls packages/db/src/migrations/*.sql | sed 's|.*/||' | LC_ALL=C sort | tail -1)
|
||||
count=$(ls packages/db/src/migrations/*.sql | wc -l | tr -d ' ')
|
||||
echo "last=${last}" >> "$GITHUB_OUTPUT"
|
||||
echo "count=${count}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Published under the same lane tag set as the self-hosted image, with a
|
||||
# `-cloud` suffix (nightly-cloud, latest-cloud, <version>-cloud,
|
||||
# sha-<short>-cloud). `:canary-cloud` follows the same retag-step
|
||||
# ownership rule as `:canary` above.
|
||||
- name: Docker meta (cloud)
|
||||
id: meta-cloud
|
||||
uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6
|
||||
with:
|
||||
images: ghcr.io/${{ github.repository }}
|
||||
flavor: |
|
||||
suffix=-cloud,onlatest=true
|
||||
tags: |
|
||||
type=raw,value=nightly,enable=${{ startsWith(github.ref, 'refs/tags/nightly/v') }}
|
||||
type=raw,value=beta,enable=${{ startsWith(github.ref, 'refs/tags/beta/v') }}
|
||||
type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
|
||||
type=semver,pattern={{version}},enable=${{ startsWith(github.ref, 'refs/tags/v') }}
|
||||
type=semver,pattern={{major}}.{{minor}},enable=${{ startsWith(github.ref, 'refs/tags/v') }}
|
||||
type=sha
|
||||
labels: |
|
||||
io.github.paperclipai.schema.last-migration=${{ steps.schema.outputs.last }}
|
||||
io.github.paperclipai.schema.migration-count=${{ steps.schema.outputs.count }}
|
||||
|
||||
- name: Build and push (cloud)
|
||||
id: build-cloud
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7
|
||||
with:
|
||||
context: .
|
||||
target: cloud
|
||||
# Space-separated sandbox-provider directory names to build into
|
||||
# the variant; add here when managed deployments need another.
|
||||
# CLOUD_BUNDLED_SERVER_DEPS names the optional peer packages the
|
||||
# variant installs from server/package.json's declared version;
|
||||
# add another name there when a managed tenant needs it.
|
||||
build-args: |
|
||||
CLOUD_BUNDLED_PLUGINS=daytona
|
||||
CLOUD_BUNDLED_SERVER_DEPS=@sentry/node
|
||||
PAPERCLIP_BUILD_VERSION=${{ steps.build-version.outputs.version }}
|
||||
PAPERCLIP_BUILD_COMMIT=${{ github.sha }}
|
||||
CLI_TOOLS_CACHE_EPOCH=${{ steps.tools-epoch.outputs.epoch }}
|
||||
# amd64 only, unlike the self-hosted image above: the cloud variant
|
||||
# is consumed exclusively by managed-deployment hosts, which run
|
||||
# amd64. The QEMU-emulated arm64 half dominated this job's wall
|
||||
# clock, and dropping it roughly halves time-to-deployable-image.
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
# Same-SHA builds serialize above; different SHAs never share a
|
||||
# writable cache ref. Registry layers are content-addressed and
|
||||
# shared even when cache manifests have separate tags.
|
||||
cache-from: ${{ steps.cloud-cache.outputs.sources }}
|
||||
cache-to: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-cloud-${{ github.sha }},mode=max
|
||||
tags: ${{ steps.meta-cloud.outputs.tags }}
|
||||
labels: ${{ steps.meta-cloud.outputs.labels }}
|
||||
|
||||
# The cloud target installs @sentry/node at the version
|
||||
# server/package.json declares, into a directory the server's own
|
||||
# module resolution walks. Verify the image this job just pushed, not
|
||||
# a local build, so a build-cache or layer-ordering regression is
|
||||
# caught before any tenant runs the image.
|
||||
|
||||
- name: Verify the pushed image resolves the declared Sentry version
|
||||
env:
|
||||
IMAGE: ghcr.io/${{ github.repository }}@${{ steps.build-cloud.outputs.digest }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
expected="$(node -e "process.stdout.write(require('./server/package.json').peerDependencies['@sentry/node'])")"
|
||||
test -n "$expected"
|
||||
|
||||
installed="$(docker run --rm --pull always \
|
||||
-v "$PWD/scripts/assert-cloud-image-sentry.mjs:/app/server/.ci-sentry-probe.mjs:ro" \
|
||||
--entrypoint node "$IMAGE" /app/server/.ci-sentry-probe.mjs)"
|
||||
|
||||
echo "Declared optional peer version: $expected"
|
||||
echo "Installed in the pushed image: $installed"
|
||||
if [ "$installed" != "$expected" ]; then
|
||||
echo "ERROR: the pushed image resolves @sentry/node@$installed, expected @sentry/node@$expected" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "The pushed image resolves the declared @sentry/node version."
|
||||
|
||||
# Verify the independently published cloud image without waiting for
|
||||
# the self-hosted manifest job. The Sentry check already pulled it.
|
||||
- name: Verify cloud PID 1 reaps orphaned processes
|
||||
env:
|
||||
IMAGE: ghcr.io/${{ github.repository }}@${{ steps.build-cloud.outputs.digest }}
|
||||
run: docker run --rm -i "$IMAGE" sh -s < scripts/assert-orphan-reaping.sh
|
||||
|
||||
# Cloud's commit resolver and preview-artifact planner use the full SHA.
|
||||
# Publish that address only after checking this build's exact digest.
|
||||
# Retagging reuses the registry manifest and does not rebuild the image.
|
||||
- name: Publish verified full-SHA cloud tag
|
||||
env:
|
||||
IMAGE: ghcr.io/${{ github.repository }}@${{ steps.build-cloud.outputs.digest }}
|
||||
FULL_SHA_TAG: ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-cloud
|
||||
run: |
|
||||
set -euo pipefail
|
||||
revision="$(docker image inspect "$IMAGE" --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}')"
|
||||
platform="$(docker image inspect "$IMAGE" --format '{{ .Os }}/{{ .Architecture }}')"
|
||||
test "$revision" = "$GITHUB_SHA"
|
||||
test "$platform" = linux/amd64
|
||||
docker buildx imagetools create --prefer-index=false --tag "$FULL_SHA_TAG" "$IMAGE"
|
||||
|
|
@ -349,8 +349,7 @@ jobs:
|
|||
# until the cgroup pid limit is exhausted and every fork() in the
|
||||
# container fails. Run against the pushed manifest rather than a local
|
||||
# build: the legs push by digest, so nothing is loaded into this
|
||||
# runner's daemon. The cloud variant is FROM production and inherits the
|
||||
# same ENTRYPOINT, so checking this image covers both.
|
||||
# runner's daemon. The independent cloud workflow checks its own image.
|
||||
- name: Verify PID 1 reaps orphaned processes
|
||||
env:
|
||||
# Through the environment, not interpolated into the script body, so
|
||||
|
|
@ -363,234 +362,15 @@ jobs:
|
|||
echo "Verifying orphan reaping in $image"
|
||||
docker run --rm -i --pull always "$image" sh -s < scripts/assert-orphan-reaping.sh
|
||||
|
||||
# The cloud variant carries built bundled plugins for managed deployments
|
||||
# (see the `cloud` stage in the Dockerfile). It runs as its own job with no
|
||||
# `needs:` on the stock publish above, so the two builds run in parallel and
|
||||
# a failure or slow build in one never gates, delays, or skips the other.
|
||||
# Both jobs share only the single top-level concurrency slot. Each job is a
|
||||
# separate runner, so this one carries its own copy of the prep steps
|
||||
# (checkout through schema labels) — the accepted cost of that isolation.
|
||||
# Master cloud builds start independently in docker-cloud.yml. Tag builds
|
||||
# and manual Docker dispatches call the same implementation, preserving the
|
||||
# release tags and the canary promotion dependency below.
|
||||
build-and-push-cloud:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
if: github.event_name != 'push' || github.ref != 'refs/heads/master'
|
||||
uses: ./.github/workflows/docker-cloud.yml
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
# Full history and tags so `git describe` below can compute the
|
||||
# release version to stamp into the image.
|
||||
fetch-depth: 0
|
||||
|
||||
# `.git` is dockerignored, so a running image cannot derive its own
|
||||
# version and otherwise reports the source package.json placeholder in
|
||||
# analytics and the debug panel. Compute it here from the pristine
|
||||
# checkout (real CalVer drift from the nearest release tag) and pass it
|
||||
# into the build. Empty when no release tag is reachable — the server
|
||||
# then keeps its existing fallbacks.
|
||||
- name: Compute build version
|
||||
id: build-version
|
||||
run: |
|
||||
set -euo pipefail
|
||||
case "${GITHUB_REF}" in
|
||||
refs/tags/nightly/v*)
|
||||
# Lane tags carry the exact published version; stamp it verbatim
|
||||
# instead of describing drift from the nearest stable tag.
|
||||
version="${GITHUB_REF#refs/tags/nightly/v}"
|
||||
;;
|
||||
refs/tags/beta/v*)
|
||||
version="${GITHUB_REF#refs/tags/beta/v}"
|
||||
;;
|
||||
*)
|
||||
version="$(git describe --tags --match 'v*' --long --dirty 2>/dev/null || true)"
|
||||
;;
|
||||
esac
|
||||
echo "version=${version}" >> "$GITHUB_OUTPUT"
|
||||
echo "Stamping build version: ${version:-<none>}"
|
||||
|
||||
# ISO week stamp for the Dockerfile's tool layer: the layer caches
|
||||
# across commits and re-pulls the @latest CLI tools when the week rolls
|
||||
# over, instead of on every build.
|
||||
- name: Compute tool cache epoch
|
||||
id: tools-epoch
|
||||
run: echo "epoch=$(date -u +%G-W%V)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v6
|
||||
with:
|
||||
version: 9.15.4
|
||||
run_install: false
|
||||
|
||||
# No dependency cache here: this workflow publishes release images, and
|
||||
# restoring a shared Actions cache into the build inputs would let a
|
||||
# poisoned cache entry reach the published artifact.
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: 24
|
||||
|
||||
- name: Refresh lockfile for Docker build context
|
||||
run: |
|
||||
set -euo pipefail
|
||||
pnpm install --resolution-only --ignore-scripts --no-frozen-lockfile
|
||||
|
||||
changed="$(git status --porcelain)"
|
||||
if [ -z "$changed" ]; then
|
||||
echo "Lockfile already matches package metadata."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if printf '%s\n' "$changed" | grep -Fvq ' pnpm-lock.yaml'; then
|
||||
echo "Unexpected files changed during lockfile refresh:"
|
||||
echo "$changed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Using refreshed pnpm-lock.yaml in the Docker build context."
|
||||
|
||||
- name: Free runner disk
|
||||
run: |
|
||||
set -euo pipefail
|
||||
echo "Disk before cleanup:"
|
||||
df -h
|
||||
|
||||
pnpm store prune || true
|
||||
sudo apt-get clean || true
|
||||
sudo rm -rf \
|
||||
/usr/share/dotnet \
|
||||
/usr/share/swift \
|
||||
/usr/local/lib/android \
|
||||
/usr/local/share/boost \
|
||||
/usr/local/share/powershell \
|
||||
/opt/ghc \
|
||||
/opt/hostedtoolcache/CodeQL \
|
||||
/opt/hostedtoolcache/PyPy \
|
||||
/opt/hostedtoolcache/Ruby || true
|
||||
docker system prune -af || true
|
||||
|
||||
echo "Disk after cleanup:"
|
||||
df -h
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
# Deployment tooling reads these labels from the registry to verify an
|
||||
# image's schema expectations against a migrator before deploying it,
|
||||
# without pulling the image. The server refuses to start when the
|
||||
# database is missing bundled migrations, so orchestrators need a cheap
|
||||
# way to check image/migrator compatibility up front.
|
||||
- name: Compute schema migration labels
|
||||
id: schema
|
||||
run: |
|
||||
set -euo pipefail
|
||||
last=$(ls packages/db/src/migrations/*.sql | sed 's|.*/||' | LC_ALL=C sort | tail -1)
|
||||
count=$(ls packages/db/src/migrations/*.sql | wc -l | tr -d ' ')
|
||||
echo "last=${last}" >> "$GITHUB_OUTPUT"
|
||||
echo "count=${count}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Published under the same lane tag set as the self-hosted image, with a
|
||||
# `-cloud` suffix (nightly-cloud, latest-cloud, <version>-cloud,
|
||||
# sha-<short>-cloud). `:canary-cloud` follows the same retag-step
|
||||
# ownership rule as `:canary` above.
|
||||
- name: Docker meta (cloud)
|
||||
id: meta-cloud
|
||||
uses: docker/metadata-action@v6
|
||||
with:
|
||||
images: ghcr.io/${{ github.repository }}
|
||||
flavor: |
|
||||
suffix=-cloud,onlatest=true
|
||||
tags: |
|
||||
type=raw,value=nightly,enable=${{ startsWith(github.ref, 'refs/tags/nightly/v') }}
|
||||
type=raw,value=beta,enable=${{ startsWith(github.ref, 'refs/tags/beta/v') }}
|
||||
type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
|
||||
type=semver,pattern={{version}},enable=${{ startsWith(github.ref, 'refs/tags/v') }}
|
||||
type=semver,pattern={{major}}.{{minor}},enable=${{ startsWith(github.ref, 'refs/tags/v') }}
|
||||
type=sha
|
||||
labels: |
|
||||
io.github.paperclipai.schema.last-migration=${{ steps.schema.outputs.last }}
|
||||
io.github.paperclipai.schema.migration-count=${{ steps.schema.outputs.count }}
|
||||
|
||||
- name: Build and push (cloud)
|
||||
id: build-cloud
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: .
|
||||
target: cloud
|
||||
# Space-separated sandbox-provider directory names to build into
|
||||
# the variant; add here when managed deployments need another.
|
||||
# CLOUD_BUNDLED_SERVER_DEPS names the optional peer packages the
|
||||
# variant installs from server/package.json's declared version;
|
||||
# add another name there when a managed tenant needs it.
|
||||
build-args: |
|
||||
CLOUD_BUNDLED_PLUGINS=daytona
|
||||
CLOUD_BUNDLED_SERVER_DEPS=@sentry/node
|
||||
PAPERCLIP_BUILD_VERSION=${{ steps.build-version.outputs.version }}
|
||||
PAPERCLIP_BUILD_COMMIT=${{ github.sha }}
|
||||
CLI_TOOLS_CACHE_EPOCH=${{ steps.tools-epoch.outputs.epoch }}
|
||||
# amd64 only, unlike the self-hosted image above: the cloud variant
|
||||
# is consumed exclusively by managed-deployment hosts, which run
|
||||
# amd64. The QEMU-emulated arm64 half dominated this job's wall
|
||||
# clock, and dropping it roughly halves time-to-deployable-image.
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
# Registry-backed BuildKit cache, separate ref from the self-hosted
|
||||
# job so the two parallel builds never clobber each other's cache
|
||||
# manifest (see the rationale on the job above).
|
||||
cache-from: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-cloud
|
||||
cache-to: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-cloud,mode=max
|
||||
tags: ${{ steps.meta-cloud.outputs.tags }}
|
||||
labels: ${{ steps.meta-cloud.outputs.labels }}
|
||||
|
||||
# The cloud target installs @sentry/node at the version
|
||||
# server/package.json declares, into a directory the server's own
|
||||
# module resolution walks. Verify the image this job just pushed, not
|
||||
# a local build, so a build-cache or layer-ordering regression is
|
||||
# caught before any tenant runs the image.
|
||||
|
||||
- name: Verify the pushed image resolves the declared Sentry version
|
||||
env:
|
||||
IMAGE: ghcr.io/${{ github.repository }}@${{ steps.build-cloud.outputs.digest }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
expected="$(node -e "process.stdout.write(require('./server/package.json').peerDependencies['@sentry/node'])")"
|
||||
test -n "$expected"
|
||||
|
||||
installed="$(docker run --rm --pull always \
|
||||
-v "$PWD/scripts/assert-cloud-image-sentry.mjs:/app/server/.ci-sentry-probe.mjs:ro" \
|
||||
--entrypoint node "$IMAGE" /app/server/.ci-sentry-probe.mjs)"
|
||||
|
||||
echo "Declared optional peer version: $expected"
|
||||
echo "Installed in the pushed image: $installed"
|
||||
if [ "$installed" != "$expected" ]; then
|
||||
echo "ERROR: the pushed image resolves @sentry/node@$installed, expected @sentry/node@$expected" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "The pushed image resolves the declared @sentry/node version."
|
||||
|
||||
# Cloud's commit resolver and preview-artifact planner use the full SHA.
|
||||
# Publish that address only after checking this build's exact digest.
|
||||
# Retagging reuses the registry manifest and does not rebuild the image.
|
||||
- name: Publish verified full-SHA cloud tag
|
||||
env:
|
||||
IMAGE: ghcr.io/${{ github.repository }}@${{ steps.build-cloud.outputs.digest }}
|
||||
FULL_SHA_TAG: ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-cloud
|
||||
run: |
|
||||
set -euo pipefail
|
||||
revision="$(docker image inspect "$IMAGE" --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}')"
|
||||
platform="$(docker image inspect "$IMAGE" --format '{{ .Os }}/{{ .Architecture }}')"
|
||||
test "$revision" = "$GITHUB_SHA"
|
||||
test "$platform" = linux/amd64
|
||||
docker buildx imagetools create --prefer-index=false --tag "$FULL_SHA_TAG" "$IMAGE"
|
||||
|
||||
# Moves the mutable `:canary` / `:canary-cloud` channel tags. Kept OUT
|
||||
# of the build jobs and serialized in its own lane, and — the load-
|
||||
|
|
|
|||
|
|
@ -35,7 +35,19 @@ docker build -t paperclip-local \
|
|||
## Cloud image addresses
|
||||
|
||||
The Docker workflow publishes the managed deployment image for Linux AMD64.
|
||||
After the pushed image passes its Sentry check, the workflow verifies its
|
||||
`Docker cloud` starts on each master push independently of the multi-platform
|
||||
self-hosted build. Different commits use separate concurrency groups and existing
|
||||
GitHub-hosted runners, so an older production or cloud build does not hold the
|
||||
new commit in a workflow queue. Available GitHub runner capacity still applies.
|
||||
Release tags and manual `Docker` dispatches call the same cloud build workflow.
|
||||
|
||||
Each commit exports to its own `buildcache-cloud-<FULL_SHA>` registry tag.
|
||||
Builds import the current commit and nine first-parent ancestors, plus the
|
||||
legacy `buildcache-cloud` fallback. This preserves reusable layers without
|
||||
letting concurrent builds overwrite one shared cache manifest. Retain recent
|
||||
cache tags if registry cleanup is configured; deleting them makes builds colder.
|
||||
|
||||
After the pushed image passes its Sentry and orphan-reaping checks, the workflow verifies its
|
||||
commit label and platform and adds `ghcr.io/paperclipai/paperclip:sha-<full-commit-sha>-cloud`.
|
||||
This address lets commit-based deployment tooling reuse the normal build.
|
||||
Existing short-SHA and release tags remain available.
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import { readFileSync, mkdtempSync, writeFileSync, rmSync } from "node:fs";
|
|||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { gzipSync } from "node:zlib";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { execFileSync, spawnSync } from "node:child_process";
|
||||
import { previewManifest, assertMetadata, validateRequest, versionFor, tarManifest, packageExists, imageExists, publishPreview, publishImage } from "./preview-artifacts.mjs";
|
||||
|
||||
const sha = "a".repeat(40);
|
||||
|
|
@ -128,9 +128,58 @@ test("commits sharing a short prefix use separate full-SHA image addresses", asy
|
|||
assert.deepEqual(urls.filter((url) => url.includes("/manifests/")), [sha, other].map((commit) => `https://ghcr.io/v2/paperclipai/paperclip/manifests/sha-${commit}-cloud`));
|
||||
});
|
||||
|
||||
test("cloud builds start per commit and preserve tag promotion dependencies", () => {
|
||||
const docker = readFileSync(new URL("../.github/workflows/docker.yml", import.meta.url), "utf8");
|
||||
const cloud = readFileSync(new URL("../.github/workflows/docker-cloud.yml", import.meta.url), "utf8");
|
||||
assert.match(cloud, /branches: \[master\]/);
|
||||
assert.match(cloud, /workflow_call:/);
|
||||
assert.match(cloud, /group: docker-cloud-\$\{\{ github.sha \}\}/);
|
||||
assert.match(cloud, /cancel-in-progress: false/);
|
||||
assert.doesNotMatch(cloud, /uses: .*@v\d\b/);
|
||||
assert.match(cloud, /cache-to: type=registry,ref=ghcr.io\/\$\{\{ github.repository \}\}:buildcache-cloud-\$\{\{ github.sha \}\},mode=max/);
|
||||
const caller = docker.split(" build-and-push-cloud:")[1].split(" promote_canary_channel:")[0];
|
||||
assert.match(caller, /if: github.event_name != 'push' \|\| github.ref != 'refs\/heads\/master'/);
|
||||
assert.match(caller, /uses: .\/.github\/workflows\/docker-cloud.yml/);
|
||||
assert.match(docker.split(" promote_canary_channel:")[1], /needs: \[merge-and-push, build-and-push-cloud\]/);
|
||||
const reaping = cloud.indexOf(" - name: Verify cloud PID 1 reaps orphaned processes");
|
||||
assert.ok(reaping > cloud.indexOf(" - name: Verify the pushed image resolves the declared Sentry version"));
|
||||
assert.ok(reaping < cloud.indexOf(" - name: Publish verified full-SHA cloud tag"));
|
||||
});
|
||||
|
||||
test("cloud cache imports are bounded, follow master ancestry, and retain the legacy fallback", () => {
|
||||
const workflow = readFileSync(new URL("../.github/workflows/docker-cloud.yml", import.meta.url), "utf8");
|
||||
const step = workflow.split(" - name: Select cloud cache ancestry")[1].split(" - name: Setup pnpm")[0];
|
||||
const script = step.split(" run: |\n")[1].split("\n").map((line) => line.replace(/^ {10}/, "")).join("\n");
|
||||
const dir = mkdtempSync(path.join(tmpdir(), "cloud-cache-test-"));
|
||||
const output = path.join(dir, "output");
|
||||
const env = { ...process.env, GIT_AUTHOR_NAME: "Test", GIT_AUTHOR_EMAIL: "test@example.test", GIT_COMMITTER_NAME: "Test", GIT_COMMITTER_EMAIL: "test@example.test" };
|
||||
const git = (...args) => execFileSync("git", ["-c", "core.hooksPath=/dev/null", "-c", "commit.gpgsign=false", ...args], { cwd: dir, env, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim();
|
||||
try {
|
||||
git("init", "--initial-branch=master");
|
||||
const commits = [];
|
||||
for (let i = 0; i < 12; i++) {
|
||||
git("commit", "--allow-empty", "-m", `main ${i}`);
|
||||
commits.unshift(git("rev-parse", "HEAD"));
|
||||
}
|
||||
git("checkout", "-b", "topic", "HEAD~1");
|
||||
git("commit", "--allow-empty", "-m", "topic");
|
||||
git("checkout", "master");
|
||||
git("merge", "--no-ff", "topic", "-m", "merge topic");
|
||||
commits.unshift(git("rev-parse", "HEAD"));
|
||||
const result = spawnSync("bash", ["-c", script], { cwd: dir, encoding: "utf8", env: { ...env, CACHE_IMAGE: "ghcr.io/paperclipai/paperclip", GITHUB_OUTPUT: output } });
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
assert.deepEqual(readFileSync(output, "utf8").trim().split("\n"), [
|
||||
"sources<<CACHE_SOURCES",
|
||||
...commits.slice(0, 10).map((commit) => `type=registry,ref=ghcr.io/paperclipai/paperclip:buildcache-cloud-${commit}`),
|
||||
"type=registry,ref=ghcr.io/paperclipai/paperclip:buildcache-cloud",
|
||||
"CACHE_SOURCES",
|
||||
]);
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test("normal cloud builds publish the checked digest only when source and platform match", () => {
|
||||
const workflow = readFileSync(new URL("../.github/workflows/docker.yml", import.meta.url), "utf8");
|
||||
const cloud = workflow.split(" build-and-push-cloud:")[1].split(" promote_canary_channel:")[0];
|
||||
const workflow = readFileSync(new URL("../.github/workflows/docker-cloud.yml", import.meta.url), "utf8");
|
||||
const cloud = workflow.split(" build-and-push-cloud:")[1];
|
||||
const verify = cloud.indexOf(" - name: Verify the pushed image resolves the declared Sentry version");
|
||||
const publish = cloud.indexOf(" - name: Publish verified full-SHA cloud tag");
|
||||
assert.ok(verify >= 0 && publish > verify);
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ import { BUNDLED_PLUGIN_CATALOG } from "../services/bundled-plugins.js";
|
|||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..", "..");
|
||||
const dockerfile = readFileSync(path.join(repoRoot, "Dockerfile"), "utf8");
|
||||
const workflow = readFileSync(path.join(repoRoot, ".github", "workflows", "docker.yml"), "utf8");
|
||||
const cloudWorkflow = readFileSync(path.join(repoRoot, ".github", "workflows", "docker-cloud.yml"), "utf8");
|
||||
|
||||
function parseList(source: string, pattern: RegExp, label: string): string[] {
|
||||
const match = source.match(pattern);
|
||||
|
|
@ -35,7 +36,7 @@ const dockerfileDefault = parseList(
|
|||
"Dockerfile",
|
||||
);
|
||||
const workflowArg = parseList(
|
||||
workflow,
|
||||
cloudWorkflow,
|
||||
/^\s*CLOUD_BUNDLED_PLUGINS=(.*)$/m,
|
||||
"docker workflow",
|
||||
);
|
||||
|
|
@ -77,16 +78,17 @@ describe("cloud image bundled plugins", () => {
|
|||
});
|
||||
|
||||
it("publishes the cloud image in its own job with no needs coupling", () => {
|
||||
// The cloud publish runs as its own top-level job so the stock/production
|
||||
// publish can never gate, delay, or skip it. Both jobs share only the
|
||||
// single top-level concurrency slot; there is deliberately no `needs:`
|
||||
// between them, so a failure in one is never coupled to the other.
|
||||
const jobsSection = workflow.slice(workflow.indexOf("\njobs:\n"));
|
||||
const caller = workflow.split(" build-and-push-cloud:")[1]?.split(" promote_canary_channel:")[0];
|
||||
expect(caller, "tag and manual builds must call the cloud workflow").toContain("uses: ./.github/workflows/docker-cloud.yml");
|
||||
expect(caller, "the reusable caller must also remain independent of production").not.toMatch(/^\s*needs:/m);
|
||||
// The reusable cloud workflow owns its job and SHA concurrency group.
|
||||
// Production publication must not gate, delay, or skip the cloud build.
|
||||
const jobsSection = cloudWorkflow.slice(cloudWorkflow.indexOf("\njobs:\n"));
|
||||
const headers = [...jobsSection.matchAll(/^ {2}([\w-]+):[^\n]*$/gm)];
|
||||
expect(
|
||||
headers.length,
|
||||
"docker.yml must declare at least two jobs under jobs:",
|
||||
).toBeGreaterThanOrEqual(2);
|
||||
"docker-cloud.yml must declare a cloud build job under jobs:",
|
||||
).toBeGreaterThanOrEqual(1);
|
||||
|
||||
// Locate the job block that carries the cloud build (target: cloud) and
|
||||
// assert it declares no `needs:` — coupling it to another job would
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ import { describe, expect, it } from "vitest";
|
|||
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..", "..");
|
||||
const dockerfile = readFileSync(path.join(repoRoot, "Dockerfile"), "utf8");
|
||||
const workflow = readFileSync(path.join(repoRoot, ".github", "workflows", "docker.yml"), "utf8");
|
||||
const workflow = readFileSync(path.join(repoRoot, ".github", "workflows", "docker-cloud.yml"), "utf8");
|
||||
const serverPackageJson = JSON.parse(
|
||||
readFileSync(path.join(repoRoot, "server", "package.json"), "utf8"),
|
||||
) as { peerDependencies?: Record<string, string> };
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import { describe, expect, it } from "vitest";
|
|||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..", "..");
|
||||
const dockerfile = readFileSync(path.join(repoRoot, "Dockerfile"), "utf8");
|
||||
const workflow = readFileSync(path.join(repoRoot, ".github", "workflows", "docker.yml"), "utf8");
|
||||
const cloudWorkflow = readFileSync(path.join(repoRoot, ".github", "workflows", "docker-cloud.yml"), "utf8");
|
||||
|
||||
/**
|
||||
* Return the text of the Dockerfile stage that starts at the named target.
|
||||
|
|
@ -68,7 +69,7 @@ describe("docker build-stamp wiring", () => {
|
|||
});
|
||||
|
||||
it("passes PAPERCLIP_BUILD_COMMIT as a build-arg for both image targets", () => {
|
||||
const argLines = [...workflow.matchAll(/^\s*PAPERCLIP_BUILD_COMMIT=.*$/gm)];
|
||||
const argLines = [...`${workflow}\n${cloudWorkflow}`.matchAll(/^\s*PAPERCLIP_BUILD_COMMIT=.*$/gm)];
|
||||
expect(
|
||||
argLines.length,
|
||||
"the docker workflow must pass PAPERCLIP_BUILD_COMMIT for the production and cloud builds",
|
||||
|
|
|
|||
Loading…
Reference in New Issue