From ca92f727c5f7e4a6e5d23d05fef188bee9066b81 Mon Sep 17 00:00:00 2001 From: Nicky Leach Date: Tue, 28 Jul 2026 15:38:01 -0700 Subject: [PATCH] ci: publish the cloud image in its own parallel job (#10408) ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The CI workflow is responsible for producing and publishing the Docker images that power Paperclip deployments > - The cloud image publish was previously coupled to the stock publish job, so a failure or delay in one path could gate the other > - That coupling makes the release pipeline less resilient than it needs to be > - This pull request gives the cloud publish its own top-level job so both publishes can run in parallel without a `needs:` dependency > - The benefit is better failure isolation and less wasted time when one publish path is slow or broken ## Linked Issues or Issue Description No public GitHub issue was found for this change. Problem statement: - The cloud image publish was implemented as trailing work inside the stock publish job. - That setup meant the cloud publish could be delayed or skipped if the stock job failed early. - The desired behavior is for the cloud publish to run independently so a failure in one publish path does not gate the other. Proposed solution: - Split the cloud publish into its own top-level workflow job. - Keep the same cloud-specific build settings and cache behavior. - Preserve the existing top-level concurrency behavior. Alternatives considered: - Keeping both publishes in one job with conditionals or later steps. Rejected because it still couples success and runtime between the two publish paths. ## What Changed - Split the cloud image publish into a separate top-level Docker workflow job. - Removed the dependency coupling so the cloud job does not need the stock job. - Expanded the drift-guard test to assert the two-job structure and the absence of `needs:` on the cloud job. ## Verification - The workflow YAML was parsed successfully and confirmed to contain two jobs: `build-and-push` and `build-and-push-cloud`. - The cloud job was confirmed to have no `needs:` entry. - The drift-guard assertions were reproduced in a dependency-free harness and passed. - PR #10408 completed GitHub Actions with all required checks green, including the e2e shards. - Greptile review completed at 5/5 with no unresolved comments. - No documentation files changed because this is a workflow/test-only change. ## Risks - The workflow now duplicates the prep steps across two runners, so any shared setup change must be kept in sync between both jobs. - The new job increases workflow surface area slightly, which can make future maintenance more verbose. - Overall risk is low because the change is limited to CI orchestration and test coverage. ## Model Used OpenAI Codex (GPT-5, tool-using code assistant) ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge Co-authored-by: Paperclip --- .github/workflows/docker.yml | 119 +++++++++++++++++- .../cloud-image-bundled-plugins.test.ts | 30 +++++ 2 files changed, 144 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index fb10c9e268..cb46bcbf2a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -155,11 +155,120 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - # The cloud variant carries built bundled plugins for managed - # deployments (see the `cloud` stage in the Dockerfile). Published - # under the same tag set with a `-cloud` suffix (sha--cloud, - # latest-cloud, -cloud). Reuses the layer cache from the - # production build, so this mostly adds the plugin-build layers. + # The cloud variant carries built bundled plugins for managed deployments + # (see the `cloud` stage in the Dockerfile). It runs as its own job with no + # `needs:` on the stock publish above, so the two builds run in parallel and + # a failure or slow build in one never gates, delays, or skips the other. + # Both jobs share only the single top-level concurrency slot. Each job is a + # separate runner, so this one carries its own copy of the prep steps + # (checkout through schema labels) — the accepted cost of that isolation. + build-and-push-cloud: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + # Full history and tags so `git describe` below can compute the + # release version to stamp into the image. + fetch-depth: 0 + + # `.git` is dockerignored, so a running image cannot derive its own + # version and otherwise reports the source package.json placeholder in + # analytics and the debug panel. Compute it here from the pristine + # checkout (real CalVer drift from the nearest release tag) and pass it + # into the build. Empty when no release tag is reachable — the server + # then keeps its existing fallbacks. + - name: Compute build version + id: build-version + run: | + set -euo pipefail + version="$(git describe --tags --match 'v*' --long --dirty 2>/dev/null || true)" + echo "version=${version}" >> "$GITHUB_OUTPUT" + echo "Stamping build version: ${version:-}" + + - name: Setup pnpm + uses: pnpm/action-setup@v6 + with: + version: 9.15.4 + run_install: false + + # No dependency cache here: this workflow publishes release images, and + # restoring a shared Actions cache into the build inputs would let a + # poisoned cache entry reach the published artifact. + - name: Setup Node.js + uses: actions/setup-node@v7 + with: + node-version: 20 + + - name: Refresh lockfile for Docker build context + run: | + set -euo pipefail + pnpm install --lockfile-only --ignore-scripts --no-frozen-lockfile + + changed="$(git status --porcelain)" + if [ -z "$changed" ]; then + echo "Lockfile already matches package metadata." + exit 0 + fi + + if printf '%s\n' "$changed" | grep -Fvq ' pnpm-lock.yaml'; then + echo "Unexpected files changed during lockfile refresh:" + echo "$changed" + exit 1 + fi + + echo "Using refreshed pnpm-lock.yaml in the Docker build context." + + - name: Free runner disk + run: | + set -euo pipefail + echo "Disk before cleanup:" + df -h + + pnpm store prune || true + sudo apt-get clean || true + sudo rm -rf \ + /usr/share/dotnet \ + /usr/share/swift \ + /usr/local/lib/android \ + /usr/local/share/boost \ + /usr/local/share/powershell \ + /opt/ghc \ + /opt/hostedtoolcache/CodeQL \ + /opt/hostedtoolcache/PyPy \ + /opt/hostedtoolcache/Ruby || true + docker system prune -af || true + + echo "Disk after cleanup:" + df -h + + - name: Login to GitHub Container Registry + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + # Deployment tooling reads these labels from the registry to verify an + # image's schema expectations against a migrator before deploying it, + # without pulling the image. The server refuses to start when the + # database is missing bundled migrations, so orchestrators need a cheap + # way to check image/migrator compatibility up front. + - name: Compute schema migration labels + id: schema + run: | + set -euo pipefail + last=$(ls packages/db/src/migrations/*.sql | sed 's|.*/||' | LC_ALL=C sort | tail -1) + count=$(ls packages/db/src/migrations/*.sql | wc -l | tr -d ' ') + echo "last=${last}" >> "$GITHUB_OUTPUT" + echo "count=${count}" >> "$GITHUB_OUTPUT" + + # Published under the same tag set with a `-cloud` suffix + # (sha--cloud, latest-cloud, -cloud). - name: Docker meta (cloud) id: meta-cloud uses: docker/metadata-action@v6 diff --git a/server/src/__tests__/cloud-image-bundled-plugins.test.ts b/server/src/__tests__/cloud-image-bundled-plugins.test.ts index 98955e0860..464fd73308 100644 --- a/server/src/__tests__/cloud-image-bundled-plugins.test.ts +++ b/server/src/__tests__/cloud-image-bundled-plugins.test.ts @@ -76,6 +76,36 @@ describe("cloud image bundled plugins", () => { expect(workflow).toMatch(/^\s*target: production$/m); }); + it("publishes the cloud image in its own job with no needs coupling", () => { + // The cloud publish runs as its own top-level job so the stock/production + // publish can never gate, delay, or skip it. Both jobs share only the + // single top-level concurrency slot; there is deliberately no `needs:` + // between them, so a failure in one is never coupled to the other. + const jobsSection = workflow.slice(workflow.indexOf("\njobs:\n")); + const headers = [...jobsSection.matchAll(/^ {2}([\w-]+):[^\n]*$/gm)]; + expect( + headers.length, + "docker.yml must declare at least two jobs under jobs:", + ).toBeGreaterThanOrEqual(2); + + // Locate the job block that carries the cloud build (target: cloud) and + // assert it declares no `needs:` — coupling it to another job would + // reintroduce the shared failure the split job exists to remove. + const cloudHeaderIdx = headers.findIndex((header, i) => { + const start = header.index ?? 0; + const end = headers[i + 1]?.index ?? jobsSection.length; + return jobsSection.slice(start, end).includes("target: cloud"); + }); + expect(cloudHeaderIdx, "one job must build the cloud target").toBeGreaterThanOrEqual(0); + const start = headers[cloudHeaderIdx].index ?? 0; + const end = headers[cloudHeaderIdx + 1]?.index ?? jobsSection.length; + const cloudJobBlock = jobsSection.slice(start, end); + expect( + cloudJobBlock, + "the cloud job must not couple to another job via needs:", + ).not.toMatch(/^\s*needs:/m); + }); + it("throttles the docker workflow with cancel-in-progress: false", () => { // Concurrency is declared at the workflow (top) level so a single group // spans the whole run, and cancel-in-progress is false so an in-flight