ci: isolate concurrent cloud build cache exports

Co-Authored-By: Paperclip <noreply@paperclip.ing>

.github/workflows/docker-cloud.yml

@@ -63,6 +63,24 @@ jobs:
         id: tools-epoch
         run: echo "epoch=$(date -u +%G-W%V)" >> "$GITHUB_OUTPUT"
 
+      # Each SHA exports its own cache. Import recent first-parent caches so
+      # a late older build cannot overwrite a newer build's cache manifest.
+      # The legacy ref keeps the first builds warm during the transition.
+      - name: Select cloud cache ancestry
+        id: cloud-cache
+        env:
+          CACHE_IMAGE: ghcr.io/${{ github.repository }}
+        run: |
+          set -euo pipefail
+          {
+            echo 'sources<<CACHE_SOURCES'
+            for commit in $(git rev-list --first-parent --max-count=10 HEAD); do
+              echo "type=registry,ref=$CACHE_IMAGE:buildcache-cloud-$commit"
+            done
+            echo "type=registry,ref=$CACHE_IMAGE:buildcache-cloud"
+            echo 'CACHE_SOURCES'
+          } >> "$GITHUB_OUTPUT"
+
       - name: Setup pnpm
         uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
         with:
@@ -188,11 +206,11 @@ jobs:
           # clock, and dropping it roughly halves time-to-deployable-image.
           platforms: linux/amd64
           push: true
-          # Registry-backed BuildKit cache, separate ref from the self-hosted
-          # job so the two parallel builds never clobber each other's cache
-          # manifest (see the rationale on the job above).
-          cache-from: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-cloud
-          cache-to: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-cloud,mode=max
+          # Same-SHA builds serialize above; different SHAs never share a
+          # writable cache ref. Registry layers are content-addressed and
+          # shared even when cache manifests have separate tags.
+          cache-from: ${{ steps.cloud-cache.outputs.sources }}
+          cache-to: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-cloud-${{ github.sha }},mode=max
           tags: ${{ steps.meta-cloud.outputs.tags }}
           labels: ${{ steps.meta-cloud.outputs.labels }}
 

doc/DOCKER.md

@@ -33,6 +33,12 @@ GitHub-hosted runners, so an older production or cloud build does not hold the
 new commit in a workflow queue. Available GitHub runner capacity still applies.
 Release tags and manual `Docker` dispatches call the same cloud build workflow.
 
+Each commit exports to its own `buildcache-cloud-<FULL_SHA>` registry tag.
+Builds import the current commit and nine first-parent ancestors, plus the
+legacy `buildcache-cloud` fallback. This preserves reusable layers without
+letting concurrent builds overwrite one shared cache manifest. Retain recent
+cache tags if registry cleanup is configured; deleting them makes builds colder.
+
 After the pushed image passes its Sentry and orphan-reaping checks, the workflow verifies its
 commit label and platform and adds `ghcr.io/paperclipai/paperclip:sha-<full-commit-sha>-cloud`.
 This address lets commit-based deployment tooling reuse the normal build.

scripts/preview-artifacts.test.mjs

@@ -4,7 +4,7 @@ import { readFileSync, mkdtempSync, writeFileSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
 import path from "node:path";
 import { gzipSync } from "node:zlib";
-import { spawnSync } from "node:child_process";
+import { execFileSync, spawnSync } from "node:child_process";
 import { previewManifest, assertMetadata, validateRequest, versionFor, tarManifest, packageExists, imageExists, publishPreview, publishImage } from "./preview-artifacts.mjs";
 
 const sha = "a".repeat(40);
@@ -135,6 +135,8 @@ test("cloud builds start per commit and preserve tag promotion dependencies", ()
   assert.match(cloud, /workflow_call:/);
   assert.match(cloud, /group: docker-cloud-\$\{\{ github.sha \}\}/);
   assert.match(cloud, /cancel-in-progress: false/);
+  assert.doesNotMatch(cloud, /uses: .*@v\d\b/);
+  assert.match(cloud, /cache-to: type=registry,ref=ghcr.io\/\$\{\{ github.repository \}\}:buildcache-cloud-\$\{\{ github.sha \}\},mode=max/);
   const caller = docker.split("  build-and-push-cloud:")[1].split("  promote_canary_channel:")[0];
   assert.match(caller, /if: github.event_name != 'push' \|\| github.ref != 'refs\/heads\/master'/);
   assert.match(caller, /uses: .\/.github\/workflows\/docker-cloud.yml/);
@@ -144,6 +146,37 @@ test("cloud builds start per commit and preserve tag promotion dependencies", ()
   assert.ok(reaping < cloud.indexOf("      - name: Publish verified full-SHA cloud tag"));
 });
 
+test("cloud cache imports are bounded, follow master ancestry, and retain the legacy fallback", () => {
+  const workflow = readFileSync(new URL("../.github/workflows/docker-cloud.yml", import.meta.url), "utf8");
+  const step = workflow.split("      - name: Select cloud cache ancestry")[1].split("      - name: Setup pnpm")[0];
+  const script = step.split("        run: |\n")[1].split("\n").map((line) => line.replace(/^ {10}/, "")).join("\n");
+  const dir = mkdtempSync(path.join(tmpdir(), "cloud-cache-test-"));
+  const output = path.join(dir, "output");
+  const env = { ...process.env, GIT_AUTHOR_NAME: "Test", GIT_AUTHOR_EMAIL: "test@example.test", GIT_COMMITTER_NAME: "Test", GIT_COMMITTER_EMAIL: "test@example.test" };
+  const git = (...args) => execFileSync("git", ["-c", "core.hooksPath=/dev/null", "-c", "commit.gpgsign=false", ...args], { cwd: dir, env, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim();
+  try {
+    git("init", "--initial-branch=master");
+    const commits = [];
+    for (let i = 0; i < 12; i++) {
+      git("commit", "--allow-empty", "-m", `main ${i}`);
+      commits.unshift(git("rev-parse", "HEAD"));
+    }
+    git("checkout", "-b", "topic", "HEAD~1");
+    git("commit", "--allow-empty", "-m", "topic");
+    git("checkout", "master");
+    git("merge", "--no-ff", "topic", "-m", "merge topic");
+    commits.unshift(git("rev-parse", "HEAD"));
+    const result = spawnSync("bash", ["-c", script], { cwd: dir, encoding: "utf8", env: { ...env, CACHE_IMAGE: "ghcr.io/paperclipai/paperclip", GITHUB_OUTPUT: output } });
+    assert.equal(result.status, 0, result.stderr);
+    assert.deepEqual(readFileSync(output, "utf8").trim().split("\n"), [
+      "sources<<CACHE_SOURCES",
+      ...commits.slice(0, 10).map((commit) => `type=registry,ref=ghcr.io/paperclipai/paperclip:buildcache-cloud-${commit}`),
+      "type=registry,ref=ghcr.io/paperclipai/paperclip:buildcache-cloud",
+      "CACHE_SOURCES",
+    ]);
+  } finally { rmSync(dir, { recursive: true, force: true }); }
+});
+
 test("normal cloud builds publish the checked digest only when source and platform match", () => {
   const workflow = readFileSync(new URL("../.github/workflows/docker-cloud.yml", import.meta.url), "utf8");
   const cloud = workflow.split("  build-and-push-cloud:")[1];