diff --git a/.github/workflows/docker-cloud.yml b/.github/workflows/docker-cloud.yml index 62025d3a85..81e608c927 100644 --- a/.github/workflows/docker-cloud.yml +++ b/.github/workflows/docker-cloud.yml @@ -65,24 +65,6 @@ jobs: id: tools-epoch run: echo "epoch=$(date -u +%G-W%V)" >> "$GITHUB_OUTPUT" - # Each SHA exports its own cache. Import recent first-parent caches so - # a late older build cannot overwrite a newer build's cache manifest. - # The legacy ref keeps the first builds warm during the transition. - - name: Select cloud cache ancestry - id: cloud-cache - env: - CACHE_IMAGE: ghcr.io/${{ github.repository }} - run: | - set -euo pipefail - { - echo 'sources<> "$GITHUB_OUTPUT" - - name: Setup pnpm uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 with: @@ -161,6 +143,14 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4 + # Mixing several historical manifests missed otherwise reusable native + # layers on fresh builders. Import the nearest available complete cache. + - name: Select cloud cache ancestry + id: cloud-cache + env: + CACHE_IMAGE: ghcr.io/${{ github.repository }} + run: node scripts/select-cloud-cache.mjs + # Deployment tooling reads these labels from the registry to verify an # image's schema expectations against a migrator before deploying it, # without pulling the image. The server refuses to start when the @@ -225,7 +215,7 @@ jobs: # Same-SHA builds serialize above; different SHAs never share a # writable cache ref. Registry layers are content-addressed and # shared even when cache manifests have separate tags. - cache-from: ${{ steps.cloud-cache.outputs.sources }} + cache-from: ${{ steps.cloud-cache.outputs.source }} cache-to: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-cloud-${{ github.sha }},mode=max tags: ${{ steps.meta-cloud.outputs.tags }} labels: ${{ steps.meta-cloud.outputs.labels }} diff --git a/.github/workflows/docker-runner-check.yml b/.github/workflows/docker-runner-check.yml index 7c3b7820b5..d595522292 100644 --- a/.github/workflows/docker-runner-check.yml +++ b/.github/workflows/docker-runner-check.yml @@ -32,7 +32,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4 # Compile the real target, then change source in a disposable context. - # Require dependency-layer reuse and changed metadata from the real binary. + # A fresh builder must import dependencies and produce changed binary metadata. # No registry credentials, external cache, or image publication. - name: Verify native build and dependency cache reuse run: bash scripts/check-docker-runner-cache.sh diff --git a/doc/DOCKER.md b/doc/DOCKER.md index 2905bfe957..04f23424b4 100644 --- a/doc/DOCKER.md +++ b/doc/DOCKER.md @@ -357,6 +357,14 @@ own native build; no cross-architecture binary is reused. No additional GitHub Actions cache is created. A cold build also installs the recipe generator and compiles dependencies, so the savings apply after those layers are available. +Cloud builds import one registry cache: the first available full-SHA cache in +the current commit's ten-entry first-parent ancestry, with the legacy cache +as a final fallback. Each build still exports its own SHA cache with +`mode=max`. In fresh-builder checks, importing several historical manifests +missed native layers that a single matching manifest reused. The selector +inspects metadata after Docker login, stops at the first available cache, and +permits a cold build if no cache can be read. + The application build inherits that stage and still runs the normal server build, including Cargo, binary staging, and generated-contract checks. Rust input file times are normalized in both stages so fresh checkouts do not force @@ -367,12 +375,14 @@ directory as before. Cache misses only cost compilation time. Pull requests that change the Dockerfile, Docker ignore rules, or Runner native inputs also build the isolated `runner-build` target in `Docker Runner check`. The check runs `bash scripts/check-docker-runner-cache.sh` against a disposable -copy of tracked source and the actual Docker ignore rules. It compiles a baseline, -changes a Rust metadata constant, and rebuilds. It requires a cached dependency -build, an unchanged dependency recipe, and changed metadata from the real binary. -It also verifies that a dependency declaration change alters the recipe. The -probe exports only small metadata files, avoiding a large image import into the -Docker daemon. It catches missing embedded inputs before the post-merge build. -It uses a GitHub-hosted runner with read-only repository access and does not -publish images or cache artifacts. Allow up to 20 minutes for its cold build and +copy of tracked source and the actual Docker ignore rules. It compiles a baseline +and exports a local cache, removes that builder, changes a Rust metadata constant, +and rebuilds on a fresh builder using only the exported cache. It requires a +cached dependency build, an unchanged dependency recipe, and changed metadata +from the real binary. It also verifies that a dependency declaration change +alters the recipe. The probe exports small metadata results instead of importing +a large test image into the Docker daemon. Temporary builders and cache files +are removed afterward. It catches missing embedded inputs before the post-merge +build. It uses a GitHub-hosted runner with read-only repository access and never +publishes images or registry caches. Allow up to 20 minutes for its cold build and source rebuild. diff --git a/package.json b/package.json index 40fc577d2c..013c1a44f4 100644 --- a/package.json +++ b/package.json @@ -59,7 +59,7 @@ "smoke:posthog-live": "node scripts/smoke/posthog-live.mjs", "smoke:pipelines-tutorial": "./scripts/smoke/pipelines-tutorial-smoke.sh", "smoke:terminal-bench-loop-skill": "node scripts/smoke/terminal-bench-loop-skill-smoke.mjs", - "test:release-registry": "node --test scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs", + "test:release-registry": "node --test scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/select-cloud-cache.test.mjs", "storybook-visual:baseline": "node scripts/storybook-visual-baseline.mjs", "test:storybook-visual": "node scripts/storybook-visual-baseline.mjs download && node scripts/storybook-visual-baseline.mjs verify && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts", "test:storybook-visual:update": "node scripts/storybook-visual-baseline.mjs download && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts --update-snapshots && node scripts/storybook-visual-baseline.mjs pack", diff --git a/scripts/check-docker-runner-cache.sh b/scripts/check-docker-runner-cache.sh index 0c8caf81d5..3456948793 100644 --- a/scripts/check-docker-runner-cache.sh +++ b/scripts/check-docker-runner-cache.sh @@ -1,10 +1,18 @@ #!/usr/bin/env bash -# Build the real Docker target twice in a disposable copy of tracked source. +# Build the real Docker target on two fresh builders using an exported cache. # Export only metadata, avoiding a multi-gigabyte test image in the daemon. set -euo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" probe_dir="$(mktemp -d "${TMPDIR:-/tmp}/paperclip-runner-cache.XXXXXX")" -trap 'rm -rf "$probe_dir"' EXIT +baseline_builder="${probe_dir##*/}-baseline" +rebuild_builder="${probe_dir##*/}-rebuild" +cleanup() { + for builder in "$baseline_builder" "$rebuild_builder"; do + docker buildx rm "$builder" >/dev/null 2>&1 || true + done + rm -rf "$probe_dir" +} +trap cleanup EXIT mkdir "$probe_dir/context" cd "$repo_root" git ls-files -z | tar -cf - --null -T - | tar -xf - -C "$probe_dir/context" @@ -21,9 +29,16 @@ FROM scratch AS recipe-proof-export COPY --from=runner-plan /tmp/runner-recipe.json /recipe.json DOCKER build_proof() { - docker buildx build --file "$probe_dir/cache-probe.Dockerfile" --target cache-proof-export --output "type=local,dest=$probe_dir/$1" --progress plain . 2>&1 | tee "$probe_dir/$1.log" + local result="$1" builder="$2" + shift 2 + docker buildx build --builder "$builder" --file "$probe_dir/cache-probe.Dockerfile" --target cache-proof-export --output "type=local,dest=$probe_dir/$result" --progress plain "$@" . 2>&1 | tee "$probe_dir/$result.log" } -build_proof baseline +docker buildx create --name "$baseline_builder" --driver docker-container +build_proof baseline "$baseline_builder" --cache-to "type=local,dest=$probe_dir/cache,mode=max" +# Removing the first builder proves the second build cannot use daemon-local +# state, and releases its disk space before importing the exported cache. +docker buildx rm "$baseline_builder" +docker buildx create --name "$rebuild_builder" --driver docker-container python3 - <<'CHECK' from pathlib import Path p=Path('packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs') @@ -31,7 +46,7 @@ s=p.read_text(); needle='paperclip-runner/runnerd-build-metadata/v1' assert s.count(needle)==1 p.write_text(s.replace(needle,needle+'-cache-probe')) CHECK -build_proof source-change +build_proof source-change "$rebuild_builder" --cache-from "type=local,src=$probe_dir/cache" python3 - <<'CHECK' import os,json,re from pathlib import Path @@ -45,12 +60,12 @@ log=(root/'source-change.log').read_text() step=re.search(r'#(\d+) \[runner-deps[^\n]+ RUN cargo chef cook',log)[1] assert f'#{step} CACHED' in log assert 'Compiling paperclip-runner-core' in log -print('PASS: unchanged dependency recipe and cached cook layer; real binary changed.') +print('PASS: fresh builder imported compiled dependencies; real binary changed.') p=Path('packages/paperclip-runner/runner/Cargo.toml') s=p.read_text(); assert 'serde_json = "1.0"' in s p.write_text(s.replace('serde_json = "1.0"','serde_json = ">=1.0.0, <2.0.0"')) CHECK -docker buildx build --file "$probe_dir/cache-probe.Dockerfile" --target recipe-proof-export --output "type=local,dest=$probe_dir/manifest-change" --progress plain . +docker buildx build --builder "$rebuild_builder" --file "$probe_dir/cache-probe.Dockerfile" --target recipe-proof-export --output "type=local,dest=$probe_dir/manifest-change" --progress plain . python3 - <<'CHECK' from pathlib import Path import os diff --git a/scripts/preview-artifacts.test.mjs b/scripts/preview-artifacts.test.mjs index 15ca6978f2..ebdd9a6122 100644 --- a/scripts/preview-artifacts.test.mjs +++ b/scripts/preview-artifacts.test.mjs @@ -4,6 +4,7 @@ import { planArtifacts } from "./preview-artifacts.mjs"; import { readFileSync, mkdtempSync, writeFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; +import { fileURLToPath } from "node:url"; import { gzipSync } from "node:zlib"; import { execFileSync, spawnSync } from "node:child_process"; import { previewManifest, assertMetadata, validateRequest, versionFor, tarManifest, packageExists, imageExists, publishPreview, publishImage } from "./preview-artifacts.mjs"; @@ -217,8 +218,13 @@ test("cloud builds bake the managed runtime identity and verify it before public test("cloud cache imports are bounded, follow master ancestry, and retain the legacy fallback", () => { const workflow = readFileSync(new URL("../.github/workflows/docker-cloud.yml", import.meta.url), "utf8"); - const step = workflow.split(" - name: Select cloud cache ancestry")[1].split(" - name: Setup pnpm")[0]; - const script = step.split(" run: |\n")[1].split("\n").map((line) => line.replace(/^ {10}/, "")).join("\n"); + const selector = workflow.indexOf(" - name: Select cloud cache ancestry"); + assert.ok(selector > workflow.indexOf(" - name: Login to GitHub Container Registry")); + assert.ok(selector > workflow.indexOf(" - name: Set up Docker Buildx")); + assert.ok(selector < workflow.indexOf(" - name: Build and push (cloud)")); + assert.match(workflow, /run: node scripts\/select-cloud-cache.mjs/); + assert.match(workflow, /cache-from: \$\{\{ steps.cloud-cache.outputs.source \}\}/); + const script = fileURLToPath(new URL("./select-cloud-cache.mjs", import.meta.url)); const dir = mkdtempSync(path.join(tmpdir(), "cloud-cache-test-")); const output = path.join(dir, "output"); const env = { ...process.env, GIT_AUTHOR_NAME: "Test", GIT_AUTHOR_EMAIL: "test@example.test", GIT_COMMITTER_NAME: "Test", GIT_COMMITTER_EMAIL: "test@example.test" }; @@ -235,14 +241,25 @@ test("cloud cache imports are bounded, follow master ancestry, and retain the le git("checkout", "master"); git("merge", "--no-ff", "topic", "-m", "merge topic"); commits.unshift(git("rev-parse", "HEAD")); - const result = spawnSync("bash", ["-c", script], { cwd: dir, encoding: "utf8", env: { ...env, CACHE_IMAGE: "ghcr.io/paperclipai/paperclip", GITHUB_OUTPUT: output } }); + const available = `ghcr.io/paperclipai/paperclip:buildcache-cloud-${commits[2]}`; + const inspections = path.join(dir, "inspections"); + writeFileSync(path.join(dir, "docker"), `#!/usr/bin/env node +const fs = require("node:fs"); +fs.appendFileSync(process.env.CACHE_INSPECTIONS, process.argv.at(-1) + "\\n"); +if (process.argv.at(-1) !== process.env.AVAILABLE_CACHE) { + process.stderr.write("manifest unknown"); + process.exit(1); +} +`, { mode: 0o755 }); + const result = spawnSync(process.execPath, [script], { + cwd: dir, encoding: "utf8", env: { + ...env, PATH: `${dir}${path.delimiter}${env.PATH}`, CACHE_IMAGE: "ghcr.io/paperclipai/paperclip", + GITHUB_OUTPUT: output, AVAILABLE_CACHE: available, CACHE_INSPECTIONS: inspections, + }, + }); assert.equal(result.status, 0, result.stderr); - assert.deepEqual(readFileSync(output, "utf8").trim().split("\n"), [ - "sources< `type=registry,ref=ghcr.io/paperclipai/paperclip:buildcache-cloud-${commit}`), - "type=registry,ref=ghcr.io/paperclipai/paperclip:buildcache-cloud", - "CACHE_SOURCES", - ]); + assert.equal(readFileSync(output, "utf8"), `source=type=registry,ref=${available}\n`); + assert.deepEqual(readFileSync(inspections, "utf8").trim().split("\n"), commits.slice(0, 3).map((commit) => `ghcr.io/paperclipai/paperclip:buildcache-cloud-${commit}`)); } finally { rmSync(dir, { recursive: true, force: true }); } }); diff --git a/scripts/select-cloud-cache.mjs b/scripts/select-cloud-cache.mjs new file mode 100644 index 0000000000..74b05a533d --- /dev/null +++ b/scripts/select-cloud-cache.mjs @@ -0,0 +1,66 @@ +#!/usr/bin/env node +import { execFileSync } from "node:child_process"; +import { appendFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; + +export function cloudCacheCandidates(image, commits) { + if (!/^ghcr\.io\/[a-z0-9._-]+\/[a-z0-9._-]+$/.test(image ?? "")) { + throw new Error("Expected a GHCR owner/repository cache image."); + } + if (!Array.isArray(commits) || commits.length === 0 || commits.some((sha) => !/^[a-f0-9]{40}$/.test(sha))) { + throw new Error("Cloud cache ancestry requires full commit SHAs."); + } + return [ + ...[...new Set(commits)].slice(0, 10).map((sha) => `${image}:buildcache-cloud-${sha}`), + `${image}:buildcache-cloud`, + ]; +} + +export async function selectCloudCache(image, commits, { + exists = registryCacheExists, + log = console.log, +} = {}) { + for (const ref of cloudCacheCandidates(image, commits)) { + try { + if (!await exists(ref)) continue; + log(`Using cloud cache: ${ref}`); + return `type=registry,ref=${ref}`; + } catch { + // Cache availability must not turn an otherwise valid build into a + // failure. A later ancestor may still be available during a rollout. + log(`Could not inspect cloud cache ${ref}; trying the next ancestor.`); + } + } + log("No cloud cache is available; this build will populate one."); + return ""; +} + +function registryCacheExists(ref) { + try { + // Use the preceding Docker login, including for private registry caches. + // Inspect metadata only: no layer download and no image execution. + execFileSync("docker", ["buildx", "imagetools", "inspect", "--raw", ref], { + timeout: 10_000, + maxBuffer: 1024 * 1024, + stdio: ["ignore", "pipe", "pipe"], + }); + return true; + } catch (error) { + if (/manifest unknown|not found|NAME_UNKNOWN/i.test(String(error.stderr ?? ""))) return false; + throw error; + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + if (!process.env.GITHUB_OUTPUT) throw new Error("GITHUB_OUTPUT is required."); + const commits = execFileSync("git", ["rev-list", "--first-parent", "--max-count=10", "HEAD"], { + encoding: "utf8", + }).trim().split("\n"); + const source = await selectCloudCache(process.env.CACHE_IMAGE, commits, { exists: registryCacheExists }); + appendFileSync(process.env.GITHUB_OUTPUT, `source=${source}\n`); + } catch (error) { + console.error(error.message); + process.exitCode = 1; + } +} diff --git a/scripts/select-cloud-cache.test.mjs b/scripts/select-cloud-cache.test.mjs new file mode 100644 index 0000000000..d160fa4f04 --- /dev/null +++ b/scripts/select-cloud-cache.test.mjs @@ -0,0 +1,65 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { cloudCacheCandidates, selectCloudCache } from "./select-cloud-cache.mjs"; + +const image = "ghcr.io/paperclipai/paperclip"; +const commits = ["a".repeat(40), "b".repeat(40), "c".repeat(40)]; +const candidates = cloudCacheCandidates(image, commits); + +test("a same-SHA rerun imports only its existing cache", async () => { + const inspected = []; + const source = await selectCloudCache(image, commits, { + exists: async (ref) => { inspected.push(ref); return true; }, log() {}, + }); + assert.equal(source, `type=registry,ref=${candidates[0]}`); + assert.deepEqual(inspected, candidates.slice(0, 1)); +}); + +test("a new merge imports only its nearest available ancestor", async () => { + const inspected = []; + const source = await selectCloudCache(image, commits, { + exists: async (ref) => { inspected.push(ref); return ref === candidates[1]; }, log() {}, + }); + assert.equal(source, `type=registry,ref=${candidates[1]}`); + assert.deepEqual(inspected, candidates.slice(0, 2)); + assert.equal(source.includes("\n"), false); +}); + +test("a still-building parent falls back to an older completed cache", async () => { + assert.equal(await selectCloudCache(image, commits, { + exists: async (ref) => ref === candidates[2], log() {}, + }), `type=registry,ref=${candidates[2]}`); +}); + +test("the legacy cache is used only if no SHA cache exists", async () => { + const inspected = []; + assert.equal(await selectCloudCache(image, commits, { + exists: async (ref) => { inspected.push(ref); return ref === candidates.at(-1); }, log() {}, + }), `type=registry,ref=${candidates.at(-1)}`); + assert.deepEqual(inspected, candidates); +}); + +test("missing caches permit a cold build", async () => { + assert.equal(await selectCloudCache(image, commits, { exists: async () => false, log() {} }), ""); +}); + +test("a failed lookup can fall back without failing image publication", async () => { + const messages = []; + assert.equal(await selectCloudCache(image, commits, { + exists: async (ref) => { + if (ref === candidates[0]) throw new Error("registry temporarily unavailable"); + return true; + }, + log: (message) => messages.push(message), + }), `type=registry,ref=${candidates[1]}`); + assert.match(messages[0], /Could not inspect cloud cache/); +}); + +test("ancestry is bounded, deduplicated, and rejects output injection", () => { + const many = Array.from({ length: 20 }, (_, i) => i.toString(16).padStart(40, "0")); + assert.equal(cloudCacheCandidates(image, many).length, 11); + assert.deepEqual(cloudCacheCandidates(image, [commits[0], commits[0]]), [candidates[0], candidates.at(-1)]); + assert.throws(() => cloudCacheCandidates(`${image}\nsource=untrusted`, commits)); + assert.throws(() => cloudCacheCandidates(image, ["master"])); + assert.throws(() => cloudCacheCandidates(image, [])); +});