From 6a7025ebe3b931928e35058a299d3fae21161eac Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Thu, 10 Sep 2026 21:02:42 -0700 Subject: [PATCH] ci: skip cloud runner cleanup when disk headroom is ample (#13191) Skip cloud runner disk cleanup when both the Docker and workspace filesystems have at least 64 GiB free. Preserve the existing cleanup for low, unavailable, or invalid measurements and verify the actual shell behavior across eight scenarios. Co-Authored-By: Paperclip --- .../tests/docker-disk-workflow.test.mjs | 65 +++++++++++++++++++ .github/workflows/docker-cloud.yml | 12 ++++ doc/DOCKER.md | 6 ++ 3 files changed, 83 insertions(+) create mode 100644 .github/scripts/tests/docker-disk-workflow.test.mjs diff --git a/.github/scripts/tests/docker-disk-workflow.test.mjs b/.github/scripts/tests/docker-disk-workflow.test.mjs new file mode 100644 index 0000000000..414af38cfd --- /dev/null +++ b/.github/scripts/tests/docker-disk-workflow.test.mjs @@ -0,0 +1,65 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; + +const workflow = readFileSync(new URL("../../workflows/docker-cloud.yml", import.meta.url), "utf8"); +const step = workflow.split(" - name: Free runner disk")[1].split(" - name: Login to GitHub Container Registry")[0]; +const script = step.split(" run: |\n")[1].split("\n").map((line) => line.replace(/^ {10}/, "")).join("\n"); +const threshold = 64 * 1024 * 1024; + +for (const { name, dockerFree, workspaceFree, dfStatus = "0", infoStatus = "0", cleanup } of [ + { name: "ample free space", dockerFree: threshold + 1, workspaceFree: threshold + 1, cleanup: false }, + { name: "exactly the headroom threshold", dockerFree: threshold, workspaceFree: threshold, cleanup: false }, + { name: "Docker filesystem below threshold", dockerFree: threshold - 1, workspaceFree: threshold + 1, cleanup: true }, + { name: "workspace filesystem below threshold", dockerFree: threshold + 1, workspaceFree: threshold - 1, cleanup: true }, + { name: "invalid Docker measurement", dockerFree: "unknown", workspaceFree: threshold + 1, cleanup: true }, + { name: "invalid workspace measurement", dockerFree: threshold + 1, workspaceFree: "unknown", cleanup: true }, + { name: "failed df command", dockerFree: threshold + 1, workspaceFree: threshold + 1, dfStatus: "1", cleanup: true }, + { name: "failed Docker inspection", dockerFree: threshold + 1, workspaceFree: threshold + 1, infoStatus: "1", cleanup: true }, +]) { + test(`cloud disk cleanup: ${name}`, () => { + const dir = mkdtempSync(path.join(tmpdir(), "cloud-disk-test-")); + const log = path.join(dir, "commands.log"); + // Every mutating command is a recording fixture; no real SDKs, caches, + // images, or directories are deleted when the workflow shell executes. + const fixture = `#!/bin/bash +printf '%s %s\\n' "\${0##*/}" "$*" >> "$COMMAND_LOG" +case "\${0##*/}" in + df) + printf 'Filesystem 1024-blocks Used Available Capacity Mounted on\\n' + if [ "$1" = '-Pk' ]; then + printf '/dev/docker 200000000 1 %s 1%% /docker\\n' "$DOCKER_FREE" + printf '/dev/workspace 200000000 1 %s 1%% /workspace\\n' "$WORKSPACE_FREE" + exit "$DF_STATUS" + fi + ;; + docker) + if [ "$1" = 'info' ]; then + printf '/docker-data\\n' + exit "$INFO_STATUS" + fi + ;; +esac +`; + try { + for (const command of ["df", "docker", "pnpm", "sudo"]) { + writeFileSync(path.join(dir, command), fixture, { mode: 0o755 }); + } + const result = spawnSync("bash", ["-c", script], { + encoding: "utf8", + env: { ...process.env, PATH: `${dir}:${process.env.PATH}`, GITHUB_WORKSPACE: "/workspace", COMMAND_LOG: log, + DOCKER_FREE: String(dockerFree), WORKSPACE_FREE: String(workspaceFree), DF_STATUS: dfStatus, INFO_STATUS: infoStatus }, + }); + assert.equal(result.status, 0, result.stderr); + const commands = readFileSync(log, "utf8"); + if (infoStatus === "0") assert.match(commands, /df -Pk \/docker-data \/workspace/); + assert.equal(commands.includes("pnpm store prune"), cleanup); + assert.equal(commands.includes("sudo rm -rf /usr/share/dotnet"), cleanup); + assert.equal(commands.includes("docker system prune -af"), cleanup); + assert.equal(result.stdout.includes("skipping cleanup"), !cleanup); + } finally { rmSync(dir, { recursive: true, force: true }); } + }); +} diff --git a/.github/workflows/docker-cloud.yml b/.github/workflows/docker-cloud.yml index 13e750e004..8fb2517bc0 100644 --- a/.github/workflows/docker-cloud.yml +++ b/.github/workflows/docker-cloud.yml @@ -120,6 +120,18 @@ jobs: echo "Disk before cleanup:" df -h + # A measured hosted cloud build started with 86 GB available. + # Keep ample headroom for BuildKit and image verification, but + # avoid minutes deleting SDKs when neither filesystem needs space. + minimum_free_kib=$((64 * 1024 * 1024)) + if docker_root="$(docker info --format '{{.DockerRootDir}}')" \ + && available_kib="$(df -Pk "$docker_root" "$GITHUB_WORKSPACE" | awk 'NR > 1 { rows++; if ($4 !~ /^[0-9]+$/) invalid = 1; if (min == "" || $4 < min) min = $4 } END { if (invalid || rows != 2) exit 1; print min }')" \ + && [[ "$available_kib" =~ ^[0-9]+$ ]] \ + && (( available_kib >= minimum_free_kib )); then + echo "At least 64 GiB is available for Docker and the workspace; skipping cleanup." + exit 0 + fi + pnpm store prune || true sudo apt-get clean || true sudo rm -rf \ diff --git a/doc/DOCKER.md b/doc/DOCKER.md index f43d9a24b1..ad8be8d9dc 100644 --- a/doc/DOCKER.md +++ b/doc/DOCKER.md @@ -47,6 +47,12 @@ legacy `buildcache-cloud` fallback. This preserves reusable layers without letting concurrent builds overwrite one shared cache manifest. Retain recent cache tags if registry cleanup is configured; deleting them makes builds colder. +Cloud CI skips SDK and cache cleanup when both the Docker data filesystem and +the checkout filesystem have at least 64 GiB available. Below that conservative +headroom threshold, or when the measurement fails, it retains the existing +cleanup. The threshold selects the fast path; it is not a new minimum disk +requirement for local builds or smaller runners. + After the pushed image passes its Sentry and orphan-reaping checks, the workflow verifies its commit label and platform and adds `ghcr.io/paperclipai/paperclip:sha--cloud`. This address lets commit-based deployment tooling reuse the normal build.