From 50bf4e3680971f6d11a7114b555460928f5d0136 Mon Sep 17 00:00:00 2001
From: Chris Sherwood <chris@crosstalksolutions.com>
Date: Mon, 13 Jul 2026 21:13:12 -0700
Subject: [PATCH] fix(updater): prune superseded images after update to reclaim
 disk (#858)

The sidecar updater pulled new image versions on every update but never
removed the old ones, so /var/lib/containerd grew unbounded across
releases (50+ GB of orphaned layers observed on long-running installs).

After a confirmed-successful recreate, prune (1) dangling layers left by
re-pulled moving tags and (2) superseded tags of the core services this
updater manages (the images in compose.yml), keeping the refs now in use.

Deliberately avoids `docker system/image prune -a`: that would delete
images for installed-but-stopped Supply Depot / curated services and
force a re-pull that fails on an offline box. Scoped strictly to
compose-managed repositories; optional/offline images are never touched.
Uses `docker rmi` without -f so anything still referenced by a container
is refused rather than force-removed. Best-effort; never fails the update.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 install/sidecar-updater/update-watcher.sh | 65 ++++++++++++++++++++++-
 1 file changed, 64 insertions(+), 1 deletion(-)

diff --git a/install/sidecar-updater/update-watcher.sh b/install/sidecar-updater/update-watcher.sh
index fbb515d..4fc29b1 100644
--- a/install/sidecar-updater/update-watcher.sh
+++ b/install/sidecar-updater/update-watcher.sh
@@ -102,12 +102,75 @@ perform_update() {
     done
     
     log "Successfully recreated all containers"
+
+    # Stage 4: Reclaim disk from superseded images (best-effort; never fails the update)
+    prune_old_images
+
     write_status "complete" 100 "System update completed successfully"
     log "System update completed successfully"
-    
+
     return 0
 }
 
+# Reclaim disk left behind by updates. Every update pulls new image versions but
+# never removed the old ones, so /var/lib/containerd grows unbounded across
+# releases (observed 50+ GB of orphaned layers on long-running installs; issue
+# #858). This runs only after a confirmed-successful recreate.
+#
+# Deliberately conservative for an offline-first appliance: we do NOT run
+# `docker system/image prune -a`, which would delete images for installed-but-
+# stopped Supply Depot / curated services and force a re-pull that fails with no
+# internet. Instead we (1) drop dangling layers and (2) remove only superseded
+# tags of the core services this updater manages (the images in compose.yml),
+# keeping the refs now in use. Optional/offline images are never touched.
+prune_old_images() {
+    write_status "pruning" 97 "Reclaiming disk from old images..."
+    log "Pruning superseded Docker images to reclaim disk space..."
+
+    # 1. Dangling (untagged) layers, e.g. the prior digest of a re-pulled moving
+    #    tag. Never referenced by any tag or container, so always safe to remove.
+    #    Docker prints "Total reclaimed space: X" to the log.
+    docker image prune -f >> "$LOG_FILE" 2>&1 || log "  WARNING: dangling image prune failed"
+
+    # 2. Superseded tags of compose-managed repositories only.
+    local in_use_raw in_use managed_repos
+    in_use_raw=$(docker compose -p "$COMPOSE_PROJECT_NAME" -f "$COMPOSE_FILE" config --images 2>/dev/null)
+    # `compose config --images` emits an untagged repo as a bare name, but
+    # `docker images` always reports `repo:tag`. Normalise bare refs to `:latest`
+    # so the in-use check below matches (otherwise the current image, e.g. the
+    # updater's own, looks superseded). Digest refs are left as-is.
+    in_use=$(echo "$in_use_raw" | while read -r r; do
+        [ -z "$r" ] && continue
+        case "${r##*/}" in
+            *:*|*@*) echo "$r" ;;
+            *) echo "$r:latest" ;;
+        esac
+    done | sort -u)
+    if [ -z "$in_use" ]; then
+        log "  Could not resolve in-use images from compose; skipped targeted cleanup"
+        log "Image cleanup complete"
+        return 0
+    fi
+
+    # Repositories we manage = the in-use refs with the tag/digest stripped off.
+    managed_repos=$(echo "$in_use" | sed 's/[:@].*$//' | sort -u)
+
+    while IFS=' ' read -r _img_id img_ref; do
+        [ -z "$img_ref" ] && continue
+        local repo="${img_ref%%:*}"
+        # Only touch repositories this updater manages.
+        echo "$managed_repos" | grep -qxF "$repo" || continue
+        # Keep any ref that is still in use by the current stack.
+        echo "$in_use" | grep -qxF "$img_ref" && continue
+        log "  Removing superseded image: $img_ref"
+        # No -f: docker refuses to remove an image still referenced by a
+        # container, which keeps us safe against removing anything in use.
+        docker rmi "$img_ref" >> "$LOG_FILE" 2>&1 || log "  Skipped $img_ref (still in use or removal failed)"
+    done < <(docker images --format '{{.ID}} {{.Repository}}:{{.Tag}}' | grep -v '<none>')
+
+    log "Image cleanup complete"
+}
+
 cleanup() {
     log "Update sidecar shutting down"
     exit 0