From 59d74b68b25b7350a56e31ee6e49ccbdaaac515d Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Thu, 10 Sep 2026 20:36:55 -0700 Subject: [PATCH] ci: cache the native Runner in a separate Docker stage (#13195) Compile the native Runner from its complete Cargo and protocol inputs in a separate cached Docker stage. Preserve Cargo validation and generated-contract checks during the normal application build, normalize input timestamps across checkouts, and compile the isolated target in PR CI. Co-Authored-By: Paperclip --- .github/workflows/docker-runner-check.yml | 37 +++++++++++++++++++++++ Dockerfile | 25 ++++++++++++++- doc/DOCKER.md | 22 ++++++++++++++ 3 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/docker-runner-check.yml diff --git a/.github/workflows/docker-runner-check.yml b/.github/workflows/docker-runner-check.yml new file mode 100644 index 0000000000..e9705c3b5e --- /dev/null +++ b/.github/workflows/docker-runner-check.yml @@ -0,0 +1,37 @@ +name: Docker Runner check + +on: + pull_request: + paths: + - .github/workflows/docker-runner-check.yml + - Dockerfile + - .dockerignore + - packages/paperclip-runner/rust-toolchain.toml + - packages/paperclip-runner/runner/** + - packages/paperclip-runner/protocol/** + +permissions: {} + +concurrency: + group: docker-runner-check-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + runner: + name: Compile isolated native Runner + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4 + # Compile the real target with the real .dockerignore. This catches new + # Cargo or embedded protocol inputs that the isolated COPY set omits. + # No registry credentials, cache imports/exports, or image publication. + - name: Compile the Runner from its isolated Docker context + run: docker buildx build --target runner-build --progress plain . diff --git a/Dockerfile b/Dockerfile index 901289778e..b349251238 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,7 +51,7 @@ COPY scripts/link-plugin-dev-sdk.mjs scripts/ RUN pnpm install --frozen-lockfile -FROM base AS build +FROM base AS rust-toolchain WORKDIR /app # Debian's packaged rust lags the ecosystem (trixie ships 1.85) and the # runner's dependency tree now requires a newer rustc. Install rustup from a @@ -83,8 +83,31 @@ RUN set -eux; \ chmod +x /tmp/rustup-init; \ /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain none; \ rm /tmp/rustup-init +# Install the package-owned compiler before any application source enters the +# stage. rustup-init above installs rustup itself, not the selected compiler. +COPY packages/paperclip-runner/rust-toolchain.toml /tmp/runner-toolchain/rust-toolchain.toml +RUN cd /tmp/runner-toolchain && rustup show + +FROM rust-toolchain AS runner-build +WORKDIR /app/packages/paperclip-runner +# Rust embeds protocol schemas and fixtures with include_str!. Keep those +# alongside the complete Cargo workspace so every compile-time input keys +# this layer. Ordinary server/UI edits can then reuse the native build. +COPY packages/paperclip-runner/rust-toolchain.toml ./ +COPY packages/paperclip-runner/runner ./runner +COPY packages/paperclip-runner/protocol ./protocol +# Cargo fingerprints source mtimes. Normalize them here and after the full +# source copy below so a fresh checkout cannot invalidate unchanged inputs. +RUN find runner protocol -type f -exec touch -d @0 {} + \ + && touch -d @0 rust-toolchain.toml \ + && cargo build --release --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --bin paperclip-runnerd + +FROM runner-build AS build +WORKDIR /app COPY --from=deps /app /app COPY . . +RUN find packages/paperclip-runner/runner packages/paperclip-runner/protocol -type f -exec touch -d @0 {} + \ + && touch -d @0 packages/paperclip-runner/rust-toolchain.toml RUN pnpm --filter @paperclipai/ui build RUN pnpm --filter @paperclipai/plugin-sdk build # The server build runs scripts/write-build-stamp.mjs, which stamps the built diff --git a/doc/DOCKER.md b/doc/DOCKER.md index c89664fd0a..e22aea05ed 100644 --- a/doc/DOCKER.md +++ b/doc/DOCKER.md @@ -316,3 +316,25 @@ Notes: - The `docker-entrypoint.sh` adjusts the container `node` user UID/GID at startup to match the values passed via `USER_UID`/`USER_GID`, avoiding permission issues on bind-mounted volumes. - Paperclip data persists via Docker volumes/bind mounts (compose) or at `~/.local/share/paperclip` (quadlet). + +## Native Runner build cache + +The image compiles the native Runner in `runner-build`, before copying the +application source. That stage includes the pinned Rust compiler, the complete +Cargo workspace and lockfile, and the protocol schemas and fixtures embedded +by Rust. Changes to those inputs rebuild the native binary. Ordinary server or +UI changes can reuse it through the existing registry cache (`mode=max`). Each +platform gets its own native build; no cross-architecture binary is reused. + +The application build inherits that stage and still runs the normal server +build, including Cargo, binary staging, and generated-contract checks. Rust +input file times are normalized in both stages so fresh checkouts do not force +Cargo to rebuild unchanged source. Changes made by build scripts still reach +Cargo's normal validation. The final application copy excludes Cargo's target +directory as before. Cache misses only cost compilation time. + +Pull requests that change the Dockerfile, Docker ignore rules, or Runner native +inputs also build the isolated `runner-build` target in `Docker Runner check`. +This compiles against the actual reduced context and catches missing embedded +inputs before the post-merge image build. It uses a GitHub-hosted runner with +read-only repository access and does not publish images or cache artifacts.