diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.yml index 607d538c41..3250469823 100644 --- a/.github/workflows/runner-full-stack-e2e.yml +++ b/.github/workflows/runner-full-stack-e2e.yml @@ -409,7 +409,10 @@ jobs: NEEDS_DAYTONA: ${{ needs.catalog.outputs.needs_daytona }} IMAGE_CONTENT_ID: ${{ needs.catalog.outputs.daytona_image_content_id }} IMAGE_TAG: ghcr.io/paperclipai/paperclip-daytona-runner:e2e-content-${{ needs.catalog.outputs.daytona_image_content_id }} + IMAGE_CACHE: ghcr.io/paperclipai/paperclip-daytona-runner:e2e-buildcache-amd64 TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} + TARGET_REF: ${{ needs.authorize.outputs.target_ref }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} run: | set -euo pipefail if [ "$NEEDS_DAYTONA" != true ]; then @@ -425,12 +428,24 @@ jobs: if docker buildx imagetools inspect "$IMAGE_TAG" >/dev/null 2>&1; then digest="$(docker buildx imagetools inspect "$IMAGE_TAG" --format '{{json .Manifest.Digest}}' | tr -d '"')" else + cache_args=( + --cache-from "type=registry,ref=${IMAGE_CACHE}" + ) + if [ "$TARGET_REF" = "refs/heads/$DEFAULT_BRANCH" ]; then + cache_args+=( + --cache-to "type=registry,ref=${IMAGE_CACHE},mode=max" + ) + echo '::notice title=Daytona image cache::Publishing cache from the trusted default-branch target' + else + echo '::notice title=Daytona image cache::Using the default-branch cache without publishing development-branch layers' + fi docker buildx build \ --platform linux/amd64 \ --build-arg "PAPERCLIP_RUNNER_CONTENT_ID=${IMAGE_CONTENT_ID}" \ --build-arg "PAPERCLIP_RUNNER_SOURCE_REVISION=${TARGET_SHA}" \ --file docker/daytona-runner/Dockerfile \ --tag "$IMAGE_TAG" \ + "${cache_args[@]}" \ --push \ . digest="$(docker buildx imagetools inspect "$IMAGE_TAG" --format '{{json .Manifest.Digest}}' | tr -d '"')" diff --git a/docker/daytona-runner/Dockerfile b/docker/daytona-runner/Dockerfile index d5157e9d1c..87d819f49c 100644 --- a/docker/daytona-runner/Dockerfile +++ b/docker/daytona-runner/Dockerfile @@ -13,24 +13,17 @@ RUN cargo build --locked --release -p paperclip-runner-core --bin paperclip-runn && strip /workspace/packages/paperclip-runner/runner/target/release/paperclip-runnerd FROM node:24-bookworm@sha256:9137a20e25879e0b557227b57e3ee4e9af4bde29eb3db66134cd1723e84f830b AS provider-pack-build -ARG PAPERCLIP_RUNNER_SOURCE_REVISION -RUN test -n "${PAPERCLIP_RUNNER_SOURCE_REVISION}" RUN corepack enable && corepack prepare pnpm@9.15.4 --activate WORKDIR /workspace COPY package.json pnpm-lock.yaml pnpm-workspace.yaml .npmrc tsconfig.base.json ./ COPY patches ./patches COPY scripts/link-plugin-dev-sdk.mjs ./scripts/link-plugin-dev-sdk.mjs -COPY \ - packages/paperclip-eval-kernel/package.json \ - packages/paperclip-eval-kernel/tsconfig.json \ - ./packages/paperclip-eval-kernel/ +COPY packages/paperclip-eval-kernel/package.json ./packages/paperclip-eval-kernel/package.json +COPY packages/paperclip-runner/package.json ./packages/paperclip-runner/package.json +RUN pnpm install --frozen-lockfile --filter '@paperclipai/paperclip-runner...' +COPY packages/paperclip-eval-kernel/tsconfig.json ./packages/paperclip-eval-kernel/tsconfig.json COPY packages/paperclip-eval-kernel/src ./packages/paperclip-eval-kernel/src -COPY \ - packages/paperclip-runner/package.json \ - packages/paperclip-runner/styles.css \ - packages/paperclip-runner/tsconfig.json \ - packages/paperclip-runner/tsconfig.surfaces.json \ - ./packages/paperclip-runner/ +COPY packages/paperclip-runner/styles.css packages/paperclip-runner/tsconfig.json packages/paperclip-runner/tsconfig.surfaces.json ./packages/paperclip-runner/ COPY packages/paperclip-runner/protocol ./packages/paperclip-runner/protocol COPY packages/paperclip-runner/runner/crates/runner-core/src/generated_acpx_sidecar_contract.rs ./packages/paperclip-runner/runner/crates/runner-core/src/generated_acpx_sidecar_contract.rs COPY \ @@ -41,7 +34,8 @@ COPY \ packages/paperclip-runner/scripts/generate-protocol-schema-module.mjs \ ./packages/paperclip-runner/scripts/ COPY packages/paperclip-runner/src ./packages/paperclip-runner/src -RUN pnpm install --frozen-lockfile --filter '@paperclipai/paperclip-runner...' +ARG PAPERCLIP_RUNNER_SOURCE_REVISION +RUN test -n "${PAPERCLIP_RUNNER_SOURCE_REVISION}" RUN pnpm --filter @paperclipai/paperclip-runner build:typescript \ && PAPERCLIP_RUNNER_SOURCE_REVISION="${PAPERCLIP_RUNNER_SOURCE_REVISION}" \ node packages/paperclip-runner/scripts/build-provider-pack.mjs /provider-pack @@ -51,13 +45,6 @@ RUN pnpm --filter @paperclipai/paperclip-runner build:typescript \ # runner-specific addition is /usr/local/bin/paperclip-runnerd below. FROM daytonaio/sandbox:0.8.0@sha256:eadf88e4391072b7ad4bed27d9cadfc9fe9d8ed375d9219d34c2ccb518f213e3 -ARG PAPERCLIP_RUNNER_CONTENT_ID -ARG PAPERCLIP_RUNNER_SOURCE_REVISION -RUN test -n "${PAPERCLIP_RUNNER_CONTENT_ID}" \ - && test -n "${PAPERCLIP_RUNNER_SOURCE_REVISION}" -LABEL io.paperclip.runner.content-id="${PAPERCLIP_RUNNER_CONTENT_ID}" \ - org.opencontainers.image.revision="${PAPERCLIP_RUNNER_SOURCE_REVISION}" - USER root ENV PAPERCLIP_RUNNER_PROVIDER_PACK_ROOT=/opt/paperclip-runner/provider-pack @@ -104,6 +91,14 @@ RUN set -eu; \ COPY --from=runnerd-build /workspace/packages/paperclip-runner/runner/target/release/paperclip-runnerd /usr/local/bin/paperclip-runnerd COPY --from=provider-pack-build /provider-pack /opt/paperclip-runner/provider-pack +# Keep revision-dependent metadata below the stable agent CLI installation +# layers. A source-only image miss can then reuse those expensive layers from +# the trusted registry cache. +ARG PAPERCLIP_RUNNER_CONTENT_ID +ARG PAPERCLIP_RUNNER_SOURCE_REVISION +RUN test -n "${PAPERCLIP_RUNNER_CONTENT_ID}" \ + && test -n "${PAPERCLIP_RUNNER_SOURCE_REVISION}" + RUN set -eu; \ chmod -R a+rX /opt/paperclip-runner/provider-pack; \ printf '%s\n' 'export PATH=/opt/paperclip-runner/provider-pack/node_modules/.bin:$PATH' \ @@ -132,3 +127,6 @@ RUN /bin/sh -lc 'set -eu; \ test "$(acpx --version)" = "0.13.1"; \ test "$(claude-agent-acp --version)" = "0.70.0"; \ test "$(codex-acp --version)" = "@agentclientprotocol/codex-acp 1.6.2"' + +LABEL io.paperclip.runner.content-id="${PAPERCLIP_RUNNER_CONTENT_ID}" \ + org.opencontainers.image.revision="${PAPERCLIP_RUNNER_SOURCE_REVISION}" diff --git a/tests/runner-e2e/daytona-image.test.ts b/tests/runner-e2e/daytona-image.test.ts index ef567ddf28..70fc7e0556 100644 --- a/tests/runner-e2e/daytona-image.test.ts +++ b/tests/runner-e2e/daytona-image.test.ts @@ -96,6 +96,18 @@ describe("runner E2E Daytona image contract", () => { expect(workflow).toContain( '--build-arg "PAPERCLIP_RUNNER_CONTENT_ID=${IMAGE_CONTENT_ID}"', ); + expect(workflow).toContain( + "IMAGE_CACHE: ghcr.io/paperclipai/paperclip-daytona-runner:e2e-buildcache-amd64", + ); + expect(workflow).toContain( + '--cache-from "type=registry,ref=${IMAGE_CACHE}"', + ); + expect(workflow).toContain( + '--cache-to "type=registry,ref=${IMAGE_CACHE},mode=max"', + ); + expect(workflow).toContain( + 'if [ "$TARGET_REF" = "refs/heads/$DEFAULT_BRANCH" ]; then', + ); expect(workflow).not.toContain("e2e-git-${{ github.sha }}"); expect(workflow).toContain("cosign sign --yes"); expect(workflow).toContain("docker logout ghcr.io"); @@ -124,6 +136,24 @@ describe("runner E2E Daytona image contract", () => { expect(workflow.indexOf("docker logout ghcr.io")).toBeLessThan( workflow.indexOf(`--format '{{json .Image}}'`), ); + const providerInstall = dockerfile.indexOf( + "RUN pnpm install --frozen-lockfile --filter '@paperclipai/paperclip-runner...'", + ); + const runnerSourceCopy = dockerfile.indexOf( + "COPY packages/paperclip-runner/src ./packages/paperclip-runner/src", + ); + const providerRevisionArg = dockerfile.indexOf( + "ARG PAPERCLIP_RUNNER_SOURCE_REVISION", + ); + const cliInstall = dockerfile.indexOf("RUN npm install -g"); + const finalMetadataArgs = dockerfile.lastIndexOf( + "ARG PAPERCLIP_RUNNER_CONTENT_ID", + ); + expect(providerInstall).toBeGreaterThan(0); + expect(providerInstall).toBeLessThan(runnerSourceCopy); + expect(providerInstall).toBeLessThan(providerRevisionArg); + expect(cliInstall).toBeGreaterThan(0); + expect(cliInstall).toBeLessThan(finalMetadataArgs); }); it("hashes the audited image dependency closure rather than the repository revision", async () => { @@ -190,7 +220,10 @@ describe("runner E2E Daytona image contract", () => { "FROM pinned\n", ); await writeFile(path.join(root, "package.json"), '{"private":true}\n'); - await writeFile(path.join(root, "pnpm-lock.yaml"), "lockfileVersion: 9\n"); + await writeFile( + path.join(root, "pnpm-lock.yaml"), + "lockfileVersion: 9\n", + ); await writeFile( path.join(root, "packages/paperclip-runner/package.json"), '{"name":"@paperclipai/paperclip-runner"}\n', @@ -225,9 +258,7 @@ describe("runner E2E Daytona image contract", () => { path.join(root, "unrelated.txt"), "does not enter the image\n", ); - expect( - await computeDaytonaImageContentId(options), - ).toBe(baseline); + expect(await computeDaytonaImageContentId(options)).toBe(baseline); for (const relativePath of [ "docker/daytona-runner/Dockerfile", diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts index c0c3b38cc9..aeceafb6db 100644 --- a/tests/runner-e2e/workflow-security.test.ts +++ b/tests/runner-e2e/workflow-security.test.ts @@ -352,6 +352,30 @@ describe("public repository paid workflow security", () => { ); expect(daytonaImageJob).toContain('echo "source_revision="'); expect(daytonaImageJob).toContain('echo "content_id="'); + expect(daytonaImageJob).toContain( + "IMAGE_CACHE: ghcr.io/paperclipai/paperclip-daytona-runner:e2e-buildcache-amd64", + ); + expect(daytonaImageJob).toContain( + "TARGET_REF: ${{ needs.authorize.outputs.target_ref }}", + ); + expect(daytonaImageJob).toContain( + "DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}", + ); + const cacheRead = daytonaImageJob.indexOf( + '--cache-from "type=registry,ref=${IMAGE_CACHE}"', + ); + const trustedTargetCheck = daytonaImageJob.indexOf( + 'if [ "$TARGET_REF" = "refs/heads/$DEFAULT_BRANCH" ]; then', + ); + const cacheWrite = daytonaImageJob.indexOf( + '--cache-to "type=registry,ref=${IMAGE_CACHE},mode=max"', + ); + expect(cacheRead).toBeGreaterThan(0); + expect(trustedTargetCheck).toBeGreaterThan(cacheRead); + expect(cacheWrite).toBeGreaterThan(trustedTargetCheck); + expect(daytonaImageJob.slice(trustedTargetCheck, cacheWrite)).not.toContain( + "secrets.", + ); const targetCodeJobs = [ fullStack.slice( fullStack.indexOf(" catalog:"),