From 77fa7d19b47609c1216fc26d9d2fe018d300fa0a Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 3 Sep 2026 08:05:10 -0500 Subject: [PATCH] ci(runner): build paid artifacts once per campaign --- .github/workflows/runner-full-stack-e2e.yml | 258 ++++++++++++++++++-- tests/runner-e2e/daytona-image.test.ts | 2 +- tests/runner-e2e/workflow-security.test.ts | 54 +++- 3 files changed, 284 insertions(+), 30 deletions(-) diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.yml index 75aadd5945..caaad4c604 100644 --- a/.github/workflows/runner-full-stack-e2e.yml +++ b/.github/workflows/runner-full-stack-e2e.yml @@ -94,6 +94,9 @@ jobs: outputs: matrix: ${{ steps.catalog.outputs.matrix }} needs_daytona: ${{ steps.catalog.outputs.needs_daytona }} + needs_runner_typescript: ${{ steps.catalog.outputs.needs_runner_typescript }} + needs_native_binaries: ${{ steps.catalog.outputs.needs_native_binaries }} + needs_remote_provider_pack: ${{ steps.catalog.outputs.needs_remote_provider_pack }} execution_ids: ${{ steps.catalog.outputs.execution_ids }} max_parallel: ${{ steps.catalog.outputs.max_parallel }} daytona_image_content_id: ${{ steps.daytona_image_content.outputs.content_id }} @@ -173,9 +176,14 @@ jobs: args+=(--all) fi catalog_json="$(pnpm --silent test:e2e:runner -- "${args[@]}")" - echo "matrix=$(jq -c '{include: .include}' <<< "$catalog_json")" >> "$GITHUB_OUTPUT" - echo "needs_daytona=$(jq -r '.needsDaytona' <<< "$catalog_json")" >> "$GITHUB_OUTPUT" - echo "execution_ids=$(jq -c '.executionIds' <<< "$catalog_json")" >> "$GITHUB_OUTPUT" + { + echo "matrix=$(jq -c '{include: .include}' <<< "$catalog_json")" + echo "needs_daytona=$(jq -r '.needsDaytona' <<< "$catalog_json")" + echo "needs_runner_typescript=$(jq -r '[.include[] | select((.profileId == "runner-opencode") or (.profileId | startswith("runner-acpx-")) or (.suiteId == "openrouter-model-breadth"))] | length > 0' <<< "$catalog_json")" + echo "needs_native_binaries=$(jq -r '[.include[] | select((.profileId | startswith("runner-")) or (.suiteId == "openrouter-model-breadth"))] | length > 0' <<< "$catalog_json")" + echo "needs_remote_provider_pack=$(jq -r '[.include[] | select((.environmentId == "daytona") and ((.profileId == "runner-opencode") or (.profileId | startswith("runner-acpx-"))))] | length > 0' <<< "$catalog_json")" + echo "execution_ids=$(jq -c '.executionIds' <<< "$catalog_json")" + } >> "$GITHUB_OUTPUT" if ! [[ "$MAX_PARALLEL" =~ ^[1-9][0-9]*$ ]] || [ "$MAX_PARALLEL" -gt 57 ]; then echo "RUNNER_E2E_MAX_PARALLEL must be an integer from 1 through 57." >&2 exit 1 @@ -284,9 +292,181 @@ jobs: echo "source_revision=$source_revision" >> "$GITHUB_OUTPUT" echo "content_id=$published_content_id" >> "$GITHUB_OUTPUT" + build_runner_artifacts: + name: Build reusable runner campaign artifacts + needs: [authorize, catalog] + if: github.event_name != 'schedule' || vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 + with: + version: 9.15.4 + + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + cache: pnpm + + - run: pnpm install --frozen-lockfile + + # build:typescript also builds the eval-kernel dependency, so the two + # TypeScript trees are compiled at most once in this campaign. + - name: Build shared TypeScript and native runner outputs + env: + NEEDS_RUNNER_TYPESCRIPT: ${{ needs.catalog.outputs.needs_runner_typescript }} + NEEDS_NATIVE_BINARIES: ${{ needs.catalog.outputs.needs_native_binaries }} + run: | + set -euo pipefail + if [ "$NEEDS_RUNNER_TYPESCRIPT" = true ]; then + pnpm --filter @paperclipai/paperclip-runner build:typescript + else + pnpm --filter @paperclipai/paperclip-eval-kernel build + fi + if [ "$NEEDS_NATIVE_BINARIES" = true ]; then + pnpm --filter @paperclipai/paperclip-runner build:runner-binaries + fi + + - name: Package immutable campaign outputs + env: + NEEDS_RUNNER_TYPESCRIPT: ${{ needs.catalog.outputs.needs_runner_typescript }} + NEEDS_NATIVE_BINARIES: ${{ needs.catalog.outputs.needs_native_binaries }} + run: | + set -euo pipefail + binary_root="packages/paperclip-runner/runner/target/debug" + binaries=( + conformance-tracer + paperclip-runnerd + fake-harness + fake-codex-app-server + fake-acpx-sidecar + ) + archive_paths=( + packages/paperclip-eval-kernel/dist + ) + if [ "$NEEDS_RUNNER_TYPESCRIPT" = true ]; then + test -d packages/paperclip-runner/dist + archive_paths+=(packages/paperclip-runner/dist) + fi + if [ "$NEEDS_NATIVE_BINARIES" = true ]; then + for binary in "${binaries[@]}"; do + test -x "$binary_root/$binary" + archive_paths+=("$binary_root/$binary") + done + fi + tar --create --gzip \ + --file runner-e2e-build-bundle.tar.gz \ + "${archive_paths[@]}" + sha256sum runner-e2e-build-bundle.tar.gz > runner-e2e-build-bundle.tar.gz.sha256 + + - name: Upload immutable shared campaign outputs + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: runner-e2e-build-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }} + path: | + runner-e2e-build-bundle.tar.gz + runner-e2e-build-bundle.tar.gz.sha256 + retention-days: 1 + compression-level: 0 + if-no-files-found: error + + build_remote_provider_pack: + name: Build reusable remote provider pack + needs: [authorize, catalog, daytona_image, build_runner_artifacts] + if: github.event_name != 'schedule' || vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + steps: + - name: No remote provider pack needed + if: needs.catalog.outputs.needs_remote_provider_pack != 'true' + run: echo "Selected cells do not require a remote provider pack." + + - if: needs.catalog.outputs.needs_remote_provider_pack == 'true' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - if: needs.catalog.outputs.needs_remote_provider_pack == 'true' + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 + with: + version: 9.15.4 + + - if: needs.catalog.outputs.needs_remote_provider_pack == 'true' + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + cache: pnpm + + - if: needs.catalog.outputs.needs_remote_provider_pack == 'true' + run: pnpm install --frozen-lockfile + + - name: Download immutable shared campaign outputs + if: needs.catalog.outputs.needs_remote_provider_pack == 'true' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: runner-e2e-build-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }} + path: runner-e2e-build + + - name: Verify and restore shared TypeScript outputs + if: needs.catalog.outputs.needs_remote_provider_pack == 'true' + run: | + set -euo pipefail + ( + cd runner-e2e-build + sha256sum --check runner-e2e-build-bundle.tar.gz.sha256 + ) + tar --extract --gzip \ + --file runner-e2e-build/runner-e2e-build-bundle.tar.gz \ + --directory "$GITHUB_WORKSPACE" + test -d packages/paperclip-eval-kernel/dist + test -d packages/paperclip-runner/dist + + - name: Assemble native remote provider pack + if: needs.catalog.outputs.needs_remote_provider_pack == 'true' + env: + # A reused image can have an older source revision with the same + # content ID. Matching that revision lets remote execution reuse the + # verified pack already installed in the immutable image. + PAPERCLIP_RUNNER_SOURCE_REVISION: ${{ needs.daytona_image.outputs.source_revision }} + run: node packages/paperclip-runner/scripts/build-provider-pack.mjs packages/paperclip-runner/provider-pack + + - name: Package verified remote provider pack + if: needs.catalog.outputs.needs_remote_provider_pack == 'true' + env: + IMAGE_SOURCE_REVISION: ${{ needs.daytona_image.outputs.source_revision }} + run: | + set -euo pipefail + test -f packages/paperclip-runner/provider-pack/provider-pack.json + jq -e \ + --arg revision "$IMAGE_SOURCE_REVISION" \ + '.schema == "paperclip-runner/remote-provider-pack/v1" and + .payload.runnerSourceRevision == $revision and + (.digest | test("^sha256:[0-9a-f]{64}$"))' \ + packages/paperclip-runner/provider-pack/provider-pack.json >/dev/null + tar --create --gzip \ + --file runner-e2e-provider-pack.tar.gz \ + packages/paperclip-runner/provider-pack + sha256sum runner-e2e-provider-pack.tar.gz > runner-e2e-provider-pack.tar.gz.sha256 + + - name: Upload immutable remote provider pack + if: needs.catalog.outputs.needs_remote_provider_pack == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: runner-e2e-provider-pack-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }} + path: | + runner-e2e-provider-pack.tar.gz + runner-e2e-provider-pack.tar.gz.sha256 + retention-days: 1 + compression-level: 0 + if-no-files-found: error + test: name: ${{ matrix.executionId }} - needs: [catalog, daytona_image] + needs: [catalog, daytona_image, build_runner_artifacts, build_remote_provider_pack] runs-on: ubuntu-latest-m timeout-minutes: ${{ matrix.timeoutMinutes }} permissions: @@ -326,12 +506,59 @@ jobs: - run: pnpm install --frozen-lockfile - - name: Build runner TypeScript prerequisites - run: pnpm --filter @paperclipai/paperclip-eval-kernel build + - name: Download immutable campaign outputs + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: runner-e2e-build-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }} + path: runner-e2e-build - - name: Build local JS-backed provider artifacts - if: matrix.environmentId == 'local' && (matrix.profileId == 'runner-opencode' || startsWith(matrix.profileId, 'runner-acpx-') || matrix.suiteId == 'openrouter-model-breadth') - run: pnpm --filter @paperclipai/paperclip-runner build:typescript + - name: Download immutable remote provider pack + if: matrix.environmentId == 'daytona' && (matrix.profileId == 'runner-opencode' || startsWith(matrix.profileId, 'runner-acpx-')) + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: runner-e2e-provider-pack-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }} + path: runner-e2e-provider-pack + + - name: Verify and restore campaign outputs + env: + NEEDS_RUNNER_TYPESCRIPT: ${{ matrix.profileId == 'runner-opencode' || startsWith(matrix.profileId, 'runner-acpx-') || matrix.suiteId == 'openrouter-model-breadth' }} + NEEDS_NATIVE_BINARY: ${{ startsWith(matrix.profileId, 'runner-') || matrix.suiteId == 'openrouter-model-breadth' }} + run: | + set -euo pipefail + ( + cd runner-e2e-build + sha256sum --check runner-e2e-build-bundle.tar.gz.sha256 + ) + tar --extract --gzip \ + --file runner-e2e-build/runner-e2e-build-bundle.tar.gz \ + --directory "$GITHUB_WORKSPACE" + test -d packages/paperclip-eval-kernel/dist + if [ "$NEEDS_RUNNER_TYPESCRIPT" = true ]; then + test -d packages/paperclip-runner/dist + fi + if [ "$NEEDS_NATIVE_BINARY" = true ]; then + test -x packages/paperclip-runner/runner/target/debug/paperclip-runnerd + fi + + - name: Verify and restore remote provider pack + if: matrix.environmentId == 'daytona' && (matrix.profileId == 'runner-opencode' || startsWith(matrix.profileId, 'runner-acpx-')) + env: + IMAGE_SOURCE_REVISION: ${{ needs.daytona_image.outputs.source_revision }} + run: | + set -euo pipefail + ( + cd runner-e2e-provider-pack + sha256sum --check runner-e2e-provider-pack.tar.gz.sha256 + ) + tar --extract --gzip \ + --file runner-e2e-provider-pack/runner-e2e-provider-pack.tar.gz \ + --directory "$GITHUB_WORKSPACE" + jq -e \ + --arg revision "$IMAGE_SOURCE_REVISION" \ + '.schema == "paperclip-runner/remote-provider-pack/v1" and + .payload.runnerSourceRevision == $revision and + (.digest | test("^sha256:[0-9a-f]{64}$"))' \ + packages/paperclip-runner/provider-pack/provider-pack.json >/dev/null - name: Qualify local provider Node interpreter if: matrix.environmentId == 'local' && (matrix.profileId == 'runner-opencode' || startsWith(matrix.profileId, 'runner-acpx-') || matrix.suiteId == 'openrouter-model-breadth') @@ -345,23 +572,10 @@ jobs: } NODE - - name: Build native remote provider pack - if: matrix.environmentId == 'daytona' && (matrix.profileId == 'runner-opencode' || startsWith(matrix.profileId, 'runner-acpx-')) - env: - # Reused images retain the source revision that was embedded in their - # provider pack. Matching it here lets the server reuse that exact - # preinstalled pack instead of uploading a duplicate to the lease. - PAPERCLIP_RUNNER_SOURCE_REVISION: ${{ needs.daytona_image.outputs.source_revision }} - run: pnpm --filter @paperclipai/paperclip-runner build:provider-pack - - name: Install pinned legacy Claude CLI if: matrix.profileId == 'legacy-claude' run: npm install --global --omit=dev @anthropic-ai/claude-code@2.1.19 - - name: Build native runner binaries - if: startsWith(matrix.profileId, 'runner-') || matrix.suiteId == 'openrouter-model-breadth' - run: pnpm --filter @paperclipai/paperclip-runner build:runner-binaries - - name: Install Chromium run: | set -euo pipefail diff --git a/tests/runner-e2e/daytona-image.test.ts b/tests/runner-e2e/daytona-image.test.ts index 9aefe4163d..19211d331d 100644 --- a/tests/runner-e2e/daytona-image.test.ts +++ b/tests/runner-e2e/daytona-image.test.ts @@ -75,7 +75,7 @@ describe("runner E2E Daytona image contract", () => { expect(workflow).toContain('.Config.User == "daytona"'); expect(workflow).toContain("PAPERCLIP_RUNNER_PROVIDER_PACK_ROOT="); expect(workflow).toContain( - "pnpm --filter @paperclipai/paperclip-runner build:provider-pack", + "node packages/paperclip-runner/scripts/build-provider-pack.mjs packages/paperclip-runner/provider-pack", ); expect(workflow).toContain( "PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH: ${{ github.workspace }}/packages/paperclip-runner/provider-pack", diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts index f6509bca3b..99bce32fdf 100644 --- a/tests/runner-e2e/workflow-security.test.ts +++ b/tests/runner-e2e/workflow-security.test.ts @@ -117,20 +117,60 @@ describe("public repository paid workflow security", () => { } }); - it("prepares every local JS-backed provider before paid execution", async () => { + it("builds runner outputs once without provider credentials and verifies them in every paid cell", async () => { const workflow = await readFile( path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"), "utf8", ); - const jsBackedLocalCondition = - "matrix.environmentId == 'local' && (matrix.profileId == 'runner-opencode' || startsWith(matrix.profileId, 'runner-acpx-') || matrix.suiteId == 'openrouter-model-breadth')"; + const buildJobStart = workflow.indexOf(" build_runner_artifacts:"); + const testJobStart = workflow.indexOf(" test:", buildJobStart); + const reportJobStart = workflow.indexOf(" report:", testJobStart); + const buildJob = workflow.slice(buildJobStart, testJobStart); + const testJob = workflow.slice(testJobStart, reportJobStart); - expect(workflow).toContain("Build local JS-backed provider artifacts"); - expect(workflow).toContain("Qualify local provider Node interpreter"); - expect(workflow.split(jsBackedLocalCondition)).toHaveLength(3); - expect(workflow).toContain( + expect(buildJobStart).toBeGreaterThan(0); + expect(testJobStart).toBeGreaterThan(buildJobStart); + expect(buildJob).toContain("needs: [authorize, catalog]"); + expect(buildJob).toContain( + "needs: [authorize, catalog, daytona_image, build_runner_artifacts]", + ); + expect(buildJob).not.toContain("environment:"); + expect(buildJob).not.toContain("secrets."); + expect(buildJob).toContain( "pnpm --filter @paperclipai/paperclip-runner build:typescript", ); + expect(buildJob).toContain( + "pnpm --filter @paperclipai/paperclip-runner build:runner-binaries", + ); + expect(buildJob).toContain( + "node packages/paperclip-runner/scripts/build-provider-pack.mjs", + ); + expect(buildJob).toContain("runner-e2e-build-bundle.tar.gz.sha256"); + expect(buildJob).toContain("runner-e2e-provider-pack.tar.gz.sha256"); + expect(buildJob).toContain( + "runner-e2e-build-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}", + ); + expect(workflow).toContain("needs_runner_typescript="); + expect(workflow).toContain("needs_native_binaries="); + expect(workflow).toContain("needs_remote_provider_pack="); + + expect(testJob).toContain( + "needs: [catalog, daytona_image, build_runner_artifacts, build_remote_provider_pack]", + ); + expect(testJob).toContain("Download immutable campaign outputs"); + expect(testJob).toContain("Download immutable remote provider pack"); + expect(testJob).toContain("sha256sum --check"); + expect(testJob.indexOf("sha256sum --check")).toBeLessThan( + testJob.indexOf("tar --extract"), + ); + expect(testJob).toContain( + "test -x packages/paperclip-runner/runner/target/debug/paperclip-runnerd", + ); + expect(testJob).toContain(".payload.runnerSourceRevision == $revision"); + expect(workflow).toContain("Qualify local provider Node interpreter"); + expect(testJob).not.toContain("build:typescript"); + expect(testJob).not.toContain("build:runner-binaries"); + expect(testJob).not.toContain("build-provider-pack.mjs"); }); it("uses environment-scoped OIDC for a no-delete history publisher", async () => {