RomanNum3ral
/
paperclip
mirror of https://github.com/paperclipai/paperclip.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ci(runner): build paid artifacts once per campaign

This commit is contained in:
Dotta 2026-09-03 08:05:10 -05:00
parent 23834d77fe
commit 77fa7d19b4
3 changed files with 284 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

258
.github/workflows/runner-full-stack-e2e.yml vendored
View File

@ -94,6 +94,9 @@ jobs:
outputs:
matrix: ${{ steps.catalog.outputs.matrix }}
needs_daytona: ${{ steps.catalog.outputs.needs_daytona }}
needs_runner_typescript: ${{ steps.catalog.outputs.needs_runner_typescript }}
needs_native_binaries: ${{ steps.catalog.outputs.needs_native_binaries }}
needs_remote_provider_pack: ${{ steps.catalog.outputs.needs_remote_provider_pack }}
execution_ids: ${{ steps.catalog.outputs.execution_ids }}
max_parallel: ${{ steps.catalog.outputs.max_parallel }}
daytona_image_content_id: ${{ steps.daytona_image_content.outputs.content_id }}
@ -173,9 +176,14 @@ jobs:
args+=(--all)
fi
catalog_json="$(pnpm --silent test:e2e:runner -- "${args[@]}")"
echo "matrix=$(jq -c '{include: .include}' <<< "$catalog_json")" >> "$GITHUB_OUTPUT"
echo "needs_daytona=$(jq -r '.needsDaytona' <<< "$catalog_json")" >> "$GITHUB_OUTPUT"
echo "execution_ids=$(jq -c '.executionIds' <<< "$catalog_json")" >> "$GITHUB_OUTPUT"
{
echo "matrix=$(jq -c '{include: .include}' <<< "$catalog_json")"
echo "needs_daytona=$(jq -r '.needsDaytona' <<< "$catalog_json")"
echo "needs_runner_typescript=$(jq -r '[.include[] | select((.profileId == "runner-opencode") or (.profileId | startswith("runner-acpx-")) or (.suiteId == "openrouter-model-breadth"))] | length > 0' <<< "$catalog_json")"
echo "needs_native_binaries=$(jq -r '[.include[] | select((.profileId | startswith("runner-")) or (.suiteId == "openrouter-model-breadth"))] | length > 0' <<< "$catalog_json")"
echo "needs_remote_provider_pack=$(jq -r '[.include[] | select((.environmentId == "daytona") and ((.profileId == "runner-opencode") or (.profileId | startswith("runner-acpx-"))))] | length > 0' <<< "$catalog_json")"
echo "execution_ids=$(jq -c '.executionIds' <<< "$catalog_json")"
} >> "$GITHUB_OUTPUT"
if ! [[ "$MAX_PARALLEL" =~ ^[1-9][0-9]*$ ]] || [ "$MAX_PARALLEL" -gt 57 ]; then
echo "RUNNER_E2E_MAX_PARALLEL must be an integer from 1 through 57." >&2
exit 1
@ -284,9 +292,181 @@ jobs:
echo "source_revision=$source_revision" >> "$GITHUB_OUTPUT"
echo "content_id=$published_content_id" >> "$GITHUB_OUTPUT"
build_runner_artifacts:
name: Build reusable runner campaign artifacts
needs: [authorize, catalog]
if: github.event_name != 'schedule' || vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true'
runs-on: ubuntu-latest
timeout-minutes: 20
permissions:
contents: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
with:
version: 9.15.4
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
cache: pnpm
- run: pnpm install --frozen-lockfile
# build:typescript also builds the eval-kernel dependency, so the two
# TypeScript trees are compiled at most once in this campaign.
- name: Build shared TypeScript and native runner outputs
env:
NEEDS_RUNNER_TYPESCRIPT: ${{ needs.catalog.outputs.needs_runner_typescript }}
NEEDS_NATIVE_BINARIES: ${{ needs.catalog.outputs.needs_native_binaries }}
run: |
set -euo pipefail
if [ "$NEEDS_RUNNER_TYPESCRIPT" = true ]; then
pnpm --filter @paperclipai/paperclip-runner build:typescript
else
pnpm --filter @paperclipai/paperclip-eval-kernel build
fi
if [ "$NEEDS_NATIVE_BINARIES" = true ]; then
pnpm --filter @paperclipai/paperclip-runner build:runner-binaries
fi
- name: Package immutable campaign outputs
env:
NEEDS_RUNNER_TYPESCRIPT: ${{ needs.catalog.outputs.needs_runner_typescript }}
NEEDS_NATIVE_BINARIES: ${{ needs.catalog.outputs.needs_native_binaries }}
run: |
set -euo pipefail
binary_root="packages/paperclip-runner/runner/target/debug"
binaries=(
conformance-tracer
paperclip-runnerd
fake-harness
fake-codex-app-server
fake-acpx-sidecar
)
archive_paths=(
packages/paperclip-eval-kernel/dist
)
if [ "$NEEDS_RUNNER_TYPESCRIPT" = true ]; then
test -d packages/paperclip-runner/dist
archive_paths+=(packages/paperclip-runner/dist)
fi
if [ "$NEEDS_NATIVE_BINARIES" = true ]; then
for binary in "${binaries[@]}"; do
test -x "$binary_root/$binary"
archive_paths+=("$binary_root/$binary")
done
fi
tar --create --gzip \
--file runner-e2e-build-bundle.tar.gz \
"${archive_paths[@]}"
sha256sum runner-e2e-build-bundle.tar.gz > runner-e2e-build-bundle.tar.gz.sha256
- name: Upload immutable shared campaign outputs
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: runner-e2e-build-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
path: |
runner-e2e-build-bundle.tar.gz
runner-e2e-build-bundle.tar.gz.sha256
retention-days: 1
compression-level: 0
if-no-files-found: error
build_remote_provider_pack:
name: Build reusable remote provider pack
needs: [authorize, catalog, daytona_image, build_runner_artifacts]
if: github.event_name != 'schedule' || vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true'
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
contents: read
steps:
- name: No remote provider pack needed
if: needs.catalog.outputs.needs_remote_provider_pack != 'true'
run: echo "Selected cells do not require a remote provider pack."
- if: needs.catalog.outputs.needs_remote_provider_pack == 'true'
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- if: needs.catalog.outputs.needs_remote_provider_pack == 'true'
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
with:
version: 9.15.4
- if: needs.catalog.outputs.needs_remote_provider_pack == 'true'
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
cache: pnpm
- if: needs.catalog.outputs.needs_remote_provider_pack == 'true'
run: pnpm install --frozen-lockfile
- name: Download immutable shared campaign outputs
if: needs.catalog.outputs.needs_remote_provider_pack == 'true'
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
name: runner-e2e-build-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
path: runner-e2e-build
- name: Verify and restore shared TypeScript outputs
if: needs.catalog.outputs.needs_remote_provider_pack == 'true'
run: |
set -euo pipefail
(
cd runner-e2e-build
sha256sum --check runner-e2e-build-bundle.tar.gz.sha256
)
tar --extract --gzip \
--file runner-e2e-build/runner-e2e-build-bundle.tar.gz \
--directory "$GITHUB_WORKSPACE"
test -d packages/paperclip-eval-kernel/dist
test -d packages/paperclip-runner/dist
- name: Assemble native remote provider pack
if: needs.catalog.outputs.needs_remote_provider_pack == 'true'
env:
# A reused image can have an older source revision with the same
# content ID. Matching that revision lets remote execution reuse the
# verified pack already installed in the immutable image.
PAPERCLIP_RUNNER_SOURCE_REVISION: ${{ needs.daytona_image.outputs.source_revision }}
run: node packages/paperclip-runner/scripts/build-provider-pack.mjs packages/paperclip-runner/provider-pack
- name: Package verified remote provider pack
if: needs.catalog.outputs.needs_remote_provider_pack == 'true'
env:
IMAGE_SOURCE_REVISION: ${{ needs.daytona_image.outputs.source_revision }}
run: |
set -euo pipefail
test -f packages/paperclip-runner/provider-pack/provider-pack.json
jq -e \
--arg revision "$IMAGE_SOURCE_REVISION" \
'.schema == "paperclip-runner/remote-provider-pack/v1" and
.payload.runnerSourceRevision == $revision and
(.digest | test("^sha256:[0-9a-f]{64}$"))' \
packages/paperclip-runner/provider-pack/provider-pack.json >/dev/null
tar --create --gzip \
--file runner-e2e-provider-pack.tar.gz \
packages/paperclip-runner/provider-pack
sha256sum runner-e2e-provider-pack.tar.gz > runner-e2e-provider-pack.tar.gz.sha256
- name: Upload immutable remote provider pack
if: needs.catalog.outputs.needs_remote_provider_pack == 'true'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: runner-e2e-provider-pack-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
path: |
runner-e2e-provider-pack.tar.gz
runner-e2e-provider-pack.tar.gz.sha256
retention-days: 1
compression-level: 0
if-no-files-found: error
test:
name: ${{ matrix.executionId }}
needs: [catalog, daytona_image]
needs: [catalog, daytona_image, build_runner_artifacts, build_remote_provider_pack]
runs-on: ubuntu-latest-m
timeout-minutes: ${{ matrix.timeoutMinutes }}
permissions:
@ -326,12 +506,59 @@ jobs:
- run: pnpm install --frozen-lockfile
- name: Build runner TypeScript prerequisites
run: pnpm --filter @paperclipai/paperclip-eval-kernel build
- name: Download immutable campaign outputs
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
name: runner-e2e-build-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
path: runner-e2e-build
- name: Build local JS-backed provider artifacts
if: matrix.environmentId == 'local' && (matrix.profileId == 'runner-opencode' || startsWith(matrix.profileId, 'runner-acpx-') || matrix.suiteId == 'openrouter-model-breadth')
run: pnpm --filter @paperclipai/paperclip-runner build:typescript
- name: Download immutable remote provider pack
if: matrix.environmentId == 'daytona' && (matrix.profileId == 'runner-opencode' || startsWith(matrix.profileId, 'runner-acpx-'))
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
name: runner-e2e-provider-pack-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
path: runner-e2e-provider-pack
- name: Verify and restore campaign outputs
env:
NEEDS_RUNNER_TYPESCRIPT: ${{ matrix.profileId == 'runner-opencode' || startsWith(matrix.profileId, 'runner-acpx-') || matrix.suiteId == 'openrouter-model-breadth' }}
NEEDS_NATIVE_BINARY: ${{ startsWith(matrix.profileId, 'runner-') || matrix.suiteId == 'openrouter-model-breadth' }}
run: |
set -euo pipefail
(
cd runner-e2e-build
sha256sum --check runner-e2e-build-bundle.tar.gz.sha256
)
tar --extract --gzip \
--file runner-e2e-build/runner-e2e-build-bundle.tar.gz \
--directory "$GITHUB_WORKSPACE"
test -d packages/paperclip-eval-kernel/dist
if [ "$NEEDS_RUNNER_TYPESCRIPT" = true ]; then
test -d packages/paperclip-runner/dist
fi
if [ "$NEEDS_NATIVE_BINARY" = true ]; then
test -x packages/paperclip-runner/runner/target/debug/paperclip-runnerd
fi
- name: Verify and restore remote provider pack
if: matrix.environmentId == 'daytona' && (matrix.profileId == 'runner-opencode' || startsWith(matrix.profileId, 'runner-acpx-'))
env:
IMAGE_SOURCE_REVISION: ${{ needs.daytona_image.outputs.source_revision }}
run: |
set -euo pipefail
(
cd runner-e2e-provider-pack
sha256sum --check runner-e2e-provider-pack.tar.gz.sha256
)
tar --extract --gzip \
--file runner-e2e-provider-pack/runner-e2e-provider-pack.tar.gz \
--directory "$GITHUB_WORKSPACE"
jq -e \
--arg revision "$IMAGE_SOURCE_REVISION" \
'.schema == "paperclip-runner/remote-provider-pack/v1" and
.payload.runnerSourceRevision == $revision and
(.digest | test("^sha256:[0-9a-f]{64}$"))' \
packages/paperclip-runner/provider-pack/provider-pack.json >/dev/null
- name: Qualify local provider Node interpreter
if: matrix.environmentId == 'local' && (matrix.profileId == 'runner-opencode' || startsWith(matrix.profileId, 'runner-acpx-') || matrix.suiteId == 'openrouter-model-breadth')
@ -345,23 +572,10 @@ jobs:
}
NODE
- name: Build native remote provider pack
if: matrix.environmentId == 'daytona' && (matrix.profileId == 'runner-opencode' || startsWith(matrix.profileId, 'runner-acpx-'))
env:
# Reused images retain the source revision that was embedded in their
# provider pack. Matching it here lets the server reuse that exact
# preinstalled pack instead of uploading a duplicate to the lease.
PAPERCLIP_RUNNER_SOURCE_REVISION: ${{ needs.daytona_image.outputs.source_revision }}
run: pnpm --filter @paperclipai/paperclip-runner build:provider-pack
- name: Install pinned legacy Claude CLI
if: matrix.profileId == 'legacy-claude'
run: npm install --global --omit=dev @anthropic-ai/claude-code@2.1.19
- name: Build native runner binaries
if: startsWith(matrix.profileId, 'runner-') || matrix.suiteId == 'openrouter-model-breadth'
run: pnpm --filter @paperclipai/paperclip-runner build:runner-binaries
- name: Install Chromium
run: |
set -euo pipefail

2
tests/runner-e2e/daytona-image.test.ts
View File

@ -75,7 +75,7 @@ describe("runner E2E Daytona image contract", () => {
expect(workflow).toContain('.Config.User == "daytona"');
expect(workflow).toContain("PAPERCLIP_RUNNER_PROVIDER_PACK_ROOT=");
expect(workflow).toContain(
"pnpm --filter @paperclipai/paperclip-runner build:provider-pack",
"node packages/paperclip-runner/scripts/build-provider-pack.mjs packages/paperclip-runner/provider-pack",
);
expect(workflow).toContain(
"PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH: ${{ github.workspace }}/packages/paperclip-runner/provider-pack",

54
tests/runner-e2e/workflow-security.test.ts
View File

@ -117,20 +117,60 @@ describe("public repository paid workflow security", () => {
}
});
it("prepares every local JS-backed provider before paid execution", async () => {
it("builds runner outputs once without provider credentials and verifies them in every paid cell", async () => {
const workflow = await readFile(
path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"),
"utf8",
);
const jsBackedLocalCondition =
"matrix.environmentId == 'local' && (matrix.profileId == 'runner-opencode' || startsWith(matrix.profileId, 'runner-acpx-') || matrix.suiteId == 'openrouter-model-breadth')";
const buildJobStart = workflow.indexOf(" build_runner_artifacts:");
const testJobStart = workflow.indexOf(" test:", buildJobStart);
const reportJobStart = workflow.indexOf(" report:", testJobStart);
const buildJob = workflow.slice(buildJobStart, testJobStart);
const testJob = workflow.slice(testJobStart, reportJobStart);
expect(workflow).toContain("Build local JS-backed provider artifacts");
expect(workflow).toContain("Qualify local provider Node interpreter");
expect(workflow.split(jsBackedLocalCondition)).toHaveLength(3);
expect(workflow).toContain(
expect(buildJobStart).toBeGreaterThan(0);
expect(testJobStart).toBeGreaterThan(buildJobStart);
expect(buildJob).toContain("needs: [authorize, catalog]");
expect(buildJob).toContain(
"needs: [authorize, catalog, daytona_image, build_runner_artifacts]",
);
expect(buildJob).not.toContain("environment:");
expect(buildJob).not.toContain("secrets.");
expect(buildJob).toContain(
"pnpm --filter @paperclipai/paperclip-runner build:typescript",
);
expect(buildJob).toContain(
"pnpm --filter @paperclipai/paperclip-runner build:runner-binaries",
);
expect(buildJob).toContain(
"node packages/paperclip-runner/scripts/build-provider-pack.mjs",
);
expect(buildJob).toContain("runner-e2e-build-bundle.tar.gz.sha256");
expect(buildJob).toContain("runner-e2e-provider-pack.tar.gz.sha256");
expect(buildJob).toContain(
"runner-e2e-build-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}",
);
expect(workflow).toContain("needs_runner_typescript=");
expect(workflow).toContain("needs_native_binaries=");
expect(workflow).toContain("needs_remote_provider_pack=");
expect(testJob).toContain(
"needs: [catalog, daytona_image, build_runner_artifacts, build_remote_provider_pack]",
);
expect(testJob).toContain("Download immutable campaign outputs");
expect(testJob).toContain("Download immutable remote provider pack");
expect(testJob).toContain("sha256sum --check");
expect(testJob.indexOf("sha256sum --check")).toBeLessThan(
testJob.indexOf("tar --extract"),
);
expect(testJob).toContain(
"test -x packages/paperclip-runner/runner/target/debug/paperclip-runnerd",
);
expect(testJob).toContain(".payload.runnerSourceRevision == $revision");
expect(workflow).toContain("Qualify local provider Node interpreter");
expect(testJob).not.toContain("build:typescript");
expect(testJob).not.toContain("build:runner-binaries");
expect(testJob).not.toContain("build-provider-pack.mjs");
});
it("uses environment-scoped OIDC for a no-delete history publisher", async () => {