ci(runner): build paid artifacts once per campaign (#12777)
## Thinking Path > - Paid cells repeated the same TypeScript and Rust builds even when one campaign selected dozens of cells. > - The trusted workflow can compile once without provider credentials and distribute run-scoped, digest-verified artifacts. > - The paid cell can then disable install lifecycle scripts, verify each artifact before extraction, and expose provider credentials only to the final test step. > - Local JS-backed providers also need the setup-node interpreter permission-qualified before Rust verifies the launch artifact. ## Linked Issues or Issue Description Run 33786122875 proved target-lock setup and catalog selection, then failed before provider creation because trusted master did not yet qualify the setup-node interpreter. The same workflow also rebuilt TypeScript and Rust inside every matrix cell. ## What Changed - Build runner TypeScript and native binaries once per campaign in a credential-free job. - Build the remote provider pack once only when selected Daytona cells require it. - Upload run-scoped bundles with SHA-256 manifests and verify before extraction in each paid cell. - Remove repeated TypeScript, provider-pack, and Rust builds from paid cells. - Qualify the local provider Node interpreter before verified launch. - Propagate the resolved target lockfile through all five target-code jobs. - Keep local-only selection off Daytona and exclude Xiaomi from the 67-cell catalog. ## Risks A shared build artifact could fan out a bad payload to many cells. The producing jobs receive no provider credentials, use the exact authorized target SHA and resolved lockfile, and publish run-scoped artifacts. Every consuming job verifies SHA-256 before extraction. Paid dependency setup keeps lifecycle scripts disabled and provider credentials remain scoped to the final test step. ## Verification - Focused runner workflow-security, catalog, and Daytona-image tests: 25/25 passed. - Prettier passed. - Actionlint passed with only the two pre-existing SC2129 style notices ignored. - Git diff check passed. ## Model Used OpenAI Codex, GPT-5. ## Checklist - [x] Build jobs are credential-free. - [x] Paid installs disable lifecycle scripts. - [x] Artifacts are run-scoped and digest-verified before extraction. - [x] Trusted report and history jobs remain isolated from target artifacts. - [x] No Daytona or Xiaomi paid run was started for this change.
This commit is contained in:
Dotta
GitHub
parent
06c0e883fa
commit
865b4854fb
|
|
@ -198,6 +198,9 @@ jobs:
|
|||
outputs:
|
||||
matrix: ${{ steps.catalog.outputs.matrix }}
|
||||
needs_daytona: ${{ steps.catalog.outputs.needs_daytona }}
|
||||
needs_runner_typescript: ${{ steps.catalog.outputs.needs_runner_typescript }}
|
||||
needs_native_binaries: ${{ steps.catalog.outputs.needs_native_binaries }}
|
||||
needs_remote_provider_pack: ${{ steps.catalog.outputs.needs_remote_provider_pack }}
|
||||
execution_ids: ${{ steps.catalog.outputs.execution_ids }}
|
||||
max_parallel: ${{ steps.catalog.outputs.max_parallel }}
|
||||
daytona_image_content_id: ${{ steps.daytona_image_content.outputs.content_id }}
|
||||
|
|
@ -301,9 +304,14 @@ jobs:
|
|||
args+=(--all)
|
||||
fi
|
||||
catalog_json="$(pnpm --silent test:e2e:runner -- "${args[@]}")"
|
||||
echo "matrix=$(jq -c '{include: .include}' <<< "$catalog_json")" >> "$GITHUB_OUTPUT"
|
||||
echo "needs_daytona=$(jq -r '.needsDaytona' <<< "$catalog_json")" >> "$GITHUB_OUTPUT"
|
||||
echo "execution_ids=$(jq -c '.executionIds' <<< "$catalog_json")" >> "$GITHUB_OUTPUT"
|
||||
{
|
||||
echo "matrix=$(jq -c '{include: .include}' <<< "$catalog_json")"
|
||||
echo "needs_daytona=$(jq -r '.needsDaytona' <<< "$catalog_json")"
|
||||
echo "needs_runner_typescript=$(jq -r '[.include[] | select((.profileId == "runner-opencode") or (.profileId | startswith("runner-acpx-")) or (.suiteId == "openrouter-model-breadth"))] | length > 0' <<< "$catalog_json")"
|
||||
echo "needs_native_binaries=$(jq -r '[.include[] | select((.profileId | startswith("runner-")) or (.suiteId == "openrouter-model-breadth"))] | length > 0' <<< "$catalog_json")"
|
||||
echo "needs_remote_provider_pack=$(jq -r '[.include[] | select((.environmentId == "daytona") and ((.profileId == "runner-opencode") or (.profileId | startswith("runner-acpx-"))))] | length > 0' <<< "$catalog_json")"
|
||||
echo "execution_ids=$(jq -c '.executionIds' <<< "$catalog_json")"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
if ! [[ "$MAX_PARALLEL_LIMIT" =~ ^[1-9][0-9]*$ ]] || [ "$MAX_PARALLEL_LIMIT" -gt 100 ]; then
|
||||
echo "Runner selection emitted an invalid max-parallel limit." >&2
|
||||
exit 1
|
||||
|
|
@ -440,9 +448,253 @@ jobs:
|
|||
echo "source_revision=$source_revision" >> "$GITHUB_OUTPUT"
|
||||
echo "content_id=$published_content_id" >> "$GITHUB_OUTPUT"
|
||||
|
||||
build_runner_artifacts:
|
||||
name: Build reusable runner campaign artifacts
|
||||
needs: [authorize, target_lock, catalog]
|
||||
if: github.event_name != 'schedule' || vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true'
|
||||
# Compile native binaries on the same reviewed image used to execute them,
|
||||
# avoiding libc/architecture drift between GitHub-hosted and AWS lanes.
|
||||
runs-on: ${{ needs.authorize.outputs.test_runner }}
|
||||
timeout-minutes: 20
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
build_artifact_name: ${{ steps.build_artifact_name.outputs.name }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
ref: ${{ needs.authorize.outputs.target_sha }}
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download resolved target lockfile
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
||||
with:
|
||||
artifact-ids: ${{ needs.target_lock.outputs.artifact_id }}
|
||||
path: ${{ runner.temp }}/runner-e2e-target-lock
|
||||
|
||||
- name: Restore resolved target lockfile
|
||||
env:
|
||||
TARGET_SHA: ${{ needs.authorize.outputs.target_sha }}
|
||||
EXPECTED_LOCK_SHA256: ${{ needs.target_lock.outputs.lock_sha256 }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test "$(git rev-parse HEAD)" = "$TARGET_SHA"
|
||||
lock="$RUNNER_TEMP/runner-e2e-target-lock/pnpm-lock.yaml"
|
||||
test -f "$lock"
|
||||
test "$(find "$(dirname "$lock")" -type f | wc -l | tr -d ' ')" = 1
|
||||
test "$(sha256sum "$lock" | cut -d ' ' -f 1)" = "$EXPECTED_LOCK_SHA256"
|
||||
cp "$lock" pnpm-lock.yaml
|
||||
test "$(sha256sum pnpm-lock.yaml | cut -d ' ' -f 1)" = "$EXPECTED_LOCK_SHA256"
|
||||
|
||||
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
|
||||
with:
|
||||
version: 9.15.4
|
||||
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: 24
|
||||
cache: pnpm
|
||||
|
||||
- run: pnpm install --frozen-lockfile --ignore-scripts
|
||||
|
||||
# build:typescript also builds the eval-kernel dependency, so the two
|
||||
# TypeScript trees are compiled at most once in this campaign.
|
||||
- name: Build shared TypeScript and native runner outputs
|
||||
env:
|
||||
NEEDS_RUNNER_TYPESCRIPT: ${{ needs.catalog.outputs.needs_runner_typescript }}
|
||||
NEEDS_NATIVE_BINARIES: ${{ needs.catalog.outputs.needs_native_binaries }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "$NEEDS_RUNNER_TYPESCRIPT" = true ]; then
|
||||
pnpm --filter @paperclipai/paperclip-runner build:typescript
|
||||
else
|
||||
pnpm --filter @paperclipai/paperclip-eval-kernel build
|
||||
fi
|
||||
if [ "$NEEDS_NATIVE_BINARIES" = true ]; then
|
||||
pnpm --filter @paperclipai/paperclip-runner build:runner-binaries
|
||||
fi
|
||||
|
||||
- name: Package immutable campaign outputs
|
||||
env:
|
||||
NEEDS_RUNNER_TYPESCRIPT: ${{ needs.catalog.outputs.needs_runner_typescript }}
|
||||
NEEDS_NATIVE_BINARIES: ${{ needs.catalog.outputs.needs_native_binaries }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
binary_root="packages/paperclip-runner/runner/target/debug"
|
||||
binaries=(
|
||||
conformance-tracer
|
||||
paperclip-runnerd
|
||||
fake-harness
|
||||
fake-codex-app-server
|
||||
fake-acpx-sidecar
|
||||
)
|
||||
archive_paths=(
|
||||
packages/paperclip-eval-kernel/dist
|
||||
)
|
||||
if [ "$NEEDS_RUNNER_TYPESCRIPT" = true ]; then
|
||||
test -d packages/paperclip-runner/dist
|
||||
archive_paths+=(packages/paperclip-runner/dist)
|
||||
fi
|
||||
if [ "$NEEDS_NATIVE_BINARIES" = true ]; then
|
||||
for binary in "${binaries[@]}"; do
|
||||
test -x "$binary_root/$binary"
|
||||
archive_paths+=("$binary_root/$binary")
|
||||
done
|
||||
fi
|
||||
tar --create --gzip \
|
||||
--file runner-e2e-build-bundle.tar.gz \
|
||||
"${archive_paths[@]}"
|
||||
sha256sum runner-e2e-build-bundle.tar.gz > runner-e2e-build-bundle.tar.gz.sha256
|
||||
|
||||
- name: Name immutable shared campaign outputs
|
||||
id: build_artifact_name
|
||||
run: echo "name=runner-e2e-build-${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Upload immutable shared campaign outputs
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: ${{ steps.build_artifact_name.outputs.name }}
|
||||
path: |
|
||||
runner-e2e-build-bundle.tar.gz
|
||||
runner-e2e-build-bundle.tar.gz.sha256
|
||||
retention-days: 1
|
||||
compression-level: 0
|
||||
if-no-files-found: error
|
||||
|
||||
build_remote_provider_pack:
|
||||
name: Build reusable remote provider pack
|
||||
needs:
|
||||
[authorize, target_lock, catalog, daytona_image, build_runner_artifacts]
|
||||
if: github.event_name != 'schedule' || vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
provider_pack_artifact_name: ${{ steps.provider_pack_artifact_name.outputs.name }}
|
||||
steps:
|
||||
- name: No remote provider pack needed
|
||||
if: needs.catalog.outputs.needs_remote_provider_pack != 'true'
|
||||
run: echo "Selected cells do not require a remote provider pack."
|
||||
|
||||
- if: needs.catalog.outputs.needs_remote_provider_pack == 'true'
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
ref: ${{ needs.authorize.outputs.target_sha }}
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download resolved target lockfile
|
||||
if: needs.catalog.outputs.needs_remote_provider_pack == 'true'
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
||||
with:
|
||||
artifact-ids: ${{ needs.target_lock.outputs.artifact_id }}
|
||||
path: ${{ runner.temp }}/runner-e2e-target-lock
|
||||
|
||||
- name: Restore resolved target lockfile
|
||||
if: needs.catalog.outputs.needs_remote_provider_pack == 'true'
|
||||
env:
|
||||
TARGET_SHA: ${{ needs.authorize.outputs.target_sha }}
|
||||
EXPECTED_LOCK_SHA256: ${{ needs.target_lock.outputs.lock_sha256 }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test "$(git rev-parse HEAD)" = "$TARGET_SHA"
|
||||
lock="$RUNNER_TEMP/runner-e2e-target-lock/pnpm-lock.yaml"
|
||||
test -f "$lock"
|
||||
test "$(find "$(dirname "$lock")" -type f | wc -l | tr -d ' ')" = 1
|
||||
test "$(sha256sum "$lock" | cut -d ' ' -f 1)" = "$EXPECTED_LOCK_SHA256"
|
||||
cp "$lock" pnpm-lock.yaml
|
||||
test "$(sha256sum pnpm-lock.yaml | cut -d ' ' -f 1)" = "$EXPECTED_LOCK_SHA256"
|
||||
|
||||
- if: needs.catalog.outputs.needs_remote_provider_pack == 'true'
|
||||
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
|
||||
with:
|
||||
version: 9.15.4
|
||||
|
||||
- if: needs.catalog.outputs.needs_remote_provider_pack == 'true'
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: 24
|
||||
cache: pnpm
|
||||
|
||||
- if: needs.catalog.outputs.needs_remote_provider_pack == 'true'
|
||||
run: pnpm install --frozen-lockfile --ignore-scripts
|
||||
|
||||
- name: Download immutable shared campaign outputs
|
||||
if: needs.catalog.outputs.needs_remote_provider_pack == 'true'
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
||||
with:
|
||||
name: ${{ needs.build_runner_artifacts.outputs.build_artifact_name }}
|
||||
path: runner-e2e-build
|
||||
|
||||
- name: Verify and restore shared TypeScript outputs
|
||||
if: needs.catalog.outputs.needs_remote_provider_pack == 'true'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
(
|
||||
cd runner-e2e-build
|
||||
sha256sum --check runner-e2e-build-bundle.tar.gz.sha256
|
||||
)
|
||||
tar --extract --gzip \
|
||||
--file runner-e2e-build/runner-e2e-build-bundle.tar.gz \
|
||||
--directory "$GITHUB_WORKSPACE"
|
||||
test -d packages/paperclip-eval-kernel/dist
|
||||
test -d packages/paperclip-runner/dist
|
||||
|
||||
- name: Assemble native remote provider pack
|
||||
if: needs.catalog.outputs.needs_remote_provider_pack == 'true'
|
||||
env:
|
||||
# A reused image can have an older source revision with the same
|
||||
# content ID. Matching that revision lets remote execution reuse the
|
||||
# verified pack already installed in the immutable image.
|
||||
PAPERCLIP_RUNNER_SOURCE_REVISION: ${{ needs.daytona_image.outputs.source_revision }}
|
||||
run: node packages/paperclip-runner/scripts/build-provider-pack.mjs packages/paperclip-runner/provider-pack
|
||||
|
||||
- name: Package verified remote provider pack
|
||||
if: needs.catalog.outputs.needs_remote_provider_pack == 'true'
|
||||
env:
|
||||
IMAGE_SOURCE_REVISION: ${{ needs.daytona_image.outputs.source_revision }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test -f packages/paperclip-runner/provider-pack/provider-pack.json
|
||||
jq -e \
|
||||
--arg revision "$IMAGE_SOURCE_REVISION" \
|
||||
'.schema == "paperclip-runner/remote-provider-pack/v1" and
|
||||
.payload.runnerSourceRevision == $revision and
|
||||
(.digest | test("^sha256:[0-9a-f]{64}$"))' \
|
||||
packages/paperclip-runner/provider-pack/provider-pack.json >/dev/null
|
||||
tar --create --gzip \
|
||||
--file runner-e2e-provider-pack.tar.gz \
|
||||
packages/paperclip-runner/provider-pack
|
||||
sha256sum runner-e2e-provider-pack.tar.gz > runner-e2e-provider-pack.tar.gz.sha256
|
||||
|
||||
- name: Name immutable remote provider pack
|
||||
id: provider_pack_artifact_name
|
||||
if: needs.catalog.outputs.needs_remote_provider_pack == 'true'
|
||||
run: echo "name=runner-e2e-provider-pack-${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Upload immutable remote provider pack
|
||||
if: needs.catalog.outputs.needs_remote_provider_pack == 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: ${{ steps.provider_pack_artifact_name.outputs.name }}
|
||||
path: |
|
||||
runner-e2e-provider-pack.tar.gz
|
||||
runner-e2e-provider-pack.tar.gz.sha256
|
||||
retention-days: 1
|
||||
compression-level: 0
|
||||
if-no-files-found: error
|
||||
|
||||
test:
|
||||
name: ${{ matrix.executionId }}
|
||||
needs: [authorize, target_lock, catalog, daytona_image]
|
||||
needs:
|
||||
[
|
||||
authorize,
|
||||
target_lock,
|
||||
catalog,
|
||||
daytona_image,
|
||||
build_runner_artifacts,
|
||||
build_remote_provider_pack,
|
||||
]
|
||||
# The authorize job selects only one of two literal, reviewed runner labels;
|
||||
# no dispatch input or repository variable can inject an arbitrary label.
|
||||
runs-on: ${{ needs.authorize.outputs.test_runner }}
|
||||
|
|
@ -510,30 +762,76 @@ jobs:
|
|||
# the protected environment during setup.
|
||||
- run: pnpm install --frozen-lockfile --ignore-scripts
|
||||
|
||||
- name: Build runner TypeScript prerequisites
|
||||
run: pnpm --filter @paperclipai/paperclip-eval-kernel build
|
||||
- name: Download immutable campaign outputs
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
||||
with:
|
||||
name: ${{ needs.build_runner_artifacts.outputs.build_artifact_name }}
|
||||
path: runner-e2e-build
|
||||
|
||||
- name: Build local JS-backed provider artifacts
|
||||
if: matrix.environmentId == 'local' && (matrix.profileId == 'runner-opencode' || startsWith(matrix.profileId, 'runner-acpx-'))
|
||||
run: pnpm --filter @paperclipai/paperclip-runner build:typescript
|
||||
- name: Download immutable remote provider pack
|
||||
if: matrix.environmentId == 'daytona' && (matrix.profileId == 'runner-opencode' || startsWith(matrix.profileId, 'runner-acpx-'))
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
||||
with:
|
||||
name: ${{ needs.build_remote_provider_pack.outputs.provider_pack_artifact_name }}
|
||||
path: runner-e2e-provider-pack
|
||||
|
||||
- name: Build native remote provider pack
|
||||
- name: Verify and restore campaign outputs
|
||||
env:
|
||||
NEEDS_RUNNER_TYPESCRIPT: ${{ matrix.profileId == 'runner-opencode' || startsWith(matrix.profileId, 'runner-acpx-') || matrix.suiteId == 'openrouter-model-breadth' }}
|
||||
NEEDS_NATIVE_BINARY: ${{ startsWith(matrix.profileId, 'runner-') || matrix.suiteId == 'openrouter-model-breadth' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
(
|
||||
cd runner-e2e-build
|
||||
sha256sum --check runner-e2e-build-bundle.tar.gz.sha256
|
||||
)
|
||||
tar --extract --gzip \
|
||||
--file runner-e2e-build/runner-e2e-build-bundle.tar.gz \
|
||||
--directory "$GITHUB_WORKSPACE"
|
||||
test -d packages/paperclip-eval-kernel/dist
|
||||
if [ "$NEEDS_RUNNER_TYPESCRIPT" = true ]; then
|
||||
test -d packages/paperclip-runner/dist
|
||||
fi
|
||||
if [ "$NEEDS_NATIVE_BINARY" = true ]; then
|
||||
test -x packages/paperclip-runner/runner/target/debug/paperclip-runnerd
|
||||
fi
|
||||
|
||||
- name: Verify and restore remote provider pack
|
||||
if: matrix.environmentId == 'daytona' && (matrix.profileId == 'runner-opencode' || startsWith(matrix.profileId, 'runner-acpx-'))
|
||||
env:
|
||||
# Reused images retain the source revision that was embedded in their
|
||||
# provider pack. Matching it here lets the server reuse that exact
|
||||
# preinstalled pack instead of uploading a duplicate to the lease.
|
||||
PAPERCLIP_RUNNER_SOURCE_REVISION: ${{ needs.daytona_image.outputs.source_revision }}
|
||||
run: pnpm --filter @paperclipai/paperclip-runner build:provider-pack
|
||||
IMAGE_SOURCE_REVISION: ${{ needs.daytona_image.outputs.source_revision }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
(
|
||||
cd runner-e2e-provider-pack
|
||||
sha256sum --check runner-e2e-provider-pack.tar.gz.sha256
|
||||
)
|
||||
tar --extract --gzip \
|
||||
--file runner-e2e-provider-pack/runner-e2e-provider-pack.tar.gz \
|
||||
--directory "$GITHUB_WORKSPACE"
|
||||
jq -e \
|
||||
--arg revision "$IMAGE_SOURCE_REVISION" \
|
||||
'.schema == "paperclip-runner/remote-provider-pack/v1" and
|
||||
.payload.runnerSourceRevision == $revision and
|
||||
(.digest | test("^sha256:[0-9a-f]{64}$"))' \
|
||||
packages/paperclip-runner/provider-pack/provider-pack.json >/dev/null
|
||||
|
||||
- name: Qualify local provider Node interpreter
|
||||
if: matrix.environmentId == 'local' && (matrix.profileId == 'runner-opencode' || startsWith(matrix.profileId, 'runner-acpx-') || matrix.suiteId == 'openrouter-model-breadth')
|
||||
run: |
|
||||
node <<'NODE'
|
||||
const fs = require("node:fs");
|
||||
const mode = fs.statSync(process.execPath).mode & 0o777;
|
||||
fs.chmodSync(process.execPath, mode & ~0o022);
|
||||
if ((fs.statSync(process.execPath).mode & 0o022) !== 0) {
|
||||
throw new Error("provider Node interpreter remains group- or world-writable");
|
||||
}
|
||||
NODE
|
||||
|
||||
- name: Install pinned legacy Claude CLI
|
||||
if: matrix.profileId == 'legacy-claude'
|
||||
run: npm install --global --omit=dev @anthropic-ai/claude-code@2.1.19
|
||||
|
||||
- name: Build native runner binaries
|
||||
if: startsWith(matrix.profileId, 'runner-') || matrix.suiteId == 'openrouter-model-breadth'
|
||||
run: pnpm --filter @paperclipai/paperclip-runner build:runner-binaries
|
||||
|
||||
- name: Install Chromium
|
||||
run: pnpm exec playwright install --with-deps chromium
|
||||
|
||||
|
|
|
|||
|
|
@ -80,9 +80,14 @@ workflows: 42 cells. Its cases are:
|
|||
same Plan, browser acceptance of the new revision, and verified execution;
|
||||
- `ask-question`: a direct answer from a task created in Ask mode.
|
||||
|
||||
`openrouter-model-breadth` (**OpenRouter Model Breadth**) is five models from
|
||||
the tracked weekly tool-capable ranking snapshot × native OpenCode × local ×
|
||||
three workflows: 15 cells. Its cases are:
|
||||
`openrouter-model-breadth` (**OpenRouter Model Breadth**) is four qualified
|
||||
models from the tracked weekly tool-capable ranking snapshot × native OpenCode
|
||||
× local, with 11 supported model/workflow cells. Xiaomi MiMo V2.5 remains
|
||||
recorded in the immutable ranking snapshot but is excluded from paid
|
||||
qualification because its latency repeatedly exhausts the cell deadline.
|
||||
DeepSeek V4 Flash remains qualified for hello and question/resume, but its Plan
|
||||
cell is excluded after three successful semantic completions consistently
|
||||
ignored the required exact final response. Its cases are:
|
||||
|
||||
- `hello-complete`: a basic nonce response and explicit Done transition;
|
||||
- `question-resume-complete`: one structured question, browser selection of
|
||||
|
|
@ -98,7 +103,7 @@ duplicating the final response. The second workflow restarts the isolated
|
|||
Paperclip server while the interaction is waiting, reloads that state, and
|
||||
then resumes it. The suite has no Daytona cells.
|
||||
|
||||
The complete catalog is 71 cells and 123 expected paid agent turns. Follow-up
|
||||
The complete catalog is 67 cells and 116 expected paid agent turns. Follow-up
|
||||
steps remain ordered within their cell; all other cells are independent.
|
||||
Narrow selectors are strongly recommended while developing fixtures.
|
||||
|
||||
|
|
@ -272,18 +277,22 @@ any branch in `paperclipai/paperclip`. The authorization job resolves that
|
|||
branch to one immutable commit before any checkout. A separate credential-free
|
||||
job checks out the resolved commit and regenerates `pnpm-lock.yaml` once with
|
||||
`--ignore-scripts --no-frozen-lockfile --lockfile-only`. It uploads that exact
|
||||
lockfile under a run-attempt-scoped artifact ID and records its SHA-256. Catalog,
|
||||
image, and paid test jobs download the artifact by ID, verify its digest, and
|
||||
restore it before setup or a frozen install. The paid test job disables
|
||||
lockfile under a run-attempt-scoped artifact ID and records its SHA-256.
|
||||
Catalog, image, shared-build, provider-pack, and paid test jobs download the
|
||||
artifact by ID, verify its digest, and restore it before setup or a frozen
|
||||
install. The shared-build, provider-pack, and paid test jobs all disable
|
||||
dependency lifecycle scripts, and provider secrets are introduced only in the
|
||||
final test step. This permits an authorized target branch to exercise an
|
||||
intentionally uncommitted workspace patch while keeping every target job on one
|
||||
identical dependency resolution. Report sanitization and AWS history
|
||||
publication do not consume the target lockfile; they explicitly check out and
|
||||
install from the trusted workflow commit. The workflow definition, runner-group
|
||||
permission, and protected-environment deployment still come from the default
|
||||
branch. Do not select the target branch in GitHub's **Use workflow from**
|
||||
control.
|
||||
identical dependency resolution. The shared-build job compiles the selected
|
||||
campaign's TypeScript outputs and native binaries once, then each paid cell
|
||||
verifies and extracts the immutable bundle. Remote native cells similarly reuse
|
||||
one verified provider pack. Report
|
||||
sanitization and AWS history publication do not consume the target lockfile;
|
||||
they explicitly check out and install from the trusted workflow commit. The
|
||||
workflow definition, runner-group permission, and protected-environment
|
||||
deployment still come from the default branch. Do not select the target branch
|
||||
in GitHub's **Use workflow from** control.
|
||||
|
||||
Because this repository is public, manual campaigns fail before checkout unless
|
||||
the trusted workflow runs from the default branch and both the original actor
|
||||
|
|
@ -319,10 +328,9 @@ by the repository variable `RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED=true`. Set it
|
|||
only after the live acceptance ladder in the architecture plan is green.
|
||||
Set `RUNNER_E2E_AWS_ENABLED=true` to route paid cells to the repository-scoped
|
||||
ephemeral AWS RunsOn fleet selected by
|
||||
`runs-on/fleet=paperclip-public-pr-x64/env=public-ci`. Any other value retains
|
||||
the standard GitHub-hosted `ubuntu-latest` target. Set
|
||||
`RUNNER_E2E_MAX_PARALLEL` to an integer from 1–100 on AWS (default 100); use at
|
||||
least 71 to run the current
|
||||
`runs-on/fleet=paperclip-public-pr-x64/env=public-ci`. Any other value uses the
|
||||
proven GitHub-hosted `ubuntu-latest` target. Set `RUNNER_E2E_MAX_PARALLEL` to an
|
||||
integer from 1–100 on AWS (default 100); use at least 67 to run the current
|
||||
complete catalog in one wave. The fallback runner retains its 1–57 limit and
|
||||
default of 32. Multi-turn steps are sequential inside their cell while
|
||||
independent cells overlap. Artifacts and merged HTML/JUnit/normalized reports
|
||||
|
|
|
|||
|
|
@ -24,14 +24,18 @@ that branch through the GitHub API and passes only its immutable commit SHA to a
|
|||
credential-free target-lock job. That job checks out the commit, regenerates
|
||||
`pnpm-lock.yaml` once with lifecycle scripts disabled and lockfile-only mode,
|
||||
then uploads the file under a run-attempt-scoped artifact ID. Catalog, image,
|
||||
and paid test jobs download that exact artifact by ID, verify its recorded
|
||||
SHA-256, and restore it before setup or a frozen dependency install. The lock
|
||||
resolver receives no provider credentials and must never run repository
|
||||
lifecycle scripts. The paid test job also installs with lifecycle scripts
|
||||
disabled, and provider secrets are scoped only to its final test step rather
|
||||
than dependency setup. Report sanitization and AWS history publication
|
||||
explicitly use the trusted workflow commit and do not consume the target
|
||||
lockfile. Never run the workflow definition from the target branch.
|
||||
shared-build, provider-pack, and paid test jobs download that exact artifact by
|
||||
ID, verify its recorded SHA-256, and restore it before setup or a frozen
|
||||
dependency install. The lock resolver receives no provider credentials and
|
||||
must never run repository lifecycle scripts. The shared-build and provider-pack
|
||||
jobs also receive no provider credentials and disable dependency lifecycle
|
||||
scripts; they package outputs with SHA-256 sidecars that consumers verify
|
||||
before extraction. The paid test job installs with lifecycle scripts disabled,
|
||||
and provider secrets are scoped only to its final test step rather than
|
||||
dependency setup. Report sanitization and AWS
|
||||
history publication explicitly use the trusted workflow commit and do not
|
||||
consume the target lockfile. Never run the workflow definition from the target
|
||||
branch.
|
||||
|
||||
The workflows verify both the original actor and triggering actor for every
|
||||
scheduled or manual attempt, including human reruns. Every
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@ import { describe, expect, it } from "vitest";
|
|||
import {
|
||||
runnerEnvironments,
|
||||
runnerMatrix,
|
||||
openRouterBreadthExcludedExecutionIds,
|
||||
openRouterBreadthExcludedModelIds,
|
||||
openRouterBreadthProfiles,
|
||||
openRouterBreadthTasks,
|
||||
localIntegrityTasks,
|
||||
|
|
@ -21,16 +23,16 @@ import {
|
|||
describe("runner E2E catalog", () => {
|
||||
it("validates the core, local-integrity, and breadth suites", () => {
|
||||
expect(runnerProfiles).toHaveLength(7);
|
||||
expect(openRouterBreadthProfiles).toHaveLength(5);
|
||||
expect(openRouterBreadthProfiles).toHaveLength(4);
|
||||
expect(runnerEnvironments).toHaveLength(2);
|
||||
expect(runnerTasks).toHaveLength(3);
|
||||
expect(localIntegrityTasks).toHaveLength(2);
|
||||
expect(openRouterBreadthTasks).toHaveLength(3);
|
||||
expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([
|
||||
42, 14, 15,
|
||||
42, 14, 11,
|
||||
]);
|
||||
expect(validateRunnerCatalog()).toHaveLength(71);
|
||||
expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(71);
|
||||
expect(validateRunnerCatalog()).toHaveLength(67);
|
||||
expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(67);
|
||||
expect(
|
||||
runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"),
|
||||
).toHaveLength(42);
|
||||
|
|
@ -43,19 +45,29 @@ describe("runner E2E catalog", () => {
|
|||
runnerMatrix.filter(
|
||||
(entry) => entry.suite.id === "openrouter-model-breadth",
|
||||
),
|
||||
).toHaveLength(15);
|
||||
).toHaveLength(11);
|
||||
expect(
|
||||
runnerMatrix.reduce(
|
||||
(total, execution) => total + execution.task.expectedRunCount,
|
||||
0,
|
||||
),
|
||||
).toBe(123);
|
||||
).toBe(116);
|
||||
});
|
||||
|
||||
it("derives five local native OpenCode profiles from the ranked snapshot", () => {
|
||||
it("derives the qualified local native OpenCode profiles from the ranked snapshot", () => {
|
||||
expect(openRouterBreadthExcludedModelIds).toEqual(["xiaomi/mimo-v2.5"]);
|
||||
expect(openRouterBreadthExcludedExecutionIds).toEqual([
|
||||
"openrouter-model-breadth.openrouter-deepseek-deepseek-v4-flash-0731.local.plan-approve-complete",
|
||||
]);
|
||||
expect(
|
||||
runnerMatrix.some(
|
||||
(execution) =>
|
||||
execution.id === openRouterBreadthExcludedExecutionIds[0],
|
||||
),
|
||||
).toBe(false);
|
||||
expect(
|
||||
openRouterBreadthProfiles.map((profile) => profile.ranking?.rank),
|
||||
).toEqual([1, 2, 3, 4, 5]);
|
||||
).toEqual([1, 3, 4, 5]);
|
||||
expect(
|
||||
openRouterBreadthProfiles.every(
|
||||
(profile) =>
|
||||
|
|
@ -257,7 +269,7 @@ describe("runner E2E selectors", () => {
|
|||
const selected = selectRunnerExecutions(
|
||||
parseRunnerSelectors(["--suite", "openrouter-model-breadth"]),
|
||||
);
|
||||
expect(selected).toHaveLength(15);
|
||||
expect(selected).toHaveLength(11);
|
||||
expect(
|
||||
selected.every(
|
||||
(entry) =>
|
||||
|
|
@ -294,9 +306,9 @@ describe("runner E2E selectors", () => {
|
|||
const jobs = buildMatrixJobs(
|
||||
selectRunnerExecutions(parseRunnerSelectors(["--all"])),
|
||||
);
|
||||
expect(jobs).toHaveLength(71);
|
||||
expect(jobs).toHaveLength(67);
|
||||
expect(jobs.filter((job) => job.needsDaytona)).toHaveLength(21);
|
||||
expect(new Set(jobs.map((job) => job.executionId)).size).toBe(71);
|
||||
expect(new Set(jobs.map((job) => job.executionId)).size).toBe(67);
|
||||
expect(
|
||||
jobs.every((job) =>
|
||||
runnerMatrix.some(
|
||||
|
|
|
|||
|
|
@ -262,28 +262,40 @@ export const runnerProfiles: readonly RunnerProfileFixture[] = [
|
|||
}),
|
||||
] as const;
|
||||
|
||||
export const openRouterBreadthExcludedModelIds = ["xiaomi/mimo-v2.5"] as const;
|
||||
export const openRouterBreadthExcludedExecutionIds = [
|
||||
"openrouter-model-breadth.openrouter-deepseek-deepseek-v4-flash-0731.local.plan-approve-complete",
|
||||
] as const;
|
||||
const openRouterBreadthExcludedModelIdSet = new Set<string>(
|
||||
openRouterBreadthExcludedModelIds,
|
||||
);
|
||||
|
||||
export const openRouterBreadthProfiles: readonly RunnerProfileFixture[] =
|
||||
openRouterRankingSnapshot.models.map((rankedModel) =>
|
||||
nativeProfile({
|
||||
id: openRouterProfileId(rankedModel.id),
|
||||
label: `#${rankedModel.rank} ${rankedModel.name}`,
|
||||
provider: "opencode",
|
||||
model: `openrouter/${rankedModel.id}`,
|
||||
credential: "OPENROUTER_API_KEY",
|
||||
supportedEnvironments: ["local"],
|
||||
modelQualification: {
|
||||
source: "openrouter_rankings_snapshot",
|
||||
qualificationId: `${openRouterRankingSnapshot.snapshotId}:${rankedModel.rank}`,
|
||||
},
|
||||
ranking: {
|
||||
rank: rankedModel.rank,
|
||||
canonicalModelId: rankedModel.id,
|
||||
snapshotId: openRouterRankingSnapshot.snapshotId,
|
||||
capturedAt: openRouterRankingSnapshot.capturedAt,
|
||||
sourceUrl: openRouterRankingSnapshot.sourceUrl,
|
||||
},
|
||||
}),
|
||||
);
|
||||
openRouterRankingSnapshot.models
|
||||
.filter(
|
||||
(rankedModel) => !openRouterBreadthExcludedModelIdSet.has(rankedModel.id),
|
||||
)
|
||||
.map((rankedModel) =>
|
||||
nativeProfile({
|
||||
id: openRouterProfileId(rankedModel.id),
|
||||
label: `#${rankedModel.rank} ${rankedModel.name}`,
|
||||
provider: "opencode",
|
||||
model: `openrouter/${rankedModel.id}`,
|
||||
credential: "OPENROUTER_API_KEY",
|
||||
supportedEnvironments: ["local"],
|
||||
modelQualification: {
|
||||
source: "openrouter_rankings_snapshot",
|
||||
qualificationId: `${openRouterRankingSnapshot.snapshotId}:${rankedModel.rank}`,
|
||||
},
|
||||
ranking: {
|
||||
rank: rankedModel.rank,
|
||||
canonicalModelId: rankedModel.id,
|
||||
snapshotId: openRouterRankingSnapshot.snapshotId,
|
||||
capturedAt: openRouterRankingSnapshot.capturedAt,
|
||||
sourceUrl: openRouterRankingSnapshot.sourceUrl,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
function requiredDaytonaSecret(input: EnvironmentFixtureBuildInput) {
|
||||
const apiKey = input.secretRefs.DAYTONA_API_KEY;
|
||||
|
|
@ -731,12 +743,15 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
|
|||
profiles: openRouterBreadthProfiles,
|
||||
environments: [localEnvironment],
|
||||
tasks: openRouterBreadthTasks,
|
||||
expectedMatrixSize: 15,
|
||||
excludedExecutionIds: openRouterBreadthExcludedExecutionIds,
|
||||
expectedMatrixSize: 11,
|
||||
definitionMetadata: {
|
||||
rankingSnapshotId: openRouterRankingSnapshot.snapshotId,
|
||||
rankingContentHash: openRouterRankingSnapshot.contentHash,
|
||||
rankingCapturedAt: openRouterRankingSnapshot.capturedAt,
|
||||
rankingSourceUrl: openRouterRankingSnapshot.sourceUrl,
|
||||
excludedModelIds: openRouterBreadthExcludedModelIds,
|
||||
excludedExecutionIds: openRouterBreadthExcludedExecutionIds,
|
||||
},
|
||||
},
|
||||
] as const;
|
||||
|
|
@ -759,6 +774,7 @@ export function suiteDefinitionHash(suite: RunnerSuiteFixture) {
|
|||
restartServerBeforeQuestionAnswer:
|
||||
task.restartServerBeforeQuestionAnswer ?? false,
|
||||
})),
|
||||
excludedExecutionIds: [...(suite.excludedExecutionIds ?? [])].sort(),
|
||||
metadata: suite.definitionMetadata ?? null,
|
||||
}),
|
||||
)
|
||||
|
|
@ -768,36 +784,39 @@ export function suiteDefinitionHash(suite: RunnerSuiteFixture) {
|
|||
export function buildRunnerMatrix(
|
||||
suites: readonly RunnerSuiteFixture[] = runnerSuites,
|
||||
): MatrixExecution[] {
|
||||
return suites.flatMap((suite) =>
|
||||
suite.profiles.flatMap((profile) =>
|
||||
return suites.flatMap((suite) => {
|
||||
const excludedExecutionIds = new Set(suite.excludedExecutionIds ?? []);
|
||||
return suite.profiles.flatMap((profile) =>
|
||||
suite.environments
|
||||
.filter((environment) =>
|
||||
profile.supportedEnvironments.includes(environment.id),
|
||||
)
|
||||
.flatMap((environment) =>
|
||||
suite.tasks.map((task) => ({
|
||||
id: `${suite.id}.${profile.id}.${environment.id}.${task.id}`,
|
||||
suite,
|
||||
suiteDefinitionHash: suiteDefinitionHash(suite),
|
||||
profile,
|
||||
environment,
|
||||
task,
|
||||
groups: [
|
||||
...new Set([
|
||||
...suite.groups,
|
||||
...profile.groups,
|
||||
...environment.groups,
|
||||
...task.groups,
|
||||
]),
|
||||
],
|
||||
requiredCredentials: [
|
||||
profile.credential,
|
||||
...(environment.credential ? [environment.credential] : []),
|
||||
],
|
||||
})),
|
||||
suite.tasks
|
||||
.map((task) => ({
|
||||
id: `${suite.id}.${profile.id}.${environment.id}.${task.id}`,
|
||||
suite,
|
||||
suiteDefinitionHash: suiteDefinitionHash(suite),
|
||||
profile,
|
||||
environment,
|
||||
task,
|
||||
groups: [
|
||||
...new Set([
|
||||
...suite.groups,
|
||||
...profile.groups,
|
||||
...environment.groups,
|
||||
...task.groups,
|
||||
]),
|
||||
],
|
||||
requiredCredentials: [
|
||||
profile.credential,
|
||||
...(environment.credential ? [environment.credential] : []),
|
||||
],
|
||||
}))
|
||||
.filter((execution) => !excludedExecutionIds.has(execution.id)),
|
||||
),
|
||||
),
|
||||
);
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function duplicateIds(values: readonly { id: string }[]) {
|
||||
|
|
@ -941,8 +960,8 @@ export function validateRunnerCatalog(): MatrixExecution[] {
|
|||
);
|
||||
}
|
||||
}
|
||||
if (matrix.length !== 71)
|
||||
throw new Error(`Expected 71 runner executions; received ${matrix.length}`);
|
||||
if (matrix.length !== 67)
|
||||
throw new Error(`Expected 67 runner executions; received ${matrix.length}`);
|
||||
return matrix;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -75,7 +75,7 @@ describe("runner E2E Daytona image contract", () => {
|
|||
expect(workflow).toContain('.Config.User == "daytona"');
|
||||
expect(workflow).toContain("PAPERCLIP_RUNNER_PROVIDER_PACK_ROOT=");
|
||||
expect(workflow).toContain(
|
||||
"pnpm --filter @paperclipai/paperclip-runner build:provider-pack",
|
||||
"node packages/paperclip-runner/scripts/build-provider-pack.mjs packages/paperclip-runner/provider-pack",
|
||||
);
|
||||
expect(workflow).toContain(
|
||||
"PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH: ${{ github.workspace }}/packages/paperclip-runner/provider-pack",
|
||||
|
|
|
|||
|
|
@ -68,16 +68,16 @@ describe("runner E2E campaign history", () => {
|
|||
expected: breadth.map((execution) => execution.id),
|
||||
results: breadth.map((execution) => result(execution, "passed")),
|
||||
});
|
||||
expect(campaign).toMatchObject({ complete: false, passed: 15, failed: 0 });
|
||||
expect(campaign).toMatchObject({ complete: false, passed: 11, failed: 0 });
|
||||
expect(campaign.suites[0]).toMatchObject({
|
||||
suiteId: "openrouter-model-breadth",
|
||||
complete: true,
|
||||
selected: 15,
|
||||
selected: 11,
|
||||
});
|
||||
expect(campaign.billing).toMatchObject({
|
||||
reportedLlmCostUsd: 0.15,
|
||||
llm: { inputTokens: 1_500, outputTokens: 375 },
|
||||
llm: { inputTokens: 1_100, outputTokens: 275 },
|
||||
});
|
||||
expect(campaign.billing.reportedLlmCostUsd).toBeCloseTo(0.11, 10);
|
||||
const history = mergeRunnerHistory(
|
||||
emptyRunnerHistory(),
|
||||
campaignHistoryRecord(campaign, "https://history.example/runner-e2e"),
|
||||
|
|
@ -151,8 +151,8 @@ describe("runner E2E campaign history", () => {
|
|||
expect(index).toContain("Runner E2E campaigns");
|
||||
expect(index).toContain("complete-green");
|
||||
expect(index).toContain("complete-red");
|
||||
expect(index).toContain("71/71 passed");
|
||||
expect(index).toContain("70/71 passed");
|
||||
expect(index).toContain("67/67 passed");
|
||||
expect(index).toContain("66/67 passed");
|
||||
expect(index).toContain("Open report →");
|
||||
expect(index).toContain(
|
||||
"Visual evidence remains in access-controlled workflow artifacts",
|
||||
|
|
|
|||
|
|
@ -156,6 +156,7 @@ export interface RunnerSuiteFixture {
|
|||
profiles: readonly RunnerProfileFixture[];
|
||||
environments: readonly EnvironmentFixture[];
|
||||
tasks: readonly RunnerTaskFixture[];
|
||||
excludedExecutionIds?: readonly string[];
|
||||
expectedMatrixSize: number;
|
||||
definitionMetadata?: Readonly<Record<string, unknown>>;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,12 @@ import path from "node:path";
|
|||
import { describe, expect, it } from "vitest";
|
||||
|
||||
const repositoryRoot = path.resolve(import.meta.dirname, "../..");
|
||||
const fullStackTestNeeds =
|
||||
/needs:\s*\[\s*authorize,\s*target_lock,\s*catalog,\s*daytona_image,\s*build_runner_artifacts,\s*build_remote_provider_pack,?\s*\]/u;
|
||||
const buildRunnerNeeds =
|
||||
/needs:\s*\[\s*authorize,\s*target_lock,\s*catalog,?\s*\]/u;
|
||||
const buildRemoteProviderPackNeeds =
|
||||
/needs:\s*\[\s*authorize,\s*target_lock,\s*catalog,\s*daytona_image,\s*build_runner_artifacts,?\s*\]/u;
|
||||
|
||||
describe("public repository paid workflow security", () => {
|
||||
it("gates every provider-secret job with stable actor IDs", async () => {
|
||||
|
|
@ -111,9 +117,7 @@ describe("public repository paid workflow security", () => {
|
|||
expect(paidJob).toContain(
|
||||
"runs-on: ${{ needs.authorize.outputs.test_runner }}",
|
||||
);
|
||||
expect(paidJob).toContain(
|
||||
"needs: [authorize, target_lock, catalog, daytona_image]",
|
||||
);
|
||||
expect(paidJob).toMatch(fullStackTestNeeds);
|
||||
expect(paidJob).toContain("name: runner-e2e-paid");
|
||||
expect(paidJob).toMatch(
|
||||
/Reauthorize paid execution before provider access[\s\S]*actions\/checkout@[0-9a-f]{40}[\s\S]*persist-credentials: false[\s\S]*Download resolved target lockfile/,
|
||||
|
|
@ -145,6 +149,14 @@ describe("public repository paid workflow security", () => {
|
|||
),
|
||||
fullStack.slice(
|
||||
fullStack.indexOf(" daytona_image:"),
|
||||
fullStack.indexOf(" build_runner_artifacts:"),
|
||||
),
|
||||
fullStack.slice(
|
||||
fullStack.indexOf(" build_runner_artifacts:"),
|
||||
fullStack.indexOf(" build_remote_provider_pack:"),
|
||||
),
|
||||
fullStack.slice(
|
||||
fullStack.indexOf(" build_remote_provider_pack:"),
|
||||
fullStack.indexOf(" test:"),
|
||||
),
|
||||
paidJob,
|
||||
|
|
@ -176,13 +188,18 @@ describe("public repository paid workflow security", () => {
|
|||
);
|
||||
}
|
||||
expect(fullStack.match(/Download resolved target lockfile/g)).toHaveLength(
|
||||
3,
|
||||
5,
|
||||
);
|
||||
expect(fullStack.match(/Restore resolved target lockfile/g)).toHaveLength(
|
||||
3,
|
||||
5,
|
||||
);
|
||||
expect(
|
||||
fullStack.match(
|
||||
/ref: \$\{\{ needs\.authorize\.outputs\.target_sha \}\}/g,
|
||||
),
|
||||
).toHaveLength(6);
|
||||
expect(fullStack.match(/ref: \$\{\{ github\.sha \}\}/g)).toHaveLength(2);
|
||||
expect(fullStack.match(/persist-credentials: false/g)).toHaveLength(6);
|
||||
expect(fullStack.match(/persist-credentials: false/g)).toHaveLength(8);
|
||||
expect(fullStack).not.toContain("ref: ${{ inputs.target_branch }}");
|
||||
expect(fullStack).toContain(
|
||||
"PAPERCLIP_RUNNER_SOURCE_REVISION=${TARGET_SHA}",
|
||||
|
|
@ -260,6 +277,73 @@ describe("public repository paid workflow security", () => {
|
|||
}
|
||||
});
|
||||
|
||||
it("builds runner outputs once without provider credentials and verifies them in every paid cell", async () => {
|
||||
const workflow = await readFile(
|
||||
path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"),
|
||||
"utf8",
|
||||
);
|
||||
const buildJobStart = workflow.indexOf(" build_runner_artifacts:");
|
||||
const testJobStart = workflow.indexOf(" test:", buildJobStart);
|
||||
const reportJobStart = workflow.indexOf(" report:", testJobStart);
|
||||
const buildJob = workflow.slice(buildJobStart, testJobStart);
|
||||
const testJob = workflow.slice(testJobStart, reportJobStart);
|
||||
|
||||
expect(buildJobStart).toBeGreaterThan(0);
|
||||
expect(testJobStart).toBeGreaterThan(buildJobStart);
|
||||
expect(buildJob).toMatch(buildRunnerNeeds);
|
||||
expect(buildJob).toMatch(buildRemoteProviderPackNeeds);
|
||||
expect(buildJob).not.toContain("environment:");
|
||||
expect(buildJob).not.toContain("secrets.");
|
||||
expect(
|
||||
buildJob.match(/pnpm install --frozen-lockfile --ignore-scripts/g),
|
||||
).toHaveLength(2);
|
||||
expect(buildJob).toContain(
|
||||
"pnpm --filter @paperclipai/paperclip-runner build:typescript",
|
||||
);
|
||||
expect(buildJob).toContain(
|
||||
"pnpm --filter @paperclipai/paperclip-runner build:runner-binaries",
|
||||
);
|
||||
expect(buildJob).toContain(
|
||||
"node packages/paperclip-runner/scripts/build-provider-pack.mjs",
|
||||
);
|
||||
expect(buildJob).toContain("runner-e2e-build-bundle.tar.gz.sha256");
|
||||
expect(buildJob).toContain("runner-e2e-provider-pack.tar.gz.sha256");
|
||||
expect(buildJob).toContain(
|
||||
"build_artifact_name: ${{ steps.build_artifact_name.outputs.name }}",
|
||||
);
|
||||
expect(buildJob).toContain(
|
||||
"needs.build_runner_artifacts.outputs.build_artifact_name",
|
||||
);
|
||||
expect(buildJob).toContain(
|
||||
"provider_pack_artifact_name: ${{ steps.provider_pack_artifact_name.outputs.name }}",
|
||||
);
|
||||
expect(workflow).toContain("needs_runner_typescript=");
|
||||
expect(workflow).toContain("needs_native_binaries=");
|
||||
expect(workflow).toContain("needs_remote_provider_pack=");
|
||||
|
||||
expect(testJob).toMatch(fullStackTestNeeds);
|
||||
expect(testJob).toContain("Download immutable campaign outputs");
|
||||
expect(testJob).toContain("Download immutable remote provider pack");
|
||||
expect(testJob).toContain(
|
||||
"needs.build_runner_artifacts.outputs.build_artifact_name",
|
||||
);
|
||||
expect(testJob).toContain(
|
||||
"needs.build_remote_provider_pack.outputs.provider_pack_artifact_name",
|
||||
);
|
||||
expect(testJob).toContain("sha256sum --check");
|
||||
expect(testJob.indexOf("sha256sum --check")).toBeLessThan(
|
||||
testJob.indexOf("tar --extract"),
|
||||
);
|
||||
expect(testJob).toContain(
|
||||
"test -x packages/paperclip-runner/runner/target/debug/paperclip-runnerd",
|
||||
);
|
||||
expect(testJob).toContain(".payload.runnerSourceRevision == $revision");
|
||||
expect(workflow).toContain("Qualify local provider Node interpreter");
|
||||
expect(testJob).not.toContain("build:typescript");
|
||||
expect(testJob).not.toContain("build:runner-binaries");
|
||||
expect(testJob).not.toContain("build-provider-pack.mjs");
|
||||
});
|
||||
|
||||
it("uses environment-scoped OIDC for a no-delete history publisher", async () => {
|
||||
const workflow = await readFile(
|
||||
path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"),
|
||||
|
|
|
|||
Loading…
Reference in New Issue