RomanNum3ral
/
paperclip
mirror of https://github.com/paperclipai/paperclip.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
paperclip/.github/workflows/runner-full-stack-e2e.yml

536 lines
22 KiB
YAML
Raw Blame History

name: Runner Full-Stack E2E
on:
schedule:
- cron: "47 8 * * 0"
workflow_dispatch:
inputs:
all:
description: "Run the complete paid matrix when no narrower selector is supplied"
type: boolean
default: true
group:
description: "Comma-separated groups (AND semantics: legacy,native,local,daytona,core,breadth)"
type: string
required: false
suite:
description: "Comma-separated suite IDs"
type: string
required: false
profile:
description: "Comma-separated runner profile fixture IDs"
type: string
required: false
environment:
description: "Comma-separated environment fixture IDs"
type: string
required: false
case:
description: "Comma-separated task case fixture IDs"
type: string
required: false
id:
description: "Comma-separated full suite.profile.environment.case IDs; exclusive with other selectors"
type: string
required: false
permissions:
contents: read
concurrency:
group: runner-full-stack-e2e-${{ github.ref }}
cancel-in-progress: false
jobs:
authorize:
name: Authorize paid campaign
if: github.event_name != 'schedule' || vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true'
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
steps:
- name: Require default branch and allowlisted numeric actor IDs
env:
GH_TOKEN: ${{ github.token }}
REPOSITORY: ${{ github.repository }}
REF: ${{ github.ref }}
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
ACTOR: ${{ github.actor }}
ACTOR_ID: ${{ github.actor_id }}
TRIGGERING_ACTOR: ${{ github.triggering_actor }}
ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
run: |
set -euo pipefail
if [ "$REF" != "refs/heads/$DEFAULT_BRANCH" ]; then
echo "Paid runner E2E campaigns may run only from the default branch." >&2
exit 1
fi
if ! jq -e 'type == "array" and length > 0 and all(.[]; type == "number" and . > 0 and floor == .)' <<< "${ALLOWED_ACTOR_IDS:-}" >/dev/null; then
echo "RUNNER_E2E_ALLOWED_ACTOR_IDS must be a non-empty JSON array of numeric GitHub user IDs." >&2
exit 1
fi
triggering_actor_id="$(gh api "users/$TRIGGERING_ACTOR" --jq .id)"
if [ "$triggering_actor_id" != "$ACTOR_ID" ] && [ "$TRIGGERING_ACTOR" = "$ACTOR" ]; then
echo "GitHub actor identity contexts disagree; refusing the paid run." >&2
exit 1
fi
candidates=("$triggering_actor_id" "$ACTOR_ID")
for candidate in "${candidates[@]}"; do
if ! jq -e --argjson candidate "$candidate" 'index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null; then
echo "The initiating GitHub account is not authorized to run paid runner E2E campaigns." >&2
exit 1
fi
done
catalog:
name: Validate catalog and select cells
needs: authorize
if: github.event_name != 'schedule' || vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true'
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
outputs:
matrix: ${{ steps.catalog.outputs.matrix }}
needs_daytona: ${{ steps.catalog.outputs.needs_daytona }}
execution_ids: ${{ steps.catalog.outputs.execution_ids }}
max_parallel: ${{ steps.catalog.outputs.max_parallel }}
daytona_image_content_id: ${{ steps.daytona_image_content.outputs.content_id }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
with:
version: 9.15.4
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
cache: pnpm
- run: pnpm install --frozen-lockfile
# The v2 contract fails closed unless every Docker FROM is digest-pinned,
# and hashes those exact base references into the immutable image tag.
- name: Compute Daytona image content ID with pinned bases
id: daytona_image_content
run: echo "content_id=$(pnpm --silent test:e2e:runner:image-id)" >> "$GITHUB_OUTPUT"
- name: Validate selectors and emit matrix
id: catalog
env:
EVENT_NAME: ${{ github.event_name }}
SELECT_ALL: ${{ inputs.all }}
SELECT_SUITE: ${{ inputs.suite }}
SELECT_GROUP: ${{ inputs.group }}
SELECT_PROFILE: ${{ inputs.profile }}
SELECT_ENVIRONMENT: ${{ inputs.environment }}
SELECT_CASE: ${{ inputs.case }}
SELECT_ID: ${{ inputs.id }}
MAX_PARALLEL: ${{ vars.RUNNER_E2E_MAX_PARALLEL || '32' }}
run: |
set -euo pipefail
args=(--matrix-json)
add_values() {
local flag="$1"
local values="$2"
local value
IFS=',' read -ra entries <<< "$values"
for value in "${entries[@]}"; do
value="${value#"${value%%[![:space:]]*}"}"
value="${value%"${value##*[![:space:]]}"}"
if [ -n "$value" ]; then
args+=("$flag" "$value")
fi
done
}
explicit=false
if [ -n "${SELECT_ID:-}" ]; then
if [ -n "${SELECT_SUITE:-}${SELECT_GROUP:-}${SELECT_PROFILE:-}${SELECT_ENVIRONMENT:-}${SELECT_CASE:-}" ]; then
echo "The id selector is exclusive with suite/group/profile/environment/case" >&2
exit 1
fi
add_values --id "$SELECT_ID"
explicit=true
else
for pair in \
"--suite:${SELECT_SUITE:-}" \
"--group:${SELECT_GROUP:-}" \
"--profile:${SELECT_PROFILE:-}" \
"--environment:${SELECT_ENVIRONMENT:-}" \
"--case:${SELECT_CASE:-}"
do
flag="${pair%%:*}"
values="${pair#*:}"
if [ -n "$values" ]; then
add_values "$flag" "$values"
explicit=true
fi
done
fi
if [ "$explicit" = false ] && { [ "$EVENT_NAME" = schedule ] || [ "${SELECT_ALL:-false}" = true ]; }; then
args+=(--all)
fi
catalog_json="$(pnpm --silent test:e2e:runner -- "${args[@]}")"
echo "matrix=$(jq -c '{include: .include}' <<< "$catalog_json")" >> "$GITHUB_OUTPUT"
echo "needs_daytona=$(jq -r '.needsDaytona' <<< "$catalog_json")" >> "$GITHUB_OUTPUT"
echo "execution_ids=$(jq -c '.executionIds' <<< "$catalog_json")" >> "$GITHUB_OUTPUT"
if ! [[ "$MAX_PARALLEL" =~ ^[1-9][0-9]*$ ]] || [ "$MAX_PARALLEL" -gt 57 ]; then
echo "RUNNER_E2E_MAX_PARALLEL must be an integer from 1 through 57." >&2
exit 1
fi
echo "max_parallel=$MAX_PARALLEL" >> "$GITHUB_OUTPUT"
daytona_image:
name: Publish verified Daytona image
needs: [authorize, catalog]
runs-on: ubuntu-latest
timeout-minutes: 45
permissions:
contents: read
packages: write
id-token: write
outputs:
image: ${{ steps.image.outputs.image }}
source_revision: ${{ steps.image.outputs.source_revision }}
content_id: ${{ steps.image.outputs.content_id }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- name: No Daytona image needed
id: local_only
if: needs.catalog.outputs.needs_daytona != 'true'
run: echo "image=" >> "$GITHUB_OUTPUT"
- name: Set up Docker Buildx
if: needs.catalog.outputs.needs_daytona == 'true'
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
- name: Log into GHCR
if: needs.catalog.outputs.needs_daytona == 'true'
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Install cosign
if: needs.catalog.outputs.needs_daytona == 'true'
uses: sigstore/cosign-installer@398d4b0eeef1380460a10c8013a76f728fb906ac # v3
- name: Reuse or publish immutable image
id: image
env:
NEEDS_DAYTONA: ${{ needs.catalog.outputs.needs_daytona }}
IMAGE_CONTENT_ID: ${{ needs.catalog.outputs.daytona_image_content_id }}
IMAGE_TAG: ghcr.io/paperclipai/paperclip-daytona-runner:e2e-content-${{ needs.catalog.outputs.daytona_image_content_id }}
run: |
set -euo pipefail
if [ "$NEEDS_DAYTONA" != true ]; then
echo "image=" >> "$GITHUB_OUTPUT"
echo "source_revision=" >> "$GITHUB_OUTPUT"
echo "content_id=" >> "$GITHUB_OUTPUT"
exit 0
fi
[[ "$IMAGE_CONTENT_ID" =~ ^[0-9a-f]{64}$ ]]
identity="^https://github.com/${GITHUB_REPOSITORY}/.github/workflows/runner-full-stack-e2e.yml@"
if docker buildx imagetools inspect "$IMAGE_TAG" >/dev/null 2>&1; then
digest="$(docker buildx imagetools inspect "$IMAGE_TAG" --format '{{json .Manifest.Digest}}' | tr -d '"')"
else
docker buildx build \
--platform linux/amd64 \
--build-arg "PAPERCLIP_RUNNER_CONTENT_ID=${IMAGE_CONTENT_ID}" \
--build-arg "PAPERCLIP_RUNNER_SOURCE_REVISION=${GITHUB_SHA}" \
--file docker/daytona-runner/Dockerfile \
--tag "$IMAGE_TAG" \
--push \
.
digest="$(docker buildx imagetools inspect "$IMAGE_TAG" --format '{{json .Manifest.Digest}}' | tr -d '"')"
cosign sign --yes "$IMAGE_TAG@$digest"
fi
cosign verify \
--certificate-identity-regexp "$identity" \
--certificate-oidc-issuer https://token.actions.githubusercontent.com \
"$IMAGE_TAG@$digest" >/dev/null
immutable="${IMAGE_TAG%:*}@$digest"
# The Daytona base image is large. The build cache plus a second full
# anonymous pull can exhaust a standard GitHub-hosted runner before
# Docker creates the tiny metadata-probe container. The pushed digest
# is already immutable, so release the local builder/cache first.
docker buildx prune --all --force >/dev/null
docker system prune --all --force >/dev/null
anonymous_config="$(mktemp -d)"
docker --config "$anonymous_config" pull "$immutable"
# The Dockerfile's final two RUN steps execute the runner metadata,
# transport-mode, provider-pack JSON, and pinned ACP binary checks as
# root and as the unprivileged Daytona user. Starting another
# container after this full pull can exhaust the hosted runner's thin
# writable layer even after pruning, so assert the published image
# configuration here without creating a redundant container.
image_config="$(docker image inspect "$immutable" \
--format '{{json .}}')"
published_content_id="$(jq -r '.Config.Labels["io.paperclip.runner.content-id"] // empty' <<< "$image_config")"
source_revision="$(jq -r '.Config.Labels["org.opencontainers.image.revision"] // empty' <<< "$image_config")"
test "$published_content_id" = "$IMAGE_CONTENT_ID"
[[ "$source_revision" =~ ^[0-9a-f]{40}$ ]]
jq -e \
'.Architecture == "amd64" and
.Os == "linux" and
.Config.User == "daytona" and
(.Config.Env | any(startswith("PAPERCLIP_RUNNER_PROVIDER_PACK_ROOT=")))' \
<<< "$image_config" >/dev/null
echo "image=$immutable" >> "$GITHUB_OUTPUT"
echo "source_revision=$source_revision" >> "$GITHUB_OUTPUT"
echo "content_id=$published_content_id" >> "$GITHUB_OUTPUT"
test:
name: ${{ matrix.executionId }}
needs: [catalog, daytona_image]
runs-on: ubuntu-latest-m
timeout-minutes: ${{ matrix.timeoutMinutes }}
permissions:
contents: read
environment:
name: runner-e2e-paid
strategy:
fail-fast: false
max-parallel: ${{ fromJSON(needs.catalog.outputs.max_parallel) }}
matrix: ${{ fromJSON(needs.catalog.outputs.matrix) }}
steps:
- name: Reauthorize paid execution before provider access
env:
GH_TOKEN: ${{ github.token }}
REF: ${{ github.ref }}
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
ACTOR_ID: ${{ github.actor_id }}
TRIGGERING_ACTOR: ${{ github.triggering_actor }}
ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
run: |
set -euo pipefail
test "$REF" = "refs/heads/$DEFAULT_BRANCH"
jq -e 'type == "array" and length > 0 and all(.[]; type == "number" and . > 0 and floor == .)' <<< "${ALLOWED_ACTOR_IDS:-}" >/dev/null
triggering_actor_id="$(gh api "users/$TRIGGERING_ACTOR" --jq .id)"
jq -e --argjson candidate "$triggering_actor_id" 'index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null
jq -e --argjson candidate "$ACTOR_ID" 'index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
with:
version: 9.15.4
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- run: pnpm install --frozen-lockfile
- name: Build runner TypeScript prerequisites
run: pnpm --filter @paperclipai/paperclip-eval-kernel build
- name: Build native remote provider pack
if: matrix.environmentId == 'daytona' && (matrix.profileId == 'runner-opencode' || startsWith(matrix.profileId, 'runner-acpx-'))
env:
# Reused images retain the source revision that was embedded in their
# provider pack. Matching it here lets the server reuse that exact
# preinstalled pack instead of uploading a duplicate to the lease.
PAPERCLIP_RUNNER_SOURCE_REVISION: ${{ needs.daytona_image.outputs.source_revision }}
run: pnpm --filter @paperclipai/paperclip-runner build:provider-pack
- name: Install pinned legacy Claude CLI
if: matrix.profileId == 'legacy-claude'
run: npm install --global --omit=dev @anthropic-ai/claude-code@2.1.19
- name: Build native runner binaries
if: startsWith(matrix.profileId, 'runner-') || matrix.suiteId == 'openrouter-model-breadth'
run: pnpm --filter @paperclipai/paperclip-runner build:runner-binaries
- name: Install Chromium
run: pnpm exec playwright install --with-deps chromium
- name: Run paid cell
env:
OPENAI_API_KEY: ${{ matrix.credentialName == 'OPENAI_API_KEY' && secrets.OPENAI_API_KEY || '' }}
ANTHROPIC_API_KEY: ${{ matrix.credentialName == 'ANTHROPIC_API_KEY' && secrets.ANTHROPIC_API_KEY || '' }}
OPENROUTER_API_KEY: ${{ matrix.credentialName == 'OPENROUTER_API_KEY' && secrets.OPENROUTER_API_KEY || '' }}
DAYTONA_API_KEY: ${{ matrix.environmentId == 'daytona' && secrets.DAYTONA_API_KEY || '' }}
PAPERCLIP_E2E_DAYTONA_IMAGE: ${{ needs.daytona_image.outputs.image }}
PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH: ${{ github.workspace }}/packages/paperclip-runner/provider-pack
PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.executionId }}
run: pnpm test:e2e:runner -- --id "${{ matrix.executionId }}"
- name: Upload access-controlled packaged cell evidence
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: runner-e2e-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.executionId }}
path: tests/runner-e2e/results/
retention-days: 30
if-no-files-found: error
report:
name: Merge and enforce campaign result
if: always() && needs.catalog.result == 'success'
needs: [catalog, daytona_image, test]
outputs:
history_source_ready: ${{ steps.history_source_ready.outputs.ready }}
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
contents: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
with:
version: 9.15.4
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
cache: pnpm
- run: pnpm install --frozen-lockfile
- name: Download cell evidence
id: download_evidence
continue-on-error: true
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
pattern: runner-e2e-${{ github.run_id }}-${{ github.run_attempt }}-*
path: downloaded-runner-e2e
merge-multiple: true
- name: Retry cell evidence download after transport failure
if: steps.download_evidence.outcome == 'failure'
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
pattern: runner-e2e-${{ github.run_id }}-${{ github.run_attempt }}-*
path: downloaded-runner-e2e
merge-multiple: true
- name: Collect blob reports
run: |
set -euo pipefail
mkdir -p merged-blob-reports
while IFS= read -r -d '' report; do
digest="$(sha256sum "$report" | cut -d ' ' -f 1)"
target="merged-blob-reports/report-${digest}.zip"
if [ ! -e "$target" ]; then
cp "$report" "$target"
fi
done < <(find downloaded-runner-e2e -path '*/blob-report/*.zip' -print0)
- name: Merge Playwright HTML and JUnit
if: always()
env:
PAPERCLIP_RUNNER_E2E_MERGED_REPORT_DIR: ${{ github.workspace }}/runner-e2e-merged-report
run: pnpm exec playwright merge-reports --config tests/runner-e2e/merge.config.ts merged-blob-reports
- name: Aggregate normalized campaign results
if: always()
env:
PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/downloaded-runner-e2e
PAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalized
PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: ${{ needs.catalog.outputs.execution_ids }}
PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-${{ github.run_attempt }}
run: |
set +e
pnpm test:e2e:runner:report
report_status=$?
set -e
cat runner-e2e-merged-report/normalized/summary.md >> "$GITHUB_STEP_SUMMARY"
exit "$report_status"
- name: Upload access-controlled merged report
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: runner-e2e-report-${{ github.run_id }}-${{ github.run_attempt }}
path: runner-e2e-merged-report/
retention-days: 30
if-no-files-found: error
- name: Verify history source report and private screenshot evidence
id: history_source_ready
if: always()
run: |
set -euo pipefail
dashboard_root="runner-e2e-merged-report/normalized"
private_screenshot="$(find "$dashboard_root" -type f -name '*.png' -print -quit 2>/dev/null || true)"
if [ -f "$dashboard_root/index.html" ] && [ -n "$private_screenshot" ]; then
echo "ready=true" >> "$GITHUB_OUTPUT"
else
echo "ready=false" >> "$GITHUB_OUTPUT"
fi
publish_history:
name: Publish pruned immutable history and landing site
needs: [catalog, report]
if: always() && needs.catalog.result == 'success' && needs.report.outputs.history_source_ready == 'true'
runs-on: ubuntu-latest
timeout-minutes: 15
concurrency:
group: runner-e2e-history-publish
cancel-in-progress: false
permissions:
contents: read
id-token: write
environment:
name: runner-e2e-history
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
with:
version: 9.15.4
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- run: pnpm install --frozen-lockfile
- name: Download access-controlled normalized campaign
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
name: runner-e2e-report-${{ github.run_id }}-${{ github.run_attempt }}
path: runner-e2e-merged-report
- name: Exchange GitHub OIDC identity for scoped AWS credentials
uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6
with:
role-to-assume: ${{ vars.RUNNER_E2E_HISTORY_AWS_ROLE_ARN }}
aws-region: ${{ vars.RUNNER_E2E_HISTORY_AWS_REGION }}
- name: Prune private evidence and publish immutable campaign history
env:
PAPERCLIP_RUNNER_E2E_REPORT_DIR: ${{ github.workspace }}/runner-e2e-merged-report/normalized
RUNNER_E2E_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_HISTORY_S3_BUCKET }}
RUNNER_E2E_HISTORY_PREFIX: ${{ vars.RUNNER_E2E_HISTORY_PREFIX || 'runner-e2e' }}
RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: ${{ vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL }}
run: pnpm test:e2e:runner:history:publish
- name: Package pruned structured dashboard for GitHub Pages
if: vars.RUNNER_FULL_STACK_E2E_PUBLISH_PAGES == 'true'
uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4
with:
path: runner-e2e-merged-report/normalized
pages:
name: Publish latest structured dashboard
needs: [report, publish_history]
if: always() && needs.report.outputs.history_source_ready == 'true' && needs.publish_history.result == 'success' && vars.RUNNER_FULL_STACK_E2E_PUBLISH_PAGES == 'true'
runs-on: ubuntu-latest
permissions:
pages: write
id-token: write
environment:
name: github-pages
url: ${{ steps.deployment.outputs.page_url }}
steps:
- name: Deploy to GitHub Pages
id: deployment
uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4
Reference in New Issue View Git Blame Copy Permalink