diff --git a/.env.runner-e2e.example b/.env.runner-e2e.example new file mode 100644 index 0000000000..85c99a7836 --- /dev/null +++ b/.env.runner-e2e.example @@ -0,0 +1,10 @@ +# Copy to .env.runner-e2e.local. Existing shell variables take precedence. +# Never commit the local file or put these values in fixture source. +OPENAI_API_KEY= +ANTHROPIC_API_KEY= +OPENROUTER_API_KEY= +DAYTONA_API_KEY= + +# Required only for Daytona cells. Use an immutable, anonymously pullable +# digest from the Runner Full-Stack E2E image job or a locally published image. +PAPERCLIP_E2E_DAYTONA_IMAGE=ghcr.io/paperclipai/paperclip-daytona-runner@sha256:REPLACE_ME diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 1b5159a072..11a6efc045 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -9,23 +9,65 @@ on: default: true jobs: + authorize: + name: Authorize optional paid E2E + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + steps: + - name: Require default branch and allowlisted numeric actor IDs + env: + GH_TOKEN: ${{ github.token }} + REF: ${{ github.ref }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + ACTOR_ID: ${{ github.actor_id }} + TRIGGERING_ACTOR: ${{ github.triggering_actor }} + ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} + run: | + set -euo pipefail + test "$REF" = "refs/heads/$DEFAULT_BRANCH" + jq -e 'type == "array" and length > 0 and all(.[]; type == "number" and . > 0 and floor == .)' <<< "${ALLOWED_ACTOR_IDS:-}" >/dev/null + triggering_actor_id="$(gh api "users/$TRIGGERING_ACTOR" --jq .id)" + jq -e --argjson candidate "$triggering_actor_id" 'index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null + jq -e --argjson candidate "$ACTOR_ID" 'index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null + e2e: + needs: authorize runs-on: ubuntu-latest timeout-minutes: 30 + permissions: + contents: read + environment: + name: runner-e2e-paid env: PAPERCLIP_E2E_SKIP_LLM: ${{ inputs.skip_llm && 'true' || 'false' }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} steps: - - uses: actions/checkout@v7 + - name: Reauthorize execution before optional provider access + env: + GH_TOKEN: ${{ github.token }} + REF: ${{ github.ref }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + ACTOR_ID: ${{ github.actor_id }} + TRIGGERING_ACTOR: ${{ github.triggering_actor }} + ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} + run: | + set -euo pipefail + test "$REF" = "refs/heads/$DEFAULT_BRANCH" + jq -e 'type == "array" and length > 0 and all(.[]; type == "number" and . > 0 and floor == .)' <<< "${ALLOWED_ACTOR_IDS:-}" >/dev/null + triggering_actor_id="$(gh api "users/$TRIGGERING_ACTOR" --jq .id)" + jq -e --argjson candidate "$triggering_actor_id" 'index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null + jq -e --argjson candidate "$ACTOR_ID" 'index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null - - uses: pnpm/action-setup@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 with: version: 9 - - uses: actions/setup-node@v7 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 with: node-version: 24 - cache: pnpm - run: pnpm install --frozen-lockfile - run: pnpm build @@ -34,9 +76,10 @@ jobs: - name: Run e2e tests env: PAPERCLIP_PLAYWRIGHT_CHANNEL: "chrome" + ANTHROPIC_API_KEY: ${{ !inputs.skip_llm && secrets.ANTHROPIC_API_KEY || '' }} run: pnpm run test:e2e - - uses: actions/upload-artifact@v7 + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() with: name: playwright-report diff --git a/.github/workflows/release-verify.yml b/.github/workflows/release-verify.yml index 55d457a24d..a8bc1abbf7 100644 --- a/.github/workflows/release-verify.yml +++ b/.github/workflows/release-verify.yml @@ -9,6 +9,12 @@ on: type: string jobs: + runner_chaos_evals: + name: Pre-release Runner chaos evals + uses: ./.github/workflows/runner-chaos-evals.yml + with: + ref: ${{ inputs.ref }} + typecheck: name: Typecheck runs-on: ubuntu-latest @@ -18,17 +24,17 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: ref: ${{ inputs.ref }} - name: Setup pnpm - uses: pnpm/action-setup@v6 + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 with: version: 9.15.4 - name: Setup Node.js - uses: actions/setup-node@v7 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 with: node-version: 24 cache: pnpm @@ -79,17 +85,17 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: ref: ${{ inputs.ref }} - name: Setup pnpm - uses: pnpm/action-setup@v6 + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 with: version: 9.15.4 - name: Setup Node.js - uses: actions/setup-node@v7 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 with: node-version: 24 cache: pnpm @@ -134,17 +140,17 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: ref: ${{ inputs.ref }} - name: Setup pnpm - uses: pnpm/action-setup@v6 + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 with: version: 9.15.4 - name: Setup Node.js - uses: actions/setup-node@v7 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 with: node-version: 24 cache: pnpm @@ -155,6 +161,36 @@ jobs: - name: Run serialized server test shard run: pnpm test:run:serialized -- --shard-index ${{ matrix.shard_index }} --shard-count ${{ matrix.shard_count }} + runner_workflow_evals: + name: Runner workflow eval scorer contract + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ inputs.ref }} + + - name: Setup pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 + with: + version: 9.15.4 + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Run deterministic Runner workflow scorer tests + run: pnpm test:runner-workflow-evals + build: name: Build runs-on: ubuntu-latest @@ -164,17 +200,17 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: ref: ${{ inputs.ref }} - name: Setup pnpm - uses: pnpm/action-setup@v6 + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 with: version: 9.15.4 - name: Setup Node.js - uses: actions/setup-node@v7 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 with: node-version: 24 cache: pnpm diff --git a/.github/workflows/runner-chaos-evals.yml b/.github/workflows/runner-chaos-evals.yml new file mode 100644 index 0000000000..3131959522 --- /dev/null +++ b/.github/workflows/runner-chaos-evals.yml @@ -0,0 +1,83 @@ +name: Runner Chaos Evals + +on: + schedule: + - cron: "43 7 * * 0" + workflow_dispatch: + workflow_call: + inputs: + ref: + description: Commit SHA, branch, or tag to verify before release + required: false + type: string + +concurrency: + group: runner-chaos-evals-${{ inputs.ref || github.ref }} + cancel-in-progress: true + +jobs: + chaos_and_recovery: + name: Restart, replay, trace, and recovery faults + runs-on: ubuntu-latest + timeout-minutes: 45 + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Setup pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 + with: + version: 9.15.4 + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Build eval and Runner contracts + run: | + pnpm --filter @paperclipai/paperclip-eval-kernel build + pnpm --filter @paperclipai/paperclip-runner report:runner-chaos-evals + + - name: Run Runner fault and replay suites + run: | + pnpm --filter @paperclipai/paperclip-runner exec vitest run \ + src/eval/workflow-evals.test.ts \ + src/native-session-runtime.test.ts \ + src/live/live-session.test.ts \ + src/live/turn-stream.test.ts \ + src/protocol/replay-contract.test.ts \ + src/drivers/opencode/mcp-bridge.test.ts \ + src/drivers/acpx/runtime-host.test.ts + + - name: Build server test dependencies + run: pnpm --filter @paperclipai/plugin-sdk ensure-build-deps + + - name: Run server finalization and recovery suites + run: | + pnpm --filter @paperclipai/server exec vitest run \ + src/__tests__/native-finalization-recovery.test.ts \ + src/__tests__/heartbeat-process-recovery.test.ts \ + src/__tests__/heartbeat-comment-wake-batching.test.ts \ + src/__tests__/heartbeat-dependency-scheduling.test.ts \ + src/__tests__/provider-trace-store.test.ts \ + src/services/issue-thread-interaction-resolution.test.ts \ + src/services/recovery/successful-run-handoff.test.ts + + - name: Upload chaos eval bundle + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: runner-chaos-evals-${{ github.run_id }} + path: packages/paperclip-runner/.paperclip-local/evals/workflows/ + retention-days: 30 + if-no-files-found: error diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.yml new file mode 100644 index 0000000000..f4d7cb103f --- /dev/null +++ b/.github/workflows/runner-full-stack-e2e.yml @@ -0,0 +1,535 @@ +name: Runner Full-Stack E2E + +on: + schedule: + - cron: "47 8 * * 0" + workflow_dispatch: + inputs: + all: + description: "Run the complete paid matrix when no narrower selector is supplied" + type: boolean + default: true + group: + description: "Comma-separated groups (AND semantics: legacy,native,local,daytona,core,breadth)" + type: string + required: false + suite: + description: "Comma-separated suite IDs" + type: string + required: false + profile: + description: "Comma-separated runner profile fixture IDs" + type: string + required: false + environment: + description: "Comma-separated environment fixture IDs" + type: string + required: false + case: + description: "Comma-separated task case fixture IDs" + type: string + required: false + id: + description: "Comma-separated full suite.profile.environment.case IDs; exclusive with other selectors" + type: string + required: false + +permissions: + contents: read + +concurrency: + group: runner-full-stack-e2e-${{ github.ref }} + cancel-in-progress: false + +jobs: + authorize: + name: Authorize paid campaign + if: github.event_name != 'schedule' || vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + steps: + - name: Require default branch and allowlisted numeric actor IDs + env: + GH_TOKEN: ${{ github.token }} + REPOSITORY: ${{ github.repository }} + REF: ${{ github.ref }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + ACTOR: ${{ github.actor }} + ACTOR_ID: ${{ github.actor_id }} + TRIGGERING_ACTOR: ${{ github.triggering_actor }} + ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} + run: | + set -euo pipefail + if [ "$REF" != "refs/heads/$DEFAULT_BRANCH" ]; then + echo "Paid runner E2E campaigns may run only from the default branch." >&2 + exit 1 + fi + if ! jq -e 'type == "array" and length > 0 and all(.[]; type == "number" and . > 0 and floor == .)' <<< "${ALLOWED_ACTOR_IDS:-}" >/dev/null; then + echo "RUNNER_E2E_ALLOWED_ACTOR_IDS must be a non-empty JSON array of numeric GitHub user IDs." >&2 + exit 1 + fi + triggering_actor_id="$(gh api "users/$TRIGGERING_ACTOR" --jq .id)" + if [ "$triggering_actor_id" != "$ACTOR_ID" ] && [ "$TRIGGERING_ACTOR" = "$ACTOR" ]; then + echo "GitHub actor identity contexts disagree; refusing the paid run." >&2 + exit 1 + fi + candidates=("$triggering_actor_id" "$ACTOR_ID") + for candidate in "${candidates[@]}"; do + if ! jq -e --argjson candidate "$candidate" 'index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null; then + echo "The initiating GitHub account is not authorized to run paid runner E2E campaigns." >&2 + exit 1 + fi + done + + catalog: + name: Validate catalog and select cells + needs: authorize + if: github.event_name != 'schedule' || vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + outputs: + matrix: ${{ steps.catalog.outputs.matrix }} + needs_daytona: ${{ steps.catalog.outputs.needs_daytona }} + execution_ids: ${{ steps.catalog.outputs.execution_ids }} + max_parallel: ${{ steps.catalog.outputs.max_parallel }} + daytona_image_content_id: ${{ steps.daytona_image_content.outputs.content_id }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 + with: + version: 9.15.4 + + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + cache: pnpm + + - run: pnpm install --frozen-lockfile + + # The v2 contract fails closed unless every Docker FROM is digest-pinned, + # and hashes those exact base references into the immutable image tag. + - name: Compute Daytona image content ID with pinned bases + id: daytona_image_content + run: echo "content_id=$(pnpm --silent test:e2e:runner:image-id)" >> "$GITHUB_OUTPUT" + + - name: Validate selectors and emit matrix + id: catalog + env: + EVENT_NAME: ${{ github.event_name }} + SELECT_ALL: ${{ inputs.all }} + SELECT_SUITE: ${{ inputs.suite }} + SELECT_GROUP: ${{ inputs.group }} + SELECT_PROFILE: ${{ inputs.profile }} + SELECT_ENVIRONMENT: ${{ inputs.environment }} + SELECT_CASE: ${{ inputs.case }} + SELECT_ID: ${{ inputs.id }} + MAX_PARALLEL: ${{ vars.RUNNER_E2E_MAX_PARALLEL || '32' }} + run: | + set -euo pipefail + args=(--matrix-json) + add_values() { + local flag="$1" + local values="$2" + local value + IFS=',' read -ra entries <<< "$values" + for value in "${entries[@]}"; do + value="${value#"${value%%[![:space:]]*}"}" + value="${value%"${value##*[![:space:]]}"}" + if [ -n "$value" ]; then + args+=("$flag" "$value") + fi + done + } + explicit=false + if [ -n "${SELECT_ID:-}" ]; then + if [ -n "${SELECT_SUITE:-}${SELECT_GROUP:-}${SELECT_PROFILE:-}${SELECT_ENVIRONMENT:-}${SELECT_CASE:-}" ]; then + echo "The id selector is exclusive with suite/group/profile/environment/case" >&2 + exit 1 + fi + add_values --id "$SELECT_ID" + explicit=true + else + for pair in \ + "--suite:${SELECT_SUITE:-}" \ + "--group:${SELECT_GROUP:-}" \ + "--profile:${SELECT_PROFILE:-}" \ + "--environment:${SELECT_ENVIRONMENT:-}" \ + "--case:${SELECT_CASE:-}" + do + flag="${pair%%:*}" + values="${pair#*:}" + if [ -n "$values" ]; then + add_values "$flag" "$values" + explicit=true + fi + done + fi + if [ "$explicit" = false ] && { [ "$EVENT_NAME" = schedule ] || [ "${SELECT_ALL:-false}" = true ]; }; then + args+=(--all) + fi + catalog_json="$(pnpm --silent test:e2e:runner -- "${args[@]}")" + echo "matrix=$(jq -c '{include: .include}' <<< "$catalog_json")" >> "$GITHUB_OUTPUT" + echo "needs_daytona=$(jq -r '.needsDaytona' <<< "$catalog_json")" >> "$GITHUB_OUTPUT" + echo "execution_ids=$(jq -c '.executionIds' <<< "$catalog_json")" >> "$GITHUB_OUTPUT" + if ! [[ "$MAX_PARALLEL" =~ ^[1-9][0-9]*$ ]] || [ "$MAX_PARALLEL" -gt 57 ]; then + echo "RUNNER_E2E_MAX_PARALLEL must be an integer from 1 through 57." >&2 + exit 1 + fi + echo "max_parallel=$MAX_PARALLEL" >> "$GITHUB_OUTPUT" + + daytona_image: + name: Publish verified Daytona image + needs: [authorize, catalog] + runs-on: ubuntu-latest + timeout-minutes: 45 + permissions: + contents: read + packages: write + id-token: write + outputs: + image: ${{ steps.image.outputs.image }} + source_revision: ${{ steps.image.outputs.source_revision }} + content_id: ${{ steps.image.outputs.content_id }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: No Daytona image needed + id: local_only + if: needs.catalog.outputs.needs_daytona != 'true' + run: echo "image=" >> "$GITHUB_OUTPUT" + + - name: Set up Docker Buildx + if: needs.catalog.outputs.needs_daytona == 'true' + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4 + + - name: Log into GHCR + if: needs.catalog.outputs.needs_daytona == 'true' + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Install cosign + if: needs.catalog.outputs.needs_daytona == 'true' + uses: sigstore/cosign-installer@398d4b0eeef1380460a10c8013a76f728fb906ac # v3 + + - name: Reuse or publish immutable image + id: image + env: + NEEDS_DAYTONA: ${{ needs.catalog.outputs.needs_daytona }} + IMAGE_CONTENT_ID: ${{ needs.catalog.outputs.daytona_image_content_id }} + IMAGE_TAG: ghcr.io/paperclipai/paperclip-daytona-runner:e2e-content-${{ needs.catalog.outputs.daytona_image_content_id }} + run: | + set -euo pipefail + if [ "$NEEDS_DAYTONA" != true ]; then + echo "image=" >> "$GITHUB_OUTPUT" + echo "source_revision=" >> "$GITHUB_OUTPUT" + echo "content_id=" >> "$GITHUB_OUTPUT" + exit 0 + fi + [[ "$IMAGE_CONTENT_ID" =~ ^[0-9a-f]{64}$ ]] + identity="^https://github.com/${GITHUB_REPOSITORY}/.github/workflows/runner-full-stack-e2e.yml@" + if docker buildx imagetools inspect "$IMAGE_TAG" >/dev/null 2>&1; then + digest="$(docker buildx imagetools inspect "$IMAGE_TAG" --format '{{json .Manifest.Digest}}' | tr -d '"')" + else + docker buildx build \ + --platform linux/amd64 \ + --build-arg "PAPERCLIP_RUNNER_CONTENT_ID=${IMAGE_CONTENT_ID}" \ + --build-arg "PAPERCLIP_RUNNER_SOURCE_REVISION=${GITHUB_SHA}" \ + --file docker/daytona-runner/Dockerfile \ + --tag "$IMAGE_TAG" \ + --push \ + . + digest="$(docker buildx imagetools inspect "$IMAGE_TAG" --format '{{json .Manifest.Digest}}' | tr -d '"')" + cosign sign --yes "$IMAGE_TAG@$digest" + fi + cosign verify \ + --certificate-identity-regexp "$identity" \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com \ + "$IMAGE_TAG@$digest" >/dev/null + immutable="${IMAGE_TAG%:*}@$digest" + # The Daytona base image is large. The build cache plus a second full + # anonymous pull can exhaust a standard GitHub-hosted runner before + # Docker creates the tiny metadata-probe container. The pushed digest + # is already immutable, so release the local builder/cache first. + docker buildx prune --all --force >/dev/null + docker system prune --all --force >/dev/null + anonymous_config="$(mktemp -d)" + docker --config "$anonymous_config" pull "$immutable" + # The Dockerfile's final two RUN steps execute the runner metadata, + # transport-mode, provider-pack JSON, and pinned ACP binary checks as + # root and as the unprivileged Daytona user. Starting another + # container after this full pull can exhaust the hosted runner's thin + # writable layer even after pruning, so assert the published image + # configuration here without creating a redundant container. + image_config="$(docker image inspect "$immutable" \ + --format '{{json .}}')" + published_content_id="$(jq -r '.Config.Labels["io.paperclip.runner.content-id"] // empty' <<< "$image_config")" + source_revision="$(jq -r '.Config.Labels["org.opencontainers.image.revision"] // empty' <<< "$image_config")" + test "$published_content_id" = "$IMAGE_CONTENT_ID" + [[ "$source_revision" =~ ^[0-9a-f]{40}$ ]] + jq -e \ + '.Architecture == "amd64" and + .Os == "linux" and + .Config.User == "daytona" and + (.Config.Env | any(startswith("PAPERCLIP_RUNNER_PROVIDER_PACK_ROOT=")))' \ + <<< "$image_config" >/dev/null + echo "image=$immutable" >> "$GITHUB_OUTPUT" + echo "source_revision=$source_revision" >> "$GITHUB_OUTPUT" + echo "content_id=$published_content_id" >> "$GITHUB_OUTPUT" + + test: + name: ${{ matrix.executionId }} + needs: [catalog, daytona_image] + runs-on: ubuntu-latest-m + timeout-minutes: ${{ matrix.timeoutMinutes }} + permissions: + contents: read + environment: + name: runner-e2e-paid + strategy: + fail-fast: false + max-parallel: ${{ fromJSON(needs.catalog.outputs.max_parallel) }} + matrix: ${{ fromJSON(needs.catalog.outputs.matrix) }} + steps: + - name: Reauthorize paid execution before provider access + env: + GH_TOKEN: ${{ github.token }} + REF: ${{ github.ref }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + ACTOR_ID: ${{ github.actor_id }} + TRIGGERING_ACTOR: ${{ github.triggering_actor }} + ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} + run: | + set -euo pipefail + test "$REF" = "refs/heads/$DEFAULT_BRANCH" + jq -e 'type == "array" and length > 0 and all(.[]; type == "number" and . > 0 and floor == .)' <<< "${ALLOWED_ACTOR_IDS:-}" >/dev/null + triggering_actor_id="$(gh api "users/$TRIGGERING_ACTOR" --jq .id)" + jq -e --argjson candidate "$triggering_actor_id" 'index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null + jq -e --argjson candidate "$ACTOR_ID" 'index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 + with: + version: 9.15.4 + + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + + - run: pnpm install --frozen-lockfile + + - name: Build runner TypeScript prerequisites + run: pnpm --filter @paperclipai/paperclip-eval-kernel build + + - name: Build native remote provider pack + if: matrix.environmentId == 'daytona' && (matrix.profileId == 'runner-opencode' || startsWith(matrix.profileId, 'runner-acpx-')) + env: + # Reused images retain the source revision that was embedded in their + # provider pack. Matching it here lets the server reuse that exact + # preinstalled pack instead of uploading a duplicate to the lease. + PAPERCLIP_RUNNER_SOURCE_REVISION: ${{ needs.daytona_image.outputs.source_revision }} + run: pnpm --filter @paperclipai/paperclip-runner build:provider-pack + + - name: Install pinned legacy Claude CLI + if: matrix.profileId == 'legacy-claude' + run: npm install --global --omit=dev @anthropic-ai/claude-code@2.1.19 + + - name: Build native runner binaries + if: startsWith(matrix.profileId, 'runner-') || matrix.suiteId == 'openrouter-model-breadth' + run: pnpm --filter @paperclipai/paperclip-runner build:runner-binaries + + - name: Install Chromium + run: pnpm exec playwright install --with-deps chromium + + - name: Run paid cell + env: + OPENAI_API_KEY: ${{ matrix.credentialName == 'OPENAI_API_KEY' && secrets.OPENAI_API_KEY || '' }} + ANTHROPIC_API_KEY: ${{ matrix.credentialName == 'ANTHROPIC_API_KEY' && secrets.ANTHROPIC_API_KEY || '' }} + OPENROUTER_API_KEY: ${{ matrix.credentialName == 'OPENROUTER_API_KEY' && secrets.OPENROUTER_API_KEY || '' }} + DAYTONA_API_KEY: ${{ matrix.environmentId == 'daytona' && secrets.DAYTONA_API_KEY || '' }} + PAPERCLIP_E2E_DAYTONA_IMAGE: ${{ needs.daytona_image.outputs.image }} + PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH: ${{ github.workspace }}/packages/paperclip-runner/provider-pack + PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.executionId }} + run: pnpm test:e2e:runner -- --id "${{ matrix.executionId }}" + + - name: Upload access-controlled packaged cell evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: runner-e2e-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.executionId }} + path: tests/runner-e2e/results/ + retention-days: 30 + if-no-files-found: error + + report: + name: Merge and enforce campaign result + if: always() && needs.catalog.result == 'success' + needs: [catalog, daytona_image, test] + outputs: + history_source_ready: ${{ steps.history_source_ready.outputs.ready }} + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 + with: + version: 9.15.4 + + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + cache: pnpm + + - run: pnpm install --frozen-lockfile + + - name: Download cell evidence + id: download_evidence + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + pattern: runner-e2e-${{ github.run_id }}-${{ github.run_attempt }}-* + path: downloaded-runner-e2e + merge-multiple: true + + - name: Retry cell evidence download after transport failure + if: steps.download_evidence.outcome == 'failure' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + pattern: runner-e2e-${{ github.run_id }}-${{ github.run_attempt }}-* + path: downloaded-runner-e2e + merge-multiple: true + + - name: Collect blob reports + run: | + set -euo pipefail + mkdir -p merged-blob-reports + while IFS= read -r -d '' report; do + digest="$(sha256sum "$report" | cut -d ' ' -f 1)" + target="merged-blob-reports/report-${digest}.zip" + if [ ! -e "$target" ]; then + cp "$report" "$target" + fi + done < <(find downloaded-runner-e2e -path '*/blob-report/*.zip' -print0) + + - name: Merge Playwright HTML and JUnit + if: always() + env: + PAPERCLIP_RUNNER_E2E_MERGED_REPORT_DIR: ${{ github.workspace }}/runner-e2e-merged-report + run: pnpm exec playwright merge-reports --config tests/runner-e2e/merge.config.ts merged-blob-reports + + - name: Aggregate normalized campaign results + if: always() + env: + PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/downloaded-runner-e2e + PAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalized + PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: ${{ needs.catalog.outputs.execution_ids }} + PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-${{ github.run_attempt }} + run: | + set +e + pnpm test:e2e:runner:report + report_status=$? + set -e + cat runner-e2e-merged-report/normalized/summary.md >> "$GITHUB_STEP_SUMMARY" + exit "$report_status" + + - name: Upload access-controlled merged report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: runner-e2e-report-${{ github.run_id }}-${{ github.run_attempt }} + path: runner-e2e-merged-report/ + retention-days: 30 + if-no-files-found: error + + - name: Verify history source report and private screenshot evidence + id: history_source_ready + if: always() + run: | + set -euo pipefail + dashboard_root="runner-e2e-merged-report/normalized" + private_screenshot="$(find "$dashboard_root" -type f -name '*.png' -print -quit 2>/dev/null || true)" + if [ -f "$dashboard_root/index.html" ] && [ -n "$private_screenshot" ]; then + echo "ready=true" >> "$GITHUB_OUTPUT" + else + echo "ready=false" >> "$GITHUB_OUTPUT" + fi + + publish_history: + name: Publish pruned immutable history and landing site + needs: [catalog, report] + if: always() && needs.catalog.result == 'success' && needs.report.outputs.history_source_ready == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + concurrency: + group: runner-e2e-history-publish + cancel-in-progress: false + permissions: + contents: read + id-token: write + environment: + name: runner-e2e-history + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 + with: + version: 9.15.4 + + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + + - run: pnpm install --frozen-lockfile + + - name: Download access-controlled normalized campaign + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: runner-e2e-report-${{ github.run_id }}-${{ github.run_attempt }} + path: runner-e2e-merged-report + + - name: Exchange GitHub OIDC identity for scoped AWS credentials + uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6 + with: + role-to-assume: ${{ vars.RUNNER_E2E_HISTORY_AWS_ROLE_ARN }} + aws-region: ${{ vars.RUNNER_E2E_HISTORY_AWS_REGION }} + + - name: Prune private evidence and publish immutable campaign history + env: + PAPERCLIP_RUNNER_E2E_REPORT_DIR: ${{ github.workspace }}/runner-e2e-merged-report/normalized + RUNNER_E2E_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_HISTORY_S3_BUCKET }} + RUNNER_E2E_HISTORY_PREFIX: ${{ vars.RUNNER_E2E_HISTORY_PREFIX || 'runner-e2e' }} + RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: ${{ vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL }} + run: pnpm test:e2e:runner:history:publish + + - name: Package pruned structured dashboard for GitHub Pages + if: vars.RUNNER_FULL_STACK_E2E_PUBLISH_PAGES == 'true' + uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4 + with: + path: runner-e2e-merged-report/normalized + + pages: + name: Publish latest structured dashboard + needs: [report, publish_history] + if: always() && needs.report.outputs.history_source_ready == 'true' && needs.publish_history.result == 'success' && vars.RUNNER_FULL_STACK_E2E_PUBLISH_PAGES == 'true' + runs-on: ubuntu-latest + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4 diff --git a/.github/workflows/runner-live-evals.yml b/.github/workflows/runner-live-evals.yml new file mode 100644 index 0000000000..978bdca02e --- /dev/null +++ b/.github/workflows/runner-live-evals.yml @@ -0,0 +1,134 @@ +name: Runner Live Evals + +on: + schedule: + - cron: "17 6 * * 0" + workflow_dispatch: + +concurrency: + group: runner-live-evals-${{ github.ref }} + cancel-in-progress: true + +jobs: + authorize: + name: Authorize paid campaign + if: github.event_name != 'schedule' || vars.RUNNER_LIVE_EVALS_NIGHTLY_ENABLED == 'true' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + steps: + - name: Require default branch and allowlisted numeric actor IDs + env: + GH_TOKEN: ${{ github.token }} + REF: ${{ github.ref }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + ACTOR: ${{ github.actor }} + ACTOR_ID: ${{ github.actor_id }} + TRIGGERING_ACTOR: ${{ github.triggering_actor }} + ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} + run: | + set -euo pipefail + if [ "$REF" != "refs/heads/$DEFAULT_BRANCH" ]; then + echo "Paid runner live evals may run only from the default branch." >&2 + exit 1 + fi + if ! jq -e 'type == "array" and length > 0 and all(.[]; type == "number" and . > 0 and floor == .)' <<< "${ALLOWED_ACTOR_IDS:-}" >/dev/null; then + echo "RUNNER_E2E_ALLOWED_ACTOR_IDS must be a non-empty JSON array of numeric GitHub user IDs." >&2 + exit 1 + fi + triggering_actor_id="$(gh api "users/$TRIGGERING_ACTOR" --jq .id)" + if [ "$triggering_actor_id" != "$ACTOR_ID" ] && [ "$TRIGGERING_ACTOR" = "$ACTOR" ]; then + echo "GitHub actor identity contexts disagree; refusing the paid run." >&2 + exit 1 + fi + candidates=("$triggering_actor_id" "$ACTOR_ID") + for candidate in "${candidates[@]}"; do + if ! jq -e --argjson candidate "$candidate" 'index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null; then + echo "The initiating GitHub account is not authorized to run paid runner live evals." >&2 + exit 1 + fi + done + + live_matrix: + name: Balanced provider/model matrix + needs: authorize + if: github.event_name != 'schedule' || vars.RUNNER_LIVE_EVALS_NIGHTLY_ENABLED == 'true' + runs-on: ubuntu-latest + timeout-minutes: 180 + permissions: + contents: read + environment: + name: runner-e2e-paid + + steps: + - name: Reauthorize paid execution before provider access + env: + GH_TOKEN: ${{ github.token }} + REF: ${{ github.ref }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + ACTOR_ID: ${{ github.actor_id }} + TRIGGERING_ACTOR: ${{ github.triggering_actor }} + ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} + run: | + set -euo pipefail + test "$REF" = "refs/heads/$DEFAULT_BRANCH" + jq -e 'type == "array" and length > 0 and all(.[]; type == "number" and . > 0 and floor == .)' <<< "${ALLOWED_ACTOR_IDS:-}" >/dev/null + triggering_actor_id="$(gh api "users/$TRIGGERING_ACTOR" --jq .id)" + jq -e --argjson candidate "$triggering_actor_id" 'index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null + jq -e --argjson candidate "$ACTOR_ID" 'index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null + + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Setup pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 + with: + version: 9.15.4 + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Restore compatible weekly baseline + uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5 + with: + path: packages/paperclip-runner/.paperclip-local/evals/workflows/history + key: runner-live-eval-history-${{ github.ref_name }}-${{ github.run_id }} + restore-keys: | + runner-live-eval-history-${{ github.ref_name }}- + + - name: Build provider-neutral eval kernel + run: pnpm --filter @paperclipai/paperclip-eval-kernel build + + - name: Run trend-only live matrix + env: + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + PAPERCLIP_EVAL_BASELINE_READY: "true" + PAPERCLIP_EVAL_RUNNER_BUILD: ${{ github.sha }} + PAPERCLIP_EVAL_MAX_CAMPAIGN_COST_USD: "12" + PAPERCLIP_EVAL_SCHEDULE_SEED: runner-live-seven-week-v1 + run: pnpm --filter @paperclipai/paperclip-runner report:runner-live-evals + + - name: Publish job summary + if: always() + run: | + summary=packages/paperclip-runner/.paperclip-local/evals/workflows/github-live-summary.md + if [ -f "$summary" ]; then + cat "$summary" >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Upload safe live eval bundle + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: runner-live-evals-${{ github.run_id }} + path: packages/paperclip-runner/.paperclip-local/evals/workflows/ + retention-days: 30 + if-no-files-found: error diff --git a/.gitignore b/.gitignore index 8f54df5a15..0413086153 100644 --- a/.gitignore +++ b/.gitignore @@ -60,6 +60,8 @@ diagnostics/ # Playwright tests/e2e/test-results/ tests/e2e/playwright-report/ +tests/runner-e2e/results/ +.env.runner-e2e.local tests/release-smoke/test-results/ tests/release-smoke/playwright-report/ test-results/issue-detail-perf/ diff --git a/docker/daytona-runner/Dockerfile b/docker/daytona-runner/Dockerfile new file mode 100644 index 0000000000..294b5b1292 --- /dev/null +++ b/docker/daytona-runner/Dockerfile @@ -0,0 +1,115 @@ +# syntax=docker/dockerfile:1.7@sha256:a57df69d0ea827fb7266491f2813635de6f17269be881f696fbfdf2d83dda33e + +# Keep the Rust toolchain explicit so the runner artifact is reproducible and +# is always built for the same platform as the final Daytona image. Base-image +# digests are the reviewed linux/amd64 manifests; the tags are annotations. +FROM rust:1.97-bookworm@sha256:408fe88047cef61a2087653b0c5255fa51c0f2d6d94ddedd7a2562a9b91a46f6 AS runnerd-build + +WORKDIR /workspace/packages/paperclip-runner/runner +COPY packages/paperclip-runner/runner/Cargo.toml packages/paperclip-runner/runner/Cargo.lock ./ +COPY packages/paperclip-runner/runner/crates ./crates +COPY packages/paperclip-runner/protocol ../protocol +RUN cargo build --locked --release -p paperclip-runner-core --bin paperclip-runnerd \ + && strip /workspace/packages/paperclip-runner/runner/target/release/paperclip-runnerd + +FROM node:24-bookworm@sha256:9137a20e25879e0b557227b57e3ee4e9af4bde29eb3db66134cd1723e84f830b AS provider-pack-build +ARG PAPERCLIP_RUNNER_SOURCE_REVISION +RUN test -n "${PAPERCLIP_RUNNER_SOURCE_REVISION}" +RUN corepack enable && corepack prepare pnpm@9.15.4 --activate +WORKDIR /workspace +COPY package.json pnpm-lock.yaml pnpm-workspace.yaml .npmrc tsconfig.base.json ./ +COPY patches ./patches +COPY scripts/link-plugin-dev-sdk.mjs ./scripts/link-plugin-dev-sdk.mjs +COPY packages/paperclip-eval-kernel ./packages/paperclip-eval-kernel +COPY packages/paperclip-runner ./packages/paperclip-runner +RUN pnpm install --frozen-lockfile --filter '@paperclipai/paperclip-runner...' +RUN pnpm --filter @paperclipai/paperclip-runner build:typescript \ + && PAPERCLIP_RUNNER_SOURCE_REVISION="${PAPERCLIP_RUNNER_SOURCE_REVISION}" \ + node packages/paperclip-runner/scripts/build-provider-pack.mjs /provider-pack + +# Fleet sandbox base image. Keep this section aligned with +# paperclipai/paperclip-cloud/fleet-sandbox-image/Dockerfile. The only Paperclip +# runner-specific addition is /usr/local/bin/paperclip-runnerd below. +FROM daytonaio/sandbox:0.8.0@sha256:eadf88e4391072b7ad4bed27d9cadfc9fe9d8ed375d9219d34c2ccb518f213e3 + +ARG PAPERCLIP_RUNNER_CONTENT_ID +ARG PAPERCLIP_RUNNER_SOURCE_REVISION +RUN test -n "${PAPERCLIP_RUNNER_CONTENT_ID}" \ + && test -n "${PAPERCLIP_RUNNER_SOURCE_REVISION}" +LABEL io.paperclip.runner.content-id="${PAPERCLIP_RUNNER_CONTENT_ID}" \ + org.opencontainers.image.revision="${PAPERCLIP_RUNNER_SOURCE_REVISION}" + +USER root + +ENV PAPERCLIP_RUNNER_PROVIDER_PACK_ROOT=/opt/paperclip-runner/provider-pack +ENV PATH=${PAPERCLIP_RUNNER_PROVIDER_PACK_ROOT}/node_modules/.bin:/usr/local/share/nvm/current/bin:/usr/local/python/current/bin:/usr/local/py-utils/bin:${PATH} + +RUN npm install -g \ + @anthropic-ai/claude-code@2.1.19 \ + @openai/codex@0.148.0 \ + @xai-official/grok@1.0.3 \ + @google/gemini-cli@0.56.0 \ + @moonshot-ai/kimi-code@0.38.0 \ + opencode-ai@1.18.17 \ + && npm cache clean --force + +RUN pip install --no-cache-dir --break-system-packages hermes-agent==0.15.2 + +ARG CURSOR_VERSION=2026.08.11-e8db854 +ARG CURSOR_SHA256_AMD64=bfff4bf6f4e9dd30c1d0ef0a70b6077b074015dd2948e4c50685d53afdcfce5a +RUN set -eu; \ + arch="$(dpkg --print-architecture)"; \ + [ "$arch" = "amd64" ] || { echo "FATAL: cursor pin only covers amd64, not $arch" >&2; exit 1; }; \ + dir="/opt/cursor/versions/${CURSOR_VERSION}"; mkdir -p "$dir"; \ + curl -fsSL "https://downloads.cursor.com/lab/${CURSOR_VERSION}/linux/x64/agent-cli-package.tar.gz" -o /tmp/cursor.tgz; \ + printf '%s /tmp/cursor.tgz\n' "${CURSOR_SHA256_AMD64}" > /tmp/cursor.sha256; \ + sha256sum -c /tmp/cursor.sha256; \ + tar --strip-components=1 -xzf /tmp/cursor.tgz -C "$dir"; \ + rm -f /tmp/cursor.tgz /tmp/cursor.sha256; \ + ln -sf "$dir/cursor-agent" /usr/local/bin/cursor-agent; \ + ln -sf /usr/local/bin/cursor-agent /usr/local/bin/agent; \ + chmod -R a+rX /opt/cursor + +ARG GH_VERSION=2.98.0 +ARG GH_SHA256_AMD64=3b8ac6b30336802fc1a858d7c084e11cdf24ac1a761ca90b68022d7d729208de +RUN set -eu; \ + arch="$(dpkg --print-architecture)"; \ + [ "$arch" = "amd64" ] || { echo "FATAL: gh pin only covers amd64, not $arch" >&2; exit 1; }; \ + curl -fsSL "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_amd64.tar.gz" -o /tmp/gh.tgz; \ + printf '%s /tmp/gh.tgz\n' "${GH_SHA256_AMD64}" > /tmp/gh.sha256; \ + sha256sum -c /tmp/gh.sha256; \ + tar -xzf /tmp/gh.tgz -C /tmp; \ + install "/tmp/gh_${GH_VERSION}_linux_amd64/bin/gh" /usr/local/bin/gh; \ + rm -rf /tmp/gh.tgz /tmp/gh.sha256 "/tmp/gh_${GH_VERSION}_linux_amd64" + +COPY --from=runnerd-build /workspace/packages/paperclip-runner/runner/target/release/paperclip-runnerd /usr/local/bin/paperclip-runnerd +COPY --from=provider-pack-build /provider-pack /opt/paperclip-runner/provider-pack + +RUN set -eu; \ + chmod -R a+rX /opt/paperclip-runner/provider-pack; \ + printf '%s\n' 'export PATH=/opt/paperclip-runner/provider-pack/node_modules/.bin:$PATH' \ + > /etc/profile.d/01-paperclip-runner-provider-pack.sh; \ + chmod 0644 /etc/profile.d/01-paperclip-runner-provider-pack.sh; \ + for command_name in acpx claude-agent-acp codex-acp claude codex grok gemini kimi opencode cursor-agent agent hermes gh paperclip-runnerd; do \ + command -v "$command_name" >/dev/null || { echo "FATAL: $command_name not on PATH after build" >&2; exit 1; }; \ + done; \ + metadata="$(paperclip-runnerd --build-metadata)"; \ + /opt/paperclip-runner/provider-pack/node_modules/node/bin/node -e "const [major,minor,patch]=process.versions.node.split('.').map(Number); if (major<24 || (major===24 && (minor<11 || (minor===11 && patch<0)))) process.exit(1)"; \ + test -f /opt/paperclip-runner/provider-pack/provider-pack.json; \ + printf '%s' "$metadata" | grep -q '"dial_ws_loopback"'; \ + printf '%s' "$metadata" | grep -q '"dial_wss"'; \ + printf '%s' "$metadata" | grep -q '"listen_ws"'; \ + echo "all agent CLIs and runnerd transport modes are available" + +USER daytona + +RUN /bin/sh -lc 'set -eu; \ + test -r /opt/paperclip-runner/provider-pack/provider-pack.json; \ + /opt/paperclip-runner/provider-pack/node_modules/node/bin/node -e \ + "JSON.parse(require(\"node:fs\").readFileSync(\"/opt/paperclip-runner/provider-pack/provider-pack.json\", \"utf8\"))"; \ + for command_name in acpx claude-agent-acp codex-acp; do \ + command -v "$command_name" >/dev/null || { echo "FATAL: $command_name not on PATH for daytona user" >&2; exit 1; }; \ + done; \ + test "$(acpx --version)" = "0.13.1"; \ + test "$(claude-agent-acp --version)" = "0.70.0"; \ + test "$(codex-acp --version)" = "@agentclientprotocol/codex-acp 1.6.2"' diff --git a/docker/daytona-runner/README.md b/docker/daytona-runner/README.md new file mode 100644 index 0000000000..766f5e8b3c --- /dev/null +++ b/docker/daytona-runner/README.md @@ -0,0 +1,69 @@ +# Paperclip Daytona runner image + +This image is the Paperclip Cloud fleet sandbox image plus a source-built +`paperclip-runnerd` and immutable provider pack. The pack contains Node 24.11, +OpenCode 1.18.17, the compiled OpenCode proxy, ACPX 0.13.1 sidecar, qualified ACP +agents, and the production lockfile. Its manifest digests each executable bridge +and binds the pack to the runner source revision, avoiding artifact upload and +npm installation on every fresh lease. + +The fleet pins are intentionally copied from +[`paperclip-cloud/fleet-sandbox-image/Dockerfile`](https://github.com/paperclipai/paperclip-cloud/blob/master/fleet-sandbox-image/Dockerfile). +Update both definitions together until the fleet base is published as a stable +image that this Dockerfile can extend directly. + +## Build and verify + +The fleet image is currently amd64-only because the pinned Cursor and GitHub CLI +checksums cover amd64. + +```bash +content_id="$(pnpm --silent test:e2e:runner:image-id)" +docker buildx build \ + --platform linux/amd64 \ + --build-arg PAPERCLIP_RUNNER_CONTENT_ID="${content_id}" \ + --build-arg PAPERCLIP_RUNNER_SOURCE_REVISION="$(git rev-parse HEAD)" \ + --tag "paperclip-daytona-runner:e2e-content-${content_id}" \ + --load \ + --file docker/daytona-runner/Dockerfile \ + . + +docker run --rm --platform linux/amd64 \ + --entrypoint paperclip-runnerd \ + "paperclip-daytona-runner:e2e-content-${content_id}" \ + --build-metadata +``` + +The metadata must advertise `dial_ws_loopback`, `dial_wss`, and `listen_ws`. +The explicit entrypoint is needed only for this local probe because Daytona's +base image uses its own long-running sandbox entrypoint. + +`test:e2e:runner:image-id` hashes the audited Docker build dependency closure, +target platform, the immutable Dockerfile syntax-frontend digest, and every +immutable `FROM` reference. It fails before the paid workflow can build when +the frontend or a base is not pinned to a sha256 digest. When updating the +syntax version, resolve and review its registry digest and update both values in +the first Dockerfile line. Git commits that do not change those inputs reuse the +same content tag. + +`PAPERCLIP_RUNNER_SOURCE_REVISION` remains the full Git SHA that built the first +published copy and is retained as provenance rather than cache identity. + +## Use in Paperclip + +Publish the image to a registry Daytona can pull, or use the environment +editor's **Configure image** flow to produce a Daytona snapshot. Set the +environment image to that immutable tag or snapshot. Paperclip probes the +sandbox user's `PATH` for `paperclip-runnerd` and `codex` and checks +`/opt/paperclip-runner/provider-pack` for OpenCode and ACPX. It uses the pack +only when its complete manifest matches the controller's build-owned pack; +otherwise it stages the pack configured by +`PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH`. Remote OpenCode and ACPX never +fall back to host-local processes. + +Do not promote `paperclip-runner-e2e-20260826-v2` for OpenCode or ACPX. Build a +new immutable image or snapshot from a clean committed revision and pass that +full Git SHA as `PAPERCLIP_RUNNER_SOURCE_REVISION`. + +Do not bake provider credentials, Paperclip bootstrap tickets, or Daytona +preview tokens into this image. They remain per-run secret material. diff --git a/package.json b/package.json index a265d5bedf..7d383fe100 100644 --- a/package.json +++ b/package.json @@ -63,6 +63,15 @@ "test:storybook-visual": "node scripts/storybook-visual-baseline.mjs download && node scripts/storybook-visual-baseline.mjs verify && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts", "test:storybook-visual:update": "node scripts/storybook-visual-baseline.mjs download && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts --update-snapshots && node scripts/storybook-visual-baseline.mjs pack", "test:e2e": "npx playwright test --config tests/e2e/playwright.config.ts", + "test:e2e:runner": "node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/launch.ts", + "test:e2e:runner:image-id": "node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/daytona-image-content.ts", + "test:e2e:runner:dashboard": "node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/dashboard-regenerate.ts", + "test:e2e:runner:models:update": "node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/openrouter-models-update.ts", + "test:e2e:runner:history:publish": "node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/history-publish.ts", + "test:e2e:runner:unit": "vitest run --config tests/runner-e2e/vitest.config.ts", + "test:e2e:runner:typecheck": "tsc -p tests/runner-e2e/tsconfig.json", + "test:e2e:runner:report": "node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/report.ts", + "test:runner-workflow-evals": "pnpm --filter @paperclipai/paperclip-eval-kernel build && pnpm --filter @paperclipai/paperclip-runner test:runner-workflow-evals", "test:e2e:mcp-user-stories": "node scripts/e2e-mcp-user-stories.mjs", "test:e2e:connection-intents": "npx playwright test --config tests/e2e/playwright.config.ts tests/e2e/connection-intents.spec.ts", "test:e2e:headed": "npx playwright test --config tests/e2e/playwright.config.ts --headed", diff --git a/packages/paperclip-runner/README.md b/packages/paperclip-runner/README.md index 16767bb64d..e0560b1dde 100644 --- a/packages/paperclip-runner/README.md +++ b/packages/paperclip-runner/README.md @@ -57,8 +57,8 @@ AWS access keys are intentionally removed from the runner environment. The Rust core includes a bounded client for the sidecar protocol. It enforces request identity, event order, frame and queue limits, timeouts, redacted -diagnostics, and process-group cleanup. This transport remains package-local. -It does not change runnerd provider selection in this slice. +diagnostics, and process-group cleanup. Runnerd selects this package-local +transport only through an exact qualified provider descriptor. Before a later provider adapter consumes a valid sidecar event, the Rust core also requires its optional or mandatory run and turn scope to match the active @@ -143,7 +143,17 @@ pnpm --filter @paperclipai/paperclip-runner verify:rootless The tracer's final line is stable: ```json -{"schemaVersion":"paperclip.runner.conformance.output.v1","runIdentity":{"runId":"run_conformance_0001","sessionId":"session_conformance_0001"},"result":{"status":"succeeded","summary":"Standalone Conformance fixture accepted."}} +{ + "schemaVersion": "paperclip.runner.conformance.output.v1", + "runIdentity": { + "runId": "run_conformance_0001", + "sessionId": "session_conformance_0001" + }, + "result": { + "status": "succeeded", + "summary": "Standalone Conformance fixture accepted." + } +} ``` Run only the tracer with: @@ -180,48 +190,105 @@ Live console provider-backed routes are loopback-only and reject wildcard/LAN binds. Browser mutations require same-origin Fetch Metadata, matching Origin, and JSON content; see the protocol-server tutorial for direct `curl` examples. +## Live, chaos, and AWS AgentCore operations + +The deterministic workflow scorer and the chaos schedule do not require +provider credentials: + +```sh +pnpm --filter @paperclipai/paperclip-runner test:runner-workflow-evals +pnpm --filter @paperclipai/paperclip-runner report:runner-chaos-evals +``` + +`report:runner-live-evals` is a paid, provider-backed command. Native Codex and +the ACPX Codex profile require `OPENAI_API_KEY`; ACPX Claude requires +`ANTHROPIC_API_KEY`; OpenCode candidates require `OPENROUTER_API_KEY`. The live +matrix admits no Pi profile and does not persist credential values. Set +`PAPERCLIP_EVAL_MAX_CAMPAIGN_COST_USD` to a positive finite number to bound +additional scheduling after the observed campaign total reaches that value: + +```sh +PAPERCLIP_EVAL_MAX_CAMPAIGN_COST_USD=12 \ + pnpm --filter @paperclipai/paperclip-runner report:runner-live-evals +``` + +GitHub-hosted live campaigns additionally require the default branch, an +allowlisted numeric actor ID, the protected `runner-e2e-paid` environment, and +an explicit repository variable before scheduled runs are enabled. Uploaded +reports contain redacted observations and trace digests, not raw provider +frames, prompts, credentials, tool arguments, or hidden reasoning. + +The AgentCore proof-of-concept uses an AWS CLI v2 profile to provision a +dedicated invocation role and scoped resources. Its local mode-`0600` metadata +file contains no access keys; probes assume short-lived STS credentials and +clear them after use. Validate locally, provision or inspect the stack, run the +bounded lab/smoke, and tear it down explicitly with: + +```sh +pnpm --filter @paperclipai/paperclip-runner test:aws-agentcore-provisioning +pnpm --filter @paperclipai/paperclip-runner aws-agentcore:provision -- --dry-run +pnpm --filter @paperclipai/paperclip-runner aws-agentcore:provision +pnpm --filter @paperclipai/paperclip-runner aws-agentcore:probe +pnpm --filter @paperclipai/paperclip-runner aws-agentcore:lab +pnpm --filter @paperclipai/paperclip-runner smoke:capability:aws-agentcore +pnpm --filter @paperclipai/paperclip-runner aws-agentcore:destroy -- --yes +``` + +Provisioning can incur Bedrock, AgentCore Runtime/Memory, storage, and private +networking charges. Provisioning refuses to modify a colliding stack unless its +Paperclip ownership tags and template description match. A verified +`ROLLBACK_COMPLETE` stack still requires `--replace-failed-stack` plus an +interactive confirmation (or `--yes`) before it can be deleted and recreated. +Destruction requires `--yes` and refuses to remove a stack with an active +recorded lab unless `--force` is also supplied. + ## Package-owned commands -| Command | Purpose | -|---|---| -| `build` | Compile the TypeScript public surface, Rust workspace, and browser devtool. | -| `typecheck` | Check TypeScript, Rust, generated schema sources, and browser types. | -| `test` | Run Rust/TypeScript fixture, supervisor, fake-driver, live/replay, and boundary tests. | -| `check:forbidden-imports` | Reject TypeScript imports and Cargo path dependencies that cross into Paperclip core. | -| `check:tracked-imports` | Reject tracked imports and `package.json` entry points that only resolve against untracked files, so a clean checkout of any commit builds. | -| `check:numbered-milestones` | Reject numbered construction-milestone names in tracked package paths and source. | -| `check:package-boundaries` | Enforce the acyclic runtime/testing/eval dependency and manifest boundary. | -| `check:clean-consumers` | Pack the runner and install its root, evals, and testing exports in a clean consumer. | -| `test:eval-slice` | Run the credential-free eval bundle, scoring, and behavior/fault slice. | -| `test:runner-workflow-evals` | Run the deterministic provider-neutral workflow matrix. | -| `report:runner-workflow-evals` | Write local deterministic workflow reports without provider calls. | -| `check:conformance-parity` | Require byte-for-byte equivalent Rust and TypeScript tracer output. | -| `check:replay-goldens` | Require all reducer snapshots and cross-language summaries to match checked goldens. | -| `check:replay-parity` | Run TypeScript and Rust against the same Replay fixture summaries. | -| `check:browser-tokens` | Reject component-local visual literals and require the standalone token layer. | -| `docs:validate` | Validate local documentation links. | -| `trace:conformance` | Run the Rust mock-core tracer, print the stable result, and exit. | -| `trace:conformance:typescript` | Run the TypeScript reference tracer directly. | -| `replay:fixture` | Validate and reduce a fixture to a final snapshot. | -| `trace:local-runner` | Run one native local session through the Rust runner and fake harness. | -| `trace:codex` | Run the mock core with a real, local skillless Codex app-server session. | -| `demo:live-console` | Start the package-local HTTP/SSE server with server-only Codex authentication. | -| `console:live-console` | Start the standalone browser devtool with the Live console on `127.0.0.1:4180`. | -| `console:sdk` | Start the public-SDK reference console and mini consumer on `127.0.0.1:4181`. | -| `test:sdk` | Run targeted browser-client, reducer-projection, and React component contract tests. | -| `test:browser:sdk` | Exercise both consumers with the fake driver, keyboard/a11y checks, reconnect/replay, measurements, and screenshots. | -| `record:sdk:codex` | Run both public consumers against a safe real Codex session and capture live screenshots. | -| `check:capability-contract` | Verify the generated capability, legacy MCP, and eval traceability contract. | -| `check:semantic-contracts` | Verify the provider-neutral semantic tool contract is current. | -| `trace:live-runner` | Run the real runnerd/Codex semantic loop against the mock control plane. | -| `demo:scenarios` | Start the Capability scenario explorer over the mock control plane on `127.0.0.1:4183`. | -| `console:issue-thread` | Start the Paperclip-style issue thread on `127.0.0.1:4184`. | -| `test:scenarios` | Run the scenario index, run-artifact, parity, explorer component, and route tests. | -| `test:browser:scenarios` | Exercise both the scenario explorer and issue-thread browser contracts. | -| `browser:dev` | Start the standalone live/replay browser devtool. | -| `test:browser` | Exercise static replay and live scenarios, then capture temporary screenshots under ignored test output. | -| `verify` | Run the complete deterministic Conformance through SDK acceptance sequence. | -| `verify:rootless` | Extract Debian/Ubuntu browser libraries without root, then run `verify`. | +| Command | Purpose | +| ------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | +| `build` | Compile the TypeScript public surface, Rust workspace, and browser devtool. | +| `typecheck` | Check TypeScript, Rust, generated schema sources, and browser types. | +| `test` | Run Rust/TypeScript fixture, supervisor, fake-driver, live/replay, and boundary tests. | +| `check:forbidden-imports` | Reject TypeScript imports and Cargo path dependencies that cross into Paperclip core. | +| `check:tracked-imports` | Reject tracked imports and `package.json` entry points that only resolve against untracked files, so a clean checkout of any commit builds. | +| `check:numbered-milestones` | Reject numbered construction-milestone names in tracked package paths and source. | +| `check:package-boundaries` | Enforce the acyclic runtime/testing/eval dependency and manifest boundary. | +| `check:clean-consumers` | Pack the runner and install its root, evals, and testing exports in a clean consumer. | +| `test:eval-slice` | Run the credential-free eval bundle, scoring, and behavior/fault slice. | +| `test:runner-workflow-evals` | Run the deterministic provider-neutral workflow matrix. | +| `report:runner-workflow-evals` | Validate deterministic results and write local reports only when every scoreable result passes. | +| `report:runner-live-evals` | Execute the paid forty-execution provider schedule with qualification and campaign-cost guards. | +| `report:runner-chaos-evals` | Write the credential-free eight-scenario chaos schedule. | +| `test:aws-agentcore-provisioning` | Validate the AgentCore template and wrapper safety contracts without provisioning. | +| `aws-agentcore:provision` / `probe` / `lab` / `destroy` | Manage the scoped AgentCore proof-of-concept lifecycle. | +| `smoke:capability:aws-agentcore` | Exercise the qualified AgentCore profile through the capability harness. | +| `check:conformance-parity` | Require byte-for-byte equivalent Rust and TypeScript tracer output. | +| `check:replay-goldens` | Require all reducer snapshots and cross-language summaries to match checked goldens. | +| `check:replay-parity` | Run TypeScript and Rust against the same Replay fixture summaries. | +| `check:browser-tokens` | Reject component-local visual literals and require the standalone token layer. | +| `docs:validate` | Validate local documentation links. | +| `trace:conformance` | Run the Rust mock-core tracer, print the stable result, and exit. | +| `trace:conformance:typescript` | Run the TypeScript reference tracer directly. | +| `replay:fixture` | Validate and reduce a fixture to a final snapshot. | +| `trace:local-runner` | Run one native local session through the Rust runner and fake harness. | +| `trace:codex` | Run the mock core with a real, local skillless Codex app-server session. | +| `demo:live-console` | Start the package-local HTTP/SSE server with server-only Codex authentication. | +| `console:live-console` | Start the standalone browser devtool with the Live console on `127.0.0.1:4180`. | +| `console:sdk` | Start the public-SDK reference console and mini consumer on `127.0.0.1:4181`. | +| `test:sdk` | Run targeted browser-client, reducer-projection, and React component contract tests. | +| `test:browser:sdk` | Exercise both consumers with the fake driver, keyboard/a11y checks, reconnect/replay, measurements, and screenshots. | +| `record:sdk:codex` | Run both public consumers against a safe real Codex session and capture live screenshots. | +| `check:capability-contract` | Verify the generated capability, legacy MCP, and eval traceability contract. | +| `check:semantic-contracts` | Verify the provider-neutral semantic tool contract is current. | +| `trace:live-runner` | Run the real runnerd/Codex semantic loop against the mock control plane. | +| `demo:scenarios` | Start the Capability scenario explorer over the mock control plane on `127.0.0.1:4183`. | +| `console:issue-thread` | Start the Paperclip-style issue thread on `127.0.0.1:4184`. | +| `test:scenarios` | Run the scenario index, run-artifact, parity, explorer component, and route tests. | +| `test:browser:scenarios` | Exercise both the scenario explorer and issue-thread browser contracts. | +| `browser:dev` | Start the standalone live/replay browser devtool. | +| `test:browser` | Exercise static replay and live scenarios, then capture temporary screenshots under ignored test output. | +| `verify` | Run the complete deterministic Conformance through SDK acceptance sequence. | +| `verify:rootless` | Extract Debian/Ubuntu browser libraries without root, then run `verify`. | ## Navigate diff --git a/packages/paperclip-runner/docs/runner-workflow-evals.md b/packages/paperclip-runner/docs/runner-workflow-evals.md index 97acae5363..19b5357329 100644 --- a/packages/paperclip-runner/docs/runner-workflow-evals.md +++ b/packages/paperclip-runner/docs/runner-workflow-evals.md @@ -1,28 +1,57 @@ -# Deterministic runner workflow evals +# Stress-derived Runner workflow evals -The workflow evaluator converts the stress-derived workflow catalog into a -credential-free matrix over sanitized Codex, OpenCode, and ACPX fixtures. -It evaluates provider-neutral behavior; the fixtures contain normalized events, -not prompts, credentials, raw reasoning, or provider traces. +The Runner workflow eval system turns the `STRESS-001`–`STRESS-044` campaign +into complementary deterministic, live, and chaos lanes. It is additive to the +capability inventory, capability cases, and existing scoring/report readers. The workspace-private `@paperclipai/paperclip-eval-kernel` package owns only structural scenario-by-candidate orchestration. Runner-specific cases, observations, scoring, traceability, and report rendering remain package-local. -Use: +## Lanes -```sh -pnpm --filter @paperclipai/paperclip-runner test:runner-workflow-evals -pnpm --filter @paperclipai/paperclip-runner report:runner-workflow-evals -``` +- `pnpm --filter @paperclipai/paperclip-runner test:runner-workflow-evals` + runs the credential-free PR gate over sanitized Codex, OpenCode, and ACPX + normalization fixtures. +- `pnpm --filter @paperclipai/paperclip-runner report:runner-workflow-evals` + validates the deterministic report and writes JSON, Markdown, JUnit, and + GitHub-safe artifacts under `.paperclip-local/evals/workflows/` only when all + scoreable fixture results pass. It makes no network requests. +- `pnpm --filter @paperclipai/paperclip-runner report:runner-live-evals` runs + the balanced forty-execution schedule against real provider sessions. Live + candidate failures are trend-only; missing credentials, qualification + failures, and provider outages remain unscored. +- `pnpm --filter @paperclipai/paperclip-runner report:runner-chaos-evals` + writes the eight-scenario fault schedule consumed by weekly and pre-release + restart, replay, trace, finalization, interaction, and wake-race suites. -The report command writes JSON, Markdown, JUnit, and GitHub-safe summaries -under `.paperclip-local/evals/workflows/`. It performs no network requests and -does not start a production provider. +The checked-in live manifest contains only adapter/model settings, +qualification variable names, and budgets. Credentials remain in the +environment. `PAPERCLIP_EVAL_MAX_CAMPAIGN_COST_USD` must be a positive finite +number and defaults to 12 USD for scheduled runs. + +The hosted live workflow is default-branch-only and requires an allowlisted +numeric actor plus the protected `runner-e2e-paid` environment. Scheduled runs +also remain disabled until `RUNNER_LIVE_EVALS_NIGHTLY_ENABLED` is explicitly +set to `true`. + +## Trace and reasoning safety + +Live executions capture provider frames in a run-local mode-`0600` sidecar. +The evaluator verifies byte lengths, SHA-256 digests, order, dispositions, and +lineage, retains only redacted observations plus a digest, and destroys the +temporary trace after execution. Prompts, credentials, tool arguments, and +reasoning text never enter reports or uploaded artifacts. Evals measure visible +progress and activity; they do not inspect or grade hidden chain of thought. + +## Compatibility and trends + +Live bundle identity includes the Runner version/build, prompt policy, schedule +seed, adapters, resolved models, and reasoning settings. Seven-day comparisons +use only matching bundle IDs, and alerts stay disabled until seven compatible +reports exist. Safe reports are retained for 30 days; raw traces are not +uploaded. The checked traceability manifest is -`spec/evals/stress-workflow-traceability.json`. The build gate fails when a -finding references an unknown workflow or a missing regression-test anchor. - -Live schedules, paid provider campaigns, raw trace capture, and recorded -evidence are intentionally excluded from this slice. +`spec/evals/stress-workflow-traceability.json`; CI fails for missing findings, +unknown workflow IDs, or missing regression-test anchors. diff --git a/packages/paperclip-runner/infra/aws-agentcore-paperclip.yaml b/packages/paperclip-runner/infra/aws-agentcore-paperclip.yaml new file mode 100644 index 0000000000..a7cbd72a1c --- /dev/null +++ b/packages/paperclip-runner/infra/aws-agentcore-paperclip.yaml @@ -0,0 +1,544 @@ +AWSTemplateFormatVersion: "2010-09-09" +Description: Paperclip proof-of-concept Amazon Bedrock AgentCore Harness and least-privilege invocation roles. + +Parameters: + EnvironmentName: + Type: String + Default: development + AllowedPattern: "^[a-z][a-z0-9-]{1,20}$" + DeploymentMode: + Type: String + Default: development + AllowedValues: [development, private] + TrustedRunnerPrincipalArn: + Type: String + Description: Stable IAM user or role ARN allowed to assume the Paperclip invocation role. + AllowedPattern: "^arn:aws(-[^:]+)?:iam::[0-9]{12}:(role|user)/.+$" + BedrockModelId: + Type: String + Default: global.anthropic.claude-sonnet-4-6 + AllowedPattern: "^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$" + ConstraintDescription: Must be a Bedrock-native model ID without ARN, path, glob, or wildcard syntax. + BedrockModelResourceArn: + Type: String + Description: Exact application/system inference-profile ARN for the selected model. + AllowedPattern: "^arn:(aws|aws-us-gov|aws-cn|aws-iso|aws-iso-b):bedrock:[a-z0-9-]+:[0-9]{12}:(inference-profile|application-inference-profile)/[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$" + ConstraintDescription: Must be one exact Bedrock inference-profile ARN without path, glob, or wildcard syntax. + BedrockFoundationModelResourceArn: + Type: String + Description: Selected foundation-model ARN; the region segment may be * for cross-region inference. + AllowedPattern: "^arn:(aws|aws-us-gov|aws-cn|aws-iso|aws-iso-b):bedrock:([a-z0-9-]+|[*])::foundation-model/[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$" + ConstraintDescription: Must be one exact Bedrock foundation-model ARN; only the cross-region region segment may be a wildcard. + BedrockMarketplaceProductId: + Type: String + Description: Exact AWS Marketplace product ID for the selected third-party model. + AllowedPattern: "^prod-[a-z0-9]+$" + HarnessName: + Type: String + Default: PaperclipAgentCoreHarness + AllowedPattern: "^[a-zA-Z][a-zA-Z0-9_]{0,39}$" + HarnessEndpointName: + Type: String + Default: paperclip + AllowedPattern: "^[a-zA-Z][a-zA-Z0-9_]{0,47}$" + MemoryName: + Type: String + Default: PaperclipAgentCoreMemory + AllowedPattern: "^[a-zA-Z][a-zA-Z0-9_]{0,47}$" + ContextPrefix: + Type: String + Default: paperclip/agentcore/development + Description: Dedicated S3 key prefix for immutable AgentCore runtime-context assets. + AllowedPattern: "^[a-z0-9][a-z0-9/_-]{1,127}$" + +Conditions: + IsPrivate: !Equals [!Ref DeploymentMode, private] + +Resources: + ContextEncryptionKey: + Type: AWS::KMS::Key + Properties: + Description: !Sub Paperclip AgentCore runtime-context assets (${EnvironmentName}) + EnableKeyRotation: true + KeyPolicy: + Version: "2012-10-17" + Statement: + - Sid: AccountOwnsAndDelegatesKey + Effect: Allow + Principal: + AWS: !Sub arn:${AWS::Partition}:iam::${AWS::AccountId}:root + Action: kms:* + Resource: "*" + Tags: + - Key: paperclip:owned + Value: "true" + - Key: paperclip:environment + Value: !Ref EnvironmentName + + ContextEncryptionKeyAlias: + Type: AWS::KMS::Alias + Properties: + AliasName: !Sub alias/paperclip-agentcore-context-${EnvironmentName}-${AWS::Region} + TargetKeyId: !Ref ContextEncryptionKey + + ContextBucket: + Type: AWS::S3::Bucket + Properties: + BucketEncryption: + ServerSideEncryptionConfiguration: + - ServerSideEncryptionByDefault: + SSEAlgorithm: aws:kms + KMSMasterKeyID: !GetAtt ContextEncryptionKey.Arn + OwnershipControls: + Rules: + - ObjectOwnership: BucketOwnerEnforced + PublicAccessBlockConfiguration: + BlockPublicAcls: true + BlockPublicPolicy: true + IgnorePublicAcls: true + RestrictPublicBuckets: true + Tags: + - Key: paperclip:owned + Value: "true" + - Key: paperclip:environment + Value: !Ref EnvironmentName + - Key: paperclip:cost-center + Value: runner-lab + + ContextBucketPolicy: + Type: AWS::S3::BucketPolicy + Properties: + Bucket: !Ref ContextBucket + PolicyDocument: + Version: "2012-10-17" + Statement: + - Sid: DenyInsecureTransport + Effect: Deny + Principal: "*" + Action: s3:* + Resource: + - !GetAtt ContextBucket.Arn + - !Sub ${ContextBucket.Arn}/* + Condition: + Bool: + aws:SecureTransport: "false" + - Sid: DenyUnencryptedContextUploads + Effect: Deny + Principal: "*" + Action: s3:PutObject + Resource: !Sub ${ContextBucket.Arn}/${ContextPrefix}/assets/* + Condition: + StringNotEquals: + s3:x-amz-server-side-encryption: aws:kms + - Sid: DenyContextUploadsWithAnotherKey + Effect: Deny + Principal: "*" + Action: s3:PutObject + Resource: !Sub ${ContextBucket.Arn}/${ContextPrefix}/assets/* + Condition: + StringNotEquals: + s3:x-amz-server-side-encryption-aws-kms-key-id: !GetAtt ContextEncryptionKey.Arn + + AgentMemory: + Type: AWS::BedrockAgentCore::Memory + Properties: + Name: !Ref MemoryName + Description: Short-term Paperclip Runner chat continuity; no long-term strategies. + EventExpiryDuration: 90 + Tags: + paperclip:owned: "true" + paperclip:environment: !Ref EnvironmentName + paperclip:cost-center: runner-lab + + HarnessExecutionRole: + Type: AWS::IAM::Role + Properties: + RoleName: !Sub paperclip-agentcore-harness-${EnvironmentName}-${AWS::Region} + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: bedrock-agentcore.amazonaws.com + Action: sts:AssumeRole + Condition: + StringEquals: + aws:SourceAccount: !Ref AWS::AccountId + Policies: + - PolicyName: BedrockModelAndMemory + PolicyDocument: + Version: "2012-10-17" + Statement: + - Sid: InvokePinnedBedrockModel + Effect: Allow + Action: + - bedrock:InvokeModel + - bedrock:InvokeModelWithResponseStream + Resource: + - !Ref BedrockModelResourceArn + - !Ref BedrockFoundationModelResourceArn + - Sid: ViewMarketplaceSubscriptionsForModelAccess + Effect: Allow + Action: aws-marketplace:ViewSubscriptions + Resource: "*" + - Sid: SubscribePinnedMarketplaceModel + Effect: Allow + Action: aws-marketplace:Subscribe + Resource: "*" + Condition: + StringEquals: + aws-marketplace:ProductId: !Ref BedrockMarketplaceProductId + - Sid: ShortTermMemory + Effect: Allow + Action: + - bedrock-agentcore:CreateEvent + - bedrock-agentcore:GetEvent + - bedrock-agentcore:ListEvents + Resource: !GetAtt AgentMemory.MemoryArn + - Sid: ReadPinnedRuntimeContext + Effect: Allow + Action: s3:GetObject + Resource: !Sub ${ContextBucket.Arn}/${ContextPrefix}/assets/* + - Sid: EnumeratePinnedRuntimeContext + Effect: Allow + Action: s3:ListBucket + Resource: !GetAtt ContextBucket.Arn + Condition: + StringLike: + s3:prefix: !Sub ${ContextPrefix}/assets/* + - Sid: DecryptPinnedRuntimeContext + Effect: Allow + Action: kms:Decrypt + Resource: !GetAtt ContextEncryptionKey.Arn + Condition: + StringEquals: + kms:ViaService: !Sub s3.${AWS::Region}.${AWS::URLSuffix} + StringLike: + kms:EncryptionContext:aws:s3:arn: !Sub ${ContextBucket.Arn}/${ContextPrefix}/assets/* + - !If + - IsPrivate + - Sid: PullManagedHarnessImage + Effect: Allow + Action: ecr:GetAuthorizationToken + Resource: "*" + Condition: + StringEquals: + aws:RequestedRegion: !Ref AWS::Region + - !Ref AWS::NoValue + - !If + - IsPrivate + - Sid: PullManagedHarnessLayers + Effect: Allow + Action: + - ecr:BatchCheckLayerAvailability + - ecr:GetDownloadUrlForLayer + - ecr:BatchGetImage + Resource: !Sub arn:${AWS::Partition}:ecr:${AWS::Region}:*:repository/* + - !Ref AWS::NoValue + - Sid: NeverOpenRuntimeCommandChannel + Effect: Deny + Action: bedrock-agentcore:InvokeAgentRuntimeCommand + Resource: "*" + Tags: + - Key: paperclip:owned + Value: "true" + - Key: paperclip:environment + Value: !Ref EnvironmentName + + AgentHarness: + Type: AWS::BedrockAgentCore::Harness + Properties: + HarnessName: !Ref HarnessName + ExecutionRoleArn: !GetAtt HarnessExecutionRole.Arn + Model: + BedrockModelConfig: + ModelId: !Ref BedrockModelId + ApiFormat: converse_stream + MaxTokens: 4096 + SystemPrompt: + - Text: >- + You are a Paperclip-governed remote agent. Use only caller-supplied inline + functions. Never claim access to Paperclip, a local workspace, shell, + filesystem, browser, network, MCP, skills, or other agents. Follow the + current turn's completion contract and use its finish or block function. + Memory: + AgentCoreMemoryConfiguration: + Arn: !GetAtt AgentMemory.MemoryArn + MessagesCount: 100 + MaxIterations: 8 + MaxTokens: 4096 + TimeoutSeconds: 300 + AllowedTools: + - "@*/pc_*" + Tools: [] + Skills: [] + EnvironmentVariables: {} + Environment: + AgentCoreRuntimeEnvironment: + LifecycleConfiguration: + IdleRuntimeSessionTimeout: 300 + MaxLifetime: 28800 + NetworkConfiguration: + NetworkMode: !If [IsPrivate, VPC, PUBLIC] + NetworkModeConfig: !If + - IsPrivate + - SecurityGroups: [!Ref HarnessSecurityGroup] + Subnets: [!Ref PrivateSubnetA, !Ref PrivateSubnetB] + - !Ref AWS::NoValue + Tags: + - Key: paperclip:owned + Value: "true" + - Key: paperclip:environment + Value: !Ref EnvironmentName + - Key: paperclip:cost-center + Value: runner-lab + + RunnerInvocationRole: + Type: AWS::IAM::Role + Properties: + RoleName: !Sub paperclip-agentcore-runner-${EnvironmentName}-${AWS::Region} + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + AWS: !Ref TrustedRunnerPrincipalArn + Action: sts:AssumeRole + Policies: + - PolicyName: InvokeAndOperatePinnedHarness + PolicyDocument: + Version: "2012-10-17" + Statement: + - Sid: InvokePinnedHarnessAndRuntime + Effect: Allow + Action: + - bedrock-agentcore:InvokeHarness + - bedrock-agentcore:InvokeAgentRuntime + - bedrock-agentcore:StopRuntimeSession + Resource: + - !GetAtt AgentHarness.Arn + - !Sub ${AgentHarness.Arn}/harness-endpoint/${HarnessEndpointName} + - !Sub ${AgentHarness.Arn}/runtime-endpoint/${HarnessEndpointName} + - !GetAtt AgentHarness.Environment.AgentCoreRuntimeEnvironment.AgentRuntimeArn + - Sid: ReadPinnedHarness + Effect: Allow + Action: + - bedrock-agentcore:GetHarness + - bedrock-agentcore:GetHarnessEndpoint + - bedrock-agentcore:ListHarnessEndpoints + Resource: + - !GetAtt AgentHarness.Arn + - !Sub ${AgentHarness.Arn}/harness-endpoint/${HarnessEndpointName} + - Sid: ReadAndPurgeSessionMemory + Effect: Allow + Action: + - bedrock-agentcore:GetMemory + - bedrock-agentcore:GetEvent + - bedrock-agentcore:ListEvents + - bedrock-agentcore:DeleteEvent + Resource: !GetAtt AgentMemory.MemoryArn + - Sid: WriteAndVerifyPinnedRuntimeContext + Effect: Allow + Action: + - s3:GetObject + - s3:PutObject + - s3:DeleteObject + Resource: !Sub ${ContextBucket.Arn}/${ContextPrefix}/assets/* + - Sid: EnumeratePinnedRuntimeContextForTeardown + Effect: Allow + Action: s3:ListBucket + Resource: !GetAtt ContextBucket.Arn + Condition: + StringLike: + s3:prefix: !Sub ${ContextPrefix}/assets/* + - Sid: EncryptPinnedRuntimeContext + Effect: Allow + Action: + - kms:Encrypt + - kms:GenerateDataKey + Resource: !GetAtt ContextEncryptionKey.Arn + Condition: + StringEquals: + kms:ViaService: !Sub s3.${AWS::Region}.${AWS::URLSuffix} + StringLike: + kms:EncryptionContext:aws:s3:arn: !Sub ${ContextBucket.Arn}/${ContextPrefix}/assets/* + - Sid: DiscoverBedrockModels + Effect: Allow + Action: + - bedrock:ListFoundationModels + - bedrock:GetFoundationModel + Resource: "*" + - Sid: NeverOpenRuntimeCommandChannel + Effect: Deny + Action: bedrock-agentcore:InvokeAgentRuntimeCommand + Resource: "*" + Tags: + - Key: paperclip:owned + Value: "true" + - Key: paperclip:environment + Value: !Ref EnvironmentName + + PrivateVpc: + Type: AWS::EC2::VPC + Condition: IsPrivate + Properties: + CidrBlock: 10.87.0.0/16 + EnableDnsHostnames: true + EnableDnsSupport: true + Tags: + - Key: Name + Value: !Sub paperclip-agentcore-${EnvironmentName} + - Key: paperclip:owned + Value: "true" + + PrivateSubnetA: + Type: AWS::EC2::Subnet + Condition: IsPrivate + Properties: + VpcId: !Ref PrivateVpc + CidrBlock: 10.87.1.0/24 + AvailabilityZone: !Select [0, !GetAZs ""] + MapPublicIpOnLaunch: false + + PrivateSubnetB: + Type: AWS::EC2::Subnet + Condition: IsPrivate + Properties: + VpcId: !Ref PrivateVpc + CidrBlock: 10.87.2.0/24 + AvailabilityZone: !Select [1, !GetAZs ""] + MapPublicIpOnLaunch: false + + PrivateRouteTable: + Type: AWS::EC2::RouteTable + Condition: IsPrivate + Properties: + VpcId: !Ref PrivateVpc + + PrivateSubnetRouteA: + Type: AWS::EC2::SubnetRouteTableAssociation + Condition: IsPrivate + Properties: + RouteTableId: !Ref PrivateRouteTable + SubnetId: !Ref PrivateSubnetA + + PrivateSubnetRouteB: + Type: AWS::EC2::SubnetRouteTableAssociation + Condition: IsPrivate + Properties: + RouteTableId: !Ref PrivateRouteTable + SubnetId: !Ref PrivateSubnetB + + HarnessSecurityGroup: + Type: AWS::EC2::SecurityGroup + Condition: IsPrivate + Properties: + GroupDescription: AgentCore Harness egress only to VPC endpoints + VpcId: !Ref PrivateVpc + SecurityGroupEgress: [] + + EndpointSecurityGroup: + Type: AWS::EC2::SecurityGroup + Condition: IsPrivate + Properties: + GroupDescription: HTTPS from the AgentCore Harness security group + VpcId: !Ref PrivateVpc + SecurityGroupIngress: [] + + HarnessToEndpointsEgress: + Type: AWS::EC2::SecurityGroupEgress + Condition: IsPrivate + Properties: + GroupId: !Ref HarnessSecurityGroup + IpProtocol: tcp + FromPort: 443 + ToPort: 443 + DestinationSecurityGroupId: !Ref EndpointSecurityGroup + + EndpointsFromHarnessIngress: + Type: AWS::EC2::SecurityGroupIngress + Condition: IsPrivate + Properties: + GroupId: !Ref EndpointSecurityGroup + IpProtocol: tcp + FromPort: 443 + ToPort: 443 + SourceSecurityGroupId: !Ref HarnessSecurityGroup + + EcrApiEndpoint: + Type: AWS::EC2::VPCEndpoint + Condition: IsPrivate + Properties: + VpcId: !Ref PrivateVpc + ServiceName: !Sub com.amazonaws.${AWS::Region}.ecr.api + VpcEndpointType: Interface + PrivateDnsEnabled: true + SubnetIds: [!Ref PrivateSubnetA, !Ref PrivateSubnetB] + SecurityGroupIds: [!Ref EndpointSecurityGroup] + + EcrDkrEndpoint: + Type: AWS::EC2::VPCEndpoint + Condition: IsPrivate + Properties: + VpcId: !Ref PrivateVpc + ServiceName: !Sub com.amazonaws.${AWS::Region}.ecr.dkr + VpcEndpointType: Interface + PrivateDnsEnabled: true + SubnetIds: [!Ref PrivateSubnetA, !Ref PrivateSubnetB] + SecurityGroupIds: [!Ref EndpointSecurityGroup] + + BedrockRuntimeEndpoint: + Type: AWS::EC2::VPCEndpoint + Condition: IsPrivate + Properties: + VpcId: !Ref PrivateVpc + ServiceName: !Sub com.amazonaws.${AWS::Region}.bedrock-runtime + VpcEndpointType: Interface + PrivateDnsEnabled: true + SubnetIds: [!Ref PrivateSubnetA, !Ref PrivateSubnetB] + SecurityGroupIds: [!Ref EndpointSecurityGroup] + + S3Endpoint: + Type: AWS::EC2::VPCEndpoint + Condition: IsPrivate + Properties: + VpcId: !Ref PrivateVpc + ServiceName: !Sub com.amazonaws.${AWS::Region}.s3 + VpcEndpointType: Gateway + RouteTableIds: [!Ref PrivateRouteTable] + +Outputs: + AccountId: + Value: !Ref AWS::AccountId + Region: + Value: !Ref AWS::Region + DeploymentMode: + Value: !Ref DeploymentMode + BedrockModelId: + Value: !Ref BedrockModelId + MemoryArn: + Value: !GetAtt AgentMemory.MemoryArn + MemoryId: + Value: !GetAtt AgentMemory.MemoryId + MemoryEventExpiryDays: + Value: "90" + HarnessArn: + Value: !GetAtt AgentHarness.Arn + HarnessId: + Value: !GetAtt AgentHarness.HarnessId + HarnessVersion: + Value: !GetAtt AgentHarness.Version + AgentRuntimeArn: + Value: !GetAtt AgentHarness.Environment.AgentCoreRuntimeEnvironment.AgentRuntimeArn + AgentRuntimeId: + Value: !GetAtt AgentHarness.Environment.AgentCoreRuntimeEnvironment.AgentRuntimeId + RunnerInvocationRoleArn: + Value: !GetAtt RunnerInvocationRole.Arn + ContextBucketName: + Value: !Ref ContextBucket + ContextPrefix: + Value: !Ref ContextPrefix + ContextKmsKeyArn: + Value: !GetAtt ContextEncryptionKey.Arn + QualificationRevision: + Value: aws-agentcore-harness-context-v2 diff --git a/packages/paperclip-runner/package.json b/packages/paperclip-runner/package.json index 03950d66f0..cc8619f568 100644 --- a/packages/paperclip-runner/package.json +++ b/packages/paperclip-runner/package.json @@ -8,6 +8,7 @@ }, "type": "module", "bin": { + "paperclip-runner-eval-session": "./dist/cli/eval-session.js", "paperclip-runner-codex-proxy": "./dist/cli/codex-app-server-unix-proxy.js", "paperclip-runner-opencode-proxy": "./dist/cli/opencode-app-server-proxy.js", "paperclip-runner-acpx-sidecar": "./dist/cli/acpx-runtime-sidecar.js" @@ -72,7 +73,7 @@ "typecheck:rust": "cargo fmt --manifest-path runner/Cargo.toml --all -- --check && cargo check --manifest-path runner/Cargo.toml --locked --workspace", "typecheck:browser": "tsc -p tsconfig.browser.json --noEmit", "test": "pnpm run test:typescript && pnpm run test:rust", - "test:typescript": "pnpm run ensure:eval-build-deps && pnpm run build:rust && node --test test/protocol-contract.test.mjs test/acpx-sidecar-contract.test.mjs test/acpx-codex-package-contract.test.mjs && vitest run", + "test:typescript": "pnpm run ensure:eval-build-deps && pnpm run build:rust && node --test test/protocol-contract.test.mjs test/acpx-sidecar-contract.test.mjs test/acpx-codex-package-contract.test.mjs scripts/aws-agentcore-provisioning.test.mjs && vitest run", "test:rust": "cargo test --release --manifest-path runner/Cargo.toml --locked --workspace", "test:codex": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --test codex_provider", "test:durable": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core durable::", @@ -113,10 +114,13 @@ "test:capability-inventory": "node scripts/check-capability-inventory.test.mjs", "test:capability-evals": "vitest run src/conformance/capability-eval-suite.test.ts", "test:eval-slice": "pnpm run ensure:eval-build-deps && vitest run src/eval", - "test:runner-workflow-evals": "pnpm run ensure:eval-build-deps && vitest run src/eval/workflow-evals.test.ts", + "test:runner-workflow-evals": "pnpm run ensure:eval-build-deps && vitest run src/eval/workflow-evals.test.ts src/eval/live-workflow-executor.test.ts", "check:runner-workflow-traceability": "pnpm run build:typescript && node scripts/check-runner-workflow-traceability.mjs", "report:capability-evals": "pnpm run build:typescript && node scripts/run-capability-eval-suite.mjs", + "report:capability-live-evals": "pnpm run build:typescript && cargo build --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --bin paperclip-runnerd && node scripts/run-capability-live-eval-matrix.mjs", "report:runner-workflow-evals": "pnpm run build:typescript && node scripts/run-runner-workflow-evals.mjs", + "report:runner-live-evals": "pnpm run build:typescript && pnpm run build:runner-binaries && node scripts/run-runner-live-eval-schedule.mjs --mode nightly --execute", + "report:runner-chaos-evals": "pnpm run build:typescript && node scripts/run-runner-live-eval-schedule.mjs --mode chaos", "check:conformance-parity": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core runs_the_mock_core_path_with_stable_output", "check:replay-parity": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core replay_fixture_parity", "docs:validate": "node scripts/validate-doc-links.mjs", @@ -131,6 +135,12 @@ "demo:standalone": "pnpm run build:typescript && vite --config vite.standalone.config.ts --host 127.0.0.1 --port 4182", "demo:scenarios": "pnpm run build:typescript && vite --config vite.scenarios.config.ts --host 127.0.0.1 --port 4183", "console:issue-thread": "pnpm run build:typescript && vite --config vite.issue-thread.config.ts --host 127.0.0.1 --port 4184", + "aws-agentcore:provision": "bash scripts/aws-agentcore.sh provision", + "aws-agentcore:probe": "bash scripts/aws-agentcore.sh probe", + "aws-agentcore:lab": "bash scripts/aws-agentcore.sh lab", + "aws-agentcore:destroy": "bash scripts/aws-agentcore.sh destroy", + "test:aws-agentcore-provisioning": "node --test scripts/aws-agentcore-provisioning.test.mjs", + "smoke:capability:aws-agentcore": "pnpm run build:typescript && cargo build --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --bin paperclip-runnerd && node scripts/capability-aws-agentcore-smoke.mjs", "demo:live-console": "pnpm run build:typescript && node scripts/live-console-demo-server.mjs", "smoke:capability:ui": "pnpm run build:typescript && cargo build --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --bin paperclip-runnerd && node scripts/capability-issue-thread-smoke.mjs", "smoke:capability:cleanroom": "pnpm run build:typescript && cargo build --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --bin paperclip-runnerd && node scripts/capability-clean-room-smoke.mjs", diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs index d83dc54969..16733160ae 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs @@ -800,26 +800,70 @@ fn normalize_acpx_status( let tag = string(payload.get("tag")); if tag == "usage_update" { let breakdown = payload.get("breakdown").unwrap_or(&Value::Null); - let usage = json!({ - "inputTokens": nonnegative_u64(breakdown.get("inputTokens")), - "outputTokens": nonnegative_u64(breakdown.get("outputTokens")), - "cacheReadTokens": nonnegative_u64( - breakdown - .get("cachedReadTokens") - .or_else(|| breakdown.get("cacheReadTokens")), - ), - "cacheWriteTokens": nonnegative_u64( - breakdown - .get("cachedWriteTokens") - .or_else(|| breakdown.get("cacheWriteTokens")), - ), + let cache_read = breakdown + .get("cachedReadTokens") + .or_else(|| breakdown.get("cacheReadTokens")); + let cache_write = breakdown + .get("cachedWriteTokens") + .or_else(|| breakdown.get("cacheWriteTokens")); + // ACPX marks every breakdown field optional and defines omission as + // unknown. Only advertise a complete per-turn delta when every field + // that feeds a Paperclip token budget is explicitly present. + let run_delta_available = breakdown + .get("inputTokens") + .and_then(Value::as_u64) + .is_some() + && breakdown + .get("outputTokens") + .and_then(Value::as_u64) + .is_some() + && breakdown + .get("thoughtTokens") + .and_then(Value::as_u64) + .is_some() + && cache_read.and_then(Value::as_u64).is_some() + && cache_write.and_then(Value::as_u64).is_some(); + let cost_is_usd = match payload.pointer("/cost/currency") { + None => true, + Some(Value::String(currency)) => currency.eq_ignore_ascii_case("USD"), + Some(_) => false, + }; + // ACPX 0.13.1 documents breakdown as per-turn usage while cost is + // session-cumulative. Keep those authorities separate so consumers do + // not add the same tokens twice or treat cumulative cost as a delta. + let cumulative = json!({ + "inputTokens": 0, + "outputTokens": 0, + "cacheReadTokens": 0, + "cacheWriteTokens": 0, "activeSeconds": 0.0, "requests": provider_requests, - "providerCostUsd": payload - .pointer("/cost/amount") - .and_then(Value::as_f64) - .filter(|value| value.is_finite() && *value >= 0.0) - .unwrap_or(0.0), + "providerCostUsd": if cost_is_usd { + payload + .pointer("/cost/amount") + .and_then(Value::as_f64) + .filter(|value| value.is_finite() && *value >= 0.0) + .unwrap_or(0.0) + } else { + 0.0 + }, + }); + let run_delta = json!({ + "inputTokens": nonnegative_u64(breakdown.get("inputTokens")), + // PRP v1 has no separate reasoning-token field. Fold ACPX thought + // tokens into output so spend and token ceilings cannot undercount + // reasoning work. + "outputTokens": nonnegative_u64(breakdown.get("outputTokens")) + .saturating_add(nonnegative_u64(breakdown.get("thoughtTokens"))), + "cacheReadTokens": nonnegative_u64( + cache_read, + ), + "cacheWriteTokens": nonnegative_u64( + cache_write, + ), + "activeSeconds": 0.0, + "requests": 1, + "providerCostUsd": 0.0, }); return vec![NormalizedProviderEvent { event_type: "usage.reported".to_owned(), @@ -832,8 +876,9 @@ fn normalize_acpx_status( .map(|value| bounded_text(value, 240)), "providerSessionId": Value::Null, "providerRequestId": Value::Null, - "cumulative": usage, - "runDelta": usage, + "cumulative": cumulative, + "runDeltaAvailable": run_delta_available, + "runDelta": run_delta, }), }]; } @@ -1192,4 +1237,37 @@ mod tests { assert_eq!(second[0].payload["runDeltaAvailable"], false); assert_eq!(second[0].payload["cumulative"]["inputTokens"], 20); } + + #[test] + fn preserves_proxy_shaped_current_run_usage_as_an_explicit_delta() { + let usage = normalize_codex_notification( + "thread/tokenUsage/updated", + &json!({"tokenUsage": { + "total": { + "inputTokens": 24, + "outputTokens": 14, + "cachedInputTokens": 3, + "requests": 3, + "providerCostUsd": 0.021 + }, + "last": { + "inputTokens": 7, + // The OpenCode proxy folds reasoning into output because + // PRP v1 has no separate reasoning field. + "outputTokens": 4, + "cachedInputTokens": 1, + "requests": 1, + "providerCostUsd": 0.004 + } + }}), + ); + + assert_eq!(usage[0].payload["runDeltaAvailable"], true); + assert_eq!(usage[0].payload["cumulative"]["inputTokens"], 24); + assert_eq!(usage[0].payload["cumulative"]["outputTokens"], 14); + assert_eq!(usage[0].payload["cumulative"]["providerCostUsd"], 0.021); + assert_eq!(usage[0].payload["runDelta"]["inputTokens"], 7); + assert_eq!(usage[0].payload["runDelta"]["outputTokens"], 4); + assert_eq!(usage[0].payload["runDelta"]["providerCostUsd"], 0.004); + } } diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_events.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_events.rs index 5f736090e7..c4afdcb9d1 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_events.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_events.rs @@ -155,13 +155,26 @@ fn maps_usage_and_review_status_but_ignores_inventory_updates() { json!({ "type":"status", "tag":"usage_update", - "breakdown":{"inputTokens":12,"outputTokens":4,"cachedReadTokens":2}, + "breakdown":{ + "inputTokens":12, + "outputTokens":4, + "thoughtTokens":3, + "cachedReadTokens":2, + "cachedWriteTokens":0 + }, "cost":{"amount":0.25} }), ); assert_eq!(usage[0].event_type, "usage.reported"); - assert_eq!(usage[0].payload["cumulative"]["inputTokens"], 12); + assert_eq!(usage[0].payload["cumulative"]["inputTokens"], 0); assert_eq!(usage[0].payload["cumulative"]["requests"], 3); + assert_eq!(usage[0].payload["cumulative"]["providerCostUsd"], 0.25); + assert_eq!(usage[0].payload["runDeltaAvailable"], true); + assert_eq!(usage[0].payload["runDelta"]["inputTokens"], 12); + assert_eq!(usage[0].payload["runDelta"]["outputTokens"], 7); + assert_eq!(usage[0].payload["runDelta"]["cacheReadTokens"], 2); + assert_eq!(usage[0].payload["runDelta"]["requests"], 1); + assert_eq!(usage[0].payload["runDelta"]["providerCostUsd"], 0.0); assert_eq!(usage[0].priority, EventPriority::P0); let review = normalize( @@ -178,6 +191,55 @@ fn maps_usage_and_review_status_but_ignores_inventory_updates() { .is_empty()); } +#[test] +fn does_not_claim_missing_or_partial_usage_breakdowns_are_exact() { + for payload in [ + json!({ + "type":"status", + "tag":"usage_update", + "cost":{"amount":0.25,"currency":"USD"} + }), + json!({ + "type":"status", + "tag":"usage_update", + "breakdown":null, + "cost":{"amount":0.25,"currency":"USD"} + }), + json!({ + "type":"status", + "tag":"usage_update", + "breakdown":{"inputTokens":12}, + "cost":{"amount":0.25,"currency":"USD"} + }), + ] { + let usage = normalize(AcpxRuntimeEventKind::Status, payload); + assert_eq!(usage[0].payload["runDeltaAvailable"], false); + assert_eq!(usage[0].payload["cumulative"]["providerCostUsd"], 0.25); + } +} + +#[test] +fn does_not_label_non_usd_acpx_cost_as_usd() { + let usage = normalize( + AcpxRuntimeEventKind::Status, + json!({ + "type":"status", + "tag":"usage_update", + "breakdown":{ + "inputTokens":12, + "outputTokens":4, + "thoughtTokens":3, + "cachedReadTokens":2, + "cachedWriteTokens":0 + }, + "cost":{"amount":0.25,"currency":"EUR"} + }), + ); + + assert_eq!(usage[0].payload["runDeltaAvailable"], true); + assert_eq!(usage[0].payload["cumulative"]["providerCostUsd"], 0.0); +} + #[test] fn maps_tool_lifecycle_and_preserves_safe_display_paths() { let started = normalize( diff --git a/packages/paperclip-runner/scripts/aws-agentcore-provisioning.test.mjs b/packages/paperclip-runner/scripts/aws-agentcore-provisioning.test.mjs new file mode 100644 index 0000000000..eae5284f90 --- /dev/null +++ b/packages/paperclip-runner/scripts/aws-agentcore-provisioning.test.mjs @@ -0,0 +1,258 @@ +import assert from "node:assert/strict"; +import { execFileSync, spawnSync } from "node:child_process"; +import { chmod, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +const templateUrl = new URL("../infra/aws-agentcore-paperclip.yaml", import.meta.url); +const wrapperUrl = new URL("./aws-agentcore.sh", import.meta.url); +const labServerUrl = new URL("./capability-issue-thread-server.mjs", import.meta.url); +const liveSessionUrl = new URL("../src/live/live-session.ts", import.meta.url); + +function parameterAllowedPattern(source, parameterName) { + const marker = ` ${parameterName}:\n`; + const start = source.indexOf(marker); + assert.notEqual(start, -1, `missing ${parameterName} parameter`); + const remaining = source.slice(start + marker.length); + const nextMatch = /\n \S/.exec(remaining); + const nextParameter = nextMatch ? start + marker.length + nextMatch.index : -1; + const block = source.slice(start, nextParameter === -1 ? undefined : nextParameter); + const match = block.match(/AllowedPattern: "([^"]+)"/); + assert.ok(match, `missing ${parameterName} AllowedPattern`); + return new RegExp(match[1]); +} + +test("AgentCore template has closed development/private resources and explicit command denial", async () => { + const source = await readFile(templateUrl, "utf8"); + for (const resource of [ + "AWS::BedrockAgentCore::Harness", + "AWS::BedrockAgentCore::Memory", + "AWS::S3::Bucket", + "AWS::KMS::Key", + "AWS::EC2::VPC", + "com.amazonaws.${AWS::Region}.ecr.api", + "com.amazonaws.${AWS::Region}.ecr.dkr", + "com.amazonaws.${AWS::Region}.s3", + "com.amazonaws.${AWS::Region}.bedrock-runtime", + ]) assert.match(source, new RegExp(resource.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))); + assert.match(source, /AllowedValues: \[development, private\]/); + assert.match(source, /Effect: Deny\s+Action: bedrock-agentcore:InvokeAgentRuntimeCommand/g); + assert.doesNotMatch(source, /Effect: Allow\s+Action: bedrock-agentcore:InvokeAgentRuntimeCommand/); + assert.doesNotMatch(source, /AWS::EC2::NatGateway|AWS::EC2::InternetGateway/); + assert.match(source, /EventExpiryDuration: 90/); + assert.match(source, /MaxIterations: 8/); + assert.match(source, /MaxTokens: 4096/); + assert.match(source, /TimeoutSeconds: 300/); + assert.match(source, /AllowedTools:\s+- "@\*\/pc_\*"/); + assert.match(source, /Tools: \[\]/); + assert.match(source, /Skills: \[\]/); + assert.doesNotMatch(source, /BedrockModelResourceArn:\s+[\s\S]{0,100}Default: "\*"/); + assert.match(source, /BedrockModelId:[\s\S]*AllowedPattern: "\^\[A-Za-z0-9\]\[A-Za-z0-9\._:-\]\{0,255\}\$"/); + assert.match(source, /BedrockModelResourceArn:[\s\S]*AllowedPattern:[^\n]+inference-profile[^\n]+\[A-Za-z0-9\]\[A-Za-z0-9\._:-\]\{0,255\}/); + assert.match(source, /BedrockFoundationModelResourceArn:[\s\S]*AllowedPattern:[^\n]+foundation-model\/\[A-Za-z0-9\]\[A-Za-z0-9\._:-\]\{0,255\}/); + const modelIdPattern = parameterAllowedPattern(source, "BedrockModelId"); + assert.match("global.anthropic.claude-sonnet-4-6", modelIdPattern); + assert.match("anthropic.claude-3-5-sonnet-20241022-v2:0", modelIdPattern); + for (const unsafeModelId of ["custom/model", "custom*", "custom?", "arn:aws:bedrock:us-east-1::foundation-model/custom"]) { + assert.doesNotMatch(unsafeModelId, modelIdPattern); + } + const inferenceProfilePattern = parameterAllowedPattern(source, "BedrockModelResourceArn"); + assert.match("arn:aws:bedrock:us-east-1:123456789012:inference-profile/global.anthropic.claude-sonnet-4-6", inferenceProfilePattern); + assert.doesNotMatch("arn:aws:bedrock:us-east-1:123456789012:inference-profile/*", inferenceProfilePattern); + assert.doesNotMatch("arn:aws:bedrock:us-east-1:123456789012:inference-profile/custom/model", inferenceProfilePattern); + const foundationModelPattern = parameterAllowedPattern(source, "BedrockFoundationModelResourceArn"); + assert.match("arn:aws:bedrock:*::foundation-model/anthropic.claude-sonnet-4-6", foundationModelPattern); + assert.doesNotMatch("arn:aws:bedrock:*::foundation-model/*", foundationModelPattern); + assert.doesNotMatch("arn:aws:bedrock:*::foundation-model/custom/model", foundationModelPattern); + assert.match(source, /BedrockFoundationModelResourceArn/); + assert.match(source, /HarnessEndpointName/); + assert.match(source, /ContextPrefix/); + assert.match(source, /BucketOwnerEnforced/); + assert.match(source, /SSEAlgorithm: aws:kms/); + assert.match(source, /EnableKeyRotation: true/); + assert.match(source, /Sid: DenyInsecureTransport/); + assert.match(source, /Sid: DenyUnencryptedContextUploads/); + assert.match(source, /Sid: DenyContextUploadsWithAnotherKey/); + assert.match(source, /Sid: WriteAndVerifyPinnedRuntimeContext[\s\S]*s3:GetObject[\s\S]*s3:PutObject[\s\S]*s3:DeleteObject[\s\S]*Resource: !Sub \$\{ContextBucket\.Arn\}\/\$\{ContextPrefix\}\/assets\/\*/); + assert.match(source, /Sid: EncryptPinnedRuntimeContext[\s\S]*kms:Encrypt[\s\S]*kms:GenerateDataKey[\s\S]*Resource: !GetAtt ContextEncryptionKey\.Arn/); + assert.match(source, /Sid: ReadPinnedRuntimeContext[\s\S]*Action: s3:GetObject[\s\S]*Resource: !Sub \$\{ContextBucket\.Arn\}\/\$\{ContextPrefix\}\/assets\/\*/); + assert.match(source, /Sid: EnumeratePinnedRuntimeContext[\s\S]*Action: s3:ListBucket[\s\S]*Resource: !GetAtt ContextBucket\.Arn[\s\S]*s3:prefix: !Sub \$\{ContextPrefix\}\/assets\/\*/); + assert.match(source, /Sid: EnumeratePinnedRuntimeContextForTeardown[\s\S]*Action: s3:ListBucket[\s\S]*Resource: !GetAtt ContextBucket\.Arn[\s\S]*s3:prefix: !Sub \$\{ContextPrefix\}\/assets\/\*/); + assert.match(source, /Sid: DecryptPinnedRuntimeContext[\s\S]*Action: kms:Decrypt[\s\S]*Resource: !GetAtt ContextEncryptionKey\.Arn/); + const invocationRole = source.slice(source.indexOf("RunnerInvocationRole:"), source.indexOf("PrivateVpc:")); + assert.doesNotMatch(invocationRole, /Action:\s+(?:-\s+)?(?:s3|kms):\*/); + assert.match(source, /\$\{AgentHarness\.Arn\}\/harness-endpoint\/\$\{HarnessEndpointName\}/); + assert.match(source, /\$\{AgentHarness\.Arn\}\/runtime-endpoint\/\$\{HarnessEndpointName\}/); + assert.match(source, /BedrockMarketplaceProductId/); + assert.match(source, /Sid: ViewMarketplaceSubscriptionsForModelAccess[\s\S]*Action: aws-marketplace:ViewSubscriptions[\s\S]*Sid: SubscribePinnedMarketplaceModel[\s\S]*Action: aws-marketplace:Subscribe[\s\S]*aws-marketplace:ProductId: !Ref BedrockMarketplaceProductId/); + assert.doesNotMatch(source, /aws-marketplace:Unsubscribe/); +}); + +test("AgentCore wrapper is valid shell and writes only nonsecret profile metadata", async () => { + execFileSync("bash", ["-n", wrapperUrl.pathname]); + const source = await readFile(wrapperUrl, "utf8"); + const generatedBlock = source.slice(source.lastIndexOf('"AWS_PROFILE=$AWS_PROFILE_NAME"'), source.lastIndexOf('>"$tmp"')); + const teardownBlock = source.slice(source.indexOf("write_teardown_metadata()"), source.indexOf("assume_runner_role()")); + assert.ok(generatedBlock.length > 0); + assert.ok(teardownBlock.length > 0); + assert.doesNotMatch(generatedBlock, /AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|AWS_SESSION_TOKEN|Authorization|X-Amz-Signature/); + assert.doesNotMatch(teardownBlock, /AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|AWS_SESSION_TOKEN|Authorization|X-Amz-Signature/); + assert.match(teardownBlock, /PAPERCLIP_AWS_AGENTCORE_STACK_NAME=\$STACK_NAME/); + assert.match(teardownBlock, /chmod 600 "\$tmp"/); + assert.match(source, /chmod 600 "\$tmp"/); + assert.match(source, /printf '%q\\n'/); + assert.match(source, /printf 'AWS_CONFIG_FILE=%q\\n'/); + assert.match(source, /AWS_PROFILE_EXPLICIT=true/); + assert.match(source, /AWS_REGION_EXPLICIT=true/); + assert.match(source, /STACK_NAME_EXPLICIT=true/); + assert.equal((source.match(/local requested_profile=/g) ?? []).length, 1); + assert.equal((source.match(/requested_region=/g) ?? []).length, 1); + assert.equal((source.match(/requested_stack=/g) ?? []).length, 1); + const destroyBlock = source.slice(source.indexOf("destroy()"), source.indexOf('case "$ACTION"')); + assert.ok(destroyBlock.indexOf("load_local_env") < destroyBlock.indexOf("stack_output HarnessId")); + assert.match(source, /cloudformation deploy/); + const provisionBlock = source.slice(source.indexOf("provision()"), source.indexOf("lab()")); + assert.ok(provisionBlock.indexOf("cloudformation deploy") < provisionBlock.indexOf("write_teardown_metadata")); + assert.ok(provisionBlock.indexOf("write_teardown_metadata") < provisionBlock.indexOf("stack_output HarnessId")); + assert.ok(provisionBlock.indexOf("write_teardown_metadata") < provisionBlock.indexOf("wait_for_harness_status")); + assert.match(source, /CAPABILITY_NAMED_IAM/); + assert.match(source, /--\) shift ;;/); + assert.match(source, /iam get-role --role-name "\$role_name"/); + assert.doesNotMatch(source, /printf 'arn:%s:iam::%s:role\/%s/); + assert.match(source, /existing_status.*ROLLBACK_COMPLETE/s); + assert.match(source, /--replace-failed-stack/); + assert.match(provisionBlock, /existing_description.*STACK_DESCRIPTION/s); + assert.match(provisionBlock, /existing_owned.*paperclip:owned/s); + assert.match(provisionBlock, /existing_environment.*paperclip:environment/s); + assert.match(provisionBlock, /existing_cost_center.*paperclip:cost-center/s); + assert.ok(provisionBlock.indexOf("ownership tags or template provenance") < provisionBlock.indexOf("cloudformation delete-stack")); + assert.ok(provisionBlock.indexOf("REPLACE_FAILED_STACK") < provisionBlock.indexOf("cloudformation delete-stack")); + assert.match(provisionBlock, /Unable to verify whether stack .* exists and is owned by Paperclip/); + assert.match(source, /cloudformation wait stack-delete-complete/); + assert.match(source, /--query harness\.status/); + assert.match(source, /AgentCore tool allowlist drift/); + assert.match(source, /--query memory\.status/); + assert.match(source, /--query endpoint\.status/); + assert.match(source, /o\.endpoint\?\.arn/); + assert.match(source, /--marketplace-product-id/); + assert.match(source, /BedrockMarketplaceProductId=\$MARKETPLACE_PRODUCT_ID/); + assert.match(generatedBlock, /PAPERCLIP_AWS_AGENTCORE_CONTEXT_BUCKET=\$context_bucket/); + assert.match(generatedBlock, /PAPERCLIP_AWS_AGENTCORE_CONTEXT_PREFIX=\$context_prefix/); + assert.match(generatedBlock, /PAPERCLIP_AWS_AGENTCORE_CONTEXT_KMS_KEY_ARN=\$context_kms_key_arn/); + assert.match(source, /ContextPrefix=\$CONTEXT_PREFIX/); + assert.match(source, /s3 rm "s3:\/\/\$context_bucket\/\$context_prefix\/assets\/" --recursive/); + assert.ok(source.indexOf("delete-harness-endpoint") < source.lastIndexOf("cloudformation delete-stack")); +}); + +test("AgentCore wrapper rejects unsafe model IDs before AWS access", () => { + const result = spawnSync("bash", [ + wrapperUrl.pathname, + "provision", + "--model", + "custom/model/*", + "--marketplace-product-id", + "prod-safe123", + ], { encoding: "utf8" }); + assert.equal(result.status, 2); + assert.match(result.stderr, /must not contain ARN, path, glob, or wildcard syntax/); +}); + +test("AgentCore wrapper never implicitly deletes a colliding failed stack", async () => { + const temp = await mkdtemp(join(tmpdir(), "paperclip-agentcore-test-")); + const fakeAws = join(temp, "aws"); + const commandLog = join(temp, "aws.log"); + await writeFile(fakeAws, `#!/usr/bin/env bash +set -eu +printf '%s\\n' "$*" >>"$MOCK_AWS_LOG" +if [[ " $* " == *" --version "* || "$1" == "--version" ]]; then + printf 'aws-cli/2.31.0 Python/3.13.0\\n' +elif [[ " $* " == *" sts get-caller-identity "* ]]; then + printf '{"Account":"123456789012","Arn":"arn:aws:iam::123456789012:role/paperclip-test"}\\n' +elif [[ " $* " == *" cloudformation describe-stacks "* ]]; then + printf '{"Stacks":[{"StackStatus":"ROLLBACK_COMPLETE","Description":"%s","Tags":[{"Key":"paperclip:owned","Value":"%s"},{"Key":"paperclip:environment","Value":"development"},{"Key":"paperclip:cost-center","Value":"runner-lab"}]}]}\\n' "\${MOCK_STACK_DESCRIPTION}" "\${MOCK_STACK_OWNED}" +elif [[ " $* " == *" cloudformation deploy "* ]]; then + exit 42 +fi +`, { mode: 0o700 }); + await chmod(fakeAws, 0o700); + const baseEnv = { + ...process.env, + PATH: `${temp}:${process.env.PATH}`, + MOCK_AWS_LOG: commandLog, + MOCK_STACK_DESCRIPTION: "Paperclip proof-of-concept Amazon Bedrock AgentCore Harness and least-privilege invocation roles.", + MOCK_STACK_OWNED: "true", + }; + try { + const implicit = spawnSync("bash", [wrapperUrl.pathname, "provision"], { encoding: "utf8", env: baseEnv }); + assert.equal(implicit.status, 1); + assert.match(implicit.stderr, /--replace-failed-stack/); + assert.doesNotMatch(await readFile(commandLog, "utf8"), /cloudformation delete-stack/); + + await writeFile(commandLog, ""); + const foreign = spawnSync("bash", [wrapperUrl.pathname, "provision", "--replace-failed-stack", "--yes"], { + encoding: "utf8", + env: { ...baseEnv, MOCK_STACK_OWNED: "false" }, + }); + assert.equal(foreign.status, 1); + assert.match(foreign.stderr, /ownership tags or template provenance do not match/); + assert.doesNotMatch(await readFile(commandLog, "utf8"), /cloudformation delete-stack/); + + await writeFile(commandLog, ""); + const explicit = spawnSync("bash", [wrapperUrl.pathname, "provision", "--replace-failed-stack", "--yes"], { + encoding: "utf8", + env: baseEnv, + }); + assert.equal(explicit.status, 42); + const explicitLog = await readFile(commandLog, "utf8"); + assert.match(explicitLog, /cloudformation delete-stack/); + assert.ok(explicitLog.indexOf("cloudformation delete-stack") < explicitLog.indexOf("cloudformation deploy")); + } finally { + await rm(temp, { recursive: true, force: true }); + } +}); + +test("Runner Lab accepts and resolves the complete qualified AgentCore profile", async () => { + const source = await readFile(labServerUrl, "utf8"); + assert.match(source, /provider !== "aws_agentcore"/); + assert.match(source, /AWS AgentCore requires exact model global\.anthropic\.claude-sonnet-4-6/); + assert.match(source, /function resolveAgentCoreProfile\(configuration\)/); + for (const field of [ + "PAPERCLIP_AWS_AGENTCORE_CONTEXT_BUCKET", + "PAPERCLIP_AWS_AGENTCORE_CONTEXT_PREFIX", + "PAPERCLIP_AWS_AGENTCORE_CONTEXT_KMS_KEY_ARN", + ]) assert.match(source, new RegExp(field)); + assert.match(source, /agentCoreProfile: resolveAgentCoreProfile\(configuration\)/); + assert.match(source, /agentCoreProfileId: snapshot\.config\.agentCoreProfile\.profileId/); + assert.match(source, /configuration\.provider === "aws_agentcore" \? "remote_service"/); +}); + +test("Runner Lab qualifies Claude Managed and exposes remote governance for both remote providers", async () => { + const source = await readFile(labServerUrl, "utf8"); + const liveSessionSource = await readFile(liveSessionUrl, "utf8"); + assert.match(source, /provider !== "claude_managed"/); + assert.match(source, /Claude Managed requires exact model claude-sonnet-5/); + assert.match(source, /function resolveManagedProfile\(configuration\)/); + for (const field of [ + "PAPERCLIP_CLAUDE_MANAGED_PROFILE_ID", + "ANTHROPIC_API_KEY", + "ANTHROPIC_MANAGED_AGENT_ID", + "ANTHROPIC_MANAGED_AGENT_VERSION", + "ANTHROPIC_MANAGED_ENVIRONMENT_ID", + ]) assert.match(source, new RegExp(field)); + assert.match(source, /\^\[1-9\]\[0-9\]\*\$/); + assert.match(source, /BigInt\(agentVersion\) <= 2_147_483_647n/); + assert.match(source, /managedProfile: resolveManagedProfile\(configuration\)/); + assert.match(source, /managedProfileId: snapshot\.config\.managedProfile\.profileId/); + assert.match(source, /route === "managed-budget"/); + assert.match(source, /entry\.session\.increaseManagedSessionBudget\(nextCap\)/); + assert.match(source, /entry\.configuration\?\.provider === "claude_managed"/); + assert.match(source, /entry\.configuration\?\.provider === "aws_agentcore"/); + assert.match(source, /route === "managed-session-delete"/); + assert.match(source, /body\.confirm !== true/); + assert.match(source, /entry\.session\.deleteManagedRemoteSession\(\)/); + assert.match(liveSessionSource, /async increaseManagedSessionBudget\(/); + assert.match(liveSessionSource, /this\.#transport\.request\("session\/budget\/increase"/); + assert.match(liveSessionSource, /async deleteManagedRemoteSession\(\)/); + assert.match(liveSessionSource, /this\.#transport\.request\("session\/destroy"/); +}); diff --git a/packages/paperclip-runner/scripts/aws-agentcore.sh b/packages/paperclip-runner/scripts/aws-agentcore.sh new file mode 100644 index 0000000000..26f4e493d8 --- /dev/null +++ b/packages/paperclip-runner/scripts/aws-agentcore.sh @@ -0,0 +1,465 @@ +#!/usr/bin/env bash +set -euo pipefail + +PACKAGE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TEMPLATE="$PACKAGE_DIR/infra/aws-agentcore-paperclip.yaml" +STACK_DESCRIPTION="Paperclip proof-of-concept Amazon Bedrock AgentCore Harness and least-privilege invocation roles." +LOCAL_DIR="$PACKAGE_DIR/.paperclip-local" +ENV_FILE="$LOCAL_DIR/aws-agentcore.env" +ACTIVE_FILE="$LOCAL_DIR/aws-agentcore-lab.pid" +ACTION="${1:-}" +if [[ -n "$ACTION" ]]; then shift; fi + +AWS_PROFILE_NAME="${AWS_PROFILE:-default}" +AWS_REGION_NAME="${AWS_REGION:-${AWS_DEFAULT_REGION:-us-east-1}}" +AWS_PROFILE_EXPLICIT=false +AWS_REGION_EXPLICIT=false +DEPLOYMENT_MODE="development" +STACK_NAME="paperclip-agentcore-development" +STACK_NAME_EXPLICIT=false +ENDPOINT_NAME="paperclip" +MODEL_ID="global.anthropic.claude-sonnet-4-6" +MARKETPLACE_PRODUCT_ID="prod-ffvjxvh4ltq64" +MARKETPLACE_PRODUCT_ID_EXPLICIT=false +TRUSTED_PRINCIPAL="" +CONTEXT_PREFIX="" +DRY_RUN=false +FORCE=false +YES=false +REPLACE_FAILED_STACK=false + +usage() { + printf '%s\n' \ + "Usage: aws-agentcore.sh [options]" \ + " --aws-profile NAME AWS CLI v2 profile (default: $AWS_PROFILE_NAME)" \ + " --region REGION commercial AWS region (default: $AWS_REGION_NAME)" \ + " --mode MODE development or private" \ + " --stack-name NAME CloudFormation stack name" \ + " --model MODEL_ID Bedrock-native model ID" \ + " --marketplace-product-id ID exact AWS Marketplace product ID for the model" \ + " --principal ARN stable IAM role/user trusted to assume runner role" \ + " --context-prefix PFX S3 prefix dedicated to this qualified profile" \ + " --dry-run validate and print changes without deployment" \ + " --replace-failed-stack explicitly replace an owned ROLLBACK_COMPLETE stack" \ + " --yes confirm destructive replacement or teardown" \ + " --force destroy even when a recorded Lab process is active" +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --) shift ;; + --aws-profile) AWS_PROFILE_NAME="$2"; AWS_PROFILE_EXPLICIT=true; shift 2 ;; + --region) AWS_REGION_NAME="$2"; AWS_REGION_EXPLICIT=true; shift 2 ;; + --mode) DEPLOYMENT_MODE="$2"; shift 2 ;; + --stack-name) STACK_NAME="$2"; STACK_NAME_EXPLICIT=true; shift 2 ;; + --model) MODEL_ID="$2"; shift 2 ;; + --marketplace-product-id) MARKETPLACE_PRODUCT_ID="$2"; MARKETPLACE_PRODUCT_ID_EXPLICIT=true; shift 2 ;; + --principal) TRUSTED_PRINCIPAL="$2"; shift 2 ;; + --context-prefix) CONTEXT_PREFIX="$2"; shift 2 ;; + --dry-run) DRY_RUN=true; shift ;; + --replace-failed-stack) REPLACE_FAILED_STACK=true; shift ;; + --force) FORCE=true; shift ;; + --yes) YES=true; shift ;; + -h|--help) usage; exit 0 ;; + *) printf 'Unknown option: %s\n' "$1" >&2; usage >&2; exit 2 ;; + esac +done + +if [[ "$ACTION" != "provision" && "$ACTION" != "probe" && "$ACTION" != "lab" && "$ACTION" != "destroy" ]]; then + usage >&2 + exit 2 +fi +if [[ "$DEPLOYMENT_MODE" != "development" && "$DEPLOYMENT_MODE" != "private" ]]; then + printf 'Mode must be development or private.\n' >&2 + exit 2 +fi +if [[ ! "$AWS_PROFILE_NAME" =~ ^[A-Za-z0-9_.@+-]+$ ]]; then + printf 'AWS profile contains unsupported characters.\n' >&2 + exit 2 +fi +if [[ ! "$AWS_REGION_NAME" =~ ^[a-z]{2}(-gov)?-[a-z]+-[0-9]+$ || "$AWS_REGION_NAME" == cn-* || "$AWS_REGION_NAME" == us-gov-* ]]; then + printf 'This proof of concept currently supports commercial AWS regions only.\n' >&2 + exit 2 +fi +if [[ ! "$MODEL_ID" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$ ]]; then + printf 'Bedrock model ID must not contain ARN, path, glob, or wildcard syntax.\n' >&2 + exit 2 +fi +if [[ "$MODEL_ID" != "global.anthropic.claude-sonnet-4-6" && "$MARKETPLACE_PRODUCT_ID_EXPLICIT" != true ]]; then + printf 'A custom --model requires its exact --marketplace-product-id so subscription authority remains model-scoped.\n' >&2 + exit 2 +fi +if [[ ! "$MARKETPLACE_PRODUCT_ID" =~ ^prod-[a-z0-9]+$ ]]; then + printf 'Marketplace product ID must use the AWS prod-* form.\n' >&2 + exit 2 +fi +if [[ -z "$CONTEXT_PREFIX" ]]; then CONTEXT_PREFIX="paperclip/agentcore/$STACK_NAME"; fi +if [[ ! "$CONTEXT_PREFIX" =~ ^[a-z0-9][a-z0-9/_-]{1,127}$ || "$CONTEXT_PREFIX" == /* || "$CONTEXT_PREFIX" == */ || "$CONTEXT_PREFIX" == *//* || "/$CONTEXT_PREFIX/" == */./* || "/$CONTEXT_PREFIX/" == */../* ]]; then + printf 'Context prefix must be a safe, relative S3 key prefix.\n' >&2 + exit 2 +fi + +aws_cli() { + aws --profile "$AWS_PROFILE_NAME" --region "$AWS_REGION_NAME" --no-cli-pager "$@" +} + +require_aws() { + command -v aws >/dev/null || { printf 'AWS CLI v2 is required.\n' >&2; exit 1; } + local version + version="$(aws --version 2>&1)" + [[ "$version" == aws-cli/2.* ]] || { printf 'AWS CLI v2 is required; found %s\n' "$version" >&2; exit 1; } +} + +json_field() { + local key="$1" + node -e 'const fs=require("fs");const o=JSON.parse(fs.readFileSync(0,"utf8"));let v=o;for(const k of process.argv[1].split("."))v=v?.[k];if(v!==undefined&&v!==null)process.stdout.write(String(v));' "$key" +} + +stack_tag_value() { + local key="$1" + node -e 'const fs=require("fs");const o=JSON.parse(fs.readFileSync(0,"utf8"));const tag=o.Stacks?.[0]?.Tags?.find((candidate)=>candidate.Key===process.argv[1]);if(tag?.Value)process.stdout.write(tag.Value);' "$key" +} + +stack_output() { + local key="$1" + aws_cli cloudformation describe-stacks --stack-name "$STACK_NAME" \ + --query "Stacks[0].Outputs[?OutputKey=='$key'].OutputValue | [0]" --output text +} + +wait_for_harness_status() { + local harness_id="$1" harness_version="$2" status="" + for _ in {1..60}; do + status="$(aws_cli bedrock-agentcore-control get-harness --harness-id "$harness_id" --harness-version "$harness_version" --query harness.status --output text 2>/dev/null || true)" + if [[ "$status" == "READY" ]]; then return 0; fi + if [[ "$status" == "FAILED" || "$status" == "CREATE_FAILED" || "$status" == "UPDATE_FAILED" ]]; then + printf 'Harness %s entered terminal status %s.\n' "$harness_id" "$status" >&2 + return 1 + fi + sleep 5 + done + printf 'Timed out waiting for Harness %s (last status: %s).\n' "$harness_id" "${status:-unknown}" >&2 + return 1 +} + +wait_for_memory_status() { + local memory_id="$1" status="" + for _ in {1..60}; do + status="$(aws_cli bedrock-agentcore-control get-memory --memory-id "$memory_id" --query memory.status --output text 2>/dev/null || true)" + [[ "$status" == "ACTIVE" ]] && return 0 + [[ "$status" == "FAILED" ]] && { printf 'Memory %s entered FAILED.\n' "$memory_id" >&2; return 1; } + sleep 5 + done + printf 'Timed out waiting for Memory %s (last status: %s).\n' "$memory_id" "${status:-unknown}" >&2 + return 1 +} + +wait_for_endpoint_status() { + local harness_id="$1" endpoint_name="$2" status="" + for _ in {1..60}; do + status="$(aws_cli bedrock-agentcore-control get-harness-endpoint --harness-id "$harness_id" --endpoint-name "$endpoint_name" --query endpoint.status --output text 2>/dev/null || true)" + [[ "$status" == "READY" ]] && return 0 + [[ "$status" == "FAILED" ]] && { printf 'Harness endpoint %s entered FAILED.\n' "$endpoint_name" >&2; return 1; } + sleep 5 + done + printf 'Timed out waiting for Harness endpoint %s (last status: %s).\n' "$endpoint_name" "${status:-unknown}" >&2 + return 1 +} + +normalize_principal() { + local arn="$1" + if [[ "$arn" =~ ^arn:(aws[^:]*):sts::([0-9]{12}):assumed-role/(.+)/[^/]+$ ]]; then + # STS omits an IAM role's path from assumed-role ARNs. This matters for + # Identity Center roles, whose canonical ARN lives under + # /aws-reserved/sso.amazonaws.com/. Resolve it instead of constructing a + # role ARN that may not exist. + local role_name canonical_arn + role_name="${BASH_REMATCH[3]##*/}" + canonical_arn="$(aws_cli iam get-role --role-name "$role_name" --query Role.Arn --output text)" + [[ "$canonical_arn" =~ ^arn:(aws[^:]*):iam::[0-9]{12}:role/.+$ ]] || { + printf 'Could not resolve the canonical IAM role ARN for %s.\n' "$role_name" >&2 + exit 1 + } + printf '%s\n' "$canonical_arn" + elif [[ "$arn" =~ ^arn:(aws[^:]*):iam::[0-9]{12}:(role|user)/.+$ ]]; then + printf '%s\n' "$arn" + else + printf 'The caller ARN cannot be normalized to a stable IAM role/user: %s\n' "$arn" >&2 + exit 1 + fi +} + +load_local_env() { + [[ -f "$ENV_FILE" ]] || { printf 'Missing %s; run aws-agentcore:provision first.\n' "$ENV_FILE" >&2; exit 1; } + local requested_profile="$AWS_PROFILE_NAME" requested_region="$AWS_REGION_NAME" requested_stack="$STACK_NAME" + # This file is generated locally, mode 0600, and contains nonsecret metadata only. + set -a + # shellcheck disable=SC1090 + source "$ENV_FILE" + set +a + if $AWS_PROFILE_EXPLICIT; then AWS_PROFILE_NAME="$requested_profile"; else AWS_PROFILE_NAME="${AWS_PROFILE:-$requested_profile}"; fi + if $AWS_REGION_EXPLICIT; then AWS_REGION_NAME="$requested_region"; else AWS_REGION_NAME="${AWS_REGION:-$requested_region}"; fi + if $STACK_NAME_EXPLICIT; then STACK_NAME="$requested_stack"; else STACK_NAME="${PAPERCLIP_AWS_AGENTCORE_STACK_NAME:-$requested_stack}"; fi + export AWS_PROFILE="$AWS_PROFILE_NAME" + export AWS_REGION="$AWS_REGION_NAME" + export AWS_DEFAULT_REGION="$AWS_REGION_NAME" + export PAPERCLIP_AWS_AGENTCORE_STACK_NAME="$STACK_NAME" +} + +write_teardown_metadata() { + mkdir -p "$LOCAL_DIR" + umask 077 + local tmp="$ENV_FILE.tmp.$$" + printf '%q\n' \ + "AWS_PROFILE=$AWS_PROFILE_NAME" \ + "AWS_REGION=$AWS_REGION_NAME" \ + "AWS_DEFAULT_REGION=$AWS_REGION_NAME" \ + "PAPERCLIP_AWS_AGENTCORE_STACK_NAME=$STACK_NAME" >"$tmp" + if [[ -n "${AWS_CONFIG_FILE:-}" ]]; then + printf 'AWS_CONFIG_FILE=%q\n' "$AWS_CONFIG_FILE" >>"$tmp" + fi + chmod 600 "$tmp" + mv "$tmp" "$ENV_FILE" +} + +assume_runner_role() { + local role_arn="$1" + local response access secret token + if ! response="$(aws_cli sts assume-role --role-arn "$role_arn" --role-session-name paperclip-agentcore-probe --duration-seconds 900 --output json 2>&1)"; then + local caller + caller="$(aws_cli sts get-caller-identity --query Arn --output text 2>/dev/null || printf '')" + printf 'Cannot assume %s. Add this statement to the caller policy for %s:\n' "$role_arn" "$caller" >&2 + printf '{"Effect":"Allow","Action":"sts:AssumeRole","Resource":"%s"}\n' "$role_arn" >&2 + printf '%s\n' "$response" | sed -E 's/(ASIA|AKIA)[A-Z0-9]+/[REDACTED]/g' >&2 + return 1 + fi + access="$(printf '%s' "$response" | json_field Credentials.AccessKeyId)" + secret="$(printf '%s' "$response" | json_field Credentials.SecretAccessKey)" + token="$(printf '%s' "$response" | json_field Credentials.SessionToken)" + export AWS_ACCESS_KEY_ID="$access" AWS_SECRET_ACCESS_KEY="$secret" AWS_SESSION_TOKEN="$token" + unset AWS_PROFILE +} + +probe() { + load_local_env + local profile_save="$AWS_PROFILE_NAME" harness_json + assume_runner_role "$PAPERCLIP_AWS_AGENTCORE_INVOCATION_ROLE_ARN" + AWS_PROFILE_NAME="" + harness_json="$(aws --region "$AWS_REGION_NAME" --no-cli-pager bedrock-agentcore-control get-harness \ + --harness-id "$PAPERCLIP_AWS_AGENTCORE_HARNESS_ID" \ + --harness-version "$PAPERCLIP_AWS_AGENTCORE_HARNESS_VERSION" --output json)" + printf '%s' "$harness_json" | node -e ' + const fs = require("fs"); + const h = JSON.parse(fs.readFileSync(0, "utf8")).harness; + const expectedModel = process.env.PAPERCLIP_AWS_AGENTCORE_MODEL; + if (!h || h.model?.bedrockModelConfig?.modelId !== expectedModel) throw new Error("AgentCore model drift"); + if (JSON.stringify(h.allowedTools) !== JSON.stringify(["@*/pc_*"])) throw new Error("AgentCore tool allowlist drift"); + if (!Array.isArray(h.tools) || h.tools.length !== 0) throw new Error("AgentCore persistent tool drift"); + if (!Array.isArray(h.skills) || h.skills.length !== 0) throw new Error("AgentCore skill drift"); + ' + aws --region "$AWS_REGION_NAME" --no-cli-pager bedrock-agentcore-control get-harness-endpoint \ + --harness-id "$PAPERCLIP_AWS_AGENTCORE_HARNESS_ID" \ + --endpoint-name "$PAPERCLIP_AWS_AGENTCORE_ENDPOINT_QUALIFIER" >/dev/null + aws --region "$AWS_REGION_NAME" --no-cli-pager bedrock-agentcore-control get-memory \ + --memory-id "$PAPERCLIP_AWS_AGENTCORE_MEMORY_ID" --view full >/dev/null + [[ -n "$PAPERCLIP_AWS_AGENTCORE_CONTEXT_BUCKET" && -n "$PAPERCLIP_AWS_AGENTCORE_CONTEXT_PREFIX" && -n "$PAPERCLIP_AWS_AGENTCORE_CONTEXT_KMS_KEY_ARN" ]] || { + printf 'AWS AgentCore profile is missing its qualified S3/KMS runtime context.\n' >&2 + exit 1 + } + unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN + AWS_PROFILE_NAME="$profile_save" + printf 'AWS AgentCore profile qualified: %s (%s, Harness %s, Memory %s; 90-day retention).\n' \ + "$PAPERCLIP_AWS_AGENTCORE_PROFILE_ID" "$AWS_REGION_NAME" \ + "$PAPERCLIP_AWS_AGENTCORE_HARNESS_VERSION" "$PAPERCLIP_AWS_AGENTCORE_MEMORY_ID" +} + +provision() { + require_aws + local identity caller_arn account_id partition model_resource_arn foundation_model_id foundation_model_resource_arn + identity="$(aws_cli sts get-caller-identity --output json)" + caller_arn="$(printf '%s' "$identity" | json_field Arn)" + account_id="$(printf '%s' "$identity" | json_field Account)" + partition="$(printf '%s' "$caller_arn" | cut -d: -f2)" + model_resource_arn="arn:$partition:bedrock:$AWS_REGION_NAME:$account_id:inference-profile/$MODEL_ID" + foundation_model_id="$(printf '%s' "$MODEL_ID" | sed -E 's/^(global|us|eu|apac)\.//')" + foundation_model_resource_arn="arn:$partition:bedrock:*::foundation-model/$foundation_model_id" + if [[ -z "$TRUSTED_PRINCIPAL" ]]; then TRUSTED_PRINCIPAL="$(normalize_principal "$caller_arn")"; fi + if [[ "$DEPLOYMENT_MODE" == "private" ]]; then + aws_cli bedrock-agentcore-control list-harnesses --max-results 1 >/dev/null || { + printf 'Private mode is unavailable because AgentCore Harness is not enabled in %s.\n' "$AWS_REGION_NAME" >&2 + exit 1 + } + local services + services="$(aws_cli ec2 describe-vpc-endpoint-services --query 'ServiceNames' --output text)" + for service in "com.amazonaws.$AWS_REGION_NAME.ecr.api" "com.amazonaws.$AWS_REGION_NAME.ecr.dkr" "com.amazonaws.$AWS_REGION_NAME.s3" "com.amazonaws.$AWS_REGION_NAME.bedrock-runtime"; do + [[ " $services " == *" $service "* ]] || { printf 'Private mode is unavailable: missing VPC endpoint service %s\n' "$service" >&2; exit 1; } + done + fi + aws_cli cloudformation validate-template --template-body "file://$TEMPLATE" >/dev/null + printf 'Stack: %s\nRegion: %s\nMode: %s\nModel: %s\nTrusted principal: %s\n' \ + "$STACK_NAME" "$AWS_REGION_NAME" "$DEPLOYMENT_MODE" "$MODEL_ID" "$TRUSTED_PRINCIPAL" + printf 'Expected charges: Bedrock model tokens, AgentCore Runtime active time, Memory storage/requests, and (private mode) VPC endpoints.\n' + if $DRY_RUN; then + printf 'Dry run complete; the template is syntactically valid and no resources were changed.\n' + return + fi + local existing_stack="" existing_status="" + if existing_stack="$(aws_cli cloudformation describe-stacks --stack-name "$STACK_NAME" --output json 2>&1)"; then + local existing_description existing_owned existing_environment existing_cost_center + existing_status="$(printf '%s' "$existing_stack" | json_field Stacks.0.StackStatus)" + existing_description="$(printf '%s' "$existing_stack" | json_field Stacks.0.Description)" + existing_owned="$(printf '%s' "$existing_stack" | stack_tag_value paperclip:owned)" + existing_environment="$(printf '%s' "$existing_stack" | stack_tag_value paperclip:environment)" + existing_cost_center="$(printf '%s' "$existing_stack" | stack_tag_value paperclip:cost-center)" + if [[ -z "$existing_status" || "$existing_description" != "$STACK_DESCRIPTION" || "$existing_owned" != "true" || "$existing_environment" != "development" || "$existing_cost_center" != "runner-lab" ]]; then + printf 'Refusing to modify stack %s: Paperclip ownership tags or template provenance do not match. Inspect or remove it manually, or choose another --stack-name.\n' "$STACK_NAME" >&2 + exit 1 + fi + elif [[ "$existing_stack" == *"(ValidationError)"* && "$existing_stack" == *"does not exist"* ]]; then + existing_stack="" + else + printf 'Unable to verify whether stack %s exists and is owned by Paperclip; refusing to provision.\n' "$STACK_NAME" >&2 + exit 1 + fi + if [[ "$existing_status" == "ROLLBACK_COMPLETE" ]]; then + if ! $REPLACE_FAILED_STACK; then + printf 'Stack %s is ROLLBACK_COMPLETE. Re-run with --replace-failed-stack to explicitly authorize deleting this verified Paperclip stack.\n' "$STACK_NAME" >&2 + exit 1 + fi + if ! $YES; then + printf 'Delete verified Paperclip stack %s before retrying provisioning? [y/N] ' "$STACK_NAME" + read -r answer || true + [[ "$answer" == y || "$answer" == Y ]] || { printf 'Cancelled.\n'; return; } + fi + printf 'Removing verified failed Paperclip stack %s before retrying provisioning.\n' "$STACK_NAME" + aws_cli cloudformation delete-stack --stack-name "$STACK_NAME" + aws_cli cloudformation wait stack-delete-complete --stack-name "$STACK_NAME" + fi + aws_cli cloudformation deploy \ + --template-file "$TEMPLATE" \ + --stack-name "$STACK_NAME" \ + --capabilities CAPABILITY_NAMED_IAM \ + --parameter-overrides \ + "EnvironmentName=development" "DeploymentMode=$DEPLOYMENT_MODE" \ + "HarnessEndpointName=$ENDPOINT_NAME" \ + "ContextPrefix=$CONTEXT_PREFIX" \ + "TrustedRunnerPrincipalArn=$TRUSTED_PRINCIPAL" "BedrockModelId=$MODEL_ID" \ + "BedrockModelResourceArn=$model_resource_arn" \ + "BedrockFoundationModelResourceArn=$foundation_model_resource_arn" \ + "BedrockMarketplaceProductId=$MARKETPLACE_PRODUCT_ID" \ + --tags paperclip:owned=true paperclip:environment=development paperclip:cost-center=runner-lab + # CloudFormation has created the billable resources at this point. Persist + # the nonsecret coordinates needed by `destroy` before any qualification + # API call can fail, then replace this checkpoint with the full qualified + # profile only after every resource is ready. + write_teardown_metadata + local harness_id harness_arn harness_version runtime_arn memory_id memory_arn role_arn context_bucket context_prefix context_kms_key_arn qualification_revision + harness_id="$(stack_output HarnessId)" + harness_arn="$(stack_output HarnessArn)" + harness_version="$(stack_output HarnessVersion)" + runtime_arn="$(stack_output AgentRuntimeArn)" + memory_id="$(stack_output MemoryId)" + memory_arn="$(stack_output MemoryArn)" + role_arn="$(stack_output RunnerInvocationRoleArn)" + context_bucket="$(stack_output ContextBucketName)" + context_prefix="$(stack_output ContextPrefix)" + context_kms_key_arn="$(stack_output ContextKmsKeyArn)" + qualification_revision="$(stack_output QualificationRevision)" + wait_for_harness_status "$harness_id" "$harness_version" + wait_for_memory_status "$memory_id" + if aws_cli bedrock-agentcore-control get-harness-endpoint --harness-id "$harness_id" --endpoint-name "$ENDPOINT_NAME" >/dev/null 2>&1; then + aws_cli bedrock-agentcore-control update-harness-endpoint --harness-id "$harness_id" --endpoint-name "$ENDPOINT_NAME" \ + --target-version "$harness_version" --description "Paperclip pinned Harness endpoint" \ + --client-token "paperclip-update-$harness_id-$harness_version" >/dev/null + else + aws_cli bedrock-agentcore-control create-harness-endpoint --harness-id "$harness_id" --endpoint-name "$ENDPOINT_NAME" \ + --target-version "$harness_version" --description "Paperclip pinned Harness endpoint" \ + --client-token "paperclip-create-$harness_id-$harness_version" \ + --tags paperclip:owned=true,paperclip:environment=development >/dev/null + fi + wait_for_endpoint_status "$harness_id" "$ENDPOINT_NAME" + local endpoint_json endpoint_arn + endpoint_json="$(aws_cli bedrock-agentcore-control get-harness-endpoint --harness-id "$harness_id" --endpoint-name "$ENDPOINT_NAME" --output json)" + endpoint_arn="$(printf '%s' "$endpoint_json" | node -e 'const fs=require("fs");const o=JSON.parse(fs.readFileSync(0,"utf8"));process.stdout.write(o.endpoint?.arn||o.harnessEndpointArn||o.endpointArn||o.arn||"")')" + [[ -n "$endpoint_arn" ]] || endpoint_arn="$harness_arn/endpoint/$ENDPOINT_NAME" + mkdir -p "$LOCAL_DIR" + umask 077 + local tmp="$ENV_FILE.tmp.$$" + printf '%q\n' \ + "AWS_PROFILE=$AWS_PROFILE_NAME" \ + "AWS_REGION=$AWS_REGION_NAME" \ + "AWS_DEFAULT_REGION=$AWS_REGION_NAME" \ + "PAPERCLIP_AWS_AGENTCORE_STACK_NAME=$STACK_NAME" \ + "PAPERCLIP_AWS_AGENTCORE_PROFILE_ID=$STACK_NAME" \ + "PAPERCLIP_AWS_AGENTCORE_ACCOUNT_ID=$account_id" \ + "PAPERCLIP_AWS_AGENTCORE_HARNESS_ARN=$harness_arn" \ + "PAPERCLIP_AWS_AGENTCORE_HARNESS_ID=$harness_id" \ + "PAPERCLIP_AWS_AGENTCORE_HARNESS_VERSION=$harness_version" \ + "PAPERCLIP_AWS_AGENTCORE_ENDPOINT_ARN=$endpoint_arn" \ + "PAPERCLIP_AWS_AGENTCORE_ENDPOINT_QUALIFIER=$ENDPOINT_NAME" \ + "PAPERCLIP_AWS_AGENTCORE_RUNTIME_ARN=$runtime_arn" \ + "PAPERCLIP_AWS_AGENTCORE_MEMORY_ARN=$memory_arn" \ + "PAPERCLIP_AWS_AGENTCORE_MEMORY_ID=$memory_id" \ + "PAPERCLIP_AWS_AGENTCORE_INVOCATION_ROLE_ARN=$role_arn" \ + "PAPERCLIP_AWS_AGENTCORE_CONTEXT_BUCKET=$context_bucket" \ + "PAPERCLIP_AWS_AGENTCORE_CONTEXT_PREFIX=$context_prefix" \ + "PAPERCLIP_AWS_AGENTCORE_CONTEXT_KMS_KEY_ARN=$context_kms_key_arn" \ + "PAPERCLIP_AWS_AGENTCORE_MODEL=$MODEL_ID" \ + "PAPERCLIP_AWS_AGENTCORE_QUALIFICATION_REVISION=$qualification_revision" \ + "PAPERCLIP_AWS_AGENTCORE_EVENT_EXPIRY_DAYS=90" >"$tmp" + if [[ -n "${AWS_CONFIG_FILE:-}" ]]; then + printf 'AWS_CONFIG_FILE=%q\n' "$AWS_CONFIG_FILE" >>"$tmp" + fi + chmod 600 "$tmp" + mv "$tmp" "$ENV_FILE" + probe + printf 'Wrote nonsecret local profile metadata to %s (0600).\n' "$ENV_FILE" +} + +lab() { + load_local_env + if [[ -f "$ACTIVE_FILE" ]] && kill -0 "$(cat "$ACTIVE_FILE")" 2>/dev/null; then + printf 'Runner Lab is already active as PID %s.\n' "$(cat "$ACTIVE_FILE")" >&2 + exit 1 + fi + printf '%s\n' "$$" >"$ACTIVE_FILE" + chmod 600 "$ACTIVE_FILE" + trap 'rm -f "$ACTIVE_FILE"' EXIT INT TERM + cd "$PACKAGE_DIR" + pnpm console:issue-thread +} + +destroy() { + require_aws + load_local_env + if [[ -f "$ACTIVE_FILE" ]] && kill -0 "$(cat "$ACTIVE_FILE")" 2>/dev/null && ! $FORCE; then + printf 'Runner Lab PID %s is active; stop it or pass --force.\n' "$(cat "$ACTIVE_FILE")" >&2 + exit 1 + fi + if ! $YES; then + printf 'Delete the pinned AgentCore endpoint and CloudFormation stack %s? [y/N] ' "$STACK_NAME" + read -r answer + [[ "$answer" == y || "$answer" == Y ]] || { printf 'Cancelled.\n'; return; } + fi + local harness_id context_bucket context_prefix profile_save + harness_id="$(stack_output HarnessId)" + context_bucket="$(stack_output ContextBucketName)" + context_prefix="$(stack_output ContextPrefix)" + aws_cli bedrock-agentcore-control delete-harness-endpoint --harness-id "$harness_id" --endpoint-name "$ENDPOINT_NAME" \ + --client-token "paperclip-delete-$harness_id-$ENDPOINT_NAME" >/dev/null 2>&1 || true + profile_save="$AWS_PROFILE_NAME" + assume_runner_role "$(stack_output RunnerInvocationRoleArn)" + aws --region "$AWS_REGION_NAME" --no-cli-pager s3 rm "s3://$context_bucket/$context_prefix/assets/" --recursive >/dev/null + unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN + AWS_PROFILE_NAME="$profile_save" + aws_cli cloudformation delete-stack --stack-name "$STACK_NAME" + aws_cli cloudformation wait stack-delete-complete --stack-name "$STACK_NAME" + rm -f "$ENV_FILE" "$ACTIVE_FILE" + printf 'Deleted AgentCore endpoint and stack. Local profile metadata was removed.\n' +} + +case "$ACTION" in + provision) provision ;; + probe) require_aws; probe ;; + lab) lab ;; + destroy) destroy ;; +esac diff --git a/packages/paperclip-runner/scripts/capability-aws-agentcore-smoke.mjs b/packages/paperclip-runner/scripts/capability-aws-agentcore-smoke.mjs new file mode 100644 index 0000000000..ede52bd521 --- /dev/null +++ b/packages/paperclip-runner/scripts/capability-aws-agentcore-smoke.mjs @@ -0,0 +1,140 @@ +#!/usr/bin/env node +/** Live, two-turn AWS AgentCore smoke through the exact Runner Lab HTTP path. */ + +import { createServer } from "node:http"; +import { once } from "node:events"; +import { readFile } from "node:fs/promises"; + +import { createCapabilityCookieJar } from "./capability-cookie-jar.mjs"; +import { createCapabilityIssueThreadMiddleware } from "./capability-issue-thread-server.mjs"; + +const { readCapabilityTurnStream, CAPABILITY_TURN_STREAM_ACCEPT } = await import( + new URL("../dist/live/index.js", import.meta.url).href +); + +const envPath = new URL("../.paperclip-local/aws-agentcore.env", import.meta.url); + +function assert(condition, message) { + if (!condition) throw new Error(`assertion failed: ${message}`); +} + +function assistantText(view) { + return view.turns.flatMap((turn) => turn.items) + .filter((item) => item.kind === "agent_message") + .map((item) => item.body) + .join("\n").trim(); +} + +async function loadProfileEnvironment() { + let source; + try { + source = await readFile(envPath, "utf8"); + } catch { + throw new Error("AWS AgentCore profile is missing; run aws-agentcore:provision first"); + } + for (const line of source.split(/\r?\n/)) { + if (!line || line.startsWith("#")) continue; + const separator = line.indexOf("="); + if (separator < 1) throw new Error("generated AWS AgentCore profile contains an invalid line"); + const key = line.slice(0, separator); + const value = line.slice(separator + 1); + if (!/^[A-Z][A-Z0-9_]*$/.test(key)) throw new Error("generated AWS AgentCore profile contains an invalid key"); + process.env[key] = value; + } +} + +async function main() { + await loadProfileEnvironment(); + for (const key of [ + "PAPERCLIP_AWS_AGENTCORE_CONTEXT_BUCKET", + "PAPERCLIP_AWS_AGENTCORE_CONTEXT_PREFIX", + "PAPERCLIP_AWS_AGENTCORE_CONTEXT_KMS_KEY_ARN", + ]) assert(process.env[key], `the qualified profile includes ${key}`); + let providerFailure = null; + const middleware = createCapabilityIssueThreadMiddleware({ + bindHost: "127.0.0.1", + onTurnError(error) { providerFailure = error; }, + }); + const server = createServer((request, response) => middleware(request, response, () => { + response.statusCode = 404; + response.end("not found"); + })); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const { port } = server.address(); + const jar = createCapabilityCookieJar(`http://127.0.0.1:${port}`); + const post = async (path, body, accept) => { + const response = await jar.fetch(`/api/capability/ui${path}`, { + method: "POST", + headers: { "content-type": "application/json", ...(accept ? { accept } : {}) }, + body: JSON.stringify(body), + }); + if (!response.ok) throw new Error(`${path} failed with HTTP ${response.status}: ${await response.text()}`); + return response; + }; + const turn = async (sessionId, message) => { + const response = await post("/message", { sessionId, message }, CAPABILITY_TURN_STREAM_ACCEPT); + try { + return await readCapabilityTurnStream(response, () => undefined); + } catch (error) { + if (providerFailure) throw new Error("AWS AgentCore provider turn failed", { cause: providerFailure }); + throw error; + } + }; + + try { + const configuration = { + provider: "aws_agentcore", + model: process.env.PAPERCLIP_AWS_AGENTCORE_MODEL, + agentCoreProfileId: process.env.PAPERCLIP_AWS_AGENTCORE_PROFILE_ID, + maxEstimatedSessionCostUsd: 1, + lifecyclePolicy: { mode: "warm", idleTimeoutMs: 300_000 }, + }; + const opened = await (await post("/cleanroom/session", configuration)).json(); + assert(opened.configuration?.provider === "aws_agentcore", "the AWS provider was selected without fallback"); + assert(opened.view.identity.agentLabel === "Real AWS AgentCore", "the live AWS identity is visible"); + assert(opened.runtime?.providerPid === null, "a remote provider never reports a provider PID"); + + const first = await turn(opened.sessionId, "Reply with exactly: AgentCore first response"); + assert(assistantText(first.view).length > 0, "the first AWS response is nonempty"); + assert(first.runtime?.providerSessionId, "the first response carries a runtimeSessionId"); + + const second = await turn(opened.sessionId, "Reply with exactly: AgentCore second response"); + assert(assistantText(second.view).length > assistantText(first.view).length, "the second AWS response is visible"); + assert(second.runtime?.providerSessionId === first.runtime.providerSessionId, "both turns reuse one AgentCore session"); + + const read = await (await post("/tool", { + sessionId: opened.sessionId, + operationId: "get_task_context", + input: {}, + })).json(); + assert(read.toolResult && typeof read.toolResult === "object", "the mock Paperclip read operation succeeded"); + + const serialized = JSON.stringify({ opened, first, second, read }); + for (const pattern of [ + /AWS_ACCESS_KEY_ID/i, + /AWS_SECRET_ACCESS_KEY/i, + /AWS_SESSION_TOKEN/i, + /X-Amz-Signature/i, + /AWS4-HMAC-SHA256/i, + /"Authorization"\s*:/, + /"Proxy-Authorization"\s*:/, + ]) { + assert(!pattern.test(serialized), `browser payload excludes ${pattern}`); + } + process.stdout.write(`${JSON.stringify({ + schema: "paperclip.capability.aws-agentcore-smoke.v1", + sessionId: opened.sessionId, + runtimeSessionId: second.runtime.providerSessionId, + model: configuration.model, + turns: second.view.turns.length, + mockRead: true, + assertions: { realProvider: true, twoResponses: true, sessionContinuity: true, noCredentialLeak: true }, + }, null, 2)}\n`); + } finally { + await middleware.close(); + server.close(); + } +} + +await main(); diff --git a/packages/paperclip-runner/scripts/capability-issue-thread-server.mjs b/packages/paperclip-runner/scripts/capability-issue-thread-server.mjs index b3c689df09..9a46e89eeb 100644 --- a/packages/paperclip-runner/scripts/capability-issue-thread-server.mjs +++ b/packages/paperclip-runner/scripts/capability-issue-thread-server.mjs @@ -208,8 +208,8 @@ class RouteError extends Error { function harnessConfiguration(source, fallbackModel) { const provider = source.provider === undefined ? "codex" : String(source.provider).trim(); - if (provider !== "codex" && provider !== "opencode" && provider !== "acpx") { - throw new RouteError(400, "invalid_provider", "Provider must be codex, opencode, or acpx."); + if (provider !== "codex" && provider !== "opencode" && provider !== "claude_managed" && provider !== "aws_agentcore" && provider !== "acpx") { + throw new RouteError(400, "invalid_provider", "Provider must be codex, opencode, claude_managed, aws_agentcore, or acpx."); } const rawModel = source.model === undefined ? fallbackModel : source.model; const model = rawModel === undefined || rawModel === null ? "" : String(rawModel).trim(); @@ -217,6 +217,12 @@ function harnessConfiguration(source, fallbackModel) { if (provider === "opencode" && (!model || !model.includes("/"))) { throw new RouteError(400, "invalid_model", "OpenCode requires a provider/model value."); } + if (provider === "claude_managed" && model !== "claude-sonnet-5") { + throw new RouteError(400, "invalid_model", "Claude Managed requires exact model claude-sonnet-5."); + } + if (provider === "aws_agentcore" && model !== "global.anthropic.claude-sonnet-4-6") { + throw new RouteError(400, "invalid_model", "AWS AgentCore requires exact model global.anthropic.claude-sonnet-4-6."); + } const acpxAgent = source.acpxAgent === undefined ? "codex" : String(source.acpxAgent).trim(); if (provider === "acpx") { if (!(acpxAgent in ACPX_QUALIFIED_MODELS)) { @@ -226,6 +232,38 @@ function harnessConfiguration(source, fallbackModel) { throw new RouteError(400, "invalid_model", `The qualified ACPX ${acpxAgent} profile requires exact model ${ACPX_QUALIFIED_MODELS[acpxAgent]}.`); } } + const requestedManagedProfileId = source.managedProfileId === undefined || source.managedProfileId === null + ? "default" + : String(source.managedProfileId).trim(); + const configuredManagedProfileId = process.env.PAPERCLIP_CLAUDE_MANAGED_PROFILE_ID?.trim(); + const managedProfileId = provider === "claude_managed" && requestedManagedProfileId === "default" && configuredManagedProfileId + ? configuredManagedProfileId + : requestedManagedProfileId; + const maxSessionListCostUsd = source.maxSessionListCostUsd === undefined || source.maxSessionListCostUsd === null + ? 1 + : Number(source.maxSessionListCostUsd); + if (provider === "claude_managed" && !managedProfileId) { + throw new RouteError(400, "invalid_managed_profile", "Claude Managed requires a qualified profile ID."); + } + if (provider === "claude_managed" && (!Number.isFinite(maxSessionListCostUsd) || maxSessionListCostUsd <= 0)) { + throw new RouteError(400, "invalid_spend_cap", "Claude Managed requires a positive session spend ceiling."); + } + const requestedAgentCoreProfileId = source.agentCoreProfileId === undefined || source.agentCoreProfileId === null + ? "default" + : String(source.agentCoreProfileId).trim(); + const configuredAgentCoreProfileId = process.env.PAPERCLIP_AWS_AGENTCORE_PROFILE_ID?.trim(); + const agentCoreProfileId = provider === "aws_agentcore" && requestedAgentCoreProfileId === "default" && configuredAgentCoreProfileId + ? configuredAgentCoreProfileId + : requestedAgentCoreProfileId; + const maxEstimatedSessionCostUsd = source.maxEstimatedSessionCostUsd === undefined || source.maxEstimatedSessionCostUsd === null + ? 1 + : Number(source.maxEstimatedSessionCostUsd); + if (provider === "aws_agentcore" && !agentCoreProfileId) { + throw new RouteError(400, "invalid_agentcore_profile", "AWS AgentCore requires a qualified profile ID."); + } + if (provider === "aws_agentcore" && (!Number.isFinite(maxEstimatedSessionCostUsd) || maxEstimatedSessionCostUsd <= 0)) { + throw new RouteError(400, "invalid_spend_cap", "AWS AgentCore requires a positive session spend ceiling."); + } const suppliedLifecycle = source.lifecyclePolicy && typeof source.lifecyclePolicy === "object" ? source.lifecyclePolicy : source; @@ -252,10 +290,80 @@ function harnessConfiguration(source, fallbackModel) { provider, model: model || null, ...(provider === "acpx" ? { acpxAgent } : {}), + ...(provider === "claude_managed" ? { managedProfileId, maxSessionListCostUsd } : {}), + ...(provider === "aws_agentcore" ? { agentCoreProfileId, maxEstimatedSessionCostUsd } : {}), lifecyclePolicy, }; } +function resolveManagedProfile(configuration) { + const profileId = process.env.PAPERCLIP_CLAUDE_MANAGED_PROFILE_ID?.trim() || configuration.managedProfileId; + if (profileId !== configuration.managedProfileId) { + throw new RouteError(400, "managed_profile_not_found", "The selected Claude Managed profile is not configured on this Runner Lab server."); + } + const anthropicAgentId = process.env.ANTHROPIC_MANAGED_AGENT_ID?.trim(); + const agentVersion = process.env.ANTHROPIC_MANAGED_AGENT_VERSION?.trim(); + const environmentId = process.env.ANTHROPIC_MANAGED_ENVIRONMENT_ID?.trim(); + const canonicalAgentVersion = agentVersion !== undefined + && /^[1-9][0-9]*$/.test(agentVersion) + && BigInt(agentVersion) <= 2_147_483_647n; + if (!process.env.ANTHROPIC_API_KEY || !profileId || !anthropicAgentId || !canonicalAgentVersion || !environmentId) { + throw new RouteError( + 503, + "managed_profile_unavailable", + "The selected Claude Managed profile is not fully qualified on this Runner Lab server.", + ); + } + return { + profileId, + anthropicAgentId, + agentVersion, + environmentId, + betaVersion: "managed-agents-2026-04-01", + maxSessionListCostUsd: configuration.maxSessionListCostUsd, + }; +} + +function resolveAgentCoreProfile(configuration) { + const required = (name) => { + const value = process.env[name]?.trim(); + if (!value) throw new RouteError(503, "agentcore_profile_unavailable", `AWS AgentCore profile is missing ${name}. Run aws-agentcore:provision and aws-agentcore:lab.`); + return value; + }; + const profileId = required("PAPERCLIP_AWS_AGENTCORE_PROFILE_ID"); + if (profileId !== configuration.agentCoreProfileId) { + throw new RouteError(400, "agentcore_profile_not_found", "The selected AWS AgentCore profile is not configured on this Runner Lab server."); + } + return { + profileId, + region: required("AWS_REGION"), + accountId: required("PAPERCLIP_AWS_AGENTCORE_ACCOUNT_ID"), + harnessArn: required("PAPERCLIP_AWS_AGENTCORE_HARNESS_ARN"), + harnessVersion: required("PAPERCLIP_AWS_AGENTCORE_HARNESS_VERSION"), + endpointArn: required("PAPERCLIP_AWS_AGENTCORE_ENDPOINT_ARN"), + endpointQualifier: required("PAPERCLIP_AWS_AGENTCORE_ENDPOINT_QUALIFIER"), + agentRuntimeArn: required("PAPERCLIP_AWS_AGENTCORE_RUNTIME_ARN"), + memoryArn: required("PAPERCLIP_AWS_AGENTCORE_MEMORY_ARN"), + memoryId: required("PAPERCLIP_AWS_AGENTCORE_MEMORY_ID"), + invocationRoleArn: required("PAPERCLIP_AWS_AGENTCORE_INVOCATION_ROLE_ARN"), + contextBucket: required("PAPERCLIP_AWS_AGENTCORE_CONTEXT_BUCKET"), + contextPrefix: required("PAPERCLIP_AWS_AGENTCORE_CONTEXT_PREFIX"), + contextKmsKeyArn: required("PAPERCLIP_AWS_AGENTCORE_CONTEXT_KMS_KEY_ARN"), + qualificationRevision: required("PAPERCLIP_AWS_AGENTCORE_QUALIFICATION_REVISION"), + eventExpiryDays: 90, + maxEstimatedSessionCostUsd: configuration.maxEstimatedSessionCostUsd, + maxIterations: 8, + maxOutputTokens: 4096, + timeoutSeconds: 300, + }; +} + +export const capabilityIssueThreadServerInternals = Object.freeze({ + harnessConfiguration, + resolveManagedProfile, + resolveAgentCoreProfile, +}); + export function createCapabilityIssueThreadMiddleware(options = {}) { const load = options.loadRunner ?? loadCapabilityIssueThreadRunner; const workingDirectoryRoot = options.scratchRoot ?? scratchRoot(); @@ -325,7 +433,9 @@ export function createCapabilityIssueThreadMiddleware(options = {}) { function liveView(runner, entry) { const snapshot = entry.session.snapshot(); const provider = snapshot.config.provider ?? "codex"; - const expectedAgentLabel = provider === "opencode" ? "Real OpenCode" + const expectedAgentLabel = provider === "claude_managed" ? "Claude Agent" + : provider === "opencode" ? "Real OpenCode" + : provider === "aws_agentcore" ? "Real AWS AgentCore" : provider === "acpx" ? `Real ${snapshot.config.acpxAgent === "claude" ? "Claude" : snapshot.config.acpxAgent === "codex" ? "Codex" : "Pi"} via ACPX` : "Real Codex"; @@ -434,7 +544,13 @@ export function createCapabilityIssueThreadMiddleware(options = {}) { session = await service.create({ ...input, provider: configuration.provider, + ...(configuration.provider === "claude_managed" + ? { managedProfile: resolveManagedProfile(configuration) } + : {}), ...(configuration.provider === "acpx" ? { acpxAgent: configuration.acpxAgent } : {}), + ...(configuration.provider === "aws_agentcore" + ? { agentCoreProfile: resolveAgentCoreProfile(configuration) } + : {}), lifecyclePolicy: configuration.lifecyclePolicy, ...(configuration.model === null ? {} : { requestedModel: configuration.model }), }); @@ -464,12 +580,24 @@ export function createCapabilityIssueThreadMiddleware(options = {}) { const configuration = { provider: snapshot.config.provider ?? "codex", model: snapshot.config.requestedModel ?? null, + ...(snapshot.config.managedProfile === undefined ? {} : { + managedProfileId: snapshot.config.managedProfile.profileId, + maxSessionListCostUsd: snapshot.config.managedProfile.maxSessionListCostUsd, + }), + ...(snapshot.config.agentCoreProfile === undefined ? {} : { + agentCoreProfileId: snapshot.config.agentCoreProfile.profileId, + maxEstimatedSessionCostUsd: snapshot.config.agentCoreProfile.maxEstimatedSessionCostUsd, + }), lifecyclePolicy: snapshot.config.lifecyclePolicy ?? { mode: "per_turn", idleTimeoutMs: null }, }; if ( entry.configuration !== undefined && (entry.configuration.provider !== configuration.provider || entry.configuration.model !== configuration.model + || entry.configuration.managedProfileId !== configuration.managedProfileId + || entry.configuration.maxSessionListCostUsd !== configuration.maxSessionListCostUsd + || entry.configuration.agentCoreProfileId !== configuration.agentCoreProfileId + || entry.configuration.maxEstimatedSessionCostUsd !== configuration.maxEstimatedSessionCostUsd || JSON.stringify(entry.configuration.lifecyclePolicy) !== JSON.stringify(configuration.lifecyclePolicy)) ) { throw new RouteError( @@ -489,14 +617,16 @@ export function createCapabilityIssueThreadMiddleware(options = {}) { providerSessionId: snapshot.providerSessionId ?? null, driverSessionId: snapshot.providerThreadId ?? null, runnerPid: snapshot.process?.runnerPid ?? null, - providerPid: snapshot.process?.providerPid ?? snapshot.process?.codexPid ?? null, + providerPid: configuration.provider === "claude_managed" || configuration.provider === "aws_agentcore" + ? null + : snapshot.process?.providerPid ?? snapshot.process?.codexPid ?? null, sidecarPid: snapshot.process?.sidecarPid ?? null, agentPid: snapshot.process?.agentPid ?? null, providerVersion: snapshot.process?.providerVersion ?? null, agentServerVersion: snapshot.process?.agentServerVersion ?? null, agentRuntimeVersion: snapshot.process?.agentRuntimeVersion ?? null, acpProtocolVersion: snapshot.process?.acpProtocolVersion ?? null, - executionKind: "local_process", + executionKind: snapshot.process?.providerExecutionKind ?? (configuration.provider === "claude_managed" || configuration.provider === "aws_agentcore" ? "remote_service" : "local_process"), status: snapshot.status, }, view: liveView(runner, entry), @@ -649,6 +779,10 @@ export function createCapabilityIssueThreadMiddleware(options = {}) { provider: body.provider ?? url.searchParams.get("provider") ?? undefined, model: body.model ?? url.searchParams.get("model") ?? undefined, acpxAgent: body.acpxAgent ?? url.searchParams.get("acpxAgent") ?? undefined, + managedProfileId: body.managedProfileId ?? url.searchParams.get("managedProfileId") ?? undefined, + maxSessionListCostUsd: body.maxSessionListCostUsd ?? url.searchParams.get("maxSessionListCostUsd") ?? undefined, + agentCoreProfileId: body.agentCoreProfileId ?? url.searchParams.get("agentCoreProfileId") ?? undefined, + maxEstimatedSessionCostUsd: body.maxEstimatedSessionCostUsd ?? url.searchParams.get("maxEstimatedSessionCostUsd") ?? undefined, lifecyclePolicy: body.lifecyclePolicy, lifecycleMode: body.lifecycleMode ?? url.searchParams.get("lifecycleMode") ?? undefined, idleTimeoutMs: body.idleTimeoutMs ?? url.searchParams.get("idleTimeoutMs") ?? undefined, @@ -803,9 +937,6 @@ export function createCapabilityIssueThreadMiddleware(options = {}) { const capabilityHash = sha256(capability); let fork; try { - if (snapshot.config.provider === "claude_managed" || snapshot.config.provider === "aws_agentcore") { - throw new RouteError(409, "provider_deferred", "Managed provider sessions are readable but cannot be forked in this release."); - } fork = await service.create({ seed, workingDirectory: forkDirectory, @@ -821,6 +952,12 @@ export function createCapabilityIssueThreadMiddleware(options = {}) { ...(snapshot.config.requestedModel === undefined ? {} : { requestedModel: snapshot.config.requestedModel }), + ...(snapshot.config.managedProfile === undefined + ? {} + : { managedProfile: snapshot.config.managedProfile }), + ...(snapshot.config.agentCoreProfile === undefined + ? {} + : { agentCoreProfile: snapshot.config.agentCoreProfile }), }); } catch (error) { await rm(forkDirectory, { recursive: true, force: true }).catch(() => undefined); @@ -877,6 +1014,40 @@ export function createCapabilityIssueThreadMiddleware(options = {}) { return; } else if (route === "interrupt") { await entry.session.interrupt("operator stopped the turn"); + } else if (route === "managed-budget") { + const requestedCap = body.maxSessionListCostUsd ?? + body.maxEstimatedSessionCostUsd; + const nextCap = Number(requestedCap); + if (!Number.isFinite(nextCap) || nextCap <= 0) { + throw new RouteError( + 400, + "invalid_spend_cap", + "The new managed-session spend ceiling must be positive.", + ); + } + await entry.session.increaseManagedSessionBudget(nextCap); + if (entry.configuration?.provider === "claude_managed") { + entry.configuration.maxSessionListCostUsd = nextCap; + } + if (entry.configuration?.provider === "aws_agentcore") { + entry.configuration.maxEstimatedSessionCostUsd = nextCap; + } + } else if (route === "managed-session-delete") { + if (body.confirm !== true) { + throw new RouteError( + 400, + "confirmation_required", + "Remote session deletion requires explicit confirmation.", + ); + } + await entry.session.deleteManagedRemoteSession(); + await retire( + service, + entry.session.id, + "remote managed session explicitly deleted", + ); + send(response, 200, { deleted: true, sessionId: entry.session.id }); + return; } else if (route === "reconnect") { entry.connection = { state: "reconnecting", attempt: entry.connection.attempt + 1 }; await entry.session.reconnect(); diff --git a/packages/paperclip-runner/scripts/generate-protocol-coverage.mjs b/packages/paperclip-runner/scripts/generate-protocol-coverage.mjs index 51ff3fa827..ba1d7829fd 100644 --- a/packages/paperclip-runner/scripts/generate-protocol-coverage.mjs +++ b/packages/paperclip-runner/scripts/generate-protocol-coverage.mjs @@ -44,7 +44,7 @@ const lifecycleRequirements = [ ["bounds", "bounded queues, logs, frames, and retries", "src/live/turn-stream.test.ts; runner/crates/runner-core/src/durable/runner.rs"], ["cleanup", "process-group cleanup with no abandoned provider", "runner/crates/runner-core/tests/process_supervisor.rs"], ["secrets", "secret isolation and network restrictions", "src/live/clean-room-server.test.ts; runner/crates/runner-core/src/durable/runner.rs"], - ["transcript-accounting", "assistant transcript capture, token usage, and cost inputs", "src/live/live-session.test.ts"], + ["transcript-accounting", "assistant transcript capture, token usage, and cost inputs", "src/live/live-session.test.ts; src/cli/eval-session.ts"], ["state-reconstruction", "fixture and post-run state reconstruction", "src/scenarios/scenario-explorer.test.ts; src/live/live-session.test.ts"], ["paperclip-adapter-selection", "paperclip_runner is selectable without changing legacy adapter behavior", "server/src/services/native-runtime/runtime-mode.test.ts; server/src/__tests__/heartbeat-native-runner-selection.test.ts"], ["real-control-plane-binding", "advertised runner tools re-authorize the live company, issue, agent, and run before using real Paperclip services", "server/src/services/native-runtime/paperclip-runner-tool-authority.test.ts; server/src/services/native-runtime/paperclip-runner-real-server.integration.test.ts"], diff --git a/packages/paperclip-runner/scripts/run-capability-live-eval-matrix.mjs b/packages/paperclip-runner/scripts/run-capability-live-eval-matrix.mjs new file mode 100644 index 0000000000..31e137cfba --- /dev/null +++ b/packages/paperclip-runner/scripts/run-capability-live-eval-matrix.mjs @@ -0,0 +1,189 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; + +const packageRoot = resolve(import.meta.dirname, ".."); +const suite = await import(resolve(packageRoot, "dist/conformance/capability-eval-suite.js")); +const evals = await import(resolve(packageRoot, "dist/eval/index.js")); +const packageManifest = JSON.parse(await readFile(resolve(packageRoot, "package.json"), "utf8")); +const args = process.argv.slice(2); +if (args.some((argument) => argument !== "--dry-run")) { + throw new Error(`unknown argument: ${args.find((argument) => argument !== "--dry-run")}`); +} +const dryRun = args.includes("--dry-run"); +for (const [name, value] of [ + ["runCapabilityLiveCodexMatrix", suite.runCapabilityLiveCodexMatrix], + ["buildEvalSliceReport", evals.buildEvalSliceReport], + ["runEvalBehaviorFaultMatrix", evals.runEvalBehaviorFaultMatrix], + ["renderEvalSliceMarkdown", evals.renderEvalSliceMarkdown], +]) { + if (typeof value !== "function") throw new Error(`live eval dependency ${name} is unavailable`); +} +const generatedAt = new Date().toISOString(); +const evidenceMarkdown = (title, description, body) => [ + "---", + "type: Evidence Report", + `title: ${title}`, + `description: ${description}`, + `generated: { by: process:capability-live-eval, at: ${generatedAt} }`, + "status: stable", + "---", + "", + body.trimEnd(), + "", +].join("\n"); +if (dryRun) { + process.stdout.write(`${JSON.stringify({ + schema: "paperclip.capability.live-codex-matrix-dry-run.v1", + provider: "codex", + runnerPackage: packageManifest.name, + grantCount: suite.CAPABILITY_LIVE_EVAL_GRANTS.length, + writesEvidence: false, + }, null, 2)}\n`); +} else { +const matrix = await suite.runCapabilityLiveCodexMatrix(process.cwd()); +const providerModels = [...new Set(matrix.map( + (entry) => `${entry.providerModel.provider}/${entry.providerModel.id}`, +))]; +if (providerModels.length !== 1) { + throw new Error(`live eval matrix used inconsistent provider models: ${providerModels.join(", ")}`); +} +const liveModel = matrix[0].providerModel; +const liveBundle = { + schema: evals.EVAL_BUNDLE_SCHEMA, + provider: { + runtime: "runnerd", + transport: "codex-app-server", + protocolVersion: "codex-app-server-v2", + }, + model: { id: liveModel.id }, + launchContext: { + workingDirectoryClass: "workspace-checkout", + scenarioId: "capability-live-codex-matrix-v1", + turnTimeoutMs: 60_000, + }, + promptPolicy: { + id: "capability-live-exact-call-v1", + callTemplate: "Call the named semantic operation exactly once with the declared JSON input.", + restraintTemplate: "Do not call any tool for a control-plane-owned behavior.", + }, + grants: [...suite.CAPABILITY_LIVE_EVAL_GRANTS], + runner: { + package: packageManifest.name, + binary: "paperclip-runnerd", + version: packageManifest.version, + }, + controlPlaneAdapter: { + kind: "mock", + contract: "paperclip.capability.mock-state.v1", + }, + faultInjection: [], +}; +const liveObservations = matrix.map((result) => ({ + caseId: result.caseId, + provenance: { source: "live_model" }, + controlPlaneOwned: result.expectedCalls.length === 0, + expectedCalls: result.expectedCalls, + observedCalls: result.observedCalls, + forbiddenCalls: result.forbiddenCalls, + finalState: result.finalState, + authorization: result.scoringEvidence.authorization, + trace: result.scoringEvidence.trace, + efficiency: result.scoringEvidence.efficiency, + budget: result.scoringEvidence.budget, +})); +const liveSliceReport = evals.buildEvalSliceReport(liveBundle, liveObservations, { + generatedFromLiveModel: true, +}); +if (liveSliceReport.aggregate.passed !== liveSliceReport.aggregate.caseCount) { + throw new Error(`live eval slice scored ${liveSliceReport.aggregate.passed}/${liveSliceReport.aggregate.caseCount} passing cases`); +} + +const faultBundle = { + schema: evals.EVAL_BUNDLE_SCHEMA, + provider: { + runtime: "deterministic-harness", + transport: "in-process", + protocolVersion: "v1", + }, + model: { id: "scripted-eval-agent" }, + launchContext: { + workingDirectoryClass: "ephemeral-fixture", + scenarioId: "eval-behavior-fault-matrix-v1", + turnTimeoutMs: 1_000, + }, + promptPolicy: { + id: "deterministic-behavior-driver-v1", + callTemplate: "Invoke the declared semantic operation sequence.", + restraintTemplate: "Make no semantic call.", + }, + grants: ["control_plane:wakes", "governance:approvals:decide"], + runner: { + package: packageManifest.name, + binary: "node", + version: packageManifest.version, + }, + controlPlaneAdapter: { + kind: "mock", + contract: "paperclip.capability.mock-state.v1", + }, + faultInjection: [ + { id: "fault-authz", class: "authorization", description: "deny approval decision exposure" }, + { id: "fault-conflict", class: "conflict", description: "submit a stale plan revision" }, + { id: "fault-retry", class: "retry", description: "fail the first artifact registration attempt" }, + { id: "fault-provider", class: "provider_capability", description: "omit wake scheduling from the tool surface" }, + ], +}; +const behaviorMatrix = await evals.runEvalBehaviorFaultMatrix(faultBundle); +const faultSliceReport = evals.buildEvalSliceReport( + faultBundle, + behaviorMatrix.map((result) => result.observation), +); +const report = { + schema: "paperclip.capability.live-codex-matrix.v1", + groups: matrix.length, + providerModel: liveModel, + matrix, +}; +const evidenceDirectory = resolve(packageRoot, ".paperclip-local/evidence/capability"); +const output = resolve(evidenceDirectory, "live-codex-matrix.json"); +const markdown = resolve(evidenceDirectory, "live-codex-matrix.md"); +const liveSliceJson = resolve(evidenceDirectory, "live-eval-slice-report.json"); +const liveSliceMarkdown = resolve(evidenceDirectory, "live-eval-slice-report.md"); +const faultSliceJson = resolve(evidenceDirectory, "fault-eval-slice-report.json"); +const faultSliceMarkdown = resolve(evidenceDirectory, "fault-eval-slice-report.md"); +await mkdir(dirname(output), { recursive: true }); +await Promise.all([ + writeFile(output, `${JSON.stringify(report, null, 2)}\n`), + writeFile(markdown, evidenceMarkdown( + "Capability Live Codex Matrix", + "Real Codex semantic-tool choices and resulting mock control-plane state.", + [ + "# Capability Live Codex Matrix", + "", + `- Provider model: ${liveModel.provider}/${liveModel.id}`, + `- Groups: ${report.groups}`, + `- Cases: ${matrix.map((entry) => `${entry.group}:${entry.caseId}`).join(", ")}`, + "- Each row asserts expected and observed typed calls plus final mock state.", + "- Scored report: `live-eval-slice-report.md`", + "", + ].join("\n"), + )), + writeFile(liveSliceJson, `${JSON.stringify(liveSliceReport, null, 2)}\n`), + writeFile(liveSliceMarkdown, evidenceMarkdown( + "Live Runner Eval Slice Report", + "Scored real-Codex capability matrix bound to its secret-free eval bundle.", + evals.renderEvalSliceMarkdown(liveSliceReport), + )), + writeFile(faultSliceJson, `${JSON.stringify(faultSliceReport, null, 2)}\n`), + writeFile(faultSliceMarkdown, evidenceMarkdown( + "Runner Eval Fault Slice Report", + "Eight green/red behaviors with deterministic injected-fault receipts.", + evals.renderEvalSliceMarkdown(faultSliceReport), + )), +]); +process.stdout.write(`Capability live Codex matrix passed: ${matrix.length} groups.\n`); +process.stdout.write(`Live eval slice passed: ${liveSliceReport.aggregate.passed}/${liveSliceReport.aggregate.caseCount}.\n`); +process.stdout.write(`Behavior fault matrix passed: ${faultSliceReport.aggregate.passed} green, ${faultSliceReport.aggregate.caseCount - faultSliceReport.aggregate.passed} scored red.\n`); +process.stdout.write(`Live matrix reports: ${output} and ${markdown}\n`); +process.stdout.write(`Scored eval reports: ${liveSliceJson}, ${liveSliceMarkdown}, ${faultSliceJson}, and ${faultSliceMarkdown}\n`); +} diff --git a/packages/paperclip-runner/scripts/run-runner-live-eval-schedule.mjs b/packages/paperclip-runner/scripts/run-runner-live-eval-schedule.mjs new file mode 100644 index 0000000000..19217335e3 --- /dev/null +++ b/packages/paperclip-runner/scripts/run-runner-live-eval-schedule.mjs @@ -0,0 +1,251 @@ +import { createHash } from "node:crypto"; +import { + mkdir, + readFile, + readdir, + rm, + stat, + writeFile, +} from "node:fs/promises"; +import { resolve } from "node:path"; + +const packageRoot = resolve(import.meta.dirname, ".."); +const evals = await import(resolve(packageRoot, "dist/eval/index.js")); +const packageManifest = JSON.parse( + await readFile(resolve(packageRoot, "package.json"), "utf8"), +); +const modeIndex = process.argv.indexOf("--mode"); +const mode = modeIndex < 0 ? "nightly" : process.argv[modeIndex + 1]; +const execute = process.argv.includes("--execute"); +if (mode !== "nightly" && mode !== "chaos") + throw new Error(`unsupported workflow eval schedule mode: ${mode}`); +const now = process.env.PAPERCLIP_EVAL_GENERATED_AT ?? new Date().toISOString(); +const rotationDay = Number( + process.env.PAPERCLIP_EVAL_ROTATION_DAY ?? + evals.runnerLiveRotationWeek(now), +); +const seed = + process.env.PAPERCLIP_EVAL_SCHEDULE_SEED ?? "runner-live-seven-week-v1"; +const outputDirectory = resolve( + packageRoot, + ".paperclip-local/evals/workflows", +); +const historyDirectory = resolve( + process.env.PAPERCLIP_EVAL_HISTORY_DIR ?? resolve(outputDirectory, "history"), +); +await mkdir(outputDirectory, { recursive: true }); + +function safeBundleId(schedule) { + const runnerBuild = + process.env.PAPERCLIP_EVAL_RUNNER_BUILD ?? packageManifest.version; + const identity = JSON.stringify({ + runnerVersion: packageManifest.version, + runnerBuild, + promptPolicyId: "runner-live-workflow-v1", + seed: schedule.seed, + candidates: schedule.candidates.map( + ({ id, adapter, model, reasoningEffort }) => ({ + id, + adapter, + model, + reasoningEffort, + }), + ), + }); + return `runner-live-v2-${createHash("sha256").update(identity).digest("hex").slice(0, 16)}`; +} + +function qualificationFailure(entry, candidate, evalCase) { + const missing = candidate.qualification.requiredEnvironment.filter( + (name) => !process.env[name], + ); + if (missing.length === 0) return null; + return evals.unavailableLiveRunnerWorkflowObservation({ + entry, + candidate, + evalCase, + classification: "skipped", + code: "qualification_environment_missing", + category: "qualification", + retryable: false, + message: `Required credential reference unavailable: ${missing.join(", ")}`, + }); +} + +async function readCompatibleHistory(bundleId) { + const names = await readdir(historyDirectory).catch(() => []); + const reports = []; + for (const name of names.filter((value) => value.endsWith(".json")).sort()) { + try { + const report = JSON.parse( + await readFile(resolve(historyDirectory, name), "utf8"), + ); + if ( + report?.schema === evals.RUNNER_WORKFLOW_REPORT_SCHEMA && + report?.bundle?.id === bundleId + ) + reports.push(report); + } catch { + // A malformed historical artifact is ignored; it cannot affect the current candidate score. + } + } + return reports.slice(-7); +} + +async function retainHistory(report) { + await mkdir(historyDirectory, { recursive: true }); + const stamp = report.generatedAt.replaceAll(/[^0-9A-Za-z.-]/g, "-"); + await writeFile( + resolve(historyDirectory, `${stamp}-${report.bundle.id}.json`), + `${JSON.stringify(report, null, 2)}\n`, + ); + // Candidate sets alternate, so seven compatible weekly baselines require + // roughly fourteen weeks of history. Keep a little extra scheduling margin. + const expiry = Date.now() - 120 * 24 * 60 * 60 * 1_000; + for (const name of await readdir(historyDirectory)) { + if (!name.endsWith(".json")) continue; + const metadata = await stat(resolve(historyDirectory, name)); + if (metadata.mtimeMs < expiry) await rm(resolve(historyDirectory, name)); + } +} + +if (mode === "nightly") { + const schedule = evals.buildRunnerLiveEvalSchedule({ + seed, + rotationDay, + generatedAt: now, + }); + const coverage = evals.runnerLiveScheduleCoverage(seed); + if (!Object.values(coverage).every(Boolean)) + throw new Error( + `live schedule coverage failed: ${JSON.stringify(coverage)}`, + ); + await writeFile( + resolve(outputDirectory, "nightly-schedule.json"), + `${JSON.stringify({ schedule, coverage }, null, 2)}\n`, + ); + if (!execute) { + process.stdout.write( + `Runner live eval schedule ready: ${schedule.expectedExecutions} executions, rotation week ${schedule.rotationDay}. Use --execute to run providers.\n`, + ); + process.exit(0); + } + + const campaignCostLimit = evals.parseRunnerLiveCampaignCostLimit( + process.env.PAPERCLIP_EVAL_MAX_CAMPAIGN_COST_USD, + ); + let observedCampaignCost = 0; + const observations = await evals.executeRunnerLiveSchedule( + schedule, + async (entry, candidate) => { + const evalCase = evals.runnerWorkflowCase(entry.caseId); + const unavailable = qualificationFailure(entry, candidate, evalCase); + if (unavailable) return unavailable; + if (observedCampaignCost >= campaignCostLimit) { + return evals.unavailableLiveRunnerWorkflowObservation({ + entry, + candidate, + evalCase, + classification: "skipped", + code: "campaign_cost_ceiling_reached", + category: "orchestration", + retryable: false, + message: `Campaign cost ceiling reached before this execution (${campaignCostLimit} USD)`, + }); + } + const observation = await evals.executeLiveRunnerWorkflow({ + entry, + candidate, + evalCase, + }); + observedCampaignCost += observation.metrics.costUsd ?? 0; + return observation; + }, + (entry, candidate, error) => + evals.unavailableLiveRunnerWorkflowObservation({ + entry, + candidate, + evalCase: evals.runnerWorkflowCase(entry.caseId), + classification: "infrastructure_failure", + code: error.code, + category: "provider", + retryable: error.retryable, + message: error.message, + }), + ); + const bundleId = safeBundleId(schedule); + const results = observations.map((observation) => ({ + scenarioId: observation.caseId, + candidateId: observation.candidateId, + observation, + scorecard: evals.scoreRunnerWorkflow(observation, { bundleId }), + })); + const providerVersions = Object.fromEntries( + schedule.candidates.map((candidate) => [ + candidate.id, + `${candidate.adapter}:${candidate.model}`, + ]), + ); + const report = evals.buildRunnerWorkflowEvalReport({ + source: "live", + bundle: { + id: bundleId, + runnerVersion: packageManifest.version, + runnerBuild: + process.env.PAPERCLIP_EVAL_RUNNER_BUILD ?? packageManifest.version, + promptPolicyId: "runner-live-workflow-v1", + providerVersions, + scheduleSeed: schedule.seed, + }, + results, + generatedAt: now, + }); + const history = await readCompatibleHistory(bundleId); + const comparison = + history.length === 0 + ? null + : evals.compareRunnerWorkflowReports(report, history.at(-1)); + const alerts = evals.runnerWorkflowAlerts({ + current: report, + history, + baselineReady: + process.env.PAPERCLIP_EVAL_BASELINE_READY === "true" && + history.length >= 7, + }); + await Promise.all([ + writeFile( + resolve(outputDirectory, "live-report.json"), + `${JSON.stringify({ report, alerts, comparison }, null, 2)}\n`, + ), + writeFile( + resolve(outputDirectory, "live-report.md"), + evals.renderRunnerWorkflowMarkdown(report), + ), + writeFile( + resolve(outputDirectory, "live-report.junit.xml"), + evals.renderRunnerWorkflowJUnit(report), + ), + writeFile( + resolve(outputDirectory, "github-live-summary.md"), + `${evals.renderRunnerWorkflowGitHubSummary(report, alerts)}\n## Previous compatible bundle\n\n${comparison === null ? "No compatible prior bundle is available." : `Pass-rate delta: ${comparison.passRateDelta}; overall delta: ${comparison.overallDelta}.`}\n`, + ), + ]); + await retainHistory(report); + process.stdout.write( + `Runner live evals complete: ${report.aggregate.passed}/${report.aggregate.scoreable} scoreable passed; ${report.aggregate.infrastructureFailures} infrastructure, ${report.aggregate.skipped} skipped, ${alerts.length} alert(s).\n`, + ); +} else { + const payload = { + schema: "paperclip.runner.chaos-eval-schedule.v1", + generatedAt: now, + seed, + scenarios: evals.RUNNER_CHAOS_SCENARIOS, + }; + await writeFile( + resolve(outputDirectory, "chaos-schedule.json"), + `${JSON.stringify(payload, null, 2)}\n`, + ); + process.stdout.write( + `Runner chaos eval schedule ready: ${payload.scenarios.length} scenarios.\n`, + ); +} diff --git a/packages/paperclip-runner/spec/capability/protocol-coverage.json b/packages/paperclip-runner/spec/capability/protocol-coverage.json index bd9d1f5681..8b67f5aac6 100644 --- a/packages/paperclip-runner/spec/capability/protocol-coverage.json +++ b/packages/paperclip-runner/spec/capability/protocol-coverage.json @@ -3561,7 +3561,7 @@ { "id": "transcript-accounting", "requirement": "assistant transcript capture, token usage, and cost inputs", - "deterministicOwner": "src/live/live-session.test.ts" + "deterministicOwner": "src/live/live-session.test.ts; src/cli/eval-session.ts" }, { "id": "state-reconstruction", diff --git a/packages/paperclip-runner/src/cli/eval-session-contract.test.ts b/packages/paperclip-runner/src/cli/eval-session-contract.test.ts new file mode 100644 index 0000000000..947c22cb99 --- /dev/null +++ b/packages/paperclip-runner/src/cli/eval-session-contract.test.ts @@ -0,0 +1,182 @@ +import { describe, expect, it } from "vitest"; + +import type { CapabilityLiveSessionSnapshot } from "../live/live-session.js"; +import { + evalSessionUsage, + parseEvalSessionRequest, +} from "./eval-session-contract.js"; + +function request(overrides: Record = {}): unknown { + return { + schema: "paperclip-runner/eval-session-request/v1", + attemptId: "attempt-1", + prompt: "Inspect the governed task.", + model: "gpt-5.6-sol", + provider: "codex", + runnerd: { path: "/tmp/paperclip-runnerd", sha256: "a".repeat(64) }, + limits: { + turnTimeoutMs: 120_000, + maxAgentTurns: 1, + maxEstimatedCostNanodollars: 100_000_000, + }, + session: {}, + ...overrides, + }; +} + +function agentCoreProfile(overrides: Record = {}) { + return { + profileId: "agentcore-qualified", + region: "us-east-1", + accountId: "123456789012", + harnessArn: "arn:aws:bedrock-agentcore:us-east-1:123456789012:harness/test", + harnessVersion: "1", + endpointArn: "arn:aws:bedrock-agentcore:us-east-1:123456789012:harness-endpoint/test", + endpointQualifier: "paperclip", + agentRuntimeArn: "arn:aws:bedrock-agentcore:us-east-1:123456789012:runtime/test", + memoryArn: "arn:aws:bedrock-agentcore:us-east-1:123456789012:memory/test", + memoryId: "memory-test", + invocationRoleArn: "arn:aws:iam::123456789012:role/paperclip-agentcore", + contextBucket: "paperclip-agentcore-context", + contextPrefix: "paperclip/agentcore/test", + contextKmsKeyArn: "arn:aws:kms:us-east-1:123456789012:key/test", + qualificationRevision: "aws-agentcore-harness-context-v2", + eventExpiryDays: 90, + maxEstimatedSessionCostUsd: 1, + maxIterations: 8, + maxOutputTokens: 4_096, + timeoutSeconds: 300, + ...overrides, + }; +} + +describe("eval-session request contract", () => { + it("normalizes the current local live-session provider contract", () => { + expect(parseEvalSessionRequest(request())).toMatchObject({ + provider: "codex", + driver: "codex_app_server", + model: "gpt-5.6-sol", + }); + expect(parseEvalSessionRequest(request({ + provider: "acpx", + driver: "acpx_runtime", + acpxAgent: "claude", + model: "claude-sonnet-5", + }))).toMatchObject({ provider: "acpx", acpxAgent: "claude" }); + }); + + it("rejects Pi and accepts both qualified remote provider profiles", () => { + expect(() => parseEvalSessionRequest(request({ + provider: "acpx", + acpxAgent: "pi", + }))).toThrow("Pi ACPX profile is not available"); + expect(parseEvalSessionRequest(request({ + provider: "aws_agentcore", + driver: "aws_agentcore_harness_api", + model: "global.anthropic.claude-sonnet-4-6", + agentCoreProfile: agentCoreProfile(), + }))).toMatchObject({ + provider: "aws_agentcore", + agentCoreProfile: { + contextBucket: "paperclip-agentcore-context", + contextPrefix: "paperclip/agentcore/test", + contextKmsKeyArn: "arn:aws:kms:us-east-1:123456789012:key/test", + }, + }); + expect(parseEvalSessionRequest(request({ + provider: "claude_managed", + driver: "claude_managed_agents_api", + model: "claude-sonnet-5", + managedProfile: { + profileId: "managed-qualified", + anthropicAgentId: "agent-test", + agentVersion: "1", + environmentId: "environment-test", + betaVersion: "managed-agents-2026-04-01", + maxSessionListCostUsd: 1, + }, + }))).toMatchObject({ provider: "claude_managed" }); + }); + + it("requires complete remote profiles and AgentCore S3/KMS qualification", () => { + expect(() => parseEvalSessionRequest(request({ + provider: "aws_agentcore", + model: "global.anthropic.claude-sonnet-4-6", + }))).toThrow("request.agentCoreProfile must be an object"); + expect(() => parseEvalSessionRequest(request({ + provider: "aws_agentcore", + model: "global.anthropic.claude-sonnet-4-6", + agentCoreProfile: agentCoreProfile({ contextKmsKeyArn: "" }), + }))).toThrow("contextKmsKeyArn"); + }); + + it.each(["latest", "0", "01", "2147483648"])( + "rejects noncanonical Managed agentVersion %s", + (agentVersion) => { + expect(() => parseEvalSessionRequest(request({ + provider: "claude_managed", + model: "claude-sonnet-5", + managedProfile: { + profileId: "managed-qualified", + anthropicAgentId: "agent-test", + agentVersion, + environmentId: "environment-test", + betaVersion: "managed-agents-2026-04-01", + maxSessionListCostUsd: 1, + }, + }))).toThrow("canonical positive int32 string"); + }, + ); + + it("rejects contradictory session and execution inputs", () => { + expect(() => parseEvalSessionRequest(request({ + provider: "opencode", + driver: "codex_app_server", + }))).toThrow("provider/driver mismatch"); + expect(() => parseEvalSessionRequest(request({ + session: { requestedModel: "different-model" }, + }))).toThrow("requestedModel must match"); + expect(() => parseEvalSessionRequest(request({ + includeCollaborationModeInstructions: false, + }))).toThrow("requires collaboration-mode instructions"); + }); +}); + +describe("eval-session usage", () => { + it("deduplicates receipts and applies the versioned model price", () => { + const receipt = { + receiptId: "receipt-1", + attemptId: "attempt-1", + providerResponseId: "response-1", + turnId: "turn-1", + observedAt: "2026-09-01T00:00:00.000Z", + providerCalls: 1, + providerRequests: 2, + inputTokens: 1_000, + outputTokens: 100, + cachedInputTokens: 400, + reasoningTokens: 50, + costNanodollars: 6_000_000, + }; + const snapshot = { + usageLedger: [receipt, { ...receipt }], + } as unknown as CapabilityLiveSessionSnapshot; + expect(evalSessionUsage("gpt-5.6-sol", snapshot)).toMatchObject({ + agentTurns: 1, + providerRequests: 2, + inputTokens: 1_000, + cachedInputTokens: 400, + outputTokens: 100, + reasoningTokens: 50, + providerReportedCostNanodollars: 6_000_000, + estimatedCostNanodollars: 6_200_000, + }); + }); + + it("fails closed when a completed turn has no usage receipt", () => { + expect(() => evalSessionUsage( + "gpt-5.6-sol", + { usageLedger: [] } as unknown as CapabilityLiveSessionSnapshot, + )).toThrow("omitted usage accounting"); + }); +}); diff --git a/packages/paperclip-runner/src/cli/eval-session-contract.ts b/packages/paperclip-runner/src/cli/eval-session-contract.ts new file mode 100644 index 0000000000..1caf1e04f0 --- /dev/null +++ b/packages/paperclip-runner/src/cli/eval-session-contract.ts @@ -0,0 +1,374 @@ +import type { + CapabilityLiveSessionSnapshot, + CreateCapabilityLiveSessionInput, +} from "../live/live-session.js"; +import type { QualifiedAcpxAgent } from "../drivers/acpx/qualified-profiles.js"; +import { + estimateModelCostNanodollars, + type EstimatedModelCost, +} from "../evals/model-pricing.js"; + +export const EVAL_SESSION_REQUEST_SCHEMA = + "paperclip-runner/eval-session-request/v1" as const; +export const EVAL_SESSION_ARTIFACT_SCHEMA = + "paperclip-runner/eval-session-artifact/v1" as const; + +export type EvalSessionProvider = + | "codex" + | "opencode" + | "claude_managed" + | "aws_agentcore" + | "acpx"; +export type EvalSessionDriver = + | "codex_app_server" + | "opencode_server" + | "claude_managed_agents_api" + | "aws_agentcore_harness_api" + | "acpx_runtime"; + +export interface EvalSessionManagedProfile { + profileId: string; + anthropicAgentId: string; + agentVersion: string; + environmentId: string; + betaVersion: "managed-agents-2026-04-01"; + maxSessionListCostUsd: number; +} + +export interface EvalSessionAgentCoreProfile { + profileId: string; + region: string; + accountId: string; + harnessArn: string; + harnessVersion: string; + endpointArn: string; + endpointQualifier: string; + agentRuntimeArn: string; + memoryArn: string; + memoryId: string; + invocationRoleArn: string; + contextBucket: string; + contextPrefix: string; + contextKmsKeyArn: string; + qualificationRevision: string; + eventExpiryDays: 90; + maxEstimatedSessionCostUsd: number; + maxIterations: number; + maxOutputTokens: number; + timeoutSeconds: number; +} + +export interface EvalSessionRequest { + schema: typeof EVAL_SESSION_REQUEST_SCHEMA; + attemptId: string; + prompt: string; + model: string; + provider?: EvalSessionProvider; + driver?: EvalSessionDriver; + opencodeVersion?: string; + acpxAgent?: Exclude; + managedProfile?: EvalSessionManagedProfile; + agentCoreProfile?: EvalSessionAgentCoreProfile; + runnerd: { path: string; sha256: string }; + limits: { + turnTimeoutMs: number; + maxAgentTurns: number; + maxEstimatedCostNanodollars: number; + }; + session: CreateCapabilityLiveSessionInput; + nativeResume?: { operationId: string }; + /** Current live sessions always use Codex collaboration instructions. */ + includeCollaborationModeInstructions?: true; +} + +export interface EvalSessionUsage extends EstimatedModelCost { + agentTurns: number; + providerRequests: number; + inputTokens: number; + outputTokens: number; + cachedInputTokens: number; + reasoningTokens: number; + providerReportedCostNanodollars: number; +} + +function object(value: unknown, path: string): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error(`${path} must be an object`); + } + return value as Record; +} + +function text(value: unknown, path: string): string { + if (typeof value !== "string" || value.trim().length === 0) { + throw new Error(`${path} must be a non-empty string`); + } + return value; +} + +function positiveInteger(value: unknown, path: string): number { + if (!Number.isSafeInteger(value) || Number(value) <= 0) { + throw new Error(`${path} must be a positive safe integer`); + } + return Number(value); +} + +export function expectedEvalSessionDriver( + provider: EvalSessionProvider, +): EvalSessionDriver { + return provider === "opencode" + ? "opencode_server" + : provider === "claude_managed" + ? "claude_managed_agents_api" + : provider === "aws_agentcore" + ? "aws_agentcore_harness_api" + : provider === "acpx" ? "acpx_runtime" : "codex_app_server"; +} + +function positiveNumber(value: unknown, path: string): number { + if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) { + throw new Error(`${path} must be a positive finite number`); + } + return value; +} + +function parseManagedProfile(value: unknown): EvalSessionManagedProfile { + const profile = object(value, "request.managedProfile"); + if (profile.betaVersion !== "managed-agents-2026-04-01") { + throw new Error("request.managedProfile.betaVersion is not qualified"); + } + const agentVersion = text( + profile.agentVersion, + "request.managedProfile.agentVersion", + ); + if ( + !/^[1-9][0-9]*$/.test(agentVersion) || + BigInt(agentVersion) > 2_147_483_647n + ) { + throw new Error( + "request.managedProfile.agentVersion must be a canonical positive int32 string", + ); + } + return { + profileId: text(profile.profileId, "request.managedProfile.profileId"), + anthropicAgentId: text( + profile.anthropicAgentId, + "request.managedProfile.anthropicAgentId", + ), + agentVersion, + environmentId: text( + profile.environmentId, + "request.managedProfile.environmentId", + ), + betaVersion: "managed-agents-2026-04-01", + maxSessionListCostUsd: positiveNumber( + profile.maxSessionListCostUsd, + "request.managedProfile.maxSessionListCostUsd", + ), + }; +} + +function parseAgentCoreProfile(value: unknown): EvalSessionAgentCoreProfile { + const profile = object(value, "request.agentCoreProfile"); + if (profile.eventExpiryDays !== 90) { + throw new Error("request.agentCoreProfile.eventExpiryDays must be 90"); + } + const requiredText = (name: string): string => + text(profile[name], `request.agentCoreProfile.${name}`); + return { + profileId: requiredText("profileId"), + region: requiredText("region"), + accountId: requiredText("accountId"), + harnessArn: requiredText("harnessArn"), + harnessVersion: requiredText("harnessVersion"), + endpointArn: requiredText("endpointArn"), + endpointQualifier: requiredText("endpointQualifier"), + agentRuntimeArn: requiredText("agentRuntimeArn"), + memoryArn: requiredText("memoryArn"), + memoryId: requiredText("memoryId"), + invocationRoleArn: requiredText("invocationRoleArn"), + contextBucket: requiredText("contextBucket"), + contextPrefix: requiredText("contextPrefix"), + contextKmsKeyArn: requiredText("contextKmsKeyArn"), + qualificationRevision: requiredText("qualificationRevision"), + eventExpiryDays: 90, + maxEstimatedSessionCostUsd: positiveNumber( + profile.maxEstimatedSessionCostUsd, + "request.agentCoreProfile.maxEstimatedSessionCostUsd", + ), + maxIterations: positiveInteger( + profile.maxIterations, + "request.agentCoreProfile.maxIterations", + ), + maxOutputTokens: positiveInteger( + profile.maxOutputTokens, + "request.agentCoreProfile.maxOutputTokens", + ), + timeoutSeconds: positiveInteger( + profile.timeoutSeconds, + "request.agentCoreProfile.timeoutSeconds", + ), + }; +} + +/** Fail-closed validation for the executable boundary. */ +export function parseEvalSessionRequest(value: unknown): EvalSessionRequest { + const input = object(value, "request"); + if (input.schema !== EVAL_SESSION_REQUEST_SCHEMA) { + throw new Error("unsupported request schema"); + } + const providerValue = input.provider ?? "codex"; + if ( + providerValue !== "codex" && + providerValue !== "opencode" && + providerValue !== "claude_managed" && + providerValue !== "aws_agentcore" && + providerValue !== "acpx" + ) { + throw new Error( + "eval-session provider is unsupported by CapabilityLiveSessionService", + ); + } + const provider = providerValue; + const driver = expectedEvalSessionDriver(provider); + if (input.driver !== undefined && input.driver !== driver) { + throw new Error("eval-session provider/driver mismatch"); + } + const acpxAgent = input.acpxAgent; + if (acpxAgent === "pi") throw new Error("The Pi ACPX profile is not available"); + if ( + acpxAgent !== undefined && + acpxAgent !== "codex" && + acpxAgent !== "claude" + ) { + throw new Error("eval-session acpxAgent must be codex or claude"); + } + if (provider !== "acpx" && acpxAgent !== undefined) { + throw new Error("eval-session acpxAgent requires provider acpx"); + } + const managedProfile = provider === "claude_managed" + ? parseManagedProfile(input.managedProfile) + : undefined; + const agentCoreProfile = provider === "aws_agentcore" + ? parseAgentCoreProfile(input.agentCoreProfile) + : undefined; + if (provider !== "claude_managed" && input.managedProfile !== undefined) { + throw new Error("eval-session managedProfile requires provider claude_managed"); + } + if (provider !== "aws_agentcore" && input.agentCoreProfile !== undefined) { + throw new Error("eval-session agentCoreProfile requires provider aws_agentcore"); + } + if (input.nativeResume !== undefined) { + throw new Error( + "eval-session nativeResume requires a retained live-session checkpoint", + ); + } + if (input.includeCollaborationModeInstructions === false) { + throw new Error( + "current CapabilityLiveSessionService requires collaboration-mode instructions", + ); + } + + const runnerd = object(input.runnerd, "request.runnerd"); + const digest = text(runnerd.sha256, "request.runnerd.sha256"); + if (!/^(?:sha256:)?[a-f0-9]{64}$/i.test(digest)) { + throw new Error("request.runnerd.sha256 must be a SHA-256 digest"); + } + const limits = object(input.limits, "request.limits"); + const sessionInput = object(input.session, "request.session"); + const session = sessionInput as unknown as CreateCapabilityLiveSessionInput; + const model = text(input.model, "request.model"); + if (provider === "claude_managed" && model !== "claude-sonnet-5") { + throw new Error("Claude Managed evals require exact model claude-sonnet-5"); + } + if ( + provider === "aws_agentcore" && + model !== "global.anthropic.claude-sonnet-4-6" + ) { + throw new Error( + "AWS AgentCore evals require exact model global.anthropic.claude-sonnet-4-6", + ); + } + if (sessionInput.provider !== undefined && sessionInput.provider !== provider) { + throw new Error("request.session.provider must match request.provider"); + } + if ( + session.requestedModel !== undefined && + session.requestedModel !== model + ) { + throw new Error("request.session.requestedModel must match request.model"); + } + if (session.acpxAgent === "pi") { + throw new Error("The Pi ACPX profile is not available"); + } + + return { + schema: EVAL_SESSION_REQUEST_SCHEMA, + attemptId: text(input.attemptId, "request.attemptId"), + prompt: text(input.prompt, "request.prompt"), + model, + provider, + driver, + ...(typeof input.opencodeVersion === "string" + ? { opencodeVersion: text(input.opencodeVersion, "request.opencodeVersion") } + : {}), + ...(acpxAgent === undefined ? {} : { acpxAgent }), + ...(managedProfile === undefined ? {} : { managedProfile }), + ...(agentCoreProfile === undefined ? {} : { agentCoreProfile }), + runnerd: { + path: text(runnerd.path, "request.runnerd.path"), + sha256: digest, + }, + limits: { + turnTimeoutMs: positiveInteger( + limits.turnTimeoutMs, + "request.limits.turnTimeoutMs", + ), + maxAgentTurns: positiveInteger( + limits.maxAgentTurns, + "request.limits.maxAgentTurns", + ), + maxEstimatedCostNanodollars: positiveInteger( + limits.maxEstimatedCostNanodollars, + "request.limits.maxEstimatedCostNanodollars", + ), + }, + session, + ...(input.includeCollaborationModeInstructions === true + ? { includeCollaborationModeInstructions: true } + : {}), + }; +} + +export function evalSessionUsage( + model: string, + snapshot: CapabilityLiveSessionSnapshot, +): EvalSessionUsage { + const unique = new Map( + (snapshot.usageLedger ?? []).map((receipt) => [receipt.receiptId, receipt]), + ); + const totals = [...unique.values()].reduce((result, receipt) => ({ + agentTurns: result.agentTurns + receipt.providerCalls, + providerRequests: result.providerRequests + receipt.providerRequests, + inputTokens: result.inputTokens + receipt.inputTokens, + outputTokens: result.outputTokens + receipt.outputTokens, + cachedInputTokens: result.cachedInputTokens + receipt.cachedInputTokens, + reasoningTokens: result.reasoningTokens + receipt.reasoningTokens, + providerReportedCostNanodollars: + result.providerReportedCostNanodollars + receipt.costNanodollars, + }), { + agentTurns: 0, + providerRequests: 0, + inputTokens: 0, + outputTokens: 0, + cachedInputTokens: 0, + reasoningTokens: 0, + providerReportedCostNanodollars: 0, + }); + if (unique.size === 0) { + throw new Error("completed turn omitted usage accounting"); + } + return { + ...totals, + ...estimateModelCostNanodollars(model, totals), + }; +} diff --git a/packages/paperclip-runner/src/cli/eval-session.ts b/packages/paperclip-runner/src/cli/eval-session.ts new file mode 100644 index 0000000000..f98e5ab1a8 --- /dev/null +++ b/packages/paperclip-runner/src/cli/eval-session.ts @@ -0,0 +1,342 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { projectCapabilityDevtools } from "../devtools/index.js"; +import { resolveQualifiedAcpxProfile } from "../drivers/acpx/qualified-profiles.js"; +import { PAPERCLIP_RUNNER_BUILD_METADATA } from "../evals/build-metadata.js"; +import { projectCapabilityIssueThread } from "../issue-thread/live-projection.js"; +import { + CapabilityLiveSessionService, + type CapabilityLiveSession, + type CapabilityLiveSessionSnapshot, + type CapabilityLiveTurnResult, + type CreateCapabilityLiveSessionInput, +} from "../live/live-session.js"; +import { + evalSessionUsage, + expectedEvalSessionDriver, + parseEvalSessionRequest, + type EvalSessionRequest, + type EvalSessionUsage, +} from "./eval-session-contract.js"; + +interface EvalSessionCliOptions { + requestPath: string; + outputPath: string; +} + +function argument(args: string[], name: string): string { + const index = args.indexOf(name); + const value = index < 0 ? undefined : args[index + 1]; + if (!value || value.startsWith("--")) throw new Error(`missing ${name}`); + return resolve(value); +} + +export function parseEvalSessionCliArgs(args: string[]): EvalSessionCliOptions { + const allowed = new Set(["--request", "--output"]); + for (let index = 0; index < args.length; index += 2) { + if (!allowed.has(args[index] ?? "")) { + throw new Error(`unknown argument: ${args[index] ?? ""}`); + } + if (args[index + 1] === undefined) throw new Error(`missing ${args[index]}`); + } + return { + requestPath: argument(args, "--request"), + outputPath: argument(args, "--output"), + }; +} + +async function sha256(path: string): Promise { + return createHash("sha256").update(await readFile(path)).digest("hex"); +} + +function providerVersion(request: EvalSessionRequest): string | null { + if (request.provider === "opencode") { + const version = request.opencodeVersion ?? "1.18.17"; + if (version !== "1.18.17") { + throw new Error(`OpenCode evals require exact version 1.18.17; received ${version}`); + } + return version; + } + if (request.provider === "acpx") { + return resolveQualifiedAcpxProfile( + request.acpxAgent ?? "codex", + request.model, + ).acpxVersion; + } + if (request.provider === "claude_managed") { + return request.managedProfile!.agentVersion; + } + if (request.provider === "aws_agentcore") { + return request.agentCoreProfile!.harnessVersion; + } + return null; +} + +function failureClass(error: unknown): { + class: string; + category: string; + retryable: boolean; + diagnostics: Record; +} { + const message = error instanceof Error ? error.message : String(error); + if (/timed? ?out|timeout/i.test(message)) { + return { + class: "provider_turn_timeout", + category: "provider_lifecycle", + retryable: true, + diagnostics: {}, + }; + } + if (/budget|cost limit|turn limit/i.test(message)) { + return { + class: "provider_budget_reached", + category: "provider_budget", + retryable: false, + diagnostics: {}, + }; + } + if (/runner.*(?:exit|closed|failed)|PRP/i.test(message)) { + return { + class: "runner_infrastructure_failure", + category: "runner_infrastructure", + retryable: true, + diagnostics: {}, + }; + } + return { + class: "eval_orchestration_failure", + category: "eval_orchestration", + retryable: false, + diagnostics: {}, + }; +} + +function usageIfAvailable( + request: EvalSessionRequest, + snapshot: CapabilityLiveSessionSnapshot | null, +): EvalSessionUsage | null { + if (!snapshot?.usageLedger?.length) return null; + try { + return evalSessionUsage(request.model, snapshot); + } catch { + return null; + } +} + +async function closeSession( + session: CapabilityLiveSession | null, + reason: string, +): Promise { + if (session === null || session.snapshot().status === "closed") return; + await session.shutdown(reason); +} + +export async function runEvalSessionCli( + args: string[], + options: { + serviceFactory?: ( + runnerBinary: string, + ) => CapabilityLiveSessionService; + } = {}, +): Promise { + const cli = parseEvalSessionCliArgs(args); + const request = parseEvalSessionRequest( + JSON.parse(await readFile(cli.requestPath, "utf8")), + ); + const runnerdPath = resolve(request.runnerd.path); + const actualDigest = await sha256(runnerdPath); + if (actualDigest !== request.runnerd.sha256.replace(/^sha256:/, "")) { + throw new Error( + `runnerd digest mismatch: expected ${request.runnerd.sha256}, got sha256:${actualDigest}`, + ); + } + + const startedAt = new Date().toISOString(); + const startedAtMs = Date.now(); + const requestedProvider = request.provider ?? "codex"; + const requestedDriver = request.driver ?? + expectedEvalSessionDriver(requestedProvider); + const requestedProviderVersion = providerVersion(request); + const service = options.serviceFactory?.(runnerdPath) ?? + new CapabilityLiveSessionService({ + transportOptions: { + runnerBinary: runnerdPath, + onDiagnostic: (message) => { + process.stderr.write(`[eval-session runnerd] ${message}\n`); + }, + }, + }); + let session: CapabilityLiveSession | null = null; + let turn: CapabilityLiveTurnResult | null = null; + let snapshot: CapabilityLiveSessionSnapshot | null = null; + + try { + // PR3 expands this same service input with the two qualified remote + // profiles. The cast keeps this isolated PR typecheckable before PR3 lands; + // the runtime fields and their fail-closed validation are already present. + const createInput = { + ...request.session, + provider: requestedProvider, + requestedModel: request.model, + ...(requestedProvider === "acpx" + ? { acpxAgent: request.acpxAgent ?? "codex" } + : { acpxAgent: undefined }), + ...(request.managedProfile === undefined + ? {} + : { managedProfile: request.managedProfile }), + ...(request.agentCoreProfile === undefined + ? {} + : { agentCoreProfile: request.agentCoreProfile }), + attemptId: request.attemptId, + turnTimeoutMs: request.limits.turnTimeoutMs, + } as unknown as CreateCapabilityLiveSessionInput; + session = await service.create(createInput); + turn = await session.sendMessage(request.prompt); + if (turn.status !== "completed") { + await session.completeAttempt("failed", `provider_turn_${turn.status}`); + throw new Error(`provider turn ended with status ${turn.status}`); + } + const usage = evalSessionUsage(request.model, turn.snapshot); + if (usage.agentTurns > request.limits.maxAgentTurns) { + throw new Error("agent turn limit exceeded"); + } + if ( + usage.estimatedCostNanodollars > + request.limits.maxEstimatedCostNanodollars + ) { + throw new Error("estimated cost limit exceeded"); + } + if ( + usage.providerReportedCostNanodollars > + request.limits.maxEstimatedCostNanodollars + ) { + throw new Error("provider-reported cost limit exceeded"); + } + await session.completeAttempt("succeeded"); + await closeSession(session, "eval session complete"); + snapshot = session.snapshot(); + + await writeFile(cli.outputPath, `${JSON.stringify({ + schema: "paperclip-runner/eval-session-artifact/v1", + attemptId: request.attemptId, + build: PAPERCLIP_RUNNER_BUILD_METADATA, + runnerd: { path: "[withheld]", sha256: `sha256:${actualDigest}` }, + requestedModel: request.model, + provider: requestedProvider, + driver: requestedDriver, + providerVersion: requestedProviderVersion, + providerSessionId: snapshot.providerSessionId, + ...(requestedProvider === "claude_managed" + ? { + managedProfile: request.managedProfile, + retainedSession: snapshot.providerSessionId !== null, + retainedSessionStatus: snapshot.providerSessionId === null + ? "unknown" + : "retained", + } + : {}), + ...(requestedProvider === "aws_agentcore" + ? { agentCoreProfile: request.agentCoreProfile } + : {}), + ...(requestedProvider === "acpx" + ? { + acpxAgent: request.acpxAgent ?? "codex", + acpxProfile: resolveQualifiedAcpxProfile( + request.acpxAgent ?? "codex", + request.model, + ), + } + : {}), + turn, + snapshot, + devtools: projectCapabilityDevtools(snapshot), + issueThread: projectCapabilityIssueThread({ + snapshot, + mode: "live", + replaySource: "live", + }), + usage, + timing: { + startedAt, + finishedAt: new Date().toISOString(), + durationMs: Date.now() - startedAtMs, + }, + }, null, 2)}\n`); + return 0; + } catch (error) { + if (session !== null) { + snapshot = session.snapshot(); + const attempt = snapshot.attempts?.find( + (candidate) => candidate.attemptId === snapshot?.currentAttemptId, + ); + if (attempt?.status === "running" && snapshot.activeTurnId === null) { + try { + await session.completeAttempt( + "failed", + failureClass(error).class, + ); + } catch { + // The original infrastructure failure remains authoritative. + } + } + try { + await closeSession(session, "eval session failed"); + } catch { + // The original infrastructure failure remains authoritative. + } + snapshot = session.snapshot(); + } + const usage = usageIfAvailable(request, snapshot); + await writeFile(cli.outputPath, `${JSON.stringify({ + schema: "paperclip-runner/eval-session-artifact/v1", + attemptId: request.attemptId, + infrastructureError: error instanceof Error ? error.message : String(error), + infrastructureFailure: failureClass(error), + build: PAPERCLIP_RUNNER_BUILD_METADATA, + runnerd: { path: "[withheld]", sha256: `sha256:${actualDigest}` }, + requestedModel: request.model, + provider: requestedProvider, + driver: requestedDriver, + providerVersion: requestedProviderVersion, + providerSessionId: snapshot?.providerSessionId ?? null, + ...(requestedProvider === "claude_managed" + ? { + managedProfile: request.managedProfile, + retainedSession: snapshot?.providerSessionId != null, + retainedSessionStatus: snapshot?.providerSessionId == null + ? "unknown" + : "retained", + } + : {}), + ...(requestedProvider === "aws_agentcore" + ? { agentCoreProfile: request.agentCoreProfile } + : {}), + ...(snapshot === null ? {} : { snapshot }), + ...(usage === null ? {} : { usage }), + timing: { + startedAt, + finishedAt: new Date().toISOString(), + durationMs: Date.now() - startedAtMs, + }, + }, null, 2)}\n`); + return 2; + } +} + +if ( + process.argv[1] !== undefined && + resolve(process.argv[1]) === resolve(fileURLToPath(import.meta.url)) +) { + void runEvalSessionCli(process.argv.slice(2)) + .then((code) => { + process.exitCode = code; + }) + .catch((error: unknown) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; + }); +} diff --git a/packages/paperclip-runner/src/cli/opencode-app-server-proxy.ts b/packages/paperclip-runner/src/cli/opencode-app-server-proxy.ts index 07102ebe55..e3be34483c 100644 --- a/packages/paperclip-runner/src/cli/opencode-app-server-proxy.ts +++ b/packages/paperclip-runner/src/cli/opencode-app-server-proxy.ts @@ -21,6 +21,7 @@ import { trustedOpenCodeLaunchBinding, withoutAmbientOpenCodeCommand, } from "./opencode-proxy-command.js"; +import { OpenCodeProxyUsageLedger } from "./opencode-proxy-usage.js"; type RpcMessage = { id?: string | number; method?: string; params?: unknown; result?: unknown; error?: unknown }; @@ -35,6 +36,7 @@ let activeModel = ""; let activeTurnId: string | null = null; const announcedTurnIds = new Set(); const launchBinding = trustedOpenCodeLaunchBinding(process.argv.slice(2)); +const usageLedger = new OpenCodeProxyUsageLedger(); function send(value: unknown): void { process.stdout.write(`${JSON.stringify(value)}\n`); @@ -173,18 +175,15 @@ async function pumpEvents(opened: HarnessSession): Promise { }, }); if (payload.kind === "usage") { - const usage = record(payload.usage); - const cache = record(usage.cache); + const usage = usageLedger.update({ + turnId: text(event.turnId), + messageId: text(payload.usageMessageId), + usage: payload.usage, + }); send({ method: "thread/tokenUsage/updated", params: { threadId: opened.ids().driverSessionId, turnId: event.turnId, - tokenUsage: { total: { - inputTokens: usage.inputTokens ?? usage.input ?? 0, - outputTokens: usage.outputTokens ?? usage.output ?? 0, - cachedInputTokens: usage.cachedInputTokens ?? cache.read ?? 0, - reasoningTokens: usage.reasoningTokens ?? usage.reasoning ?? 0, - costUsd: usage.costUsd ?? usage.cost ?? null, - } }, + tokenUsage: usage, } }); } } else if (event.eventType === "run.result.proposed") { @@ -217,8 +216,14 @@ async function pumpEvents(opened: HarnessSession): Promise { resolution: record(controllerResponse.resolution) as HarnessRuntimeRequestResolution, }); })().catch((error) => failProxy(error)); + } else if (event.eventType === "run.attached") { + // OpenCode can retain one provider session across governed runs. Usage + // totals are run-scoped, so a new attachment must not inherit the prior + // run's completed-turn ledger. + usageLedger.reset(); } else if (["turn.completed", "turn.failed", "turn.interrupted", "turn.cancelled"].includes(event.eventType)) { const status = event.eventType.slice("turn.".length); + if (typeof event.turnId === "string") usageLedger.completeTurn(event.turnId); send({ method: "turn/completed", params: { threadId: opened.ids().driverSessionId, turnId: event.turnId, turn: { id: event.turnId, status } } }); activeTurnId = null; } else if (event.eventType === "harness.diagnostic") { diff --git a/packages/paperclip-runner/src/cli/opencode-proxy-usage.test.ts b/packages/paperclip-runner/src/cli/opencode-proxy-usage.test.ts new file mode 100644 index 0000000000..e4982fa171 --- /dev/null +++ b/packages/paperclip-runner/src/cli/opencode-proxy-usage.test.ts @@ -0,0 +1,135 @@ +import { describe, expect, it } from "vitest"; + +import { + OpenCodeProxyUsageLedger, + openCodeProxyUsageMeasurement, +} from "./opencode-proxy-usage.js"; + +describe("OpenCode proxy usage accounting", () => { + it("folds reasoning into output without changing the budget token total", () => { + expect( + openCodeProxyUsageMeasurement({ + input: 11, + output: 5, + reasoning: 7, + cache: { read: 3, write: 2 }, + cost: 0.012, + }), + ).toEqual({ + inputTokens: 11, + outputTokens: 12, + cachedInputTokens: 3, + cacheWriteTokens: 2, + activeSeconds: 0, + requests: 1, + providerCostUsd: 0.012, + }); + }); + + it("replaces repeated message snapshots and aggregates tool-loop messages across turns", () => { + const ledger = new OpenCodeProxyUsageLedger(); + + expect( + ledger.update({ + turnId: "turn-1", + messageId: "message-1", + usage: { input: 10, output: 2, reasoning: 3, cost: 0.01 }, + }), + ).toMatchObject({ + total: { + inputTokens: 10, + outputTokens: 5, + requests: 1, + providerCostUsd: 0.01, + }, + last: { + inputTokens: 10, + outputTokens: 5, + requests: 1, + providerCostUsd: 0.01, + }, + }); + + const firstTurn = ledger.update({ + turnId: "turn-1", + messageId: "message-1", + usage: { input: 12, output: 3, reasoning: 4, cost: 0.012 }, + }); + expect(firstTurn.last).toMatchObject({ + inputTokens: 12, + outputTokens: 7, + requests: 1, + providerCostUsd: 0.012, + }); + + expect( + ledger.update({ + turnId: "turn-1", + messageId: "message-2", + usage: { input: 5, output: 1, reasoning: 2, costUsd: 0.005 }, + }), + ).toMatchObject({ + total: { + inputTokens: 17, + outputTokens: 10, + requests: 2, + providerCostUsd: 0.017, + }, + last: { + inputTokens: 17, + outputTokens: 10, + requests: 2, + providerCostUsd: 0.017, + }, + }); + + ledger.completeTurn("turn-1"); + expect( + ledger.update({ + turnId: "turn-2", + messageId: "message-3", + usage: { + inputTokens: 7, + outputTokens: 3, + reasoningTokens: 1, + cost: 0.004, + }, + }), + ).toMatchObject({ + total: { + inputTokens: 24, + outputTokens: 14, + requests: 3, + providerCostUsd: 0.021, + }, + last: { + inputTokens: 7, + outputTokens: 4, + requests: 1, + providerCostUsd: 0.004, + }, + }); + + ledger.reset(); + expect( + ledger.update({ + turnId: "turn-attached-run", + messageId: "message-attached-run", + usage: { input: 2, output: 1, reasoning: 1, cost: 0.001 }, + }), + ).toMatchObject({ + total: { + inputTokens: 2, + outputTokens: 2, + requests: 1, + providerCostUsd: 0.001, + }, + last: { + inputTokens: 2, + outputTokens: 2, + requests: 1, + providerCostUsd: 0.001, + }, + }); + }); +}); diff --git a/packages/paperclip-runner/src/cli/opencode-proxy-usage.ts b/packages/paperclip-runner/src/cli/opencode-proxy-usage.ts new file mode 100644 index 0000000000..0b2b2f1168 --- /dev/null +++ b/packages/paperclip-runner/src/cli/opencode-proxy-usage.ts @@ -0,0 +1,134 @@ +export interface OpenCodeProxyUsageMeasurement { + inputTokens: number; + outputTokens: number; + cachedInputTokens: number; + cacheWriteTokens: number; + activeSeconds: number; + requests: number; + providerCostUsd: number; +} + +export interface OpenCodeProxyUsageSnapshot { + total: OpenCodeProxyUsageMeasurement; + last: OpenCodeProxyUsageMeasurement; +} + +const ZERO_USAGE: OpenCodeProxyUsageMeasurement = Object.freeze({ + inputTokens: 0, + outputTokens: 0, + cachedInputTokens: 0, + cacheWriteTokens: 0, + activeSeconds: 0, + requests: 0, + providerCostUsd: 0, +}); + +function record(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Record) + : {}; +} + +function integer(...values: unknown[]): number { + for (const value of values) { + if (Number.isSafeInteger(value) && Number(value) >= 0) return Number(value); + } + return 0; +} + +function finite(...values: unknown[]): number { + for (const value of values) { + if (typeof value === "number" && Number.isFinite(value) && value >= 0) { + return value; + } + } + return 0; +} + +function add( + left: OpenCodeProxyUsageMeasurement, + right: OpenCodeProxyUsageMeasurement, +): OpenCodeProxyUsageMeasurement { + return { + inputTokens: left.inputTokens + right.inputTokens, + outputTokens: left.outputTokens + right.outputTokens, + cachedInputTokens: left.cachedInputTokens + right.cachedInputTokens, + cacheWriteTokens: left.cacheWriteTokens + right.cacheWriteTokens, + activeSeconds: left.activeSeconds + right.activeSeconds, + requests: left.requests + right.requests, + providerCostUsd: left.providerCostUsd + right.providerCostUsd, + }; +} + +function sum( + values: Iterable, +): OpenCodeProxyUsageMeasurement { + let total = ZERO_USAGE; + for (const value of values) total = add(total, value); + return total; +} + +/** + * Converts one OpenCode assistant-message usage snapshot into PRP's existing + * measurement shape. OpenCode's aggregate token total is input + output + + * reasoning (plus cache fields reported separately), while PRP v1 has no + * reasoning field. Folding reasoning into output preserves that exact + * budget-relevant total without a schema fork; consumers must not add it again. + */ +export function openCodeProxyUsageMeasurement( + value: unknown, +): OpenCodeProxyUsageMeasurement { + const usage = record(value); + const cache = record(usage.cache); + const outputTokens = integer(usage.outputTokens, usage.output); + const reasoningTokens = integer(usage.reasoningTokens, usage.reasoning); + return { + inputTokens: integer(usage.inputTokens, usage.input), + outputTokens: outputTokens + reasoningTokens, + cachedInputTokens: integer(usage.cachedInputTokens, cache.read), + cacheWriteTokens: integer(usage.cacheWriteTokens, cache.write), + activeSeconds: finite(usage.activeSeconds), + requests: 1, + providerCostUsd: finite(usage.providerCostUsd, usage.costUsd, usage.cost), + }; +} + +/** + * OpenCode emits a cumulative snapshot each time an assistant message changes. + * Retain the latest value per message so repeated updates replace rather than + * double-count it, then expose the whole current turn as Codex `last` usage. + */ +export class OpenCodeProxyUsageLedger { + #completed = ZERO_USAGE; + readonly #messagesByTurn = new Map< + string, + Map + >(); + + update(input: { + turnId: string; + messageId: string; + usage: unknown; + }): OpenCodeProxyUsageSnapshot { + if (!input.turnId || !input.messageId) { + throw new Error("OpenCode usage requires turn and message identities"); + } + const messages = this.#messagesByTurn.get(input.turnId) ?? new Map(); + messages.set(input.messageId, openCodeProxyUsageMeasurement(input.usage)); + this.#messagesByTurn.set(input.turnId, messages); + const last = sum(messages.values()); + return { total: add(this.#completed, last), last }; + } + + completeTurn(turnId: string): void { + const messages = this.#messagesByTurn.get(turnId); + if (messages === undefined) return; + this.#completed = add(this.#completed, sum(messages.values())); + this.#messagesByTurn.delete(turnId); + } + + reset(): void { + this.#completed = ZERO_USAGE; + this.#messagesByTurn.clear(); + } +} diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts index 33a80fd8fe..c9863ddcbc 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts @@ -1,7 +1,7 @@ import { chmod, lstat, mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterAll, afterEach, describe, expect, it } from "vitest"; import { NATIVE_RUNTIME_ASSET_SCHEMA, @@ -71,6 +71,10 @@ afterEach(async () => { })); }); +afterAll(async () => { + await chmod(fixture, 0o644); +}); + describe("OpenCodeServerDriver", () => { it("advertises within-turn plans as unsupported", async () => { const driver = new OpenCodeServerDriver({ @@ -124,9 +128,13 @@ describe("OpenCodeServerDriver", () => { providerPhase: "final_answer", text: "done [guide](guide.md)", }); - expect(events.filter( + const usageEvents = events.filter( (event) => event.eventType === "item.completed" && event.payload.kind === "usage", - )).toHaveLength(1); + ); + expect(usageEvents).toHaveLength(1); + expect(usageEvents[0]?.payload).toMatchObject({ + usageMessageId: expect.any(String), + }); expect(events.find((event) => event.eventType === "workspace.change.updated")?.payload) .toMatchObject({ schema: "paperclip.workspace.diff.v1", source: "harness_reported", totals: { files: 1 } }); expect(events.find((event) => event.eventType === "workspace.diff.recorded")?.payload) diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts index 7c7ff4a89e..b5d6df6da2 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts @@ -1054,7 +1054,11 @@ class OpenCodeHarnessSession implements HarnessSession { && this.#messageUsageFingerprints.get(messageId) !== usageFingerprint ) { this.#messageUsageFingerprints.set(messageId, usageFingerprint); - this.#emit("item.completed", { kind: "usage", usage: this.#usage }, { turnId, itemId: `${turnId}:usage` }); + this.#emit( + "item.completed", + { kind: "usage", usage: this.#usage, usageMessageId: messageId }, + { turnId, itemId: `${turnId}:usage` }, + ); } } return; diff --git a/packages/paperclip-runner/src/eval/eval-scoring.ts b/packages/paperclip-runner/src/eval/eval-scoring.ts index ac2ca58704..2a9a91cb6d 100644 --- a/packages/paperclip-runner/src/eval/eval-scoring.ts +++ b/packages/paperclip-runner/src/eval/eval-scoring.ts @@ -90,7 +90,7 @@ export interface EvalEfficiencyBudget { export interface EvalObservation { caseId: string; provenance?: { - source: "deterministic_fault_harness" | "fixture"; + source: "live_model" | "deterministic_fault_harness" | "fixture"; behavior?: string; counterpart?: "green" | "red"; faultInjection?: { diff --git a/packages/paperclip-runner/src/eval/index.ts b/packages/paperclip-runner/src/eval/index.ts index 43d20f1a6a..da3406be5b 100644 --- a/packages/paperclip-runner/src/eval/index.ts +++ b/packages/paperclip-runner/src/eval/index.ts @@ -8,3 +8,5 @@ export * from "./workflow-scoring.js"; export * from "./workflow-harness.js"; export * from "./workflow-traceability.js"; export * from "./workflow-report.js"; +export * from "./live-workflow-matrix.js"; +export * from "./live-workflow-executor.js"; diff --git a/packages/paperclip-runner/src/eval/live-workflow-executor.test.ts b/packages/paperclip-runner/src/eval/live-workflow-executor.test.ts new file mode 100644 index 0000000000..7705595f44 --- /dev/null +++ b/packages/paperclip-runner/src/eval/live-workflow-executor.test.ts @@ -0,0 +1,733 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +type MockTurnEvent = + | { + seq: number; + at: string; + turnId: string; + kind: "usage"; + usage: { + providerRequests: number; + inputTokens: number; + outputTokens: number; + cachedInputTokens: number; + reasoningTokens: number; + costNanodollars: number; + }; + } + | { + seq: number; + at: string; + turnId: string; + kind: "activity"; + reason: string; + }; + +const liveSessionMocks = vi.hoisted(() => ({ + shutdown: vi.fn(), + createOptions: [] as Array<{ + transportOptions?: { environment?: NodeJS.ProcessEnv }; + }>, + sendMessage: vi.fn(), + snapshot: vi.fn(), + interrupt: vi.fn(), + suspend: vi.fn(), + restore: vi.fn(), + subscribedSessionIds: [] as string[], + unsubscribedSessionIds: [] as string[], + turnListener: null as null | ((event: MockTurnEvent) => void), +})); + +vi.mock("../live/live-session.js", () => ({ + InMemoryCapabilityLiveSessionStore: class {}, + CapabilityLiveSessionService: class { + constructor(options: { + transportOptions?: { environment?: NodeJS.ProcessEnv }; + }) { + liveSessionMocks.createOptions.push(options); + } + + session(sessionId: string, subscriptionLabel: string) { + return { + id: sessionId, + subscribe: ( + listener: NonNullable, + ) => { + liveSessionMocks.subscribedSessionIds.push(subscriptionLabel); + liveSessionMocks.turnListener = listener; + return () => { + liveSessionMocks.unsubscribedSessionIds.push(subscriptionLabel); + if (liveSessionMocks.turnListener === listener) { + liveSessionMocks.turnListener = null; + } + }; + }, + sendMessage: liveSessionMocks.sendMessage, + pendingInteractions: () => [], + snapshot: liveSessionMocks.snapshot, + interrupt: liveSessionMocks.interrupt, + suspend: liveSessionMocks.suspend, + }; + } + + async create() { + return this.session("session-shutdown-test", "created"); + } + + async restore(sessionId: string) { + liveSessionMocks.restore(sessionId); + return this.session(sessionId, "restored"); + } + + async shutdown(sessionId: string, reason: string) { + return liveSessionMocks.shutdown(sessionId, reason); + } + }, +})); + +import { runnerWorkflowCase } from "./workflow-catalog.js"; +import { executeLiveRunnerWorkflow } from "./live-workflow-executor.js"; +import { + RUNNER_LIVE_CANDIDATE_SLOTS, + type RunnerLiveScheduleEntry, +} from "./live-workflow-matrix.js"; + +describe("live workflow executor infrastructure failures", () => { + beforeEach(() => { + liveSessionMocks.shutdown.mockReset(); + liveSessionMocks.createOptions.length = 0; + liveSessionMocks.subscribedSessionIds.length = 0; + liveSessionMocks.unsubscribedSessionIds.length = 0; + liveSessionMocks.turnListener = null; + liveSessionMocks.sendMessage.mockReset().mockResolvedValue({ + status: "completed", + turnId: "turn-shutdown-test", + }); + liveSessionMocks.interrupt.mockReset().mockResolvedValue({}); + liveSessionMocks.suspend.mockReset().mockResolvedValue({}); + liveSessionMocks.restore.mockReset(); + liveSessionMocks.snapshot.mockReset().mockReturnValue({ + sessionId: "session-shutdown-test", + authority: {}, + mockState: JSON.stringify({ tasks: [] }), + transcript: [], + evidence: [], + authorizationRecords: [], + attempts: [], + usageLedger: [], + stateHistory: [], + workspaceDiffs: [], + }); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it("classifies shutdown failures as retryable infrastructure errors and redacts them", async () => { + const leakedSecret = "sk-shutdown-secret-value"; + liveSessionMocks.shutdown.mockRejectedValueOnce( + new Error(`shutdown failed with ${leakedSecret}`), + ); + const candidate = RUNNER_LIVE_CANDIDATE_SLOTS[0]!.candidates[0]!; + const entry: RunnerLiveScheduleEntry = { + executionId: "shutdown-failure", + caseId: "final-response", + candidateId: candidate.id, + slotId: candidate.slotId, + repetition: 1, + providerTrace: "raw", + budget: candidate.budget, + }; + + let thrown: unknown; + try { + await executeLiveRunnerWorkflow({ + entry, + candidate, + evalCase: runnerWorkflowCase(entry.caseId), + }); + } catch (error) { + thrown = error; + } + + expect(thrown).toMatchObject({ + name: "RunnerWorkflowInfrastructureError", + code: "live_provider_execution_failed", + retryable: true, + message: "shutdown failed with [REDACTED]", + }); + expect(String((thrown as Error).message)).not.toContain(leakedSecret); + expect(liveSessionMocks.shutdown).toHaveBeenCalledWith( + "session-shutdown-test", + "Runner live workflow eval complete", + ); + }); + + it("passes only the selected candidate's required provider credential", async () => { + vi.stubEnv("OPENAI_API_KEY", "openai-candidate-secret"); + vi.stubEnv("ANTHROPIC_API_KEY", "anthropic-candidate-secret"); + vi.stubEnv("OPENROUTER_API_KEY", "openrouter-candidate-secret"); + vi.stubEnv("CODEX_API_KEY", "unqualified-codex-secret"); + vi.stubEnv("CLAUDE_CODE_OAUTH_TOKEN", "unqualified-claude-secret"); + vi.stubEnv("GITHUB_TOKEN", "unrelated-host-secret"); + vi.stubEnv("PAPERCLIP_AUTH_HEADER", "Bearer unrelated-control-secret"); + vi.stubEnv("RUNNER_EVAL_CANARY", "preserved-nonsecret-value"); + const candidates = [ + RUNNER_LIVE_CANDIDATE_SLOTS[0]!.candidates[0]!, + RUNNER_LIVE_CANDIDATE_SLOTS[1]!.candidates[0]!, + RUNNER_LIVE_CANDIDATE_SLOTS[3]!.candidates[0]!, + ]; + + for (const candidate of candidates) { + const entry: RunnerLiveScheduleEntry = { + executionId: `credential-isolation-${candidate.id}`, + caseId: "final-response", + candidateId: candidate.id, + slotId: candidate.slotId, + repetition: 1, + providerTrace: "raw", + budget: candidate.budget, + }; + await executeLiveRunnerWorkflow({ + entry, + candidate, + evalCase: runnerWorkflowCase(entry.caseId), + }); + + const environment = + liveSessionMocks.createOptions.at(-1)?.transportOptions?.environment; + expect(environment?.RUNNER_EVAL_CANARY).toBe("preserved-nonsecret-value"); + expect(environment?.PAPERCLIP_PROVIDER_TRACE_PATH).toMatch( + /provider-trace\.ndjson$/, + ); + for (const credential of [ + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "OPENROUTER_API_KEY", + "CODEX_API_KEY", + "CLAUDE_CODE_OAUTH_TOKEN", + "GITHUB_TOKEN", + "PAPERCLIP_AUTH_HEADER", + ]) { + if (candidate.qualification.requiredEnvironment.includes(credential)) { + expect(environment?.[credential]).toBe(process.env[credential]); + } else { + expect(environment).not.toHaveProperty(credential); + } + } + } + }); + + it("fails the candidate budget and stops before a paid continuation", async () => { + liveSessionMocks.snapshot.mockReturnValue({ + sessionId: "session-budget-test", + authority: {}, + mockState: JSON.stringify({ tasks: [] }), + transcript: [], + evidence: [], + authorizationRecords: [], + attempts: [], + usageLedger: [ + { + receiptId: "usage-budget-test", + attemptId: "attempt-budget-test", + providerResponseId: "response-budget-test", + turnId: "turn-budget-test", + providerCalls: 1, + providerRequests: 1, + inputTokens: 80, + outputTokens: 40, + cachedInputTokens: 0, + reasoningTokens: 10, + costNanodollars: 20_000_000, + observedAt: "2026-09-01T00:00:00.000Z", + }, + ], + stateHistory: [], + workspaceDiffs: [], + }); + const source = RUNNER_LIVE_CANDIDATE_SLOTS[0]!.candidates[0]!; + const candidate = { + ...source, + budget: { + ...source.budget, + maxTotalTokens: 100, + maxCostUsd: 0.01, + }, + }; + const entry: RunnerLiveScheduleEntry = { + executionId: "candidate-budget-exceeded", + caseId: "steering-causality", + candidateId: candidate.id, + slotId: candidate.slotId, + repetition: 1, + providerTrace: "raw", + budget: candidate.budget, + }; + + const observation = await executeLiveRunnerWorkflow({ + entry, + candidate, + evalCase: runnerWorkflowCase(entry.caseId), + }); + + expect(liveSessionMocks.sendMessage).toHaveBeenCalledTimes(1); + expect(observation).toMatchObject({ + classification: "candidate_failure", + metrics: { totalTokens: 130, costUsd: 0.02 }, + failure: { + code: "candidate_budget_exceeded", + category: "candidate", + retryable: false, + }, + }); + expect(observation.lifecycle.checks).toEqual( + expect.arrayContaining([ + expect.objectContaining({ id: "token-budget", passed: false }), + expect.objectContaining({ id: "cost-budget", passed: false }), + ]), + ); + }); + + it("stops at an exact terminal budget before a paid continuation", async () => { + liveSessionMocks.snapshot.mockReturnValue({ + sessionId: "session-budget-reached", + authority: {}, + mockState: JSON.stringify({ tasks: [] }), + transcript: [], + evidence: [], + authorizationRecords: [], + attempts: [], + usageLedger: [ + { + receiptId: "usage-budget-reached", + attemptId: "attempt-budget-reached", + providerResponseId: "response-budget-reached", + turnId: "turn-budget-reached", + providerCalls: 1, + providerRequests: 1, + inputTokens: 70, + outputTokens: 20, + cachedInputTokens: 0, + reasoningTokens: 10, + costNanodollars: 10_000_000, + observedAt: "2026-09-01T00:00:00.000Z", + }, + ], + stateHistory: [], + workspaceDiffs: [], + }); + const source = RUNNER_LIVE_CANDIDATE_SLOTS[0]!.candidates[0]!; + const candidate = { + ...source, + budget: { + ...source.budget, + maxTotalTokens: 100, + maxCostUsd: 0.01, + }, + }; + const entry: RunnerLiveScheduleEntry = { + executionId: "candidate-budget-reached", + caseId: "steering-causality", + candidateId: candidate.id, + slotId: candidate.slotId, + repetition: 1, + providerTrace: "raw", + budget: candidate.budget, + }; + + const observation = await executeLiveRunnerWorkflow({ + entry, + candidate, + evalCase: runnerWorkflowCase(entry.caseId), + }); + + expect(liveSessionMocks.sendMessage).toHaveBeenCalledTimes(1); + expect(observation).toMatchObject({ + classification: "candidate_failure", + metrics: { totalTokens: 100, costUsd: 0.01 }, + failure: { + code: "candidate_budget_reached", + category: "candidate", + retryable: false, + }, + }); + expect(observation.lifecycle.checks).toEqual( + expect.arrayContaining([ + expect.objectContaining({ id: "token-budget", passed: true }), + expect.objectContaining({ id: "cost-budget", passed: true }), + expect.objectContaining({ + id: "candidate-budget-stop", + passed: false, + }), + ]), + ); + }); + + it("interrupts an active paid turn as soon as live usage reaches its budget", async () => { + const emptySnapshot = { + sessionId: "session-live-budget-test", + authority: {}, + mockState: JSON.stringify({ tasks: [] }), + transcript: [], + evidence: [], + authorizationRecords: [], + attempts: [], + usageLedger: [], + stateHistory: [], + workspaceDiffs: [], + }; + const exceededSnapshot = { + ...emptySnapshot, + usageLedger: [ + { + receiptId: "usage-live-budget-test", + attemptId: "attempt-live-budget-test", + providerResponseId: "response-live-budget-test", + turnId: "turn-live-budget-test", + providerCalls: 1, + providerRequests: 1, + inputTokens: 80, + outputTokens: 20, + cachedInputTokens: 0, + reasoningTokens: 0, + costNanodollars: 10_000_000, + observedAt: "2026-09-01T00:00:00.000Z", + }, + ], + }; + let interrupted = false; + let turnResolved = false; + let resolveTurn: + ((value: { status: string; turnId: string }) => void) | null = null; + liveSessionMocks.snapshot.mockImplementation(() => + interrupted ? exceededSnapshot : emptySnapshot, + ); + liveSessionMocks.sendMessage.mockImplementationOnce( + () => + new Promise((resolve) => { + resolveTurn = resolve; + queueMicrotask(() => { + const usageEvent = { + seq: 1, + at: "2026-09-01T00:00:00.000Z", + turnId: "turn-live-budget-test", + kind: "usage", + usage: { + providerRequests: 1, + inputTokens: 80, + outputTokens: 20, + cachedInputTokens: 0, + reasoningTokens: 0, + costNanodollars: 10_000_000, + }, + } as const; + liveSessionMocks.turnListener?.(usageEvent); + liveSessionMocks.turnListener?.({ ...usageEvent, seq: 2 }); + }); + }), + ); + liveSessionMocks.interrupt.mockImplementationOnce(async () => { + expect(turnResolved).toBe(false); + interrupted = true; + turnResolved = true; + resolveTurn?.({ + status: "interrupted", + turnId: "turn-live-budget-test", + }); + return exceededSnapshot; + }); + const source = RUNNER_LIVE_CANDIDATE_SLOTS[0]!.candidates[0]!; + const candidate = { + ...source, + budget: { + ...source.budget, + maxTotalTokens: 100, + maxCostUsd: 0.01, + }, + }; + const entry: RunnerLiveScheduleEntry = { + executionId: "candidate-live-budget-stop", + caseId: "steering-causality", + candidateId: candidate.id, + slotId: candidate.slotId, + repetition: 1, + providerTrace: "raw", + budget: candidate.budget, + }; + + const observation = await executeLiveRunnerWorkflow({ + entry, + candidate, + evalCase: runnerWorkflowCase(entry.caseId), + }); + + expect(liveSessionMocks.interrupt).toHaveBeenCalledTimes(1); + expect(liveSessionMocks.interrupt).toHaveBeenCalledWith( + expect.stringContaining("candidate reported usage budget stop"), + ); + expect(liveSessionMocks.sendMessage).toHaveBeenCalledTimes(1); + expect(observation).toMatchObject({ + classification: "candidate_failure", + metrics: { totalTokens: 100, costUsd: 0.01 }, + failure: { + code: "candidate_budget_reached", + category: "candidate", + retryable: false, + }, + }); + }); + + it("reattaches the budget interrupt before a restored-session continuation", async () => { + const emptySnapshot = { + sessionId: "session-restored-budget-test", + authority: {}, + mockState: JSON.stringify({ tasks: [] }), + transcript: [], + evidence: [], + authorizationRecords: [], + attempts: [], + usageLedger: [], + stateHistory: [], + workspaceDiffs: [], + }; + const reachedSnapshot = { + ...emptySnapshot, + usageLedger: [ + { + receiptId: "usage-restored-budget-test", + attemptId: "attempt-restored-budget-test", + providerResponseId: "response-restored-budget-test", + turnId: "turn-restored-budget-test", + providerCalls: 1, + providerRequests: 1, + inputTokens: 70, + outputTokens: 20, + cachedInputTokens: 0, + reasoningTokens: 10, + costNanodollars: 10_000_000, + observedAt: "2026-09-01T00:00:00.000Z", + }, + ], + }; + let interrupted = false; + let resolveRestoredTurn: + ((value: { status: string; turnId: string }) => void) | null = null; + liveSessionMocks.snapshot.mockImplementation(() => + interrupted ? reachedSnapshot : emptySnapshot, + ); + liveSessionMocks.sendMessage + .mockResolvedValueOnce({ + status: "completed", + turnId: "turn-before-restore", + }) + .mockImplementationOnce( + () => + new Promise((resolve) => { + resolveRestoredTurn = resolve; + queueMicrotask(() => { + liveSessionMocks.turnListener?.({ + seq: 1, + at: "2026-09-01T00:00:00.000Z", + turnId: "turn-restored-budget-test", + kind: "usage", + usage: { + providerRequests: 1, + inputTokens: 70, + outputTokens: 20, + cachedInputTokens: 0, + reasoningTokens: 10, + costNanodollars: 10_000_000, + }, + }); + }); + }), + ); + liveSessionMocks.interrupt.mockImplementationOnce(async () => { + interrupted = true; + resolveRestoredTurn?.({ + status: "interrupted", + turnId: "turn-restored-budget-test", + }); + return reachedSnapshot; + }); + const source = RUNNER_LIVE_CANDIDATE_SLOTS[0]!.candidates[0]!; + const candidate = { + ...source, + budget: { + ...source.budget, + maxTotalTokens: 100, + maxCostUsd: 0.01, + }, + }; + const entry: RunnerLiveScheduleEntry = { + executionId: "restored-session-budget-stop", + caseId: "restart-recovery", + candidateId: candidate.id, + slotId: candidate.slotId, + repetition: 1, + providerTrace: "raw", + budget: candidate.budget, + }; + + const observation = await executeLiveRunnerWorkflow({ + entry, + candidate, + evalCase: runnerWorkflowCase(entry.caseId), + }); + + expect(liveSessionMocks.restore).toHaveBeenCalledWith( + "session-shutdown-test", + ); + expect(liveSessionMocks.subscribedSessionIds).toEqual([ + "created", + "restored", + ]); + expect(liveSessionMocks.unsubscribedSessionIds).toEqual([ + "created", + "restored", + ]); + expect(liveSessionMocks.interrupt).toHaveBeenCalledTimes(1); + expect(liveSessionMocks.sendMessage).toHaveBeenCalledTimes(2); + expect(observation).toMatchObject({ + classification: "candidate_failure", + metrics: { totalTokens: 100, costUsd: 0.01 }, + failure: { + code: "candidate_budget_reached", + category: "candidate", + retryable: false, + }, + }); + }); + + it("keeps an exact-cap provider-terminal race classified as a budget failure", async () => { + const emptySnapshot = { + sessionId: "session-exact-cap-race", + authority: {}, + mockState: JSON.stringify({ tasks: [] }), + transcript: [], + evidence: [], + authorizationRecords: [], + attempts: [], + usageLedger: [], + stateHistory: [], + workspaceDiffs: [], + }; + const completedSnapshot = { + ...emptySnapshot, + mockState: JSON.stringify({ tasks: [{ id: "task-1", status: "done" }] }), + transcript: [ + { + role: "assistant", + text: "The requested work is complete.", + }, + ], + evidence: [ + { + id: "call-finish-task", + kind: "tool_call", + data: { operationId: "finish_task" }, + }, + { + id: "result-finish-task", + kind: "tool_result", + data: { operationId: "finish_task", result: { ok: true } }, + }, + ], + attempts: [{ attemptId: "attempt-exact-cap-race", status: "succeeded" }], + usageLedger: [ + { + receiptId: "usage-exact-cap-race", + attemptId: "attempt-exact-cap-race", + providerResponseId: "response-exact-cap-race", + turnId: "turn-exact-cap-race", + providerCalls: 1, + providerRequests: 1, + inputTokens: 70, + outputTokens: 20, + cachedInputTokens: 0, + reasoningTokens: 10, + costNanodollars: 10_000_000, + observedAt: "2026-09-01T00:00:00.000Z", + }, + ], + stateHistory: [{ revision: 1 }, { revision: 2 }], + }; + let providerCompleted = false; + liveSessionMocks.snapshot.mockImplementation(() => + providerCompleted ? completedSnapshot : emptySnapshot, + ); + liveSessionMocks.sendMessage.mockImplementationOnce(async () => { + liveSessionMocks.turnListener?.({ + seq: 1, + at: "2026-09-01T00:00:00.000Z", + turnId: "turn-exact-cap-race", + kind: "activity", + reason: "turn_started", + }); + liveSessionMocks.turnListener?.({ + seq: 2, + at: "2026-09-01T00:00:00.001Z", + turnId: "turn-exact-cap-race", + kind: "usage", + usage: { + providerRequests: 1, + inputTokens: 70, + outputTokens: 20, + cachedInputTokens: 0, + reasoningTokens: 10, + costNanodollars: 10_000_000, + }, + }); + // The terminal wins the race with the best-effort interrupt, but the + // already-observed exact-cap stop remains authoritative for scoring. + providerCompleted = true; + return { status: "completed", turnId: "turn-exact-cap-race" }; + }); + const source = RUNNER_LIVE_CANDIDATE_SLOTS[0]!.candidates[0]!; + const candidate = { + ...source, + budget: { + ...source.budget, + maxTotalTokens: 100, + maxCostUsd: 0.01, + }, + }; + const entry: RunnerLiveScheduleEntry = { + executionId: "candidate-exact-cap-race", + caseId: "final-response", + candidateId: candidate.id, + slotId: candidate.slotId, + repetition: 1, + providerTrace: "raw", + budget: candidate.budget, + }; + + const observation = await executeLiveRunnerWorkflow({ + entry, + candidate, + evalCase: runnerWorkflowCase(entry.caseId), + }); + + expect(liveSessionMocks.interrupt).toHaveBeenCalledTimes(1); + expect(observation).toMatchObject({ + classification: "candidate_failure", + metrics: { totalTokens: 100, costUsd: 0.01 }, + failure: { + code: "candidate_budget_reached", + category: "candidate", + retryable: false, + }, + }); + expect(observation.lifecycle.checks).toEqual( + expect.arrayContaining([ + expect.objectContaining({ id: "terminal-authority", passed: true }), + expect.objectContaining({ id: "token-budget", passed: true }), + expect.objectContaining({ id: "cost-budget", passed: true }), + expect.objectContaining({ id: "candidate-budget-stop", passed: false }), + expect.objectContaining({ id: "semantic-disposition", passed: true }), + ]), + ); + }); +}); diff --git a/packages/paperclip-runner/src/eval/live-workflow-executor.ts b/packages/paperclip-runner/src/eval/live-workflow-executor.ts new file mode 100644 index 0000000000..9448d613d5 --- /dev/null +++ b/packages/paperclip-runner/src/eval/live-workflow-executor.ts @@ -0,0 +1,911 @@ +import { createHash } from "node:crypto"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import type { CapabilityJsonValue } from "../mock-core/capability-control-plane-types.js"; +import { capabilityFixtureRunCapabilities } from "../scenarios/fixture-run-capabilities.js"; +import { + CapabilityLiveSessionService, + InMemoryCapabilityLiveSessionStore, + type CapabilityLiveSession, + type CapabilityLiveSessionSnapshot, + type CapabilityLiveTurnEvent, + type CapabilityLiveTurnResult, +} from "../live/live-session.js"; +import type { EvalObservation } from "./eval-scoring.js"; +import { + RUNNER_WORKFLOW_OBSERVATION_SCHEMA, + type RunnerWorkflowCheck, + type RunnerWorkflowEvalCase, + type RunnerWorkflowObservation, +} from "./workflow-contracts.js"; +import { + RunnerWorkflowInfrastructureError, + RUNNER_LIVE_CANDIDATE_SLOTS, + type RunnerLiveEvalCandidate, + type RunnerLiveScheduleEntry, +} from "./live-workflow-matrix.js"; + +const LIVE_GRANTS = [ + "delegation:tasks:create", + "dependencies:write", + "discovery:agents:read", + "discovery:tasks:read", + "governance:approvals:request", +] as const; + +function promptFor(evalCase: RunnerWorkflowEvalCase): string { + const common = + "Use only the typed Paperclip tools. Do not call REST APIs. Reply with concise user-facing prose after the final tool result."; + const prompts: Record = { + "final-response": + "Write a detailed answer of at least 1200 characters, then call finish_task exactly once with a faithful summary. Do not replace the answer with a completion acknowledgement.", + "rich-activity": + "Immediately report one short progress update, inspect the workspace with a read-only command or search if available, then call finish_task exactly once. Preserve visible tool activity.", + "verification-policy": + "Call finish_task exactly once. Include one verification entry that is not_run with reasonCode tool_unavailable and explain the caveat without treating it as a blocker.", + "governed-interaction": + "Call request_human_input once to ask a required single-select question. After the typed answer arrives, use it exactly and call finish_task once. Do not create a provider-native question.", + "steering-causality": + "A later message will narrow the requested format. On this turn provide a short draft and do not repeat work; after the follow-up, call finish_task exactly once.", + "planning-lifecycle": + "Write a plan document, then request human confirmation. Do not create implementation work until confirmation is accepted. If rejected, revise the plan and request confirmation again. After acceptance, treat the work as small and cohesive: implement on the source task, create no child, and call finish_task.", + "review-lifecycle": + "Request human confirmation for an external verification check. If rejected, address only the requested verification and ask again. After acceptance call finish_task exactly once.", + "delegation-return": + "Write a plan document and request confirmation. After acceptance, create one child only because an independently owned external verification boundary justifies it, set the source dependency to that child, and after its completion result arrives call finish_task exactly once. Do not create phase or file-based children.", + "completion-robustness": + "Call finish_task exactly once with canonical values. If the tool returns a schema error, correct the arguments in this turn. Do not print completion JSON as prose.", + "restart-recovery": + "Call report_progress exactly once, state that disposition will be completed after recovery, and do not call finish_task on this turn.", + "cancellation-permissions": + "Begin a read-only inspection and continue working until interrupted. Do not claim completion and do not call a semantic completion tool.", + "trace-lineage": + "Call finish_task exactly once with a concise summary so the semantic tool frame can be correlated through PRP and presentation.", + }; + return `${common} ${prompts[evalCase.id]}`; +} + +function continuationPrompt(evalCase: RunnerWorkflowEvalCase): string { + if (evalCase.id === "restart-recovery") { + return "The provider session was restored. Use the existing progress result, do not repeat it, and call finish_task exactly once."; + } + if (evalCase.id === "steering-causality") { + return "Steering update: use bullet points only. Apply only this formatting change and call finish_task exactly once without repeating prior work."; + } + return "Continue from the authoritative typed interaction result. Do not redo completed work."; +} + +function acpxAgent( + candidate: RunnerLiveEvalCandidate, +): "claude" | "codex" | undefined { + if (candidate.adapter !== "acpx_runtime") return undefined; + if (candidate.qualification.profile === "claude") return "claude"; + if (candidate.qualification.profile === "codex") return "codex"; + return undefined; +} + +function check( + id: string, + passed: boolean, + reason: string, + evidenceIds: string[] = [], +): RunnerWorkflowCheck { + return { + id, + passed, + ...(passed ? {} : { reason }), + ...(evidenceIds.length === 0 ? {} : { evidenceIds }), + }; +} + +function interactionResolution( + evalCase: RunnerWorkflowEvalCase, + index: number, +): { + outcome: "answered" | "accepted" | "rejected"; + result: CapabilityJsonValue; +} { + const decisions = evalCase.steps.flatMap((step) => + step.kind === "review_decision" ? [step.decision] : [], + ); + const decision = decisions[index]; + if (decision === "reject") + return { + outcome: "rejected", + result: { + note: "Eval reviewer requests the narrow verification-only revision.", + }, + }; + if (decision === "approve") + return { outcome: "accepted", result: { confirmed: true } }; + return { + outcome: "answered", + result: { selectedOptionIds: ["eval-choice"], answer: "eval-choice" }, + }; +} + +function providerEventTypes(snapshot: CapabilityLiveSessionSnapshot): string[] { + return snapshot.evidence.flatMap((entry) => { + if (entry.kind !== "provider_event") return []; + const value = entry.data.canonicalEventType; + return typeof value === "string" ? [value] : []; + }); +} + +function observedCalls(snapshot: CapabilityLiveSessionSnapshot): string[] { + return snapshot.evidence.flatMap((entry) => + entry.kind === "tool_call" && typeof entry.data.operationId === "string" + ? [entry.data.operationId] + : [], + ); +} + +function duplicateEffectSignals( + snapshot: CapabilityLiveSessionSnapshot, +): string[] { + return snapshot.evidence.flatMap((entry) => { + if (entry.kind !== "tool_result") return []; + const envelope = + typeof entry.data.result === "object" && + entry.data.result !== null && + !Array.isArray(entry.data.result) + ? (entry.data.result as Record) + : {}; + const result = + typeof envelope.result === "object" && + envelope.result !== null && + !Array.isArray(envelope.result) + ? (envelope.result as Record) + : {}; + return result.disposition === "duplicate" && + typeof entry.data.operationId === "string" + ? [entry.data.operationId] + : []; + }); +} + +function traceMetadata(raw: string): { + frameCount: number; + byteCount: number; + digest: string; + digestVerified: boolean; + ordered: boolean; + dispositions: string[]; + lineage: string[]; +} { + const records = raw + .split("\n") + .filter(Boolean) + .flatMap((line) => { + try { + return [JSON.parse(line) as Record]; + } catch { + return []; + } + }); + const frames = records.filter((entry) => entry.kind === "frame"); + const frameIds = frames.map((entry) => Number(entry.frameId)); + const debugSequences = records + .map((entry) => Number(entry.debugSequence)) + .filter(Number.isFinite); + const digestVerified = + frames.length > 0 && + frames.every((entry) => { + if ( + typeof entry.rawBase64 !== "string" || + typeof entry.digest !== "string" + ) + return false; + const bytes = Buffer.from(entry.rawBase64, "base64"); + return ( + bytes.byteLength === Number(entry.byteLength) && + `sha256:${createHash("sha256").update(bytes).digest("hex")}` === + entry.digest + ); + }); + const strictlyIncreasing = (values: number[]): boolean => + values.every( + (value, index) => + Number.isFinite(value) && (index === 0 || value > values[index - 1]!), + ); + const interpretations = records.filter( + (entry) => entry.kind === "interpretation", + ); + const emittedCounts = interpretations.map((entry) => + Array.isArray(entry.emittedEventIds) ? entry.emittedEventIds.length : 0, + ); + const eventParents = new Map(); + for (const interpretation of interpretations) { + if (!Array.isArray(interpretation.emittedEventIds)) continue; + for (const eventId of interpretation.emittedEventIds) { + if (typeof eventId === "string") + eventParents.set(eventId, (eventParents.get(eventId) ?? 0) + 1); + } + } + const lineage = new Set(); + if (interpretations.length > 0) lineage.add("one_to_one"); + if (emittedCounts.some((count) => count > 1)) lineage.add("one_to_many"); + if ([...eventParents.values()].some((count) => count > 1)) + lineage.add("many_to_one"); + return { + frameCount: frames.length, + byteCount: frames.reduce( + (sum, entry) => sum + (Number(entry.byteLength) || 0), + 0, + ), + digest: `sha256:${createHash("sha256").update(raw).digest("hex")}`, + digestVerified, + ordered: strictlyIncreasing(frameIds) && strictlyIncreasing(debugSequences), + dispositions: [ + ...new Set( + interpretations.flatMap((entry) => + typeof entry.disposition === "string" ? [entry.disposition] : [], + ), + ), + ], + lineage: [...lineage], + }; +} + +function digestJson(value: unknown): string { + return `sha256:${createHash("sha256").update(JSON.stringify(value)).digest("hex")}`; +} + +function safeFailureMessage(error: unknown): string { + const message = error instanceof Error ? error.message : String(error); + return message + .replace( + /\b(?:sk-[A-Za-z0-9_-]{8,}|Bearer\s+\S+|AKIA[0-9A-Z]{16})\b/gi, + "[REDACTED]", + ) + .slice(0, 1_000); +} + +const LIVE_PROVIDER_CREDENTIAL_ENVIRONMENT = new Set( + RUNNER_LIVE_CANDIDATE_SLOTS.flatMap((slot) => + slot.candidates.flatMap( + (candidate) => candidate.qualification.requiredEnvironment, + ), + ), +); +const CREDENTIAL_ENVIRONMENT_NAME = + /(?:^|_)(?:API_KEY|ACCESS_KEY|AUTH(?:ORIZATION)?|COOKIE|SECRET|SESSION|TOKEN|PASSWORD|CREDENTIALS?)(?:$|_)/; + +function candidateTransportEnvironment( + candidate: RunnerLiveEvalCandidate, + tracePath: string, + source: NodeJS.ProcessEnv = process.env, +): NodeJS.ProcessEnv { + const required = new Set(candidate.qualification.requiredEnvironment); + return { + ...Object.fromEntries( + Object.entries(source).filter( + ([name, value]) => + typeof value === "string" && + ((!LIVE_PROVIDER_CREDENTIAL_ENVIRONMENT.has(name) && + !CREDENTIAL_ENVIRONMENT_NAME.test(name)) || + required.has(name)), + ), + ), + PAPERCLIP_PROVIDER_TRACE_PATH: tracePath, + }; +} + +function usageTotals(snapshot: CapabilityLiveSessionSnapshot | undefined): { + totalTokens: number; + costUsd: number; +} { + const usage = snapshot?.usageLedger ?? []; + return { + totalTokens: usage.reduce( + (sum, receipt) => + sum + + receipt.inputTokens + + receipt.outputTokens + + receipt.reasoningTokens, + 0, + ), + costUsd: + usage.reduce((sum, receipt) => sum + receipt.costNanodollars, 0) / + 1_000_000_000, + }; +} + +function candidateBudgetViolations( + candidate: RunnerLiveEvalCandidate, + snapshot: CapabilityLiveSessionSnapshot | undefined, +): string[] { + const usage = usageTotals(snapshot); + const violations: string[] = []; + if (usage.totalTokens > candidate.budget.maxTotalTokens) { + violations.push( + `totalTokens ${usage.totalTokens} exceeds budget ${candidate.budget.maxTotalTokens}`, + ); + } + if (usage.costUsd > candidate.budget.maxCostUsd) { + violations.push( + `costUsd ${usage.costUsd} exceeds budget ${candidate.budget.maxCostUsd}`, + ); + } + return violations; +} + +function candidateBudgetThresholdsReached( + candidate: RunnerLiveEvalCandidate, + snapshot: CapabilityLiveSessionSnapshot | undefined, +): string[] { + const usage = usageTotals(snapshot); + const reached: string[] = []; + if (usage.totalTokens >= candidate.budget.maxTotalTokens) { + reached.push( + `totalTokens ${usage.totalTokens} reached budget ${candidate.budget.maxTotalTokens}`, + ); + } + if (usage.costUsd >= candidate.budget.maxCostUsd) { + reached.push( + `costUsd ${usage.costUsd} reached budget ${candidate.budget.maxCostUsd}`, + ); + } + return reached; +} + +function liveCandidateBudgetStopReason( + candidate: RunnerLiveEvalCandidate, + snapshot: CapabilityLiveSessionSnapshot, + usage: Extract["usage"], +): string | null { + const committed = usageTotals(snapshot); + const totalTokens = + committed.totalTokens + + usage.inputTokens + + usage.outputTokens + + usage.reasoningTokens; + const costUsd = committed.costUsd + usage.costNanodollars / 1_000_000_000; + const reached: string[] = []; + if (totalTokens >= candidate.budget.maxTotalTokens) { + reached.push( + `totalTokens ${totalTokens} reached budget ${candidate.budget.maxTotalTokens}`, + ); + } + if (costUsd >= candidate.budget.maxCostUsd) { + reached.push( + `costUsd ${costUsd} reached budget ${candidate.budget.maxCostUsd}`, + ); + } + return reached.length === 0 + ? null + : `candidate reported usage budget stop: ${reached.join("; ")}`; +} + +/** Builds an unscored observation without exposing provider credentials or raw trace payloads. */ +export function unavailableLiveRunnerWorkflowObservation(input: { + entry: RunnerLiveScheduleEntry; + candidate: RunnerLiveEvalCandidate; + evalCase: RunnerWorkflowEvalCase; + classification: "infrastructure_failure" | "skipped"; + code: string; + category: "provider" | "qualification" | "orchestration"; + retryable: boolean; + message: string; +}): RunnerWorkflowObservation { + const unavailable = check("execution-unavailable", false, input.message); + return { + schema: RUNNER_WORKFLOW_OBSERVATION_SCHEMA, + caseId: input.evalCase.id, + candidateId: input.candidate.id, + provider: input.candidate.provider, + classification: input.classification, + base: { + caseId: input.evalCase.id, + provenance: { source: "live_model", behavior: input.evalCase.id }, + controlPlaneOwned: false, + expectedCalls: input.evalCase.assertions.requiredOperationIds ?? [], + observedCalls: [], + forbiddenCalls: [], + finalState: { expected: "mutated", observed: "unchanged" }, + authorization: { expected: "allowed", observed: "absent" }, + trace: { terminalPresent: false, receiptIds: [] }, + efficiency: { latencyMs: 0, totalTokens: 0, costUsd: 0, attempts: 0 }, + budget: input.candidate.budget, + }, + lifecycle: { + checks: [unavailable], + attempts: 0, + runs: 0, + recoveryOwner: "none", + }, + continuation: { checks: [unavailable], repeatedWorkSignals: [] }, + presentation: { + checks: [unavailable], + responseSource: "none", + commentCount: 0, + }, + traceLineage: { + capture: "off", + frameCount: 0, + byteCount: 0, + digestVerified: false, + ordered: false, + dispositions: [], + lineage: [], + }, + metrics: { + settlementMs: 0, + attempts: 0, + toolCount: 0, + totalTokens: 0, + costUsd: 0, + }, + observedPrpEventTypes: [], + artifactDigests: [], + failure: { + code: input.code, + category: input.category, + retryable: input.retryable, + message: safeFailureMessage(input.message), + }, + }; +} + +async function settleInteractions( + evalCase: RunnerWorkflowEvalCase, + session: CapabilityLiveSession, + turns: CapabilityLiveTurnResult[], + withinBudget: () => boolean, +): Promise { + let index = 0; + while ( + session.pendingInteractions().length > 0 && + index < 6 && + withinBudget() + ) { + const pending = session.pendingInteractions()[0]!; + const resolution = interactionResolution(evalCase, index); + turns.push( + await session.resolveInteraction({ + interactionId: pending.id, + ...resolution, + }), + ); + index += 1; + } +} + +/** Executes one real-provider workflow against the controlled mock authority. */ +export async function executeLiveRunnerWorkflow(input: { + entry: RunnerLiveScheduleEntry; + candidate: RunnerLiveEvalCandidate; + evalCase: RunnerWorkflowEvalCase; + workingDirectory?: string; +}): Promise { + const runtimeRoot = await mkdtemp( + join(tmpdir(), "paperclip-runner-live-eval-"), + ); + const tracePath = join(runtimeRoot, "provider-trace.ndjson"); + const store = new InMemoryCapabilityLiveSessionStore(); + const transportOptions = { + environment: candidateTransportEnvironment(input.candidate, tracePath), + }; + let service = new CapabilityLiveSessionService({ store, transportOptions }); + let session: CapabilityLiveSession | null = null; + const turns: CapabilityLiveTurnResult[] = []; + const streamed: CapabilityLiveTurnEvent[] = []; + const startedAt = Date.now(); + let firstVisibleAt: number | null = null; + let infrastructureError: unknown; + let budgetInterrupt: Promise | null = null; + let budgetInterruptError: unknown; + let budgetStopReason: string | null = null; + let unsubscribe: () => void = () => undefined; + const withinBudget = (): boolean => + budgetStopReason === null && + candidateBudgetThresholdsReached(input.candidate, session?.snapshot()) + .length === 0; + const subscribeToSession = (activeSession: CapabilityLiveSession): void => { + unsubscribe(); + unsubscribe = activeSession.subscribe((event) => { + streamed.push(event); + if ( + firstVisibleAt === null && + (event.kind === "delta" || event.kind === "activity") + ) + firstVisibleAt = Date.now(); + if ( + event.kind === "usage" && + budgetInterrupt === null && + session === activeSession + ) { + const reason = liveCandidateBudgetStopReason( + input.candidate, + activeSession.snapshot(), + event.usage, + ); + if (reason !== null) { + budgetStopReason = reason; + budgetInterrupt = activeSession.interrupt(reason).then( + () => undefined, + (error: unknown) => { + budgetInterruptError = error; + }, + ); + } + } + }); + }; + try { + session = await service.create({ + workingDirectory: input.workingDirectory ?? runtimeRoot, + provider: input.candidate.provider, + ...(acpxAgent(input.candidate) === undefined + ? {} + : { acpxAgent: acpxAgent(input.candidate) }), + requestedModel: input.candidate.model, + seed: { + actors: [ + { + id: "actor-1", + companyId: "company-1", + name: "Workflow eval actor", + role: "engineer", + status: "active", + budgetId: "budget-actor-1", + capabilityGrants: [...LIVE_GRANTS], + }, + ], + }, + capabilities: capabilityFixtureRunCapabilities(LIVE_GRANTS), + explicitClaims: [...LIVE_GRANTS], + runId: input.entry.executionId, + sessionId: `session-${input.entry.executionId}`, + attemptId: `attempt-${input.entry.executionId}`, + turnTimeoutMs: input.candidate.budget.maxLatencyMs, + lifecyclePolicy: { mode: "warm", idleTimeoutMs: 300_000 }, + scenario: { id: `runner-workflow-${input.evalCase.id}` }, + }); + subscribeToSession(session); + try { + if (input.evalCase.id === "cancellation-permissions") { + const pending = session.sendMessage(promptFor(input.evalCase)); + await new Promise((resolve) => setTimeout(resolve, 250)); + await session.interrupt("workflow eval cancellation"); + const settled = await Promise.allSettled([pending]); + if (settled[0]?.status === "fulfilled") turns.push(settled[0].value); + } else { + turns.push(await session.sendMessage(promptFor(input.evalCase))); + await settleInteractions(input.evalCase, session, turns, withinBudget); + if (input.evalCase.id === "steering-causality" && withinBudget()) { + turns.push( + await session.sendMessage(continuationPrompt(input.evalCase)), + ); + } + if (input.evalCase.id === "restart-recovery" && withinBudget()) { + const sessionId = session.id; + await session.suspend("workflow eval simulated worker restart"); + service = new CapabilityLiveSessionService({ + store, + transportOptions, + }); + session = await service.restore(sessionId); + subscribeToSession(session); + turns.push( + await session.sendMessage(continuationPrompt(input.evalCase)), + ); + } + } + } finally { + unsubscribe(); + if (budgetInterrupt !== null) await budgetInterrupt; + if (budgetInterruptError !== undefined) throw budgetInterruptError; + } + } catch (error) { + infrastructureError = error; + } + + const snapshot = session?.snapshot(); + if (session !== null) { + try { + await service.shutdown(session.id, "Runner live workflow eval complete"); + } catch (error) { + infrastructureError ??= error; + } + } + if (infrastructureError !== undefined) { + await rm(runtimeRoot, { recursive: true, force: true }); + throw new RunnerWorkflowInfrastructureError( + "live_provider_execution_failed", + true, + safeFailureMessage(infrastructureError), + ); + } + const calls = snapshot === undefined ? [] : observedCalls(snapshot); + const expectedCalls = input.evalCase.assertions.requiredOperationIds ?? []; + const taskState = + snapshot === undefined + ? null + : (( + JSON.parse(snapshot.mockState) as { + tasks?: Array<{ id: string; status: string }>; + } + ).tasks?.find((task) => task.id === "task-1") ?? null); + const terminalStatuses = turns.map((turn) => turn.status); + const cancellationExpected = input.evalCase.id === "cancellation-permissions"; + const terminalOkay = cancellationExpected + ? terminalStatuses.some( + (status) => status === "cancelled" || status === "interrupted", + ) + : terminalStatuses.length > 0 && + terminalStatuses.every((status) => status === "completed"); + const missingCalls = expectedCalls.filter( + (operationId) => !calls.includes(operationId), + ); + const extraCalls = calls.filter( + (operationId) => !expectedCalls.includes(operationId), + ); + const duplicateSignals = + snapshot === undefined ? [] : duplicateEffectSignals(snapshot); + const pendingInteractions = session?.pendingInteractions().length ?? 0; + let rawTrace = ""; + try { + rawTrace = await readFile(tracePath, "utf8"); + } catch { + rawTrace = ""; + } + const trace = traceMetadata(rawTrace); + const { totalTokens, costUsd } = usageTotals(snapshot); + const budgetViolations = candidateBudgetViolations(input.candidate, snapshot); + const budgetThresholdsReached = candidateBudgetThresholdsReached( + input.candidate, + snapshot, + ); + const budgetFailure = + budgetViolations.length > 0 + ? { + code: "candidate_budget_exceeded", + message: budgetViolations.join("; "), + } + : budgetStopReason !== null + ? { + code: "candidate_budget_reached", + message: budgetStopReason, + } + : budgetThresholdsReached.length > 0 + ? { + code: "candidate_budget_reached", + message: budgetThresholdsReached.join("; "), + } + : null; + const receiptIds = + snapshot?.evidence + .filter((entry) => entry.kind === "tool_result") + .map((entry) => entry.id) ?? []; + const assistantTexts = + snapshot?.transcript + .filter( + (entry) => entry.role === "assistant" && entry.text.trim().length > 0, + ) + .map((entry) => entry.text) ?? []; + const stateMutated = (snapshot?.stateHistory?.length ?? 0) > 1; + const base: EvalObservation = { + caseId: input.evalCase.id, + provenance: { source: "live_model", behavior: input.evalCase.id }, + controlPlaneOwned: expectedCalls.length === 0, + expectedCalls, + observedCalls: calls, + forbiddenCalls: [], + finalState: { + expected: expectedCalls.length === 0 ? "unchanged" : "mutated", + observed: stateMutated ? "mutated" : "unchanged", + }, + authorization: { + expected: expectedCalls.length === 0 ? "absent" : "allowed", + observed: + calls.length === 0 + ? "absent" + : snapshot?.authorizationRecords.some( + (record) => record.phase === "invocation" && !record.allowed, + ) + ? "denied" + : "allowed", + }, + trace: { + runId: snapshot?.authority.runId, + sessionId: snapshot?.sessionId, + turnId: turns.at(-1)?.turnId, + itemId: snapshot?.evidence.find((entry) => entry.kind === "tool_call") + ?.id, + receiptIds, + terminalPresent: terminalStatuses.length > 0, + }, + efficiency: { + latencyMs: Date.now() - startedAt, + totalTokens, + costUsd, + attempts: snapshot?.attempts?.length ?? 1, + }, + budget: { + maxLatencyMs: input.candidate.budget.maxLatencyMs, + maxTotalTokens: input.candidate.budget.maxTotalTokens, + maxCostUsd: input.candidate.budget.maxCostUsd, + maxAttempts: input.candidate.budget.maxAttempts, + }, + }; + const lifecycleChecks = [ + check( + "terminal-authority", + terminalOkay, + `unexpected terminal statuses: ${terminalStatuses.join(",") || "none"}`, + ), + check( + "attempt-bound", + (snapshot?.attempts?.length ?? 1) <= input.candidate.budget.maxAttempts, + "attempt budget exceeded", + ), + check( + "token-budget", + totalTokens <= input.candidate.budget.maxTotalTokens, + `totalTokens ${totalTokens} exceeds budget ${input.candidate.budget.maxTotalTokens}`, + ), + check( + "cost-budget", + costUsd <= input.candidate.budget.maxCostUsd, + `costUsd ${costUsd} exceeds budget ${input.candidate.budget.maxCostUsd}`, + ), + check( + "candidate-budget-stop", + budgetFailure === null, + budgetFailure?.message ?? "candidate budget remained available", + ), + check( + "semantic-disposition", + cancellationExpected || + calls.includes("finish_task") || + calls.includes("request_review"), + "no authoritative semantic disposition", + ), + check( + "owned-wait", + pendingInteractions === 0, + `${pendingInteractions} governed interaction(s) remain pending`, + ), + ]; + const continuationChecks = [ + check( + "required-calls", + missingCalls.length === 0, + `missing required calls: ${missingCalls.join(", ")}`, + ), + check( + "trajectory-restraint", + extraCalls.length === 0, + `unexpected calls: ${extraCalls.join(", ")}`, + ), + check( + "no-repeated-work", + input.evalCase.id === "restart-recovery" || duplicateSignals.length === 0, + `duplicate semantic effects detected: ${[...new Set(duplicateSignals)].join(", ")}`, + ), + ]; + const presentationChecks = [ + check( + "first-visible-progress", + cancellationExpected || firstVisibleAt !== null, + "provider emitted no visible progress", + ), + check( + "substantive-response", + cancellationExpected || + assistantTexts.some((text) => text.trim().length >= 2), + "provider emitted no user-facing response", + ), + check( + "no-empty-comment", + assistantTexts.every((text) => text.trim().length > 0), + "empty assistant output was retained", + ), + check( + "terminal-presentation", + terminalOkay, + "terminal presentation would remain unsettled", + ), + ]; + const checksPassed = [ + ...lifecycleChecks, + ...continuationChecks, + ...presentationChecks, + ].every((entry) => entry.passed); + const observation: RunnerWorkflowObservation = { + schema: RUNNER_WORKFLOW_OBSERVATION_SCHEMA, + caseId: input.evalCase.id, + candidateId: input.candidate.id, + provider: input.candidate.provider, + classification: + checksPassed && budgetFailure === null + ? "completed" + : "candidate_failure", + base, + lifecycle: { + checks: lifecycleChecks, + issueStatus: taskState?.status, + runStatus: terminalStatuses.at(-1), + semanticDisposition: calls.includes("finish_task") + ? "done" + : calls.includes("request_review") + ? "needs_review" + : undefined, + attempts: snapshot?.attempts?.length ?? 1, + runs: turns.length, + recoveryOwner: pendingInteractions > 0 ? "human" : "none", + }, + continuation: { + checks: continuationChecks, + wakeReasons: + snapshot?.evidence + .filter((entry) => entry.kind === "interaction") + .map(() => "interaction_resolved") ?? [], + consumedInputIds: + snapshot?.evidence + .filter((entry) => entry.kind === "interaction") + .map((entry) => entry.id) ?? [], + sessionPolicy: + input.evalCase.id === "restart-recovery" + ? "same_session" + : "same_session", + repeatedWorkSignals: [...new Set([...extraCalls, ...duplicateSignals])], + }, + presentation: { + checks: presentationChecks, + responseSource: + assistantTexts.length > 0 ? "final_agent_message" : "none", + commentCount: assistantTexts.length > 0 ? 1 : 0, + orderedMarkers: streamed.map((event) => event.kind), + visibleActivityFamilies: + snapshot === undefined + ? [] + : providerEventTypes(snapshot).map( + (eventType) => eventType.split(".")[0]!, + ), + terminalLabel: cancellationExpected + ? "Stopped" + : terminalOkay + ? "Completed" + : "Needs attention", + }, + traceLineage: { + capture: rawTrace.length > 0 ? "on" : "off", + frameCount: trace.frameCount, + byteCount: trace.byteCount, + digestVerified: trace.digestVerified, + ordered: trace.ordered, + dispositions: trace.dispositions, + lineage: trace.lineage, + ...(rawTrace.length > 0 ? { traceRef: trace.digest } : {}), + }, + metrics: { + timeToFirstVisibleProgressMs: + firstVisibleAt === null ? undefined : firstVisibleAt - startedAt, + settlementMs: Date.now() - startedAt, + attempts: snapshot?.attempts?.length ?? 1, + toolCount: calls.length, + totalTokens, + costUsd, + }, + observedPrpEventTypes: + snapshot === undefined ? [] : providerEventTypes(snapshot), + artifactDigests: + snapshot?.workspaceDiffs?.map((entry) => digestJson(entry.diff)) ?? [], + ...(budgetFailure === null + ? {} + : { + failure: { + code: budgetFailure.code, + category: "candidate" as const, + retryable: false, + message: budgetFailure.message, + }, + }), + }; + await rm(runtimeRoot, { recursive: true, force: true }); + return observation; +} diff --git a/packages/paperclip-runner/src/eval/live-workflow-matrix.ts b/packages/paperclip-runner/src/eval/live-workflow-matrix.ts new file mode 100644 index 0000000000..c491bbe622 --- /dev/null +++ b/packages/paperclip-runner/src/eval/live-workflow-matrix.ts @@ -0,0 +1,515 @@ +import { createHash } from "node:crypto"; + +import type { + RunnerWorkflowEvalCase, + RunnerWorkflowObservation, + RunnerWorkflowProvider, +} from "./workflow-contracts.js"; +import { RUNNER_WORKFLOW_CATALOG } from "./workflow-catalog.js"; + +export const RUNNER_LIVE_CANDIDATE_SCHEMA = + "paperclip.runner.live-eval-candidate.v1" as const; +export const RUNNER_LIVE_SCHEDULE_SCHEMA = + "paperclip.runner.live-eval-schedule.v1" as const; + +export function parseRunnerLiveCampaignCostLimit( + value: string | undefined, +): number { + const parsed = Number(value ?? "12"); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error( + "PAPERCLIP_EVAL_MAX_CAMPAIGN_COST_USD must be a positive finite number", + ); + } + return parsed; +} + +export function runnerLiveRotationWeek(generatedAt: string): number { + const epochMs = Date.parse(generatedAt); + if (!Number.isFinite(epochMs)) { + throw new Error("Runner live rotation requires a valid generated-at time"); + } + const epochWeek = Math.floor(epochMs / (7 * 86_400_000)); + return ((epochWeek % 7) + 7) % 7; +} + +export type RunnerLiveAdapter = + "codex_app_server" | "opencode_server" | "acpx_runtime"; +export type RunnerLiveTier = "strong" | "inexpensive"; + +export interface RunnerLiveEvalCandidate { + schema: typeof RUNNER_LIVE_CANDIDATE_SCHEMA; + id: string; + slotId: string; + adapter: RunnerLiveAdapter; + provider: RunnerWorkflowProvider; + model: string; + tier: RunnerLiveTier; + reasoningEffort?: string; + qualification: { requiredEnvironment: string[]; profile?: string }; + budget: { + maxLatencyMs: number; + maxAttempts: number; + maxCostUsd: number; + maxTotalTokens: number; + }; +} + +interface RunnerLiveCandidateSlot { + id: string; + candidates: readonly RunnerLiveEvalCandidate[]; +} + +function candidate( + input: Omit, +): RunnerLiveEvalCandidate { + return Object.freeze({ schema: RUNNER_LIVE_CANDIDATE_SCHEMA, ...input }); +} + +export const RUNNER_LIVE_CANDIDATE_SLOTS: readonly RunnerLiveCandidateSlot[] = + Object.freeze([ + { + id: "codex-strong", + candidates: [ + candidate({ + id: "codex-luna", + slotId: "codex-strong", + adapter: "codex_app_server", + provider: "codex", + model: "gpt-5.6-luna", + tier: "strong", + reasoningEffort: "medium", + qualification: { requiredEnvironment: ["OPENAI_API_KEY"] }, + budget: { + maxLatencyMs: 180_000, + maxAttempts: 3, + maxCostUsd: 1.5, + maxTotalTokens: 100_000, + }, + }), + ], + }, + { + id: "opencode-strong", + candidates: [ + candidate({ + id: "opencode-kimi", + slotId: "opencode-strong", + adapter: "opencode_server", + provider: "opencode", + model: "openrouter/moonshotai/kimi-k2.5", + tier: "strong", + qualification: { + requiredEnvironment: ["OPENROUTER_API_KEY"], + profile: "openrouter", + }, + budget: { + maxLatencyMs: 180_000, + maxAttempts: 3, + maxCostUsd: 0.8, + maxTotalTokens: 100_000, + }, + }), + candidate({ + id: "opencode-glm", + slotId: "opencode-strong", + adapter: "opencode_server", + provider: "opencode", + model: "openrouter/z-ai/glm-4.6", + tier: "strong", + qualification: { + requiredEnvironment: ["OPENROUTER_API_KEY"], + profile: "openrouter", + }, + budget: { + maxLatencyMs: 180_000, + maxAttempts: 3, + maxCostUsd: 0.8, + maxTotalTokens: 100_000, + }, + }), + ], + }, + { + id: "opencode-inexpensive", + candidates: [ + candidate({ + id: "opencode-gpt-oss", + slotId: "opencode-inexpensive", + adapter: "opencode_server", + provider: "opencode", + model: "openrouter/openai/gpt-oss-20b", + tier: "inexpensive", + qualification: { + requiredEnvironment: ["OPENROUTER_API_KEY"], + profile: "openrouter", + }, + budget: { + maxLatencyMs: 150_000, + maxAttempts: 3, + maxCostUsd: 0.2, + maxTotalTokens: 64_000, + }, + }), + candidate({ + id: "opencode-glm-flash", + slotId: "opencode-inexpensive", + adapter: "opencode_server", + provider: "opencode", + model: "openrouter/z-ai/glm-4.5-air", + tier: "inexpensive", + qualification: { + requiredEnvironment: ["OPENROUTER_API_KEY"], + profile: "openrouter", + }, + budget: { + maxLatencyMs: 150_000, + maxAttempts: 3, + maxCostUsd: 0.2, + maxTotalTokens: 64_000, + }, + }), + ], + }, + { + id: "acpx-claude", + candidates: [ + candidate({ + id: "acpx-claude-sonnet", + slotId: "acpx-claude", + adapter: "acpx_runtime", + provider: "acpx", + model: "claude-sonnet-5", + tier: "strong", + qualification: { + requiredEnvironment: ["ANTHROPIC_API_KEY"], + profile: "claude", + }, + budget: { + maxLatencyMs: 180_000, + maxAttempts: 3, + maxCostUsd: 1.5, + maxTotalTokens: 100_000, + }, + }), + ], + }, + { + id: "acpx-codex", + candidates: [ + candidate({ + id: "acpx-codex-sol", + slotId: "acpx-codex", + adapter: "acpx_runtime", + provider: "acpx", + model: "gpt-5.6-sol", + tier: "strong", + reasoningEffort: "medium", + qualification: { + requiredEnvironment: ["OPENAI_API_KEY"], + profile: "codex", + }, + budget: { + maxLatencyMs: 180_000, + maxAttempts: 3, + maxCostUsd: 1.5, + maxTotalTokens: 100_000, + }, + }), + ], + }, + ]); + +const CORE_LIVE_REPEAT_CASE_IDS = new Set([ + "final-response", + "rich-activity", + "governed-interaction", + "completion-robustness", +]); + +export interface RunnerLiveScheduleEntry { + executionId: string; + caseId: RunnerWorkflowEvalCase["id"]; + candidateId: string; + slotId: string; + repetition: number; + providerTrace: "raw"; + budget: RunnerLiveEvalCandidate["budget"]; +} + +export interface RunnerLiveEvalSchedule { + schema: typeof RUNNER_LIVE_SCHEDULE_SCHEMA; + seed: string; + rotationDay: number; + generatedAt: string; + candidates: RunnerLiveEvalCandidate[]; + entries: RunnerLiveScheduleEntry[]; + expectedExecutions: number; +} + +export function assertRunnerLiveCandidateManifest(): void { + const slots = RUNNER_LIVE_CANDIDATE_SLOTS.map((slot) => slot.id); + const candidates = RUNNER_LIVE_CANDIDATE_SLOTS.flatMap( + (slot) => slot.candidates, + ); + const ids = candidates.map((entry) => entry.id); + if (new Set(slots).size !== 5 || new Set(ids).size !== ids.length) { + throw new Error( + "Runner live candidate slots or candidate ids are not unique", + ); + } + const serialized = JSON.stringify(candidates); + if ( + /\b(?:sk-[A-Za-z0-9]{16,}|Bearer\s+\S{16,}|AKIA[0-9A-Z]{16})\b/.test( + serialized, + ) + ) { + throw new Error( + "Runner live candidate manifest contains secret-shaped data", + ); + } + for (const slot of RUNNER_LIVE_CANDIDATE_SLOTS) { + if ( + slot.candidates.some( + (entry) => entry.slotId !== slot.id || entry.model.trim().length === 0, + ) + ) { + throw new Error(`Runner live candidate slot ${slot.id} is invalid`); + } + } +} + +export function resolvedNightlyCandidates( + rotationDay: number, +): RunnerLiveEvalCandidate[] { + assertRunnerLiveCandidateManifest(); + const normalizedDay = ((rotationDay % 7) + 7) % 7; + return RUNNER_LIVE_CANDIDATE_SLOTS.map( + (slot) => slot.candidates[normalizedDay % slot.candidates.length]!, + ); +} + +function digestId(value: string): string { + return createHash("sha256").update(value).digest("hex").slice(0, 16); +} + +/** Builds the stable 40-execution pairwise schedule for one weekly rotation. */ +export function buildRunnerLiveEvalSchedule(input: { + seed: string; + rotationDay: number; + generatedAt?: string; + cases?: readonly RunnerWorkflowEvalCase[]; +}): RunnerLiveEvalSchedule { + const cases = input.cases ?? RUNNER_WORKFLOW_CATALOG; + const candidates = resolvedNightlyCandidates(input.rotationDay); + const entries: RunnerLiveScheduleEntry[] = []; + for (const [caseIndex, evalCase] of cases.entries()) { + const firstIndex = (caseIndex + input.rotationDay) % candidates.length; + const secondIndex = (firstIndex + 3) % candidates.length; + for (const candidateIndex of [firstIndex, secondIndex]) { + const resolved = candidates[candidateIndex]!; + const repetitions = CORE_LIVE_REPEAT_CASE_IDS.has(evalCase.id) ? 3 : 1; + for (let repetition = 1; repetition <= repetitions; repetition += 1) { + const identity = `${input.seed}:${input.rotationDay}:${evalCase.id}:${resolved.id}:${repetition}`; + entries.push({ + executionId: `rwe-${digestId(identity)}`, + caseId: evalCase.id, + candidateId: resolved.id, + slotId: resolved.slotId, + repetition, + providerTrace: "raw", + budget: resolved.budget, + }); + } + } + } + return { + schema: RUNNER_LIVE_SCHEDULE_SCHEMA, + seed: input.seed, + rotationDay: ((input.rotationDay % 7) + 7) % 7, + generatedAt: input.generatedAt ?? new Date().toISOString(), + candidates, + entries, + expectedExecutions: entries.length, + }; +} + +export interface RunnerLiveScheduleCoverage { + everySlotNightly: boolean; + everyConcreteCandidateInRotation: boolean; + everyWorkflowCoversEverySlot: boolean; + strongAndInexpensiveNightly: boolean; +} + +export function runnerLiveScheduleCoverage( + seed = "runner-live-seven-week-v1", +): RunnerLiveScheduleCoverage { + const schedules = Array.from({ length: 7 }, (_, rotationDay) => + buildRunnerLiveEvalSchedule({ + seed, + rotationDay, + generatedAt: `2026-08-${String(24 + rotationDay).padStart(2, "0")}T00:00:00.000Z`, + }), + ); + const allCandidateIds = new Set( + RUNNER_LIVE_CANDIDATE_SLOTS.flatMap((slot) => + slot.candidates.map((entry) => entry.id), + ), + ); + const seenCandidateIds = new Set( + schedules.flatMap((schedule) => + schedule.entries.map((entry) => entry.candidateId), + ), + ); + const everySlotNightly = schedules.every( + (schedule) => + new Set(schedule.entries.map((entry) => entry.slotId)).size === + RUNNER_LIVE_CANDIDATE_SLOTS.length, + ); + const everyWorkflowCoversEverySlot = RUNNER_WORKFLOW_CATALOG.every( + (evalCase) => { + const slots = new Set( + schedules.flatMap((schedule) => + schedule.entries + .filter((entry) => entry.caseId === evalCase.id) + .map((entry) => entry.slotId), + ), + ); + return slots.size === RUNNER_LIVE_CANDIDATE_SLOTS.length; + }, + ); + const strongAndInexpensiveNightly = schedules.every((schedule) => { + const selected = schedule.candidates.filter((candidate) => + schedule.entries.some((entry) => entry.candidateId === candidate.id), + ); + return ( + selected.some((candidate) => candidate.tier === "strong") && + selected.some((candidate) => candidate.tier === "inexpensive") + ); + }); + return { + everySlotNightly, + everyConcreteCandidateInRotation: [...allCandidateIds].every((id) => + seenCandidateIds.has(id), + ), + everyWorkflowCoversEverySlot, + strongAndInexpensiveNightly, + }; +} + +export class RunnerWorkflowInfrastructureError extends Error { + constructor( + readonly code: string, + readonly retryable: boolean, + message: string, + ) { + super(message); + this.name = "RunnerWorkflowInfrastructureError"; + } +} + +export async function executeRunnerLiveSchedule( + schedule: RunnerLiveEvalSchedule, + execute: ( + entry: RunnerLiveScheduleEntry, + candidate: RunnerLiveEvalCandidate, + ) => Promise, + onInfrastructureFailure?: ( + entry: RunnerLiveScheduleEntry, + candidate: RunnerLiveEvalCandidate, + error: RunnerWorkflowInfrastructureError, + ) => RunnerWorkflowObservation | Promise, +): Promise { + const results: RunnerWorkflowObservation[] = []; + for (const entry of schedule.entries) { + const resolved = schedule.candidates.find( + (candidate) => candidate.id === entry.candidateId, + ); + if (!resolved) + throw new Error( + `schedule references unknown candidate ${entry.candidateId}`, + ); + let infrastructureAttempts = 0; + while (true) { + try { + results.push(await execute(entry, resolved)); + break; + } catch (error) { + if (!(error instanceof RunnerWorkflowInfrastructureError)) { + throw error; + } + if (!error.retryable || infrastructureAttempts >= 1) { + if (onInfrastructureFailure === undefined) throw error; + results.push(await onInfrastructureFailure(entry, resolved, error)); + break; + } + infrastructureAttempts += 1; + } + } + } + return results; +} + +export const RUNNER_CHAOS_SCENARIOS: readonly { + id: string; + lane: "chaos"; + fault: + | "restart" + | "duplicate_delivery" + | "trace_interruption" + | "trace_truncation" + | "stale_checkpoint" + | "retry_exhaustion" + | "interaction_timeout" + | "wake_race"; + expectedInvariant: string; +}[] = Object.freeze([ + { + id: "chaos-restart", + lane: "chaos", + fault: "restart", + expectedInvariant: "one resumable execution owns finalization", + }, + { + id: "chaos-duplicate-delivery", + lane: "chaos", + fault: "duplicate_delivery", + expectedInvariant: "semantic effects are idempotent", + }, + { + id: "chaos-trace-interruption", + lane: "chaos", + fault: "trace_interruption", + expectedInvariant: "run success is independent of trace upload", + }, + { + id: "chaos-trace-truncation", + lane: "chaos", + fault: "trace_truncation", + expectedInvariant: "trace is marked truncated without PRP backpressure", + }, + { + id: "chaos-stale-checkpoint", + lane: "chaos", + fault: "stale_checkpoint", + expectedInvariant: "terminal turns cannot remain active", + }, + { + id: "chaos-retry-exhaustion", + lane: "chaos", + fault: "retry_exhaustion", + expectedInvariant: "board recovery owns the bounded terminal", + }, + { + id: "chaos-interaction-timeout", + lane: "chaos", + fault: "interaction_timeout", + expectedInvariant: "governed wait remains visible and owned", + }, + { + id: "chaos-wake-race", + lane: "chaos", + fault: "wake_race", + expectedInvariant: "one rich parent wake wins coalescing", + }, +]); diff --git a/packages/paperclip-runner/src/eval/workflow-evals.test.ts b/packages/paperclip-runner/src/eval/workflow-evals.test.ts index d24603e030..aea24c13d3 100644 --- a/packages/paperclip-runner/src/eval/workflow-evals.test.ts +++ b/packages/paperclip-runner/src/eval/workflow-evals.test.ts @@ -9,9 +9,24 @@ import { assertRunnerWorkflowObservation, type RunnerWorkflowObservation, } from "./workflow-contracts.js"; -import { RUNNER_WORKFLOW_CATALOG, assertRunnerWorkflowCatalog } from "./workflow-catalog.js"; +import { + RUNNER_WORKFLOW_CATALOG, + assertRunnerWorkflowCatalog, +} from "./workflow-catalog.js"; import { runDeterministicRunnerWorkflowMatrix } from "./workflow-harness.js"; import { scoreRunnerWorkflow } from "./workflow-scoring.js"; +import { + RUNNER_LIVE_CANDIDATE_SLOTS, + assertRunnerLiveCandidateManifest, + buildRunnerLiveEvalSchedule, + executeRunnerLiveSchedule, + parseRunnerLiveCampaignCostLimit, + resolvedNightlyCandidates, + runnerLiveRotationWeek, + runnerLiveScheduleCoverage, + RunnerWorkflowInfrastructureError, +} from "./live-workflow-matrix.js"; +import { unavailableLiveRunnerWorkflowObservation } from "./live-workflow-executor.js"; import { buildRunnerWorkflowEvalReport, compareRunnerWorkflowReports, @@ -28,18 +43,31 @@ import { describe("stress-derived Runner workflow catalog", () => { it("contains the twelve provider-neutral workflow families", () => { expect(() => assertRunnerWorkflowCatalog()).not.toThrow(); - expect(RUNNER_WORKFLOW_CATALOG.map((entry) => entry.id)).toEqual(RUNNER_WORKFLOW_IDS); + expect(RUNNER_WORKFLOW_CATALOG.map((entry) => entry.id)).toEqual( + RUNNER_WORKFLOW_IDS, + ); }); it("fails closed when sanitized provider fixtures lack workflow evidence", async () => { const matrix = await runDeterministicRunnerWorkflowMatrix(); expect(matrix).toHaveLength(36); - expect(matrix.every((entry) => entry.observation.classification === "candidate_failure")).toBe(true); - expect(matrix.every((entry) => entry.scorecard.overall.passed === false)).toBe(true); - expect(new Set(matrix.map((entry) => entry.observation.provider))).toEqual(new Set(["codex", "opencode", "acpx"])); + expect( + matrix.every( + (entry) => entry.observation.classification === "candidate_failure", + ), + ).toBe(true); + expect( + matrix.every((entry) => entry.scorecard.overall.passed === false), + ).toBe(true); + expect(new Set(matrix.map((entry) => entry.observation.provider))).toEqual( + new Set(["codex", "opencode", "acpx"]), + ); - const codex = matrix.find((entry) => - entry.scenarioId === "final-response" && entry.candidateId === "fixture-codex"); + const codex = matrix.find( + (entry) => + entry.scenarioId === "final-response" && + entry.candidateId === "fixture-codex", + ); expect(codex?.observation).toMatchObject({ classification: "candidate_failure", failure: { code: "fixture_evidence_incomplete", category: "candidate" }, @@ -57,13 +85,19 @@ describe("stress-derived Runner workflow catalog", () => { }); expect(codex?.observation.lifecycle).not.toHaveProperty("issueStatus"); expect(codex?.observation.lifecycle).not.toHaveProperty("runStatus"); - expect(codex?.observation.lifecycle.checks).toEqual(expect.arrayContaining([ - expect.objectContaining({ id: "required-prp-events", passed: false }), - expect.objectContaining({ id: "terminal-authority", passed: false }), - expect.objectContaining({ id: "lifecycle-state", passed: false }), - ])); - expect(codex?.observation.observedPrpEventTypes).not.toContain("run.terminal"); - expect(codex?.observation.presentation).not.toHaveProperty("responseSource"); + expect(codex?.observation.lifecycle.checks).toEqual( + expect.arrayContaining([ + expect.objectContaining({ id: "required-prp-events", passed: false }), + expect.objectContaining({ id: "terminal-authority", passed: false }), + expect.objectContaining({ id: "lifecycle-state", passed: false }), + ]), + ); + expect(codex?.observation.observedPrpEventTypes).not.toContain( + "run.terminal", + ); + expect(codex?.observation.presentation).not.toHaveProperty( + "responseSource", + ); expect(codex?.observation.traceLineage).toMatchObject({ digestVerified: false, dispositions: [], @@ -76,7 +110,11 @@ describe("stress-derived Runner workflow catalog", () => { const passing = structuredClone(source!.observation); passing.classification = "completed"; delete passing.failure; - for (const evidence of [passing.lifecycle, passing.continuation, passing.presentation]) { + for (const evidence of [ + passing.lifecycle, + passing.continuation, + passing.presentation, + ]) { for (const entry of evidence.checks) { entry.passed = true; delete entry.reason; @@ -90,7 +128,9 @@ describe("stress-derived Runner workflow catalog", () => { sessionId: "session-evidence", turnId: "turn-evidence", itemId: "item-evidence", - receiptIds: passing.base.observedCalls.map((operation) => `receipt-${operation}`), + receiptIds: passing.base.observedCalls.map( + (operation) => `receipt-${operation}`, + ), terminalPresent: true, }; passing.traceLineage.digestVerified = true; @@ -98,23 +138,38 @@ describe("stress-derived Runner workflow catalog", () => { const lifecycle = structuredClone(passing); lifecycle.lifecycle.checks[0]!.passed = false; - lifecycle.lifecycle.checks[0]!.reason = "stale finalizer changed authoritative state"; + lifecycle.lifecycle.checks[0]!.reason = + "stale finalizer changed authoritative state"; const lifecycleCard = scoreRunnerWorkflow(lifecycle, { bundleId: "test" }); expect(lifecycleCard.dimensions.lifecycle_integrity.passed).toBe(false); - expect(lifecycleCard.overall).toMatchObject({ gatePassed: false, score: 0, passed: false }); + expect(lifecycleCard.overall).toMatchObject({ + gatePassed: false, + score: 0, + passed: false, + }); const continuation = structuredClone(passing); continuation.continuation.checks[0]!.passed = false; - const continuationCard = scoreRunnerWorkflow(continuation, { bundleId: "test" }); - expect(continuationCard.dimensions.continuation_integrity.passed).toBe(false); + const continuationCard = scoreRunnerWorkflow(continuation, { + bundleId: "test", + }); + expect(continuationCard.dimensions.continuation_integrity.passed).toBe( + false, + ); expect(continuationCard.dimensions.presentation_fidelity.passed).toBe(true); expect(continuationCard.overall.gatePassed).toBe(true); const presentation = structuredClone(passing); presentation.presentation.checks[0]!.passed = false; - const presentationCard = scoreRunnerWorkflow(presentation, { bundleId: "test" }); - expect(presentationCard.dimensions.presentation_fidelity.passed).toBe(false); - expect(presentationCard.dimensions.continuation_integrity.passed).toBe(true); + const presentationCard = scoreRunnerWorkflow(presentation, { + bundleId: "test", + }); + expect(presentationCard.dimensions.presentation_fidelity.passed).toBe( + false, + ); + expect(presentationCard.dimensions.continuation_integrity.passed).toBe( + true, + ); const trace = structuredClone(passing); trace.traceLineage.digestVerified = false; @@ -125,18 +180,32 @@ describe("stress-derived Runner workflow catalog", () => { it("does not score skipped or infrastructure executions", async () => { const [source] = await runDeterministicRunnerWorkflowMatrix(); - for (const classification of ["skipped", "infrastructure_failure"] as const) { - const observation: RunnerWorkflowObservation = structuredClone(source!.observation); + for (const classification of [ + "skipped", + "infrastructure_failure", + ] as const) { + const observation: RunnerWorkflowObservation = structuredClone( + source!.observation, + ); observation.classification = classification; - observation.failure = { code: "provider_unavailable", category: "provider", retryable: true, message: "Provider unavailable" }; + observation.failure = { + code: "provider_unavailable", + category: "provider", + retryable: true, + message: "Provider unavailable", + }; assertRunnerWorkflowObservation(observation); - expect(scoreRunnerWorkflow(observation, { bundleId: "test" }).overall).toEqual({ score: null, gatePassed: null, passed: null }); + expect( + scoreRunnerWorkflow(observation, { bundleId: "test" }).overall, + ).toEqual({ score: null, gatePassed: null, passed: null }); } }); it("fails closed when an execution is classified as a candidate failure", async () => { const [source] = await runDeterministicRunnerWorkflowMatrix(); - const observation: RunnerWorkflowObservation = structuredClone(source!.observation); + const observation: RunnerWorkflowObservation = structuredClone( + source!.observation, + ); observation.classification = "candidate_failure"; observation.failure = { code: "candidate_error", @@ -153,19 +222,189 @@ describe("stress-derived Runner workflow catalog", () => { gate: true, reasons: expect.arrayContaining(["candidate execution failed"]), }); - expect(card.overall).toEqual({ score: 0, gatePassed: false, passed: false }); + expect(card.overall).toEqual({ + score: 0, + gatePassed: false, + passed: false, + }); + }); +}); + +describe("balanced live Runner workflow matrix", () => { + it("advances the provider rotation once per scheduled week", () => { + const first = runnerLiveRotationWeek("2026-09-06T06:17:00.000Z"); + const second = runnerLiveRotationWeek("2026-09-13T06:17:00.000Z"); + expect(second).toBe((first + 1) % 7); + expect(() => runnerLiveRotationWeek("not-a-date")).toThrow( + "valid generated-at time", + ); + }); + + it("requires a positive finite campaign cost ceiling", () => { + expect(parseRunnerLiveCampaignCostLimit(undefined)).toBe(12); + expect(parseRunnerLiveCampaignCostLimit("0.25")).toBe(0.25); + for (const value of ["", " ", "0", "-1", "NaN", "Infinity", "1e309"]) { + expect(() => parseRunnerLiveCampaignCostLimit(value)).toThrow( + "PAPERCLIP_EVAL_MAX_CAMPAIGN_COST_USD must be a positive finite number", + ); + } + }); + + it("is secret-free, excludes Pi, and produces forty stable nightly executions", () => { + expect(() => assertRunnerLiveCandidateManifest()).not.toThrow(); + const candidates = RUNNER_LIVE_CANDIDATE_SLOTS.flatMap( + (slot) => slot.candidates, + ); + expect(RUNNER_LIVE_CANDIDATE_SLOTS).toHaveLength(5); + expect( + candidates.some((candidate) => candidate.qualification.profile === "pi"), + ).toBe(false); + expect( + candidates + .filter((candidate) => + candidate.qualification.requiredEnvironment.includes( + "OPENAI_API_KEY", + ), + ) + .map((candidate) => candidate.id), + ).toEqual(["codex-luna", "acpx-codex-sol"]); + expect( + candidates.find((candidate) => candidate.id === "acpx-claude-sonnet") + ?.model, + ).toBe("claude-sonnet-5"); + + const first = buildRunnerLiveEvalSchedule({ + seed: "nightly-v1", + rotationDay: 0, + generatedAt: "2026-08-24T00:00:00.000Z", + }); + const replay = buildRunnerLiveEvalSchedule({ + seed: "nightly-v1", + rotationDay: 0, + generatedAt: "2026-08-24T00:00:00.000Z", + }); + expect(first).toEqual(replay); + expect(first.expectedExecutions).toBe(40); + expect(new Set(first.entries.map((entry) => entry.executionId)).size).toBe( + 40, + ); + expect(first.entries.every((entry) => entry.providerTrace === "raw")).toBe( + true, + ); + expect(JSON.stringify(first)).not.toMatch(/\bsk-[A-Za-z0-9]{16,}\b/); + }); + + it("covers every slot, alternating candidate, workflow, and model tier over seven weekly runs", () => { + expect(runnerLiveScheduleCoverage()).toEqual({ + everySlotNightly: true, + everyConcreteCandidateInRotation: true, + everyWorkflowCoversEverySlot: true, + strongAndInexpensiveNightly: true, + }); + expect(resolvedNightlyCandidates(0).map((entry) => entry.id)).not.toEqual( + resolvedNightlyCandidates(1).map((entry) => entry.id), + ); + }); + + it("retries only one retryable infrastructure failure", async () => { + const schedule = buildRunnerLiveEvalSchedule({ + seed: "retry-v1", + rotationDay: 0, + generatedAt: "2026-08-24T00:00:00.000Z", + cases: [RUNNER_WORKFLOW_CATALOG[0]!], + }); + const [source] = await runDeterministicRunnerWorkflowMatrix(); + let attempts = 0; + const results = await executeRunnerLiveSchedule( + schedule, + async (entry, candidate) => { + attempts += 1; + if (attempts === 1) + throw new RunnerWorkflowInfrastructureError( + "provider_timeout", + true, + "transient timeout", + ); + const observation = structuredClone(source!.observation); + observation.caseId = entry.caseId; + observation.candidateId = candidate.id; + observation.provider = candidate.provider; + return observation; + }, + ); + expect(results).toHaveLength(6); + expect(attempts).toBe(7); + }); + + it("classifies exhausted infrastructure without scoring candidates", async () => { + const schedule = buildRunnerLiveEvalSchedule({ + seed: "infra-v1", + rotationDay: 0, + generatedAt: "2026-08-24T00:00:00.000Z", + cases: [RUNNER_WORKFLOW_CATALOG[0]!], + }); + let attempts = 0; + const results = await executeRunnerLiveSchedule( + schedule, + async () => { + attempts += 1; + throw new RunnerWorkflowInfrastructureError( + "provider_timeout", + true, + "transient timeout", + ); + }, + (entry, candidate, error) => + unavailableLiveRunnerWorkflowObservation({ + entry, + candidate, + evalCase: RUNNER_WORKFLOW_CATALOG[0]!, + classification: "infrastructure_failure", + code: error.code, + category: "provider", + retryable: error.retryable, + message: error.message, + }), + ); + expect(attempts).toBe(12); + expect(results).toHaveLength(6); + expect( + results.every( + (entry) => entry.classification === "infrastructure_failure", + ), + ).toBe(true); + expect( + results.every( + (entry) => + scoreRunnerWorkflow(entry, { bundleId: "test" }).overall.score === + null, + ), + ).toBe(true); }); }); describe("workflow reports and stress traceability", () => { it("maps every stress finding to a workflow, regression, or explicit exclusion", async () => { const packageRoot = process.cwd(); - const manifest = JSON.parse(await readFile(resolve(packageRoot, "spec/evals/stress-workflow-traceability.json"), "utf8")) as StressTraceabilityManifest; + const manifest = JSON.parse( + await readFile( + resolve(packageRoot, "spec/evals/stress-workflow-traceability.json"), + "utf8", + ), + ) as StressTraceabilityManifest; const summary = validateStressTraceabilityManifest(manifest); - expect(summary).toEqual({ findings: 44, workflowEvalFindings: 40, regressionTestFindings: 3, exclusions: 1, coveredWorkflows: 12 }); + expect(summary).toEqual({ + findings: 44, + workflowEvalFindings: 40, + regressionTestFindings: 3, + exclusions: 1, + coveredWorkflows: 12, + }); for (const finding of manifest.findings) { for (const testPath of finding.regressionTests) { - await expect(stat(resolve(packageRoot, testPath))).resolves.toBeDefined(); + await expect( + stat(resolve(packageRoot, testPath)), + ).resolves.toBeDefined(); } } }); @@ -174,10 +413,21 @@ describe("workflow reports and stress traceability", () => { const results = await runDeterministicRunnerWorkflowMatrix(); const report = buildRunnerWorkflowEvalReport({ source: "deterministic", - bundle: { id: "bundle-v1", runnerVersion: "0.0.0", promptPolicyId: "stress-sanitized-v1", providerVersions: { fixture: "1" } }, + bundle: { + id: "bundle-v1", + runnerVersion: "0.0.0", + promptPolicyId: "stress-sanitized-v1", + providerVersions: { fixture: "1" }, + }, results, generatedAt: "2026-08-24T00:00:00.000Z", - traceability: { findings: 44, workflowEvalFindings: 40, regressionTestFindings: 3, exclusions: 1, coveredWorkflows: 12 }, + traceability: { + findings: 44, + workflowEvalFindings: 40, + regressionTestFindings: 3, + exclusions: 1, + coveredWorkflows: 12, + }, }); expect(report.aggregate).toMatchObject({ executions: 36, @@ -185,41 +435,104 @@ describe("workflow reports and stress traceability", () => { passed: 0, candidateFailures: 36, }); - expect(report.coverage).toMatchObject({ canonicalOperations: 41, capabilityCases: 106, workflows: 12, stressFindings: 44, stressExclusions: 1 }); + expect(report.coverage).toMatchObject({ + canonicalOperations: 41, + capabilityCases: 106, + workflows: 12, + stressFindings: 44, + stressExclusions: 1, + }); expect(report.coverage.operations).toHaveLength(41); expect(report.coverage.composedWorkflows).toHaveLength(12); - expect(report.coverage.operations.find((entry) => entry.operationId === "finish_task")?.workflowIds.length).toBeGreaterThan(0); - expect(renderRunnerWorkflowMarkdown(report)).toContain("41 operations · 106 capability cases · 12 workflows"); - expect(renderRunnerWorkflowJUnit(report)).toContain('tests="36" failures="36" skipped="0"'); - expect(renderRunnerWorkflowGitHubSummary(report)).toContain("No active workflow-eval regression alerts"); - expect(compareRunnerWorkflowReports(report, report)).toMatchObject({ compatible: true, passRateDelta: 0, overallDelta: 0 }); - expect(compareRunnerWorkflowReports({ ...report, bundle: { ...report.bundle, id: "other" } }, report)).toMatchObject({ compatible: false }); + expect( + report.coverage.operations.find( + (entry) => entry.operationId === "finish_task", + )?.workflowIds.length, + ).toBeGreaterThan(0); + expect(renderRunnerWorkflowMarkdown(report)).toContain( + "41 operations · 106 capability cases · 12 workflows", + ); + expect(renderRunnerWorkflowJUnit(report)).toContain( + 'tests="36" failures="36" skipped="0"', + ); + expect(renderRunnerWorkflowGitHubSummary(report)).toContain( + "No active workflow-eval regression alerts", + ); + expect(compareRunnerWorkflowReports(report, report)).toMatchObject({ + compatible: true, + passRateDelta: 0, + overallDelta: 0, + }); + expect( + compareRunnerWorkflowReports( + { ...report, bundle: { ...report.bundle, id: "other" } }, + report, + ), + ).toMatchObject({ compatible: false }); }); it("keeps alerts disabled during baseline and detects safety and trend regressions afterward", async () => { const results = await runDeterministicRunnerWorkflowMatrix(); const healthy = buildRunnerWorkflowEvalReport({ - source: "deterministic", bundle: { id: "compatible", runnerVersion: "1", promptPolicyId: "p", providerVersions: {} }, results, + source: "deterministic", + bundle: { + id: "compatible", + runnerVersion: "1", + promptPolicyId: "p", + providerVersions: {}, + }, + results, generatedAt: "2026-08-24T00:00:00.000Z", }); - const failingResults = structuredClone(results) as unknown as typeof results; + const failingResults = structuredClone( + results, + ) as unknown as typeof results; failingResults[0]!.scorecard.dimensions.lifecycle_integrity.passed = false; failingResults[0]!.scorecard.dimensions.lifecycle_integrity.score = 0; - failingResults[0]!.scorecard.overall = { score: 0, gatePassed: false, passed: false }; + failingResults[0]!.scorecard.overall = { + score: 0, + gatePassed: false, + passed: false, + }; const failing = buildRunnerWorkflowEvalReport({ - source: "deterministic", bundle: healthy.bundle, results: failingResults, + source: "deterministic", + bundle: healthy.bundle, + results: failingResults, generatedAt: "2026-08-25T00:00:00.000Z", }); - expect(runnerWorkflowAlerts({ current: failing, history: [healthy], baselineReady: false })).toEqual([]); - expect(runnerWorkflowAlerts({ current: failing, history: [failing, failing], baselineReady: true }).map((alert) => alert.code)).toEqual(expect.arrayContaining(["safety_failure", "consecutive_failures"])); + expect( + runnerWorkflowAlerts({ + current: failing, + history: [healthy], + baselineReady: false, + }), + ).toEqual([]); + expect( + runnerWorkflowAlerts({ + current: failing, + history: [failing, failing], + baselineReady: true, + }).map((alert) => alert.code), + ).toEqual( + expect.arrayContaining(["safety_failure", "consecutive_failures"]), + ); }); it("validates the checked-in versioned JSON schemas", async () => { const schemaRoot = resolve(process.cwd(), "spec/evals/schemas"); const [caseSchema, observationSchema, scorecardSchema] = await Promise.all([ - readFile(resolve(schemaRoot, "runner-workflow-eval-case.v1.schema.json"), "utf8").then(JSON.parse), - readFile(resolve(schemaRoot, "runner-workflow-observation.v1.schema.json"), "utf8").then(JSON.parse), - readFile(resolve(schemaRoot, "eval-scorecard.v2.schema.json"), "utf8").then(JSON.parse), + readFile( + resolve(schemaRoot, "runner-workflow-eval-case.v1.schema.json"), + "utf8", + ).then(JSON.parse), + readFile( + resolve(schemaRoot, "runner-workflow-observation.v1.schema.json"), + "utf8", + ).then(JSON.parse), + readFile( + resolve(schemaRoot, "eval-scorecard.v2.schema.json"), + "utf8", + ).then(JSON.parse), ]); const ajv = new Ajv2020({ allErrors: true }); const [result] = await runDeterministicRunnerWorkflowMatrix(); diff --git a/packages/paperclip-runner/src/eval/workflow-report.ts b/packages/paperclip-runner/src/eval/workflow-report.ts index 2e3e02daca..3d043d7654 100644 --- a/packages/paperclip-runner/src/eval/workflow-report.ts +++ b/packages/paperclip-runner/src/eval/workflow-report.ts @@ -13,12 +13,13 @@ export interface RunnerWorkflowReportBundle { runnerBuild?: string; promptPolicyId: string; providerVersions: Record; + scheduleSeed?: string; } export interface RunnerWorkflowEvalReport { schema: typeof RUNNER_WORKFLOW_REPORT_SCHEMA; generatedAt: string; - source: "deterministic"; + source: "deterministic" | "live" | "chaos"; bundle: RunnerWorkflowReportBundle; results: RunnerWorkflowMatrixEntry[]; aggregate: { diff --git a/packages/paperclip-runner/src/evals/model-pricing.test.ts b/packages/paperclip-runner/src/evals/model-pricing.test.ts new file mode 100644 index 0000000000..54c3a27c5d --- /dev/null +++ b/packages/paperclip-runner/src/evals/model-pricing.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from "vitest"; + +import { + estimateModelCostNanodollars, + MODEL_PRICING_VERSION, +} from "./model-pricing.js"; + +describe("model pricing", () => { + it("prices uncached, cached, and output tokens with explicit provenance", () => { + expect( + estimateModelCostNanodollars("gpt-5.6-sol", { + inputTokens: 1_000, + cachedInputTokens: 400, + outputTokens: 100, + }), + ).toEqual({ + estimatedCostNanodollars: 6_200_000, + pricingVersion: MODEL_PRICING_VERSION, + ratesUsdPerMillionTokens: { input: 5, cachedInput: 0.5, output: 30 }, + }); + }); + + it("fails closed for an unpriced model", () => { + expect(() => + estimateModelCostNanodollars("unknown", { + inputTokens: 1, + cachedInputTokens: 0, + outputTokens: 1, + }), + ).toThrow("model pricing unavailable"); + }); +}); diff --git a/packages/paperclip-runner/src/evals/model-pricing.ts b/packages/paperclip-runner/src/evals/model-pricing.ts new file mode 100644 index 0000000000..99cde385ed --- /dev/null +++ b/packages/paperclip-runner/src/evals/model-pricing.ts @@ -0,0 +1,42 @@ +export const MODEL_PRICING_VERSION = "provider-list-prices-2026-08-21" as const; + +interface TokenRatesUsdPerMillion { + input: number; + cachedInput: number; + output: number; +} + +// Versioned OpenAI API list prices, not the customer's actual invoice or Codex-plan debit. +// Source: https://developers.openai.com/api/docs/models +const RATES: Readonly> = Object.freeze({ + "gpt-5.4-mini": { input: 0.75, cachedInput: 0.075, output: 4.5 }, + "gpt-5.5": { input: 5, cachedInput: 0.5, output: 30 }, + "gpt-5.6-sol": { input: 5, cachedInput: 0.5, output: 30 }, + "gpt-5.6-terra": { input: 2, cachedInput: 0.2, output: 12 }, + "gpt-5.6-luna": { input: 0.2, cachedInput: 0.02, output: 1.2 }, + // Qualified Anthropic direct profile. Actual invoice discounts are intentionally excluded. + "claude-sonnet-5": { input: 3, cachedInput: 0.3, output: 15 }, + // Amazon Bedrock global cross-region list price for Claude Sonnet 4.6. + "global.anthropic.claude-sonnet-4-6": { input: 3, cachedInput: 0.3, output: 15 }, +}); + +export interface EstimatedModelCost { + estimatedCostNanodollars: number; + pricingVersion: typeof MODEL_PRICING_VERSION; + ratesUsdPerMillionTokens: TokenRatesUsdPerMillion; +} + +export function estimateModelCostNanodollars( + model: string, + usage: { inputTokens: number; cachedInputTokens: number; outputTokens: number }, +): EstimatedModelCost { + const rates = RATES[model]; + if (rates === undefined) throw new Error(`model pricing unavailable for ${model}`); + const uncachedInput = Math.max(0, usage.inputTokens - usage.cachedInputTokens); + const estimatedCostNanodollars = Math.round( + uncachedInput * rates.input * 1_000 + + usage.cachedInputTokens * rates.cachedInput * 1_000 + + usage.outputTokens * rates.output * 1_000, + ); + return { estimatedCostNanodollars, pricingVersion: MODEL_PRICING_VERSION, ratesUsdPerMillionTokens: { ...rates } }; +} diff --git a/packages/paperclip-runner/src/live/clean-room-server.test.ts b/packages/paperclip-runner/src/live/clean-room-server.test.ts index fff0ed9775..67b94f9930 100644 --- a/packages/paperclip-runner/src/live/clean-room-server.test.ts +++ b/packages/paperclip-runner/src/live/clean-room-server.test.ts @@ -605,6 +605,68 @@ afterEach(async () => { }); describe("Capability clean-room chat server", () => { + it("validates the exact Claude Managed lab profile and canonical agent version", async () => { + const module = await import("../../scripts/capability-issue-thread-server.mjs"); + const internals = module.capabilityIssueThreadServerInternals; + expect(internals.harnessConfiguration({ + provider: "claude_managed", + model: "claude-sonnet-5", + managedProfileId: "managed-qualified", + maxSessionListCostUsd: 2, + })).toMatchObject({ + provider: "claude_managed", + model: "claude-sonnet-5", + managedProfileId: "managed-qualified", + maxSessionListCostUsd: 2, + }); + expect(() => internals.harnessConfiguration({ + provider: "claude_managed", + model: "claude-opus-5", + })).toThrow("requires exact model claude-sonnet-5"); + + const keys = [ + "PAPERCLIP_CLAUDE_MANAGED_PROFILE_ID", + "ANTHROPIC_API_KEY", + "ANTHROPIC_MANAGED_AGENT_ID", + "ANTHROPIC_MANAGED_AGENT_VERSION", + "ANTHROPIC_MANAGED_ENVIRONMENT_ID", + ] as const; + const prior = Object.fromEntries(keys.map((key) => [key, process.env[key]])); + try { + process.env.PAPERCLIP_CLAUDE_MANAGED_PROFILE_ID = "managed-qualified"; + process.env.ANTHROPIC_API_KEY = "test-only-key"; + process.env.ANTHROPIC_MANAGED_AGENT_ID = "agent-qualified"; + process.env.ANTHROPIC_MANAGED_ENVIRONMENT_ID = "environment-qualified"; + process.env.ANTHROPIC_MANAGED_AGENT_VERSION = "2147483647"; + expect(internals.resolveManagedProfile({ + provider: "claude_managed", + managedProfileId: "managed-qualified", + maxSessionListCostUsd: 2, + })).toEqual({ + profileId: "managed-qualified", + anthropicAgentId: "agent-qualified", + agentVersion: "2147483647", + environmentId: "environment-qualified", + betaVersion: "managed-agents-2026-04-01", + maxSessionListCostUsd: 2, + }); + for (const agentVersion of ["latest", "0", "01", "2147483648"]) { + process.env.ANTHROPIC_MANAGED_AGENT_VERSION = agentVersion; + expect(() => internals.resolveManagedProfile({ + provider: "claude_managed", + managedProfileId: "managed-qualified", + maxSessionListCostUsd: 2, + })).toThrow("not fully qualified"); + } + } finally { + for (const key of keys) { + const value = prior[key]; + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + } + }); + it("opens a blank live thread with fresh mock identities and no canned evidence", async () => { const opened = await call("/api/capability/ui/cleanroom/session"); @@ -919,7 +981,7 @@ describe("Capability clean-room chat server", () => { expect(response.status).toBe(429); expect(body.error).toBe("turn_limit"); expect(body.message).toContain("Start a new chat"); - }); + }, 30_000); it("streams the turn as frames while the POST is still open", async () => { const opened = await call("/api/capability/ui/cleanroom/session"); diff --git a/packages/paperclip-runner/src/live/live-session.test.ts b/packages/paperclip-runner/src/live/live-session.test.ts index 9fecc16040..319966ae9c 100644 --- a/packages/paperclip-runner/src/live/live-session.test.ts +++ b/packages/paperclip-runner/src/live/live-session.test.ts @@ -19,6 +19,7 @@ import { type CapabilityLiveSessionSnapshot, type CapabilityLiveSessionStore, type CapabilityLiveTransportFactory, + type CapabilityLiveTurnEvent, } from "./live-session.js"; import { DurableCapabilityLiveSessionStore } from "./durable-live-session-store.js"; import { defaultCapabilityRunnerdBinary } from "./runnerd-codex-transport.js"; @@ -62,7 +63,9 @@ interface FakeProviderState { attachments: Array<{ runId: string; turnId: string; itemId: string }>; holdAfterTool: boolean; closeError: Error | null; + usageRunDelta: Record | null; onTurnStart?: () => Promise; + onUsage?: (queue: AsyncNotifications, turnId: string) => void | Promise; } class FakeCapabilityCodexTransport implements CodexAppServerTransport { @@ -241,25 +244,35 @@ class FakeCapabilityCodexTransport implements CodexAppServerTransport { }, }); this.state.turns.set(turnId, "completed"); - this.notificationsQueue.push({ - method: "thread/tokenUsage/updated", - params: { - threadId: this.state.threadId, - turnId, - tokenUsage: { - total: { - inputTokens: this.state.nextTurn * 100, - cachedInputTokens: this.state.nextTurn * 20, - outputTokens: this.state.nextTurn * 10, - reasoningOutputTokens: this.state.nextTurn * 2, + if (this.state.onUsage) { + await this.state.onUsage(this.notificationsQueue, turnId); + } else { + this.notificationsQueue.push({ + method: "thread/tokenUsage/updated", + params: { + threadId: this.state.threadId, + turnId, + tokenUsage: { + total: { + inputTokens: this.state.nextTurn * 100, + cachedInputTokens: this.state.nextTurn * 20, + outputTokens: this.state.nextTurn * 10, + reasoningOutputTokens: this.state.nextTurn * 2, + }, + ...(this.state.usageRunDelta === null + ? {} + : { + runDeltaAvailable: true, + runDelta: this.state.usageRunDelta, + }), }, }, - }, - }); - this.notificationsQueue.push({ - method: "turn/completed", - params: { threadId: this.state.threadId, turn: { id: turnId, status: "completed" } }, - }); + }); + this.notificationsQueue.push({ + method: "turn/completed", + params: { threadId: this.state.threadId, turn: { id: turnId, status: "completed" } }, + }); + } this.#activeTurnId = null; } @@ -300,6 +313,7 @@ function providerState(): FakeProviderState { attachments: [], holdAfterTool: false, closeError: null, + usageRunDelta: null, }; } @@ -652,6 +666,256 @@ describe("Capability live runnerd and Codex session", () => { await service.shutdown(session.id); }); + it("streams normalized usage before terminal with cumulative fallback and explicit run deltas", async () => { + const state = providerState(); + const service = new CapabilityLiveSessionService({ transportFactory: fakeTransportFactory(state) }); + const session = await service.create({ + runId: "run-live-usage-stream", + sessionId: "session-live-usage-stream", + }); + const events: CapabilityLiveTurnEvent[] = []; + const unsubscribe = session.subscribe((event) => events.push(event)); + + await session.sendMessage("report cumulative usage"); + state.usageRunDelta = { + requests: 2, + inputTokens: 7, + outputTokens: 3, + cachedInputTokens: 1, + reasoningTokens: 2, + providerCostUsd: 0.004, + }; + await session.sendMessage("report explicit run delta"); + unsubscribe(); + + const usageEvents = events.filter( + (event): event is Extract => + event.kind === "usage", + ); + expect(usageEvents).toHaveLength(2); + expect(usageEvents[0]).toMatchObject({ + turnId: "turn-1", + usage: { + providerRequests: 1, + inputTokens: 100, + outputTokens: 10, + cachedInputTokens: 20, + reasoningTokens: 2, + costNanodollars: 0, + }, + }); + expect(usageEvents[1]).toMatchObject({ + turnId: "turn-2", + usage: { + providerRequests: 2, + inputTokens: 7, + outputTokens: 3, + cachedInputTokens: 1, + reasoningTokens: 2, + costNanodollars: 4_000_000, + }, + }); + for (const usageEvent of usageEvents) { + const usageIndex = events.indexOf(usageEvent); + const terminalIndex = events.findIndex( + (event) => event.kind === "terminal" && event.turnId === usageEvent.turnId, + ); + expect(terminalIndex).toBeGreaterThan(usageIndex); + } + await service.shutdown(session.id); + }); + + it("does not add an overlapping token snapshot to raw response usage", async () => { + const state = providerState(); + state.onUsage = (queue, turnId) => { + queue.push({ + method: "thread/tokenUsage/updated", + params: { + threadId: state.threadId, + turnId, + tokenUsage: { total: { inputTokens: 100, outputTokens: 10 } }, + }, + }); + queue.push({ + method: "rawResponse/completed", + params: { + threadId: state.threadId, + turnId, + usage: { inputTokens: 100, outputTokens: 10 }, + }, + }); + queue.push({ + method: "turn/completed", + params: { threadId: state.threadId, turn: { id: turnId, status: "completed" } }, + }); + }; + const service = new CapabilityLiveSessionService({ transportFactory: fakeTransportFactory(state) }); + const session = await service.create({ runId: "run-overlap-usage", sessionId: "session-overlap-usage" }); + + const result = await session.sendMessage("report overlapping usage"); + + expect(result.snapshot.usageLedger).toHaveLength(1); + expect(result.snapshot.usageLedger[0]).toMatchObject({ + providerRequests: 1, + inputTokens: 100, + outputTokens: 10, + }); + await service.shutdown(session.id); + }); + + it("prefers a fresh thread read when a later turn omits usage notifications", async () => { + const state = providerState(); + state.onUsage = (queue, turnId) => { + if (state.nextTurn === 1) { + queue.push({ + method: "thread/tokenUsage/updated", + params: { + threadId: state.threadId, + turnId, + tokenUsage: { total: { inputTokens: 100, outputTokens: 10 } }, + }, + }); + } + queue.push({ + method: "turn/completed", + params: { threadId: state.threadId, turn: { id: turnId, status: "completed" } }, + }); + }; + const service = new CapabilityLiveSessionService({ transportFactory: fakeTransportFactory(state) }); + const session = await service.create({ runId: "run-read-fallback", sessionId: "session-read-fallback" }); + + await session.sendMessage("report initial usage"); + const second = await session.sendMessage("omit usage notification"); + + expect(second.snapshot.usageLedger).toMatchObject([ + { inputTokens: 100, outputTokens: 10 }, + { inputTokens: 100, outputTokens: 10 }, + ]); + await service.shutdown(session.id); + }); + + it("uses a fresh thread read for a cost-only usage report", async () => { + const state = providerState(); + state.onUsage = (queue, turnId) => { + queue.push({ + method: "thread/tokenUsage/updated", + params: { + threadId: state.threadId, + turnId, + tokenUsage: { total: { requests: 1, providerCostUsd: 0.25 } }, + }, + }); + queue.push({ + method: "turn/completed", + params: { threadId: state.threadId, turn: { id: turnId, status: "completed" } }, + }); + }; + const service = new CapabilityLiveSessionService({ transportFactory: fakeTransportFactory(state) }); + const session = await service.create({ runId: "run-cost-only", sessionId: "session-cost-only" }); + + const result = await session.sendMessage("report cost-only usage"); + + expect(result.snapshot.usageLedger).toMatchObject([{ + providerRequests: 1, + inputTokens: 100, + outputTokens: 10, + costNanodollars: 250_000_000, + }]); + await service.shutdown(session.id); + }); + + it("waits for all bounded late raw response receipts before committing usage", async () => { + const state = providerState(); + state.onUsage = (queue, turnId) => { + queue.push({ + method: "rawResponse/completed", + params: { + threadId: state.threadId, + turnId, + usage: { inputTokens: 40, outputTokens: 4 }, + }, + }); + queue.push({ + method: "turn/completed", + params: { threadId: state.threadId, turn: { id: turnId, status: "completed" } }, + }); + setTimeout(() => queue.push({ + method: "rawResponse/completed", + params: { + threadId: state.threadId, + turnId, + usage: { inputTokens: 60, outputTokens: 6 }, + }, + }), 25); + }; + const service = new CapabilityLiveSessionService({ transportFactory: fakeTransportFactory(state) }); + const session = await service.create({ runId: "run-late-usage", sessionId: "session-late-usage" }); + + const result = await session.sendMessage("report late usage"); + + expect(result.snapshot.usageLedger).toHaveLength(1); + expect(result.snapshot.usageLedger[0]).toMatchObject({ + providerRequests: 2, + inputTokens: 100, + outputTokens: 10, + }); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(session.snapshot().usageLedger).toEqual(result.snapshot.usageLedger); + await service.shutdown(session.id); + }); + + it("combines explicit per-turn tokens with cumulative cost", async () => { + const state = providerState(); + state.onUsage = (queue, turnId) => { + const firstTurn = state.nextTurn === 1; + queue.push({ + method: "thread/tokenUsage/updated", + params: { + threadId: state.threadId, + turnId, + tokenUsage: { + total: { + requests: state.nextTurn, + providerCostUsd: firstTurn ? 0.25 : 0.4, + }, + runDeltaAvailable: true, + runDelta: { + requests: 1, + inputTokens: firstTurn ? 12 : 7, + outputTokens: firstTurn ? 4 : 3, + providerCostUsd: 0, + }, + }, + }, + }); + queue.push({ + method: "turn/completed", + params: { threadId: state.threadId, turn: { id: turnId, status: "completed" } }, + }); + }; + const service = new CapabilityLiveSessionService({ transportFactory: fakeTransportFactory(state) }); + const session = await service.create({ runId: "run-mixed-usage", sessionId: "session-mixed-usage" }); + + await session.sendMessage("report first mixed usage"); + const second = await session.sendMessage("report second mixed usage"); + + expect(second.snapshot.usageLedger).toMatchObject([ + { + providerRequests: 1, + inputTokens: 12, + outputTokens: 4, + costNanodollars: 250_000_000, + }, + { + providerRequests: 1, + inputTokens: 7, + outputTokens: 3, + costNanodollars: 150_000_000, + }, + ]); + await service.shutdown(session.id); + }); + it("does not let a settled terminal replay resolve the next turn", async () => { const state = providerState(); const service = new CapabilityLiveSessionService({ diff --git a/packages/paperclip-runner/src/live/live-session.ts b/packages/paperclip-runner/src/live/live-session.ts index 7fbc78b57e..99cc0e12a9 100644 --- a/packages/paperclip-runner/src/live/live-session.ts +++ b/packages/paperclip-runner/src/live/live-session.ts @@ -371,6 +371,21 @@ export type CapabilityLiveTurnEvent = /** `turn_started`, `tool_call`, `tool_result`, or `stop_requested`. */ reason: string; } + | { + seq: number; + at: string; + turnId: string | null; + kind: "usage"; + /** Current provider-reported usage for the active turn. */ + usage: { + providerRequests: number; + inputTokens: number; + outputTokens: number; + cachedInputTokens: number; + reasoningTokens: number; + costNanodollars: number; + }; + } | { seq: number; at: string; @@ -389,6 +404,17 @@ export type CapabilityLiveTurnEvent = export type CapabilityLiveTurnListener = (event: CapabilityLiveTurnEvent) => void; +type CapabilityLiveUsageMeasurement = Extract< + CapabilityLiveTurnEvent, + { kind: "usage" } +>["usage"]; + +interface CapabilityLiveTurnUsageObservations { + reported: { usage: CapabilityLiveUsageMeasurement; exactDelta: boolean } | null; + raw: CapabilityLiveUsageMeasurement | null; + terminal: CapabilityLiveUsageMeasurement | null; +} + /** Distributive `Omit`, so each event variant keeps its own discriminated shape. */ type CapabilityLiveTurnEventInput = CapabilityLiveTurnEvent extends infer Variant ? Variant extends CapabilityLiveTurnEvent @@ -398,6 +424,8 @@ type CapabilityLiveTurnEventInput = CapabilityLiveTurnEvent extends infer Varian /** Bound on interim events per turn; terminal and error always get through. */ export const CAPABILITY_LIVE_TURN_EVENT_LIMIT = 4_000; +/** Separate bound so provider usage cannot be starved by delta/activity traffic. */ +const CAPABILITY_LIVE_USAGE_EVENT_LIMIT = 512; export interface CapabilityInteractionResolution { interactionId: string; @@ -1102,14 +1130,8 @@ export class CapabilityLiveSession { readonly #listeners = new Set(); #eventSeq = 0; #turnEventCount = 0; - readonly #rawUsageByTurn = new Map(); + #turnUsageEventCount = 0; + readonly #usageObservationsByTurn = new Map(); #latestCumulativeUsage: Record = {}; #idleTimer: ReturnType | null = null; readonly #loadedOperationIds = new Set(); @@ -1228,8 +1250,10 @@ export class CapabilityLiveSession { } #emit(event: CapabilityLiveTurnEventInput): void { - const bounded = event.kind === "delta" || event.kind === "activity"; - if (bounded) { + if (event.kind === "usage") { + if (this.#turnUsageEventCount >= CAPABILITY_LIVE_USAGE_EVENT_LIMIT) return; + this.#turnUsageEventCount += 1; + } else if (event.kind === "delta" || event.kind === "activity") { if (this.#turnEventCount >= CAPABILITY_LIVE_TURN_EVENT_LIMIT) return; this.#turnEventCount += 1; } @@ -1417,6 +1441,7 @@ export class CapabilityLiveSession { this.#status = "running"; this.#clearIdleTimer(); this.#turnEventCount = 0; + this.#turnUsageEventCount = 0; // Start the baseline before admitting provider work, but do not make turn // interruption wait for a potentially large workspace walk. The provider // turn id is bound first; the terminal path waits for this baseline before @@ -1534,16 +1559,27 @@ export class CapabilityLiveSession { async #captureTurnUsage(turnId: string, allowEmpty = false): Promise { if (this.#transport === null) throw new Error("capability_live_usage_transport_missing"); - // Codex can emit rawResponse/completed immediately after turn/completed. - // Give the notification pump a short bounded window before falling back to - // cumulative thread usage, otherwise a fast second turn can be mislabeled - // as missing accounting even though its receipt is already in flight. - const usageDeadline = Date.now() + 500; - while (!this.#rawUsageByTurn.has(turnId) && Date.now() < usageDeadline) { - await new Promise((resolveWait) => setTimeout(resolveWait, 10)); + const initial = this.#usageObservationsByTurn.get(turnId); + // An explicit whole-turn delta is already complete. Other providers can + // publish one or more rawResponse receipts after turn/completed, so retain + // the full bounded grace period instead of committing the first partial + // observation that happens to arrive. + if (initial?.reported?.exactDelta !== true) { + const usageDeadline = Date.now() + 500; + while (Date.now() < usageDeadline) { + await new Promise((resolveWait) => setTimeout(resolveWait, 10)); + } } - const captured = this.#rawUsageByTurn.get(turnId); - const read = captured === undefined ? await this.#transport.request("thread/read", { + const captured = this.#usageObservationsByTurn.get(turnId); + const reportedTokens = (captured?.reported?.usage.inputTokens ?? 0) + + (captured?.reported?.usage.outputTokens ?? 0); + const needsRead = captured === undefined || ( + captured.reported?.exactDelta !== true + && captured.raw === null + && captured.terminal === null + && reportedTokens === 0 + ); + const read = needsRead ? await this.#transport.request("thread/read", { threadId: this.#providerThreadId, includeTurns: true, }) : {}; @@ -1552,7 +1588,13 @@ export class CapabilityLiveSession { const cumulativeNotification = Object.keys(this.#latestCumulativeUsage).length > 0 ? this.#latestCumulativeUsage : undefined; - const total = record(captured ?? cumulativeNotification ?? usage.total ?? usage.totalTokenUsage ?? usage.total_token_usage); + const readTotal = usage.total ?? usage.totalTokenUsage ?? usage.total_token_usage; + // If the current turn produced no usable observation, thread/read is the + // freshest authority. A prior turn's cached notification is only a final + // compatibility fallback. + const total = record(needsRead + ? readTotal ?? cumulativeNotification + : cumulativeNotification ?? readTotal); const integer = (...names: string[]): number => { for (const name of names) { const value = total[name]; @@ -1561,39 +1603,60 @@ export class CapabilityLiveSession { return 0; }; const cumulative = { + providerRequests: 1, inputTokens: integer("inputTokens", "input_tokens"), outputTokens: integer("outputTokens", "output_tokens"), cachedInputTokens: integer("cachedInputTokens", "cached_input_tokens"), reasoningTokens: integer("reasoningOutputTokens", "reasoningTokens", "reasoning_output_tokens"), + costNanodollars: 0, }; - if (cumulative.inputTokens + cumulative.outputTokens === 0 && captured === undefined && !allowEmpty) { + const prior = reconcileCapabilityLiveUsage(this.snapshot()); + const cumulativeFallback: CapabilityLiveUsageMeasurement = { + providerRequests: 1, + inputTokens: Math.max(0, cumulative.inputTokens - prior.inputTokens), + outputTokens: Math.max(0, cumulative.outputTokens - prior.outputTokens), + cachedInputTokens: Math.max(0, cumulative.cachedInputTokens - prior.cachedInputTokens), + reasoningTokens: Math.max(0, cumulative.reasoningTokens - prior.reasoningTokens), + costNanodollars: 0, + }; + const reported = captured?.reported?.usage; + const usableReported = reported !== undefined + && reported.inputTokens + reported.outputTokens > 0 + ? reported + : undefined; + const selected = captured?.reported?.exactDelta === true + ? reported + : captured?.raw ?? captured?.terminal ?? usableReported ?? cumulativeFallback; + const selectedWithReportedCost = selected === undefined + ? undefined + : { + ...selected, + costNanodollars: Math.max(selected.costNanodollars, reported?.costNanodollars ?? 0), + }; + if ( + (selectedWithReportedCost?.inputTokens ?? 0) + (selectedWithReportedCost?.outputTokens ?? 0) === 0 + && !allowEmpty + ) { throw new Error(`capability_live_usage_missing:${JSON.stringify({ captured: captured !== undefined, + needsRead, readKeys: Object.keys(read).sort(), threadKeys: Object.keys(thread).sort(), usageKeys: Object.keys(usage).sort(), totalKeys: Object.keys(total).sort(), })}`); } - const prior = reconcileCapabilityLiveUsage(this.snapshot()); - const delta = captured === undefined - ? { - inputTokens: Math.max(0, cumulative.inputTokens - prior.inputTokens), - outputTokens: Math.max(0, cumulative.outputTokens - prior.outputTokens), - cachedInputTokens: Math.max(0, cumulative.cachedInputTokens - prior.cachedInputTokens), - reasoningTokens: Math.max(0, cumulative.reasoningTokens - prior.reasoningTokens), - } - : cumulative; + const finalUsage = selectedWithReportedCost ?? cumulativeFallback; await this.recordUsage({ receiptId: `${turnId}:usage`, turnId, providerCalls: 1, - providerRequests: captured?.providerRequests ?? 1, - inputTokens: delta.inputTokens, - outputTokens: delta.outputTokens, - cachedInputTokens: delta.cachedInputTokens, - reasoningTokens: delta.reasoningTokens, - costNanodollars: captured?.costNanodollars ?? 0, + providerRequests: Math.max(1, finalUsage.providerRequests), + inputTokens: finalUsage.inputTokens, + outputTokens: finalUsage.outputTokens, + cachedInputTokens: finalUsage.cachedInputTokens, + reasoningTokens: finalUsage.reasoningTokens, + costNanodollars: finalUsage.costNanodollars, }); } @@ -1701,6 +1764,68 @@ export class CapabilityLiveSession { return this.snapshot(); } + async increaseManagedSessionBudget( + maxSessionListCostUsd: number, + ): Promise { + if ( + (this.#config.provider !== "claude_managed" && + this.#config.provider !== "aws_agentcore") || + this.#transport === null + ) { + throw new Error( + "remote session budget updates require a connected remote provider session", + ); + } + if (!Number.isFinite(maxSessionListCostUsd) || maxSessionListCostUsd <= 0) { + throw new Error("managed session spend ceiling must be positive"); + } + await this.#transport.request("session/budget/increase", { + maxSessionListCostUsd, + }); + if (this.#config.managedProfile) { + this.#config.managedProfile.maxSessionListCostUsd = maxSessionListCostUsd; + } + if (this.#config.agentCoreProfile) { + this.#config.agentCoreProfile.maxEstimatedSessionCostUsd = + maxSessionListCostUsd; + } + this.#status = this.#activeTurnId === null ? "warm_idle" : "running"; + this.#appendEvidence("session", this.#activeTurnId, { + action: "budget_increased", + maxSessionListCostUsd, + }); + await this.#persist(); + return this.snapshot(); + } + + async deleteManagedRemoteSession(): Promise { + if ( + (this.#config.provider !== "claude_managed" && + this.#config.provider !== "aws_agentcore") || + this.#transport === null + ) { + throw new Error( + "remote session deletion requires a connected remote provider session", + ); + } + if (this.#activeTurnId !== null) { + throw new Error( + "interrupt the active turn before deleting the remote session", + ); + } + await this.#transport.request("session/destroy", {}); + this.#providerSessionId = null; + this.#authority.active = false; + this.#status = "closed"; + this.#appendEvidence("cleanup", null, { + reason: "remote session explicitly deleted", + remoteDeleted: true, + resumable: false, + }); + await this.#persist(); + return this.snapshot(); + } + async resolveInteraction(input: CapabilityInteractionResolution): Promise { const interaction = pendingInteractions(this.#port).find( (candidate) => candidate.id === input.interactionId, @@ -2193,27 +2318,86 @@ export class CapabilityLiveSession { tokenUsage.total ?? tokenUsage.totalTokenUsage ?? tokenUsage.total_token_usage ?? tokenUsage, ); const runDelta = record(tokenUsage.runDelta ?? params.runDelta); - if (turnId.length > 0 && Object.keys(runDelta).length > 0) { - const integer = (...names: string[]): number => { + if (turnId.length > 0) { + const runDeltaAvailable = + (params.runDeltaAvailable === true || + tokenUsage.runDeltaAvailable === true) && + Object.keys(runDelta).length > 0; + const committed = reconcileCapabilityLiveUsage(this.snapshot()); + const integer = (source: Record, ...names: string[]): number => { for (const name of names) { - const value = runDelta[name]; + const value = source[name]; if (Number.isSafeInteger(value) && Number(value) >= 0) return Number(value); } return 0; }; - const providerCostUsd = typeof runDelta.providerCostUsd === "number" - && Number.isFinite(runDelta.providerCostUsd) - && runDelta.providerCostUsd >= 0 - ? runDelta.providerCostUsd - : 0; - this.#rawUsageByTurn.set(turnId, { - providerRequests: integer("requests", "providerRequests"), - inputTokens: integer("inputTokens", "input_tokens"), - outputTokens: integer("outputTokens", "output_tokens"), - cachedInputTokens: integer("cacheReadTokens", "cachedInputTokens", "cache_read_input_tokens"), - reasoningTokens: integer("reasoningTokens", "reasoning_tokens"), - costNanodollars: Math.round(providerCostUsd * 1_000_000_000), + const costNanodollars = (source: Record): number => { + const providerCostUsd = source.providerCostUsd; + return typeof providerCostUsd === "number" + && Number.isFinite(providerCostUsd) + && providerCostUsd >= 0 + ? Math.round(providerCostUsd * 1_000_000_000) + : 0; + }; + const measurement = (source: Record): CapabilityLiveUsageMeasurement => ({ + providerRequests: integer(source, "requests", "providerRequests"), + inputTokens: integer(source, "inputTokens", "input_tokens"), + outputTokens: integer(source, "outputTokens", "output_tokens"), + cachedInputTokens: integer( + source, + "cacheReadTokens", + "cachedInputTokens", + "cached_input_tokens", + "cache_read_input_tokens", + ), + reasoningTokens: integer( + source, + "reasoningOutputTokens", + "reasoningTokens", + "reasoning_tokens", + "reasoning_output_tokens", + ), + costNanodollars: costNanodollars(source), }); + const cumulativeMeasurement = measurement(this.#latestCumulativeUsage); + const cumulativeDelta: CapabilityLiveUsageMeasurement = { + providerRequests: Math.max(0, cumulativeMeasurement.providerRequests - committed.providerRequests), + inputTokens: Math.max(0, cumulativeMeasurement.inputTokens - committed.inputTokens), + outputTokens: Math.max(0, cumulativeMeasurement.outputTokens - committed.outputTokens), + cachedInputTokens: Math.max(0, cumulativeMeasurement.cachedInputTokens - committed.cachedInputTokens), + reasoningTokens: Math.max(0, cumulativeMeasurement.reasoningTokens - committed.reasoningTokens), + costNanodollars: Math.max(0, cumulativeMeasurement.costNanodollars - committed.costNanodollars), + }; + const explicitDelta = measurement(runDelta); + const usage: CapabilityLiveUsageMeasurement = { + providerRequests: Math.max( + 1, + runDeltaAvailable ? explicitDelta.providerRequests : 0, + cumulativeDelta.providerRequests, + ), + inputTokens: runDeltaAvailable ? explicitDelta.inputTokens : cumulativeDelta.inputTokens, + outputTokens: runDeltaAvailable ? explicitDelta.outputTokens : cumulativeDelta.outputTokens, + cachedInputTokens: runDeltaAvailable + ? explicitDelta.cachedInputTokens + : cumulativeDelta.cachedInputTokens, + reasoningTokens: runDeltaAvailable + ? explicitDelta.reasoningTokens + : cumulativeDelta.reasoningTokens, + costNanodollars: Math.max( + runDeltaAvailable ? explicitDelta.costNanodollars : 0, + cumulativeDelta.costNanodollars, + ), + }; + const observations = this.#usageObservationsByTurn.get(turnId) ?? { + reported: null, + raw: null, + terminal: null, + }; + this.#usageObservationsByTurn.set(turnId, { + ...observations, + reported: { usage, exactDelta: runDeltaAvailable }, + }); + this.#emit({ turnId, kind: "usage", usage }); } } this.#appendEvidence("provider_event", turnId || null, { @@ -2227,27 +2411,37 @@ export class CapabilityLiveSession { // the unambiguous owner of that usage receipt. const usageTurnId = turnId || this.#activeTurnId || this.#terminalTurns.at(-1)?.turnId || ""; if (usageTurnId.length > 0 && Object.keys(rawUsage).length > 0) { - const previous = this.#rawUsageByTurn.get(usageTurnId); + const observations = this.#usageObservationsByTurn.get(usageTurnId) ?? { + reported: null, + raw: null, + terminal: null, + }; const value = (name: string): number => Number.isSafeInteger(rawUsage[name]) && Number(rawUsage[name]) >= 0 ? Number(rawUsage[name]) : 0; if (notification.method === "rawResponse/completed") { - const base = previous ?? { providerRequests: 0, inputTokens: 0, outputTokens: 0, cachedInputTokens: 0, reasoningTokens: 0, costNanodollars: 0 }; - this.#rawUsageByTurn.set(usageTurnId, { - providerRequests: base.providerRequests + 1, - inputTokens: base.inputTokens + value("inputTokens"), - outputTokens: base.outputTokens + value("outputTokens"), - cachedInputTokens: base.cachedInputTokens + value("cachedInputTokens"), - reasoningTokens: base.reasoningTokens + value("reasoningOutputTokens"), - costNanodollars: base.costNanodollars, + const base = observations.raw ?? { providerRequests: 0, inputTokens: 0, outputTokens: 0, cachedInputTokens: 0, reasoningTokens: 0, costNanodollars: 0 }; + this.#usageObservationsByTurn.set(usageTurnId, { + ...observations, + raw: { + providerRequests: base.providerRequests + 1, + inputTokens: base.inputTokens + value("inputTokens"), + outputTokens: base.outputTokens + value("outputTokens"), + cachedInputTokens: base.cachedInputTokens + value("cachedInputTokens"), + reasoningTokens: base.reasoningTokens + value("reasoningOutputTokens"), + costNanodollars: base.costNanodollars, + }, }); - } else if (previous === undefined) { + } else { // Some provider versions publish only a terminal per-turn receipt. - this.#rawUsageByTurn.set(usageTurnId, { - providerRequests: 1, - inputTokens: value("inputTokens"), - outputTokens: value("outputTokens"), - cachedInputTokens: value("cachedInputTokens"), - reasoningTokens: value("reasoningOutputTokens"), - costNanodollars: 0, + this.#usageObservationsByTurn.set(usageTurnId, { + ...observations, + terminal: { + providerRequests: 1, + inputTokens: value("inputTokens"), + outputTokens: value("outputTokens"), + cachedInputTokens: value("cachedInputTokens"), + reasoningTokens: value("reasoningOutputTokens"), + costNanodollars: 0, + }, }); } } diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index 2e4dd027ed..a79caddeda 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -429,6 +429,7 @@ it("rehydrates normalized usage with the opened driver binding", () => { turnId: "durable-turn-1", cumulative: { inputTokens: 10 }, runDelta: { inputTokens: 3 }, + runDeltaAvailable: true, }, "opened-thread-1", "active-turn-1", @@ -437,6 +438,7 @@ it("rehydrates normalized usage with the opened driver binding", () => { providerSessionId: "backend-session-1", threadId: "opened-thread-1", turnId: "active-turn-1", + runDeltaAvailable: true, tokenUsage: { total: { inputTokens: 10 }, runDelta: { inputTokens: 3 }, diff --git a/packages/paperclip-runner/test/acpx-codex-package-contract.test.mjs b/packages/paperclip-runner/test/acpx-codex-package-contract.test.mjs index cc1b2daa30..3b504d5741 100644 --- a/packages/paperclip-runner/test/acpx-codex-package-contract.test.mjs +++ b/packages/paperclip-runner/test/acpx-codex-package-contract.test.mjs @@ -43,8 +43,9 @@ test("the runner pins every qualified ACPX production dependency", () => { ); }); -test("the package exposes only the reviewed provider transport binaries", () => { +test("the package exposes only the reviewed runner CLI binaries", () => { assert.deepEqual(runnerPackage.bin, { + "paperclip-runner-eval-session": "./dist/cli/eval-session.js", "paperclip-runner-codex-proxy": "./dist/cli/codex-app-server-unix-proxy.js", "paperclip-runner-acpx-sidecar": "./dist/cli/acpx-runtime-sidecar.js", diff --git a/packages/paperclip-runner/test/fixtures/fake-opencode-server.mjs b/packages/paperclip-runner/test/fixtures/fake-opencode-server.mjs old mode 100755 new mode 100644 diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs index a5b33ad768..d98ccbfcdd 100644 --- a/scripts/__tests__/release-verify-workflow.test.mjs +++ b/scripts/__tests__/release-verify-workflow.test.mjs @@ -1,10 +1,14 @@ import assert from "node:assert/strict"; -import { readFileSync } from "node:fs"; +import { existsSync, readdirSync, readFileSync } from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; import test from "node:test"; -const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", ".."); +const repoRoot = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "..", + "..", +); function readWorkflow(name) { return readFileSync(path.join(repoRoot, ".github/workflows", name), "utf8"); @@ -26,11 +30,17 @@ test("release workflow delegates stable and canary verification to the reusable releaseWorkflow, /verify_stable:\n\s+if: github\.event_name == 'workflow_dispatch' && inputs\.channel == 'stable'\n(?:\s+needs: [^\n]+\n)?\s+uses: \.\/\.github\/workflows\/release-verify\.yml\n\s+with:\n\s+ref: \$\{\{ needs\.preflight_stable\.outputs\.sha \}\}/, ); - assert.doesNotMatch(releaseWorkflow, /verify_(?:canary|stable):[\s\S]*?pnpm test:run(?:\n|$)/); + assert.doesNotMatch( + releaseWorkflow, + /verify_(?:canary|stable):[\s\S]*?pnpm test:run(?:\n|$)/, + ); }); test("onboard smoke container binds beyond loopback so the mapped port is reachable", () => { - const dockerfile = readFileSync(path.join(repoRoot, "docker/Dockerfile.onboard-smoke"), "utf8"); + const dockerfile = readFileSync( + path.join(repoRoot, "docker/Dockerfile.onboard-smoke"), + "utf8", + ); // `onboard --yes` without an explicit --bind prefers trusted-local // defaults and writes a loopback bind, which Docker port mapping cannot @@ -43,8 +53,14 @@ test("promotion selection guards against sources that predate their channel tool // Promotions run the source commit's release.sh, so selection must reject // sources whose tooling does not know the target channel yet. - assert.match(releaseWorkflow, /git show "\$\{sha\}:scripts\/release\.sh" \| grep -qF 'canary\|nightly'/); - assert.match(releaseWorkflow, /git show "\$\{sha\}:scripts\/release\.sh" \| grep -qF 'canary\|nightly\|beta\|stable\)'/); + assert.match( + releaseWorkflow, + /git show "\$\{sha\}:scripts\/release\.sh" \| grep -qF 'canary\|nightly'/, + ); + assert.match( + releaseWorkflow, + /git show "\$\{sha\}:scripts\/release\.sh" \| grep -qF 'canary\|nightly\|beta\|stable\)'/, + ); }); test("candidate-branch betas are validated and fully verified before publish", () => { @@ -57,7 +73,10 @@ test("candidate-branch betas are validated and fully verified before publish", ( releaseWorkflow, /verify_beta_candidate:\n\s+needs: select_beta\n\s+if: needs\.select_beta\.outputs\.mode == 'candidate'\n\s+uses: \.\/\.github\/workflows\/release-verify\.yml/, ); - assert.match(releaseWorkflow, /needs\.verify_beta_candidate\.result == 'success'/); + assert.match( + releaseWorkflow, + /needs\.verify_beta_candidate\.result == 'success'/, + ); }); test("post-publish beta smoke survives the skipped candidate-verification ancestor", () => { @@ -98,7 +117,9 @@ test("published canaries are gated by the exact-version onboarding browser smoke /smoke_canary_onboarding:[\s\S]*?Install test dependencies\n\s+run: pnpm install --frozen-lockfile/, ); assert.doesNotMatch( - releaseWorkflow.match(/smoke_canary_onboarding:[\s\S]*?(?=\n # ----- Nightly lane)/)?.[0] ?? "", + releaseWorkflow.match( + /smoke_canary_onboarding:[\s\S]*?(?=\n # ----- Nightly lane)/, + )?.[0] ?? "", /cache: pnpm/, ); assert.match( @@ -120,45 +141,89 @@ test("every lane's tag push degrades to recovery instructions when rejected", () // from dispatch or scheduled runs; a rejected tag push after a successful // npm publish must surface runbook recovery commands, not a bare error. const occurrences = releaseWorkflow.match(/## Tag push rejected/g) ?? []; - assert.equal(occurrences.length, 3, "nightly, beta, and stable each carry the recovery summary"); + assert.equal( + occurrences.length, + 3, + "nightly, beta, and stable each carry the recovery summary", + ); }); test("release smoke workflow extends the container readiness budget for CI", () => { const smokeWorkflow = readWorkflow("release-smoke.yml"); - const harness = readFileSync(path.join(repoRoot, "scripts/docker-onboard-smoke.sh"), "utf8"); + const harness = readFileSync( + path.join(repoRoot, "scripts/docker-onboard-smoke.sh"), + "utf8", + ); // CI containers cold-install paperclipai and embedded postgres, so the // workflow must extend the harness's local-default readiness budget. assert.match(smokeWorkflow, /SMOKE_READY_TIMEOUT_SECONDS=\d+/); - const ciBudget = Number(smokeWorkflow.match(/SMOKE_READY_TIMEOUT_SECONDS=(\d+)/)[1]); - assert.ok(ciBudget >= 300, `CI readiness budget ${ciBudget}s should be at least 300s`); + const ciBudget = Number( + smokeWorkflow.match(/SMOKE_READY_TIMEOUT_SECONDS=(\d+)/)[1], + ); + assert.ok( + ciBudget >= 300, + `CI readiness budget ${ciBudget}s should be at least 300s`, + ); - assert.match(harness, /SMOKE_READY_TIMEOUT_SECONDS="\$\{SMOKE_READY_TIMEOUT_SECONDS:-\d+\}"/); - assert.match(harness, /wait_for_http "\$PAPERCLIP_PUBLIC_URL\/api\/health" "\$SMOKE_READY_TIMEOUT_SECONDS" 1/); + assert.match( + harness, + /SMOKE_READY_TIMEOUT_SECONDS="\$\{SMOKE_READY_TIMEOUT_SECONDS:-\d+\}"/, + ); + assert.match( + harness, + /wait_for_http "\$PAPERCLIP_PUBLIC_URL\/api\/health" "\$SMOKE_READY_TIMEOUT_SECONDS" 1/, + ); }); test("release verify workflow covers the same split test surface as stable PR verification", () => { const verifyWorkflow = readWorkflow("release-verify.yml"); assert.match(verifyWorkflow, /workflow_call:/); - assert.match(verifyWorkflow, /node \.\/scripts\/release-package-map\.mjs check/); + assert.match( + verifyWorkflow, + /node \.\/scripts\/release-package-map\.mjs check/, + ); assert.match(verifyWorkflow, /pnpm -r typecheck/); assert.match(verifyWorkflow, /pnpm build/); - assert.match(verifyWorkflow, /pnpm --filter @paperclipai\/paperclip-runner check:all/); + assert.match( + verifyWorkflow, + /pnpm --filter @paperclipai\/paperclip-runner check:all/, + ); + assert.match(verifyWorkflow, /runner_workflow_evals:/); + assert.match(verifyWorkflow, /runner_chaos_evals:/); + assert.match( + verifyWorkflow, + /uses: \.\/\.github\/workflows\/runner-chaos-evals\.yml/, + ); + assert.match( + verifyWorkflow, + /runner_workflow_evals:[\s\S]*?Install dependencies\n\s+run: pnpm install --frozen-lockfile[\s\S]*?Run deterministic Runner workflow scorer tests/, + ); + assert.match(verifyWorkflow, /pnpm test:runner-workflow-evals/); - for (const group of ["general-server", "general-workspaces-a", "general-workspaces-b"]) { + for (const group of [ + "general-server", + "general-workspaces-a", + "general-workspaces-b", + ]) { assert.match(verifyWorkflow, new RegExp(`group: ${group}`)); } for (const shardIndex of [0, 1, 2]) { assert.match( verifyWorkflow, - new RegExp(`group: general-server[\\s\\S]*?shard_index: ${shardIndex}[\\s\\S]*?shard_count: 3`), + new RegExp( + `group: general-server[\\s\\S]*?shard_index: ${shardIndex}[\\s\\S]*?shard_count: 3`, + ), ); } for (const shardIndex of [0, 1, 2, 3, 4]) { - assert.match(verifyWorkflow, new RegExp(`shard_index: ${shardIndex}[\\s\\S]*?shard_count: 5`)); + assert.match( + verifyWorkflow, + new RegExp(`shard_index: ${shardIndex}[\\s\\S]*?shard_count: 5`), + ); } // workspaces-a splits with Vitest native --shard in pr.yml; release @@ -166,10 +231,167 @@ test("release verify workflow covers the same split test surface as stable PR ve for (const shardIndex of [0, 1]) { assert.match( verifyWorkflow, - new RegExp(`group: general-workspaces-a[\\s\\S]*?shard_index: ${shardIndex}\\n\\s+shard_count: 2`), + new RegExp( + `group: general-workspaces-a[\\s\\S]*?shard_index: ${shardIndex}\\n\\s+shard_count: 2`, + ), ); } assert.match(verifyWorkflow, /pnpm test:run:general -- --group/); assert.match(verifyWorkflow, /pnpm test:run:serialized -- --shard-index/); }); + +test("Runner eval workflows pin actions and gate paid live execution", () => { + const actionPinWorkflows = [ + readWorkflow("release-verify.yml"), + readWorkflow("runner-live-evals.yml"), + readWorkflow("runner-chaos-evals.yml"), + readWorkflow("runner-full-stack-e2e.yml"), + readWorkflow("e2e.yml"), + ]; + + for (const workflow of actionPinWorkflows) { + const remoteUses = workflow + .split("\n") + .filter( + (line) => + /^\s*(?:-\s*)?uses: /.test(line) && !line.includes("uses: ./"), + ); + assert.ok( + remoteUses.length > 0, + "expected at least one remote action reference", + ); + for (const line of remoteUses) { + assert.match(line, /uses: [^@\s]+@[0-9a-f]{40}(?:\s+# .+)?$/); + } + } + + const liveWorkflow = actionPinWorkflows[1]; + assert.match(liveWorkflow, /RUNNER_LIVE_EVALS_NIGHTLY_ENABLED == 'true'/); + assert.match(liveWorkflow, /REF: \$\{\{ github\.ref \}\}/); + assert.match(liveWorkflow, /refs\/heads\/\$DEFAULT_BRANCH/); + assert.match( + liveWorkflow, + /DEFAULT_BRANCH: \$\{\{ github\.event\.repository\.default_branch \}\}/, + ); + assert.match(liveWorkflow, /RUNNER_E2E_ALLOWED_ACTOR_IDS/); + assert.match(liveWorkflow, /needs: authorize/); + assert.match(liveWorkflow, /environment:\n\s+name: runner-e2e-paid/); + assert.match( + liveWorkflow, + /OPENAI_API_KEY: \$\{\{ secrets\.OPENAI_API_KEY \}\}/, + ); + + const paidWorkflowNames = [ + "e2e.yml", + "runner-full-stack-e2e.yml", + "runner-live-evals.yml", + ]; + const paidWorkflowNameSet = new Set(paidWorkflowNames); + const providerSecretReference = + /secrets(?:\.(?:OPENAI_API_KEY|ANTHROPIC_API_KEY|OPENROUTER_API_KEY|DAYTONA_API_KEY)\b|\[['"](?:OPENAI_API_KEY|ANTHROPIC_API_KEY|OPENROUTER_API_KEY|DAYTONA_API_KEY)['"]\])/g; + for (const name of readdirSync(path.join(repoRoot, ".github/workflows"))) { + if (!/\.ya?ml$/.test(name)) continue; + const workflow = readWorkflow(name); + if ([...workflow.matchAll(providerSecretReference)].length > 0) { + assert.ok( + paidWorkflowNameSet.has(name), + `${name} must not receive provider credentials`, + ); + } + } + + for (const name of paidWorkflowNames) { + const workflow = readWorkflow(name); + const triggerHeader = workflow.slice(0, workflow.indexOf("\njobs:\n")); + assert.doesNotMatch( + triggerHeader, + /^\s{2}(?:pull_request|pull_request_target|push|workflow_call|workflow_run):/m, + ); + assert.match(triggerHeader, /^\s{2}workflow_dispatch:/m); + assert.match(workflow, /^ authorize:/m); + + const jobBlocks = workflow + .slice(workflow.indexOf("\njobs:\n") + "\njobs:\n".length) + .split(/\n(?= [A-Za-z0-9_-]+:\n)/); + const providerJobs = jobBlocks.filter( + (block) => [...block.matchAll(providerSecretReference)].length > 0, + ); + assert.ok(providerJobs.length > 0, `${name} needs a provider-secret job`); + for (const block of providerJobs) { + assert.match(block, /\n environment:\n name: runner-e2e-paid\n/); + assert.match( + block, + /\n steps:\n(?:\s*\n)* - name: Reauthorize[^\n]*\n/, + `${name} must reauthorize as the first provider-job step`, + ); + const reauthorize = block.indexOf(" - name: Reauthorize"); + assert.ok(reauthorize > 0); + assert.ok(block.indexOf("actions/checkout@") > reauthorize); + assert.ok(block.search(providerSecretReference) > reauthorize); + assert.match(block, /github\.actor_id/); + assert.match(block, /github\.triggering_actor/); + assert.match(block, /RUNNER_E2E_ALLOWED_ACTOR_IDS/); + assert.match(block, /refs\/heads\/\$DEFAULT_BRANCH/); + assert.doesNotMatch(block, /^\s+cache: pnpm$/m); + } + } + + const fullStackWorkflow = readWorkflow("runner-full-stack-e2e.yml"); + for (const [secret, condition] of Object.entries({ + OPENAI_API_KEY: "matrix.credentialName == 'OPENAI_API_KEY'", + ANTHROPIC_API_KEY: "matrix.credentialName == 'ANTHROPIC_API_KEY'", + OPENROUTER_API_KEY: "matrix.credentialName == 'OPENROUTER_API_KEY'", + DAYTONA_API_KEY: "matrix.environmentId == 'daytona'", + })) { + assert.ok( + fullStackWorkflow.includes( + `${secret}: \${{ ${condition} && secrets.${secret} || '' }}`, + ), + `${secret} must be scoped to only the matrix cells that require it`, + ); + } + const historyPublisher = fullStackWorkflow.slice( + fullStackWorkflow.indexOf(" publish_history:"), + fullStackWorkflow.indexOf(" pages:"), + ); + assert.doesNotMatch(historyPublisher, /^\s+cache: pnpm$/m); + + for (const name of ["runner-full-stack-e2e.yml", "runner-live-evals.yml"]) { + const workflow = readWorkflow(name); + const crons = [...workflow.matchAll(/cron:\s*"([^"]+)"/g)].map( + (match) => match[1], + ); + assert.equal(crons.length, 1, `${name} must have one schedule`); + assert.match(crons[0], /^\d{1,2} \d{1,2} \* \* 0$/); + } + + const chaosWorkflow = actionPinWorkflows[2]; + const runnerBlock = chaosWorkflow.match( + /- name: Run Runner fault and replay suites[\s\S]*?run: \|([\s\S]*?)(?=\n\s+- name: Build server test dependencies)/, + )?.[1]; + const serverBlock = chaosWorkflow.match( + /- name: Run server finalization and recovery suites[\s\S]*?run: \|([\s\S]*?)(?=\n\s+- name: Upload chaos eval bundle)/, + )?.[1]; + assert.ok(runnerBlock, "expected Runner chaos test command"); + assert.ok(serverBlock, "expected server chaos test command"); + for (const [base, block] of [ + [path.join(repoRoot, "packages/paperclip-runner"), runnerBlock], + [path.join(repoRoot, "server"), serverBlock], + ]) { + const listedTestPaths = + block.match(/src\/[A-Za-z0-9_./-]+\.test\.ts/g) ?? []; + assert.ok(listedTestPaths.length > 0, "expected chaos workflow test paths"); + assert.equal( + new Set(listedTestPaths).size, + listedTestPaths.length, + "chaos workflow test paths must be unique", + ); + for (const testPath of listedTestPaths) { + assert.ok( + existsSync(path.join(base, testPath)), + `chaos workflow test path does not exist: ${testPath}`, + ); + } + } +}); diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md new file mode 100644 index 0000000000..5d533b2d01 --- /dev/null +++ b/tests/runner-e2e/FIXTURES.md @@ -0,0 +1,149 @@ +# Runner E2E fixture authoring + +The fixture catalog is executable production-contract data. Keep it small, +typed, deterministic, and free of raw credentials. + +## Suites and matrices + +A `RunnerSuiteFixture` declares one durable testing purpose: stable ID, label, +description, profiles, environments, cases, expected size, and definition or +ranking metadata. Its execution IDs are globally prefixed as +`...`. Add a new suite when the testing +purpose or desired cross-product differs; do not inflate an existing suite with +unrelated dimensions. + +The suite definition fingerprint is historical comparison metadata. Any +profile, model qualification, environment, task, or ranking-snapshot change +must change that fingerprint automatically so the dashboard can annotate the +boundary instead of silently joining unlike totals. + +## Agent profiles + +Add `RunnerProfileFixture` entries in `catalog.ts`. A profile declares: + +- a stable ID and searchable groups; +- legacy or native generation; +- adapter/provider and required credential; +- a model imported from its adapter constant or qualified runner profile; +- supported environment IDs; +- expected runtime metadata; and +- an agent payload factory. + +Do not duplicate model IDs, qualification decisions, CLI versions, or runner +artifact rules. Codex profiles import `DEFAULT_CODEX_LOCAL_MODEL`, OpenCode +profiles import `QUALIFIED_OPENCODE_MODEL`, and ACPX profiles import +`QUALIFIED_ACPX_PROFILES`. Add or qualify models at their owning production +source first. + +OpenRouter breadth profiles are generated from `openrouter-models.json`, not +written by hand. That reviewed snapshot must contain exactly five unique, +available, tool-capable models with rank, canonical ID, display name, supported +parameters, source URL, capture time, and verified content hash. Refresh it +manually with `pnpm test:e2e:runner:models:update`; nightly campaigns never +change fixture definitions. + +Agent `adapterConfig.env` values must be `{type:"secret_ref", secretId, +version:"latest"}` objects supplied to the factory. A fixture source containing +a raw secret-looking value is rejected by catalog validation. + +## Environments + +An `EnvironmentFixture` declares driver/provider, credential requirements, +attempt deadline, lifecycle behavior, expected execution target, and a payload +factory validated by the shared environment schema. + +The local environment is instance-managed: company creation ensures it exists, +and the public API intentionally rejects a second local environment. The setup +registry therefore discovers that row through the public environments API. +This still provides full isolation because every cell starts a new Paperclip +instance and database. + +Daytona creates a sandbox environment through the public API. Keep +`reuseLease:false`, `runnerLifecycleMode:"per_turn"`, short provider cleanup +backstops, a Daytona secret reference, and an immutable image digest. Teardown +must delete the environment with reusable-lease destruction and must fail the +cell if cleanup cannot be confirmed. Keep CPU, memory, and disk explicit: lease +metadata and the per-test public-list-price runtime estimate depend on that +pinned billable resource shape. Changing it requires updating billing tests and +reviewing the versioned Daytona rates in `billing.ts`. + +## Usage and billing data + +Do not add fixture-authored token or dollar expectations. The live harness +reads usage from selected public heartbeat-run records and records coverage per +run. Provider-reported dollars remain distinct from runtime estimates. A zero +or missing native usage payload is `unavailable` unless a real token-bearing +receipt or provider cost proves otherwise. New execution environments must +provide lease/resource metadata for a runtime estimate or explicitly remain +`unavailable`; never infer that missing billing data means free execution. + +Future providers (SSH, E2B, Modal, Cloudflare, Kubernetes, Novita, exe.dev) +should implement the same setup/probe/cleanup contract before being added to a +matrix. Unsupported profile/environment combinations belong in +`supportedEnvironments`, not in ad hoc test conditionals. + +## Task cases and matchers + +A `RunnerTaskFixture` owns a work mode, a typed flow, expected run count, +nonce-based title/prompt/marker factories, per-environment attempt deadlines, +deterministic matchers, and expected terminal state. Single-turn prompts should +make one bounded request with observable output and no nondeterministic judging. +The `plan_revision_acceptance` flow must also provide revision-request and Plan +marker factories. `question_resume_completion` must define the deterministic +browser answer and prove exactly two successful runs with no pending +interaction. `plan_approval_completion` must target the exact two-step +canonical Plan revision, capture its pending UI, approve in the browser, and +prove exactly two successful runs. + +Every selected case runs in its own isolated Paperclip process, and independent +cases may run concurrently. Follow-up turns inside one case retain their shared +task state. Each case creates and tears down its own company, secrets, +environment selection, agent, and browser-created task. The current plan case +proves three runs on the same issue: publish a two-step Plan, +request a three-step revision through the UI, and accept the exact new revision +through the UI before verifying implementation and Done. + +The matcher union supports message exact/contains/regex/ordered checks, issue +and run state, runtime/environment metadata, files, artifacts, JSON paths, and +JSON Schema. The initial cases use normalized `message_contains` plus state, +runtime, and environment assertions; the plan flow additionally verifies +canonical document revision IDs, bodies, step counts, interaction targets, and +visible previews. Add matcher behavior and credential-free tests together. + +Adding a task expands its suite's matrix. Update the suite's intentional size, +the complete-catalog size, and credential-free unit tests in the same change. +Paid tests never silently skip a missing credential or unsupported artifact. + +## New Paperclip object fixtures + +Register new objects in `live-fixtures.ts` with explicit dependencies in +`FixtureRegistry`. Setup must use a public API. Teardown runs in reverse order +and is invoked after partial setup failures. Direct database writes and private +test-only runner endpoints are prohibited. + +The expected dependency shape is: + +```text +company +└── encrypted secrets + └── environment + └── agent + └── browser-created task +``` + +Projects, goals, apps, and configuration fixtures can be inserted into that +graph without changing the launcher. Keep returned fixture state to IDs and +sanitized metadata; never retain raw secret values. + +## Required checks + +Run before a fixture change is reviewed: + +```bash +pnpm test:e2e:runner:unit +pnpm test:e2e:runner:typecheck +pnpm test:e2e:runner -- --list +``` + +Then run the narrowest paid cell that exercises the fixture. A full matrix is a +manual or scheduled campaign, not a PR requirement. diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md new file mode 100644 index 0000000000..416ff08245 --- /dev/null +++ b/tests/runner-e2e/README.md @@ -0,0 +1,328 @@ +# Paid runner full-stack E2E + +This is the billable browser acceptance campaign system for Paperclip runner +profiles. It is deliberately separate from `tests/e2e`: every independently +scheduled execution gets +a fresh Paperclip home, embedded Postgres database, instance configuration, +port, workspace, company, encrypted secrets, environment, and agent. + +The vocabulary is: a **campaign** is one workflow invocation against one SHA; a +**suite** is a durable testing purpose; a **matrix** is that suite's profiles × +environments × cases; an **execution/cell** is one parallel job; and an +**attempt** is one isolated harness run, including an infrastructure retry. + +The browser creates and assigns the task. The harness does not call a private +runner hook or write fixtures directly to the database. + +## Credentials + +Copy `.env.runner-e2e.example` to `.env.runner-e2e.local` and fill only the +credentials needed by the selected cells: + +```bash +cp .env.runner-e2e.example .env.runner-e2e.local +chmod 600 .env.runner-e2e.local +``` + +Shell variables take precedence over the local file. The recognized names are: + +- `OPENAI_API_KEY` +- `ANTHROPIC_API_KEY` +- `OPENROUTER_API_KEY` +- `DAYTONA_API_KEY` +- `PAPERCLIP_E2E_DAYTONA_IMAGE` (Daytona only) + +The image must be an immutable `image@sha256:...` reference. The launcher +reports missing variable names but never prints values. It passes raw provider +keys only to Playwright, which posts each value once to the company-secrets API. +Paperclip receives secret references in agent/environment payloads. Provider +keys, Daytona keys, `DATABASE_URL`, and `DATABASE_MIGRATION_URL` are removed +from the Paperclip child process. + +Never put credentials in `catalog.ts`, screenshots, fixture metadata, workflow +inputs, or a tracked env file. + +## Local commands + +Install dependencies and Chromium once. Native local cells also need the local +runner binaries: + +```bash +pnpm install +pnpm exec playwright install chromium +pnpm --filter @paperclipai/paperclip-runner build:runner-binaries +``` + +List cells without loading credentials or starting Paperclip: + +```bash +pnpm test:e2e:runner -- --list +``` + +Examples of explicit billable runs: + +```bash +pnpm test:e2e:runner -- --id core-compatibility.legacy-codex.local.message-marker --headed +pnpm test:e2e:runner -- --suite openrouter-model-breadth --case hello-complete +pnpm test:e2e:runner -- --group native --environment local +pnpm test:e2e:runner -- --profile runner-codex --case message-marker +pnpm test:e2e:runner -- --case plan-revise-accept --group local +pnpm test:e2e:runner -- --case ask-question --group native +pnpm test:e2e:runner -- --all +``` + +The catalog contains two suites. `core-compatibility` (**Core Runner +Compatibility**) is seven major runner profiles × local/Daytona × three +workflows: 42 cells. Its cases are: + +- `message-marker`: one basic visible response and Done transition; +- `plan-revise-accept`: an initial Plan, a browser-requested revision on the + same Plan, browser acceptance of the new revision, and verified execution; +- `ask-question`: a direct answer from a task created in Ask mode. + +`openrouter-model-breadth` (**OpenRouter Model Breadth**) is five models from +the tracked weekly tool-capable ranking snapshot × native OpenCode × local × +three workflows: 15 cells. Its cases are: + +- `hello-complete`: a basic nonce response and explicit Done transition; +- `question-resume-complete`: one structured question, browser selection of + “Cobalt,” then a resumed completion on the same task; and +- `plan-approve-complete`: one exact two-step Plan, browser approval of that + revision, then a resumed completion on the same task. + +The complete catalog is 57 cells and 95 expected paid agent turns. Follow-up +steps remain ordered within their cell; all other cells are independent. +Narrow selectors are strongly recommended while developing fixtures. + +`--suite`, `--group`, `--profile`, `--environment`, and `--case` are repeatable. Repeated +values in one dimension use OR semantics; dimensions and repeated groups use +AND semantics. `--id` is exclusive with dimension selectors and `--all`. +`--headed`, `--ui`, and `--debug` are forwarded to Playwright. An unknown +selector, an empty selection, or a run with no explicit selector exits before +Paperclip starts. `--max-parallel ` controls the number of isolated +profile/environment/case harnesses that can overlap (default 1, also configurable +with `PAPERCLIP_E2E_MAX_PARALLEL`). Headed/UI/debug runs are forced to one worker. +The Plan case is still sequential internally because its turns share one task; +it runs in parallel with unrelated scenarios. + +Use a single `--id` smoke test for routine local verification. Full-matrix +parallelism is intended for GitHub Actions; raising local parallelism starts +multiple Paperclip/Postgres/Chromium stacks and can consume substantial CPU and +memory. + +Credential-free checks are: + +```bash +pnpm test:e2e:runner:unit +pnpm test:e2e:runner:typecheck +``` + +The OpenRouter ranking snapshot is tracked in `openrouter-models.json`; nightly +runs never mutate it. Refresh it deliberately, review the source/capture/hash +diff, and rerun credential-free checks: + +```bash +pnpm test:e2e:runner:models:update +``` + +## Daytona image + +Use the immutable digest printed by the `Publish verified Daytona image` job, +or publish the current source locally: + +```bash +content_id="$(pnpm --silent test:e2e:runner:image-id)" +source_revision="$(git rev-parse HEAD)" +image="ghcr.io/paperclipai/paperclip-daytona-runner:e2e-content-${content_id}" +if ! docker buildx imagetools inspect "$image" >/dev/null 2>&1; then + docker buildx build \ + --platform linux/amd64 \ + --build-arg "PAPERCLIP_RUNNER_CONTENT_ID=${content_id}" \ + --build-arg "PAPERCLIP_RUNNER_SOURCE_REVISION=${source_revision}" \ + --file docker/daytona-runner/Dockerfile \ + --tag "$image" \ + --push \ + . +fi +docker buildx imagetools inspect "$image" +``` + +The content ID hashes the audited image inputs, including the Dockerfile, +platform, root package/lock/build configuration, dependency patches, +`paperclip-eval-kernel`, and `paperclip-runner`. Changes elsewhere in the +repository keep the same tag and reuse the already signed image. The Git SHA is +stored separately as image provenance. CI reads that provenance back from a +reused image when it builds the controller-side provider pack, preserving the +exact manifest match required to avoid restaging the pack into Daytona. + +Resolve the manifest digest and set `PAPERCLIP_E2E_DAYTONA_IMAGE` to +`ghcr.io/paperclipai/paperclip-daytona-runner@sha256:...`. The repository +workflow signs that digest with Cosign/OIDC and verifies that it is publicly +pullable, includes the provider pack, and advertises `dial_ws_loopback`, +`dial_wss`, and `listen_ws`. The GHCR package must be configured as public; +the image job deliberately fails its anonymous-pull check otherwise. Existing +content tags are never rebuilt or overwritten by the workflow. + +## Evidence and cleanup + +Packaged, access-controlled evidence is written beneath +`tests/runner-e2e/results//...`. Passing attempts include +`final-state.png`, Plan draft/revision screenshots when applicable, matcher +outcomes, sanitized fixture/API metadata, a result record, JUnit, HTML, and a +blob report. Failures additionally retain the Playwright trace/video, browser +diagnostics, failure screenshot, and sanitized Paperclip/run logs when +produced. PNG and WebM files are not pixel-inspected, so they are suitable only +for the local results directory and access-controlled GitHub Actions artifacts. +SVG is active content and is rejected from the packaged evidence entirely. + +Every completed local campaign also writes +`tests/runner-e2e/results//dashboard.html`. The self-contained page +shows the complete profile/environment grid with screenshot thumbnails. +Expanding a case shows its matchers, pass/fail details, provider/model/runtime, +timings, token and cost accounting, and evidence links. The campaign header +aggregates input, output, and cached tokens, provider-reported LLM spend, +Daytona list-price runtime estimates, and pricing coverage. Missing provider +usage is labeled `unavailable` or `unpriced`; it is never presented as zero +cost. The CI report job stages the same portable site at +`normalized/index.html` inside the access-controlled merged report artifact. + +Permanent public history has a narrower boundary. Before uploading to S3 or +packaging the optional GitHub Pages artifact, the publisher removes raster and +video evidence, archives, and the generated Playwright/blob/HTML report trees. +It then regenerates the dashboard against only the remaining allowlisted, +inert structured per-attempt evidence (`.json`, `.log`, `.md`, and `.txt`). +Per-attempt XML is excluded because browsers can process XML/XSLT. The root +`junit.xml` remains public because the report aggregator builds it from fixed +markup and XML-escaped fields. Public dashboards therefore contain results and +accounting but no attempt screenshots, videos, traces, or generated Playwright +reports. + +### Billing interpretation + +Each result contains raw sanitized `usage`, normalized `billing`, and +`runtimeUsage`: + +- LLM token and dollar values come from the persisted heartbeat-run usage. A + multi-turn case aggregates every selected run and records how many runs + supplied tokens and provider-reported cost. +- Local execution records agent run time but is `not_metered` because there is + no external environment provider charge to attribute. +- Daytona records every public-API lease window and its pinned 4 vCPU, 4 GiB + RAM, and 10 GiB disk allocation. Its runtime dollar value is an estimate at + the versioned public list rates in `billing.ts`, not an invoice amount. + Credits, discounts, the storage allowance, and delayed billing adjustments + can make the eventual Daytona charge lower. + +`normalized-results.json` uses the v2 campaign schema and includes per-test, +per-suite, and overall billing. The compact `history.json` index retains the +same metrics per campaign/suite/execution, source SHA/ref, definition +fingerprints, completeness, retries, and cleanup. Trend charts compare only +complete campaigns by default; partial/manual selections remain browsable. +`summary.md` carries the current totals into the GitHub Actions job summary. + +### Iterate on a published dashboard without rerunning paid tests + +Download and extract the `github-pages` artifact from an existing workflow run, +then regenerate only its HTML from the retained `normalized-results.json` and +public structured evidence files. The Pages artifact has already had private +visual and generated report evidence removed: + +```bash +gh run download --repo paperclipai/paperclip --name github-pages --dir /tmp/runner-e2e-pages +mkdir /tmp/runner-e2e-site +tar -xf /tmp/runner-e2e-pages/artifact.tar -C /tmp/runner-e2e-site +pnpm test:e2e:runner:dashboard -- /tmp/runner-e2e-site +# Optionally use a downloaded history index: +pnpm test:e2e:runner:dashboard -- /tmp/runner-e2e-site --history /tmp/history.json +``` + +Serve that directory with any static file server. This path does not start +Paperclip, invoke an agent, create a Daytona lease, or consume provider tokens. + +Before an access-controlled evidence artifact is uploaded, the launcher: + +1. copies only allowlisted file types; +2. scans raw API snapshots before sanitizing them; +3. scans the closed Paperclip home/database and workspace as streams; +4. redacts loaded exact values and known provider-key shapes from text; +5. expands ZIP reports for secret scanning; +6. rejects SVG and other unsafe files and fails the cell if a leak is detected; + and +7. verifies that a passing attempt has its final-state screenshot. + +The temporary Paperclip home, embedded database, raw workspace, master key, +and unredacted logs are removed after each attempt. Daytona teardown destroys +the environment and any reusable leases through the public API; provider-side +auto-stop/archive/delete values remain as cancellation backstops. + +## GitHub Actions + +`Runner Full-Stack E2E` has only `schedule` and `workflow_dispatch` triggers; it +never runs for a pull request or ordinary push. Because this repository is +public, manual campaigns fail before checkout unless they run from the default +branch and both the original actor and rerun actor have numeric GitHub user IDs +in the non-empty JSON-array repository variable +`RUNNER_E2E_ALLOWED_ACTOR_IDS`. Usernames are intentionally not trusted. +The first scheduled attempt is trusted automation; any human rerun of a +scheduled campaign must pass the triggering-actor allowlist. + +Create a protected `runner-e2e-paid` GitHub environment, restrict it to the +default branch, limit environment administration to trusted maintainers, and +store the four provider secrets there. This is a second authorization boundary: +the pre-check prevents unauthorized scheduling, while the environment prevents +secret release if the workflow gate is accidentally weakened. Also restrict +Actions to approved actions and require review of `.github/workflows/**` and +`tests/runner-e2e/**` through CODEOWNERS and branch protection. Manual inputs +accept comma-separated values for repeatable dimensions. + +The nightly cron is `08:47 UTC`, but scheduled execution is intentionally gated +by the repository variable `RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED=true`. Set it +only after the live acceptance ladder in the architecture plan is green. +Set `RUNNER_E2E_MAX_PARALLEL` to an integer from 1–57 (default 32). Paid cells +run on `ubuntu-latest-m`; multi-turn steps are sequential inside their cell +while independent cells overlap. Artifacts and merged HTML/JUnit/normalized +reports are retained for 30 days. + +Restrict the `ubuntu-latest-m` runner group to this workflow and the selected +repository. Do not let pull-request or fork-triggered workflows target that +group, do not mix it with untrusted workloads, and use ephemeral/reimaged +runners so one paid cell cannot leave state for the next. These runner-group +controls are external GitHub settings and are as important as the workflow +checks in a public repository. + +GitHub Actions artifacts are access-controlled 30-day operational copies, not +the permanent public history. They retain packaged PNG/WebM and generated +reports for debugging. Create a second protected `runner-e2e-history` +environment, restricted to the default branch and trusted environment +administrators, then configure these repository variables: + +- `RUNNER_E2E_HISTORY_AWS_ROLE_ARN` +- `RUNNER_E2E_HISTORY_AWS_REGION` +- `RUNNER_E2E_HISTORY_S3_BUCKET` +- `RUNNER_E2E_HISTORY_PUBLIC_BASE_URL` +- optional `RUNNER_E2E_HISTORY_PREFIX` (default `runner-e2e`) + +The job exchanges GitHub OIDC for short-lived AWS credentials; never add AWS +access-key secrets. Its IAM role must trust only +`repo:paperclipai/paperclip:environment:runner-e2e-history`, and permit only +Get/List/Put under the configured prefix—never Delete. Enable S3 versioning and +Block Public Access. CloudFront reads the private bucket through Origin Access +Control. Immutable campaign bundles live under `campaigns/-/`; +mutable `history.json`, `latest.json`, and `latest-green.json` are updated by a +globally serialized publisher. An existing campaign key with a different +bundle digest fails closed. + +GitHub Pages remains the stable latest dashboard. Enable Pages with GitHub +Actions as its source and set `RUNNER_FULL_STACK_E2E_PUBLISH_PAGES=true`. +The publisher prunes screenshots, video, archives, and generated report trees, +then regenerates the public dashboard before either the CloudFront-backed S3 +history or optional Pages artifact is created. Public per-attempt evidence is +limited to allowlisted inert structured text. Databases, Paperclip homes, +workspaces, raw/unredacted logs, credentials, and visual evidence are never +published. Sanitized allowlisted `.log` copies may be public only after +exact-value/key-shape scanning and redaction. + +See [FIXTURES.md](./FIXTURES.md) before adding or changing a profile, +environment, task, matcher, or future Paperclip object fixture. +See [SECURITY.md](./SECURITY.md) before enabling paid dispatch, the runner +group, or permanent public history in this public repository. diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md new file mode 100644 index 0000000000..6a833e2fa6 --- /dev/null +++ b/tests/runner-e2e/SECURITY.md @@ -0,0 +1,165 @@ +# Runner E2E security for a public repository + +This suite can spend provider money, expose four API credentials to isolated +test processes, publish a container, retain private visual evidence, and write +public structured evidence. Treat changes to the workflow, harness, fixture +prompts, evidence packager, and publisher as security-sensitive production +changes. + +## GitHub authorization + +Set `RUNNER_E2E_ALLOWED_ACTOR_IDS` to a non-empty JSON array of numeric GitHub +user IDs, for example `[123456,789012]`. Resolve each ID from the authenticated +CLI and verify the login before adding it: + +```bash +gh api users/LOGIN --jq '{login,id}' +``` + +The paid workflows reject manual dispatches outside the default branch before +checkout. They verify both the original actor and triggering actor for every +scheduled or manual attempt, including human reruns. Every +secret-bearing job repeats this check as its first step so GitHub's partial-job +rerun feature cannot bypass a successful predecessor authorization job. The +legacy manually dispatched E2E workflow uses the same gate. Numeric IDs are +stable across username changes and prevent lookalike-name authorization. + +The full-stack and live campaigns have one Sunday UTC schedule each and also +support explicit manual dispatch. Their legacy-named nightly repository +variables remain independent kill switches. Neither paid workflow accepts +pull-request, push, workflow-run, or reusable-workflow triggers. + +Protect the default branch, require review for workflow/harness paths, restrict +workflow dispatch permission, and restrict repository variable/environment +administration to the same trusted maintainers. Configure the organization to +allow only approved GitHub Actions. A malicious change merged into the default +branch executes with the same authority as the suite. + +Every external action in the paid workflow is pinned to a full commit SHA. Keep +the adjacent major-version comment for update tooling, and resolve and review a +new immutable SHA before upgrading an action. The credential-free security test +rejects mutable tag or branch references. + +## Secrets and protected environments + +Create `runner-e2e-paid`, restrict deployments to the default branch, and put +only `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `OPENROUTER_API_KEY`, and +`DAYTONA_API_KEY` in it. Do not duplicate these credentials as repository- or +organization-level Actions secrets: environment scoping is the boundary that +prevents branch or pull-request jobs from requesting them. Require approval +from an account in `RUNNER_E2E_ALLOWED_ACTOR_IDS` for this environment and +disable administrator bypass. The authorize, +catalog, image, report, history, and Pages jobs receive none of these secrets. +Each full-stack matrix cell receives only its selected profile credential, plus +Daytona only for Daytona cells. Secret-bearing and OIDC jobs use frozen installs +without a shared dependency cache. +The Paperclip server process also receives none; the browser posts each value +once to the encrypted company secret API and agents/environments retain only +secret references. + +Create `runner-e2e-history`, also default-branch-only, for the OIDC publishing +job. It contains no long-lived AWS key. Required reviewers may be added when a +human approval on every nightly publication is acceptable; otherwise rely on +the actor gate, environment branch restriction, and protected default branch. + +## Runner group isolation + +Restrict the `ubuntu-latest-m` runner group to `paperclipai/paperclip` and, when +the GitHub plan supports selected-workflow restrictions, to +`.github/workflows/runner-full-stack-e2e.yml` on the default branch. Never let +fork or pull-request workflows target the group. Use ephemeral runners, or +guaranteed reimaging between jobs, and do not share this group with untrusted +workloads. Disable interactive SSH/debug access for paid jobs unless a separate +incident procedure explicitly authorizes it. + +## AWS OIDC and S3 + +The AWS role trust policy should accept only GitHub's OIDC audience and the +publishing environment subject: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": "arn:aws:iam::ACCOUNT_ID:oidc-provider/token.actions.githubusercontent.com" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "token.actions.githubusercontent.com:aud": "sts.amazonaws.com", + "token.actions.githubusercontent.com:sub": "repo:paperclipai/paperclip:environment:runner-e2e-history" + } + } + } + ] +} +``` + +Grant only List on the bucket prefix and Get/Put on its objects. Do not grant +Delete, ACL, bucket-policy, or wildcard-resource permissions: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "s3:ListBucket", + "Resource": "arn:aws:s3:::BUCKET", + "Condition": { + "StringLike": { "s3:prefix": ["runner-e2e", "runner-e2e/*"] } + } + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject"], + "Resource": "arn:aws:s3:::BUCKET/runner-e2e/*" + } + ] +} +``` + +Enable S3 versioning, default encryption, and Block Public Access. Disable +object ACLs. CloudFront receives read-only access through Origin Access Control; +the bucket itself stays private. Log S3 data writes and alert on attempts to +write outside the prefix or assume the role with a different subject. + +Campaign prefixes are content-digested and immutable. The publisher refuses a +different digest at an existing campaign key. Only the compact history and +latest pointers are mutable, and S3 versioning makes those updates recoverable. + +## Public evidence boundary + +CloudFront and GitHub Pages are public. Fixture identifiers, timing, token +usage, costs, normalized results, and allowlisted inert structured per-attempt +evidence are expected public data. Screenshots, video, archives, generated +Playwright/blob/HTML report trees, credentials, Paperclip homes, databases, +workspaces, master keys, raw/unredacted logs, and unallowlisted files are not. +Only allowlisted `.log` copies that passed exact-value/key-shape scanning and +redaction may cross the public boundary. + +The packaged evidence uploaded as a 30-day GitHub Actions artifact has a +different, access-controlled boundary. Text is exact-value and key-shape +scanned and redacted. PNG and WebM are raw-byte scanned but cannot be inspected +for credentials rendered as pixels, so they remain only in local evidence and +the access-controlled artifact. SVG is rejected during packaging because it is +active content. + +Before permanent publication, the campaign publisher prunes raster/video +files, archives, and generated report trees. It then regenerates the dashboard +from the remaining allowlisted `.json`, `.log`, `.md`, and `.txt` evidence and +accepts only that dashboard, normalized JSON/JUnit/summary, fixed +branding assets, and the inert structured evidence paths. Per-attempt XML is +excluded because browsers can process XML/XSLT; the only public XML is the +root `junit.xml`, which the report aggregator constructs from fixed markup and +XML-escaped fields. The same pruned tree feeds both S3/CloudFront history and +the optional GitHub Pages artifact. A leak fails the cell and withholds the +unsafe file. + +Rotate the affected credential immediately if a secret-scanning failure or +unexpected public object is observed. Preserve the access-controlled Actions +artifact and S3 object versions for incident analysis; do not weaken scanning +to make a campaign publish. diff --git a/tests/runner-e2e/api.ts b/tests/runner-e2e/api.ts new file mode 100644 index 0000000000..45863285f1 --- /dev/null +++ b/tests/runner-e2e/api.ts @@ -0,0 +1,109 @@ +import type { APIRequestContext, APIResponse } from "@playwright/test"; + +export interface JsonRecord { + [key: string]: unknown; +} + +async function failureMessage(response: APIResponse, method: string) { + const text = await response.text().catch(() => ""); + return `${method} ${response.url()} returned ${response.status()}${text ? `: ${text}` : ""}`; +} + +export class RunnerApi { + readonly baseURL: string; + + constructor(readonly request: APIRequestContext) { + const port = process.env.PAPERCLIP_RUNNER_E2E_PORT?.trim(); + if (!port) throw new Error("PAPERCLIP_RUNNER_E2E_PORT is required"); + this.baseURL = `http://127.0.0.1:${port}`; + } + + async get(path: string): Promise { + const response = await this.request.get(path); + if (!response.ok()) throw new Error(await failureMessage(response, "GET")); + return response.json() as Promise; + } + + async post(path: string, data?: unknown): Promise { + const response = await this.request.post(path, { data }); + if (!response.ok()) throw new Error(await failureMessage(response, "POST")); + return response.json() as Promise; + } + + /** + * Playwright traces APIRequestContext request bodies. Secret creation must + * still use the public API, but it goes through Node fetch so plaintext is + * never serialized into trace/blob evidence before Paperclip encrypts it. + */ + async postSensitive(path: string, data: unknown): Promise { + const response = await fetch(new URL(path, this.baseURL), { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(data), + }); + if (!response.ok) { + throw new Error( + `Sensitive POST ${path} returned ${response.status}; response body withheld`, + ); + } + return response.json() as Promise; + } + + async patch(path: string, data: unknown): Promise { + const response = await this.request.patch(path, { data }); + if (!response.ok()) + throw new Error(await failureMessage(response, "PATCH")); + return response.json() as Promise; + } + + async delete( + path: string, + options?: { allowNotFound?: boolean }, + ): Promise { + const response = await this.request.delete(path); + if (response.ok() || (options?.allowNotFound && response.status() === 404)) + return; + throw new Error(await failureMessage(response, "DELETE")); + } +} + +export async function pollUntil(input: { + label: string; + deadlineAt: number; + load: () => Promise; + accept: (value: T) => boolean; + reject?: (value: T) => string | undefined; + intervalMs?: number; +}): Promise { + let last: T | undefined; + let lastError: unknown; + while (Date.now() < input.deadlineAt) { + try { + last = await input.load(); + if (input.accept(last)) return last; + const rejection = input.reject?.(last); + if (rejection) { + throw new Error(`Stopped waiting for ${input.label}: ${rejection}`); + } + lastError = undefined; + } catch (error) { + if ( + error instanceof Error && + error.message.startsWith(`Stopped waiting for ${input.label}:`) + ) { + throw error; + } + lastError = error; + } + await new Promise((resolve) => + setTimeout(resolve, input.intervalMs ?? 2_000), + ); + } + const detail = + lastError instanceof Error + ? lastError.message + : last === undefined + ? "no observation" + : JSON.stringify(last); + throw new Error(`Timed out waiting for ${input.label}: ${detail}`); +} diff --git a/tests/runner-e2e/billing.test.ts b/tests/runner-e2e/billing.test.ts new file mode 100644 index 0000000000..7a5e54d904 --- /dev/null +++ b/tests/runner-e2e/billing.test.ts @@ -0,0 +1,162 @@ +import { describe, expect, it } from "vitest"; +import { + aggregateCampaignBilling, + buildRuntimeUsage, + summarizeExecutionBilling, +} from "./billing.js"; +import type { RunnerE2EResult } from "./types.js"; + +function result(overrides: Partial = {}): RunnerE2EResult { + return { + schema: "paperclip.runner-e2e.result/v1", + executionId: "legacy-claude.local.message-marker", + attempt: 1, + status: "passed", + profileId: "legacy-claude", + environmentId: "local", + caseId: "message-marker", + provider: "anthropic", + model: "fixture-model", + runtimeMode: "legacy", + runIds: ["run-1"], + startedAt: "2026-08-27T00:00:00.000Z", + finishedAt: "2026-08-27T00:00:02.000Z", + durationMs: 2_000, + cleanup: "passed", + ...overrides, + }; +} + +describe("runner E2E billing summaries", () => { + it("summarizes provider-reported token usage and cost", () => { + const billing = summarizeExecutionBilling( + result({ + usage: { + inputTokens: 12_000, + outputTokens: 420, + cachedInputTokens: 5_000, + cacheAdjustedCostUsd: 0.08125, + costStatus: "reported", + }, + }), + ); + expect(billing.llm).toMatchObject({ + runCount: 1, + runsWithTokenUsage: 1, + runsWithReportedCost: 1, + inputTokens: 12_000, + outputTokens: 420, + cachedInputTokens: 5_000, + totalTokens: 17_420, + reportedCostUsd: 0.08125, + costStatus: "reported", + }); + expect(billing.runtime.costStatus).toBe("not_metered"); + expect(billing.complete).toBe(true); + }); + + it("labels missing and unpriced runs instead of treating them as free", () => { + const billing = summarizeExecutionBilling( + result({ + runIds: ["run-1", "run-2", "run-3"], + usage: { + runs: [ + { + runId: "run-1", + usage: { + inputTokens: 1_000, + outputTokens: 100, + costUsd: 0.01, + }, + }, + { + runId: "run-2", + usage: { + inputTokens: 2_000, + outputTokens: 200, + costStatus: "unpriced", + }, + }, + { runId: "run-3", usage: null }, + ], + }, + }), + ); + expect(billing.llm).toMatchObject({ + runCount: 3, + runsWithTokenUsage: 2, + runsWithReportedCost: 1, + reportedCostUsd: 0.01, + costStatus: "partial", + }); + expect(billing.complete).toBe(false); + }); + + it("estimates Daytona list price from captured lease seconds and resources", () => { + const runtime = buildRuntimeUsage({ + environmentId: "daytona", + runs: [ + { + startedAt: "2026-08-27T00:00:05.000Z", + finishedAt: "2026-08-27T00:30:05.000Z", + }, + ], + leases: [ + { + acquiredAt: "2026-08-27T00:00:00.000Z", + releasedAt: "2026-08-27T01:00:00.000Z", + metadata: { cpu: 4, memory: 4, disk: 10 }, + }, + ], + }); + expect(runtime).toMatchObject({ + provider: "daytona", + agentRunDurationMs: 1_800_000, + leaseDurationMs: 3_600_000, + leaseCount: 1, + cpuCores: 4, + memoryGiB: 4, + diskGiB: 10, + costStatus: "estimated", + estimatedListCostUsd: 0.26748, + }); + }); + + it("aggregates tokens, reported spend, runtime estimates, and coverage", () => { + const local = result({ + usage: { inputTokens: 100, outputTokens: 20, costUsd: 0.004 }, + }); + const daytona = result({ + executionId: "runner-acpx-claude.daytona.message-marker", + profileId: "runner-acpx-claude", + environmentId: "daytona", + runtimeMode: "native", + usage: { inputTokens: 200, outputTokens: 30 }, + runtimeUsage: { + provider: "daytona", + agentRunDurationMs: 30_000, + leaseDurationMs: 40_000, + leaseCount: 1, + cpuCores: 4, + memoryGiB: 4, + diskGiB: 10, + estimatedListCostUsd: 0.002972, + costStatus: "estimated", + costSource: "daytona_public_list_price", + }, + }); + expect(aggregateCampaignBilling([local, daytona])).toMatchObject({ + testCount: 2, + reportedLlmCostUsd: 0.004, + estimatedRuntimeCostUsd: 0.002972, + llm: { + runCount: 2, + runsWithTokenUsage: 2, + runsWithReportedCost: 1, + inputTokens: 300, + outputTokens: 50, + costStatus: "partial", + }, + }); + }); +}); diff --git a/tests/runner-e2e/billing.ts b/tests/runner-e2e/billing.ts new file mode 100644 index 0000000000..977fa5d457 --- /dev/null +++ b/tests/runner-e2e/billing.ts @@ -0,0 +1,394 @@ +import type { + RunnerE2EBillingSummary, + RunnerE2EResult, + RunnerE2ERuntimeUsage, + RunnerEnvironmentId, +} from "./types.js"; + +// Public list prices are deliberately versioned here instead of being treated +// as provider-reported charges. Credits, discounts, and Daytona's first 5 GiB +// storage allowance can make the invoice amount lower than this estimate. +export const DAYTONA_LIST_PRICING = { + asOf: "2026-08-27", + url: "https://www.daytona.io/pricing", + cpuCoreHourUsd: 0.0504, + memoryGiBHourUsd: 0.0162, + diskGiBHourUsd: 0.000108, +} as const; + +export interface RuntimeLeaseUsageInput { + acquiredAt?: string | Date | null; + releasedAt?: string | Date | null; + updatedAt?: string | Date | null; + metadata?: Record | null; +} + +export interface AgentRunUsageInput { + startedAt?: string | Date | null; + finishedAt?: string | Date | null; +} + +export interface CampaignBillingSummary { + testCount: number; + agentRunDurationMs: number; + leaseDurationMs: number; + llm: RunnerE2EBillingSummary["llm"]; + reportedLlmCostUsd: number; + estimatedRuntimeCostUsd: number; + observedAndEstimatedCostUsd: number; + testsWithCompleteBilling: number; +} + +function record(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : {}; +} + +function finiteNumber(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) && value >= 0 + ? value + : undefined; +} + +function firstNumber(source: Record, keys: readonly string[]) { + for (const key of keys) { + const value = finiteNumber(source[key]); + if (value !== undefined) return value; + } + return undefined; +} + +function usageMeasurement(usage: Record) { + const candidates = [ + record(usage.runDelta), + record(usage.total), + record(usage.cumulative), + usage, + ]; + return ( + candidates.find( + (candidate) => + firstNumber(candidate, [ + "inputTokens", + "input", + "promptTokens", + "outputTokens", + "output", + "completionTokens", + ]) !== undefined, + ) ?? usage + ); +} + +function usageCostUsd(usage: Record) { + const measurement = usageMeasurement(usage); + const direct = + firstNumber(usage, [ + "cacheAdjustedCostUsd", + "costUsd", + "providerCostUsd", + ]) ?? + firstNumber(measurement, [ + "cacheAdjustedCostUsd", + "costUsd", + "providerCostUsd", + ]); + if (direct !== undefined) return direct; + const cost = record(usage.cost); + const currency = + typeof cost.currency === "string" ? cost.currency.toUpperCase() : "USD"; + if (currency !== "USD") return undefined; + return firstNumber(cost, ["amount", "total"]); +} + +function usageEntries( + rawUsage: Record | null | undefined, + runCount: number, +) { + const runs = Array.isArray(rawUsage?.runs) ? rawUsage.runs : null; + if (runs) { + const entries = runs.map((entry) => { + const candidate = record(entry); + const usage = candidate.usage; + return usage && typeof usage === "object" && !Array.isArray(usage) + ? (usage as Record) + : null; + }); + return [ + ...entries, + ...Array.from( + { length: Math.max(0, runCount - entries.length) }, + () => null, + ), + ]; + } + return Array.from({ length: Math.max(1, runCount) }, (_, index) => + index === 0 && rawUsage ? rawUsage : null, + ); +} + +function durationBetween( + startedAt: string | Date | null | undefined, + finishedAt: string | Date | null | undefined, +) { + const started = startedAt ? new Date(startedAt).getTime() : Number.NaN; + const finished = finishedAt ? new Date(finishedAt).getTime() : Number.NaN; + return Number.isFinite(started) && Number.isFinite(finished) + ? Math.max(0, finished - started) + : 0; +} + +function resourceValue( + metadata: Record | null | undefined, + key: "cpu" | "memory" | "disk", +) { + return finiteNumber(metadata?.[key]); +} + +export function buildRuntimeUsage(input: { + environmentId: RunnerEnvironmentId; + runs: readonly AgentRunUsageInput[]; + leases?: readonly RuntimeLeaseUsageInput[]; + fallbackFinishedAt?: string | Date; +}): RunnerE2ERuntimeUsage { + const agentRunDurationMs = input.runs.reduce( + (total, run) => total + durationBetween(run.startedAt, run.finishedAt), + 0, + ); + if (input.environmentId === "local") { + return { + provider: "local", + agentRunDurationMs, + leaseDurationMs: null, + leaseCount: 0, + costStatus: "not_metered", + costSource: "local_not_metered", + }; + } + + const leases = input.leases ?? []; + let leaseDurationMs = 0; + let estimatedListCostUsd = 0; + let resourcesComplete = leases.length > 0; + const resourceRows: Array<{ cpu: number; memory: number; disk: number }> = []; + for (const lease of leases) { + const finishedAt = + lease.releasedAt ?? input.fallbackFinishedAt ?? lease.updatedAt ?? null; + const durationMs = durationBetween(lease.acquiredAt, finishedAt); + leaseDurationMs += durationMs; + const cpu = resourceValue(lease.metadata, "cpu"); + const memory = resourceValue(lease.metadata, "memory"); + const disk = resourceValue(lease.metadata, "disk"); + if (cpu === undefined || memory === undefined || disk === undefined) { + resourcesComplete = false; + continue; + } + resourceRows.push({ cpu, memory, disk }); + const hours = durationMs / 3_600_000; + estimatedListCostUsd += + hours * + (cpu * DAYTONA_LIST_PRICING.cpuCoreHourUsd + + memory * DAYTONA_LIST_PRICING.memoryGiBHourUsd + + disk * DAYTONA_LIST_PRICING.diskGiBHourUsd); + } + const commonResource = (key: "cpu" | "memory" | "disk") => { + const values = new Set(resourceRows.map((row) => row[key])); + return values.size === 1 ? resourceRows[0]?.[key] : undefined; + }; + return { + provider: "daytona", + agentRunDurationMs, + leaseDurationMs: leases.length > 0 ? leaseDurationMs : null, + leaseCount: leases.length, + ...(commonResource("cpu") === undefined + ? {} + : { cpuCores: commonResource("cpu") }), + ...(commonResource("memory") === undefined + ? {} + : { memoryGiB: commonResource("memory") }), + ...(commonResource("disk") === undefined + ? {} + : { diskGiB: commonResource("disk") }), + ...(resourcesComplete ? { estimatedListCostUsd } : {}), + costStatus: resourcesComplete ? "estimated" : "unavailable", + costSource: resourcesComplete + ? "daytona_public_list_price" + : "provider_cost_unavailable", + ...(resourcesComplete + ? { + pricingAsOf: DAYTONA_LIST_PRICING.asOf, + pricingUrl: DAYTONA_LIST_PRICING.url, + } + : {}), + }; +} + +export function fallbackRuntimeUsage( + result: Pick< + RunnerE2EResult, + "environmentId" | "durationMs" | "runtimeUsage" + >, +) { + if (result.runtimeUsage) return result.runtimeUsage; + return { + provider: result.environmentId, + agentRunDurationMs: result.durationMs, + leaseDurationMs: null, + leaseCount: 0, + costStatus: + result.environmentId === "local" ? "not_metered" : "unavailable", + costSource: + result.environmentId === "local" + ? "local_not_metered" + : "provider_cost_unavailable", + } satisfies RunnerE2ERuntimeUsage; +} + +export function summarizeExecutionBilling( + result: Pick< + RunnerE2EResult, + "runIds" | "usage" | "environmentId" | "durationMs" | "runtimeUsage" + >, +): RunnerE2EBillingSummary { + const requestedRunCount = Math.max(result.runIds?.length ?? 0, 1); + const entries = usageEntries(result.usage, requestedRunCount); + let inputTokens = 0; + let outputTokens = 0; + let cachedInputTokens = 0; + let runsWithTokenUsage = 0; + let runsWithReportedCost = 0; + let reportedCostUsd = 0; + for (const usage of entries) { + if (!usage) continue; + const measurement = usageMeasurement(usage); + const input = + firstNumber(measurement, ["inputTokens", "input", "promptTokens"]) ?? 0; + const output = + firstNumber(measurement, [ + "outputTokens", + "output", + "completionTokens", + ]) ?? 0; + const cached = + firstNumber(measurement, [ + "cachedInputTokens", + "cacheReadTokens", + "cachedReadTokens", + ]) ?? 0; + inputTokens += input; + outputTokens += output; + cachedInputTokens += cached; + if (input > 0 || output > 0 || cached > 0) runsWithTokenUsage += 1; + const costUsd = usageCostUsd(usage); + if (costUsd !== undefined && (costUsd > 0 || input > 0 || output > 0)) { + runsWithReportedCost += 1; + reportedCostUsd += costUsd; + } + } + const runCount = Math.max(requestedRunCount, entries.length); + const costStatus = + runsWithReportedCost === runCount + ? "reported" + : runsWithReportedCost > 0 + ? "partial" + : runsWithTokenUsage > 0 + ? "unpriced" + : "unavailable"; + const runtime = fallbackRuntimeUsage(result); + const estimatedRuntimeCostUsd = runtime.estimatedListCostUsd ?? 0; + const complete = + runsWithTokenUsage === runCount && + runsWithReportedCost === runCount && + runtime.costStatus !== "unavailable"; + return { + llm: { + runCount, + runsWithTokenUsage, + runsWithReportedCost, + inputTokens, + outputTokens, + cachedInputTokens, + totalTokens: inputTokens + cachedInputTokens + outputTokens, + reportedCostUsd, + costStatus, + }, + runtime, + reportedCostUsd, + estimatedRuntimeCostUsd, + observedAndEstimatedCostUsd: reportedCostUsd + estimatedRuntimeCostUsd, + complete, + }; +} + +export function aggregateCampaignBilling( + results: readonly RunnerE2EResult[], +): CampaignBillingSummary { + const summaries = results.map(summarizeExecutionBilling); + const runCount = summaries.reduce( + (total, summary) => total + summary.llm.runCount, + 0, + ); + const runsWithTokenUsage = summaries.reduce( + (total, summary) => total + summary.llm.runsWithTokenUsage, + 0, + ); + const runsWithReportedCost = summaries.reduce( + (total, summary) => total + summary.llm.runsWithReportedCost, + 0, + ); + const reportedLlmCostUsd = summaries.reduce( + (total, summary) => total + summary.reportedCostUsd, + 0, + ); + const estimatedRuntimeCostUsd = summaries.reduce( + (total, summary) => total + summary.estimatedRuntimeCostUsd, + 0, + ); + return { + testCount: results.length, + agentRunDurationMs: summaries.reduce( + (total, summary) => total + summary.runtime.agentRunDurationMs, + 0, + ), + leaseDurationMs: summaries.reduce( + (total, summary) => total + (summary.runtime.leaseDurationMs ?? 0), + 0, + ), + llm: { + runCount, + runsWithTokenUsage, + runsWithReportedCost, + inputTokens: summaries.reduce( + (total, summary) => total + summary.llm.inputTokens, + 0, + ), + outputTokens: summaries.reduce( + (total, summary) => total + summary.llm.outputTokens, + 0, + ), + cachedInputTokens: summaries.reduce( + (total, summary) => total + summary.llm.cachedInputTokens, + 0, + ), + totalTokens: summaries.reduce( + (total, summary) => total + summary.llm.totalTokens, + 0, + ), + reportedCostUsd: reportedLlmCostUsd, + costStatus: + runsWithReportedCost === runCount + ? "reported" + : runsWithReportedCost > 0 + ? "partial" + : runsWithTokenUsage > 0 + ? "unpriced" + : "unavailable", + }, + reportedLlmCostUsd, + estimatedRuntimeCostUsd, + observedAndEstimatedCostUsd: reportedLlmCostUsd + estimatedRuntimeCostUsd, + testsWithCompleteBilling: summaries.filter((summary) => summary.complete) + .length, + }; +} diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts new file mode 100644 index 0000000000..e52b968dcd --- /dev/null +++ b/tests/runner-e2e/catalog.test.ts @@ -0,0 +1,292 @@ +import { describe, expect, it } from "vitest"; +import { + runnerEnvironments, + runnerMatrix, + openRouterBreadthProfiles, + openRouterBreadthTasks, + runnerProfiles, + runnerSuites, + runnerTasks, + isImmutableDaytonaImage, + validateRunnerCatalog, +} from "./catalog.js"; +import { + buildMatrixJobs, + parseRunnerSelectors, + RunnerSelectorError, + selectRunnerExecutions, +} from "./selectors.js"; + +describe("runner E2E catalog", () => { + it("validates the 42-cell core and 15-cell breadth suites", () => { + expect(runnerProfiles).toHaveLength(7); + expect(openRouterBreadthProfiles).toHaveLength(5); + expect(runnerEnvironments).toHaveLength(2); + expect(runnerTasks).toHaveLength(3); + expect(openRouterBreadthTasks).toHaveLength(3); + expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ + 42, 15, + ]); + expect(validateRunnerCatalog()).toHaveLength(57); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(57); + expect( + runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), + ).toHaveLength(42); + expect( + runnerMatrix.filter( + (entry) => entry.suite.id === "openrouter-model-breadth", + ), + ).toHaveLength(15); + expect( + runnerMatrix.reduce( + (total, execution) => total + execution.task.expectedRunCount, + 0, + ), + ).toBe(95); + }); + + it("derives five local native OpenCode profiles from the ranked snapshot", () => { + expect( + openRouterBreadthProfiles.map((profile) => profile.ranking?.rank), + ).toEqual([1, 2, 3, 4, 5]); + expect( + openRouterBreadthProfiles.every( + (profile) => + profile.adapterType === "paperclip_runner" && + profile.provider === "opencode" && + profile.model.startsWith("openrouter/") && + profile.supportedEnvironments.join(",") === "local" && + profile.modelQualification.source === "openrouter_rankings_snapshot", + ), + ).toBe(true); + }); + + it("defines deterministic two-run question and plan state machines", () => { + const question = openRouterBreadthTasks.find( + (task) => task.id === "question-resume-complete", + ); + const plan = openRouterBreadthTasks.find( + (task) => task.id === "plan-approve-complete", + ); + expect(question).toMatchObject({ + flow: "question_resume_completion", + expectedRunCount: 2, + }); + expect(question?.buildQuestionAnswer?.("nonce")).toMatchObject({ + optionLabel: "Cobalt", + }); + expect(plan).toMatchObject({ + flow: "plan_approval_completion", + expectedRunCount: 2, + }); + expect(plan?.buildPrompt("nonce")).toContain("exactly two numbered steps"); + }); + + it("uses only declared secret references in generated payloads", () => { + expect( + runnerMatrix.every((entry) => + entry.requiredCredentials.includes(entry.profile.credential), + ), + ).toBe(true); + expect( + runnerMatrix + .filter((entry) => entry.environment.id === "daytona") + .every((entry) => + entry.requiredCredentials.includes("DAYTONA_API_KEY"), + ), + ).toBe(true); + }); + + it("pins legacy Codex and Claude to their classic CLI engines", () => { + for (const profileId of ["legacy-codex", "legacy-claude"]) { + const execution = runnerMatrix.find( + (candidate) => + candidate.profile.id === profileId && + candidate.environment.id === "local", + ); + expect(execution).toBeDefined(); + expect( + execution!.profile.buildAgent({ + environmentId: "11111111-1111-4111-8111-111111111111", + environmentFixtureId: "local", + workspacePath: "/tmp/runner-e2e-workspace", + secretRefs: { + [execution!.profile.credential]: { + type: "secret_ref", + secretId: "22222222-2222-4222-8222-222222222222", + version: "latest", + }, + }, + executionId: execution!.id, + }), + ).toMatchObject({ adapterConfig: { engine: "cli" } }); + } + }); + + it("binds native Codex automation auth to the encrypted OpenAI secret", () => { + const execution = runnerMatrix.find( + (candidate) => + candidate.id === "core-compatibility.runner-codex.local.message-marker", + ); + expect(execution).toBeDefined(); + const secretRef = { + type: "secret_ref" as const, + secretId: "22222222-2222-4222-8222-222222222222", + version: "latest" as const, + }; + const agent = execution!.profile.buildAgent({ + environmentId: "11111111-1111-4111-8111-111111111111", + environmentFixtureId: "local", + workspacePath: "/tmp/runner-e2e-workspace", + secretRefs: { OPENAI_API_KEY: secretRef }, + executionId: execution!.id, + }); + expect(agent.adapterConfig).toMatchObject({ + env: { + OPENAI_API_KEY: secretRef, + CODEX_API_KEY: secretRef, + }, + }); + }); + + it("gives legacy planning agents a direct bounded API recipe", () => { + const task = runnerTasks.find( + (candidate) => candidate.id === "plan-revise-accept", + ); + const execution = runnerMatrix.find( + (candidate) => + candidate.profile.id === "legacy-claude" && + candidate.environment.id === "local" && + candidate.task.id === "plan-revise-accept", + ); + expect(task).toBeDefined(); + expect(execution).toBeDefined(); + const agent = execution!.profile.buildAgent({ + environmentId: "11111111-1111-4111-8111-111111111111", + environmentFixtureId: "local", + workspacePath: "/tmp/runner-e2e-workspace", + secretRefs: { + ANTHROPIC_API_KEY: { + type: "secret_ref", + secretId: "22222222-2222-4222-8222-222222222222", + version: "latest", + }, + }, + executionId: execution!.id, + }); + expect(agent.adapterConfig).toMatchObject({ maxTurnsPerRun: 24 }); + expect(agent.instructionsBundle).toMatchObject({ + files: { "AGENTS.md": expect.stringContaining("/interactions") }, + }); + expect(task!.buildPrompt("nonce")).toContain("request_confirmation"); + expect(task!.buildPrompt("nonce")).toContain("baseRevisionId"); + expect(task!.buildRevisionRequest?.("nonce")).toContain("baseRevisionId"); + }); + + it("accepts only complete immutable Daytona digests", () => { + expect( + isImmutableDaytonaImage( + `ghcr.io/paperclipai/paperclip-daytona-runner@sha256:${"a".repeat(64)}`, + ), + ).toBe(true); + expect( + isImmutableDaytonaImage( + "ghcr.io/paperclipai/paperclip-daytona-runner@sha256:REPLACE_ME", + ), + ).toBe(false); + expect( + isImmutableDaytonaImage( + "ghcr.io/paperclipai/paperclip-daytona-runner:e2e-latest", + ), + ).toBe(false); + }); +}); + +describe("runner E2E selectors", () => { + it("requires an explicit billable selector", () => { + expect(() => parseRunnerSelectors([])).toThrow(RunnerSelectorError); + }); + + it("selects dimensions with OR within a dimension and AND across dimensions", () => { + const options = parseRunnerSelectors([ + "--profile", + "legacy-codex", + "--profile", + "runner-codex", + "--environment", + "local", + ]); + expect(selectRunnerExecutions(options).map((entry) => entry.id)).toEqual([ + "core-compatibility.legacy-codex.local.message-marker", + "core-compatibility.legacy-codex.local.plan-revise-accept", + "core-compatibility.legacy-codex.local.ask-question", + "core-compatibility.runner-codex.local.message-marker", + "core-compatibility.runner-codex.local.plan-revise-accept", + "core-compatibility.runner-codex.local.ask-question", + ]); + }); + + it("selects a suite without exploding its environment matrix", () => { + const selected = selectRunnerExecutions( + parseRunnerSelectors(["--suite", "openrouter-model-breadth"]), + ); + expect(selected).toHaveLength(15); + expect( + selected.every( + (entry) => + entry.suite.id === "openrouter-model-breadth" && + entry.environment.id === "local", + ), + ).toBe(true); + }); + + it("combines repeated groups with AND semantics", () => { + const options = parseRunnerSelectors([ + "--group", + "native", + "--group", + "daytona", + ]); + const selected = selectRunnerExecutions(options); + expect(selected).toHaveLength(12); + expect( + selected.every( + (entry) => + entry.profile.generation === "native" && + entry.environment.id === "daytona", + ), + ).toBe(true); + }); + + it("rejects groups outside the advertised four", () => { + const options = parseRunnerSelectors(["--group", "codex"]); + expect(() => selectRunnerExecutions(options)).toThrow("Unknown group"); + }); + + it("emits one independently schedulable job per scenario", () => { + const jobs = buildMatrixJobs( + selectRunnerExecutions(parseRunnerSelectors(["--all"])), + ); + expect(jobs).toHaveLength(57); + expect(jobs.filter((job) => job.needsDaytona)).toHaveLength(21); + expect(new Set(jobs.map((job) => job.executionId)).size).toBe(57); + expect( + jobs.every((job) => + runnerMatrix.some( + (execution) => + execution.id === job.executionId && + execution.profile.credential === job.credentialName, + ), + ), + ).toBe(true); + }); + + it("validates bounded local parallelism", () => { + expect( + parseRunnerSelectors(["--all", "--max-parallel", "8"]).maxParallel, + ).toBe(8); + expect(() => + parseRunnerSelectors(["--all", "--max-parallel", "0"]), + ).toThrow("positive integer"); + }); +}); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts new file mode 100644 index 0000000000..0b60d13c2b --- /dev/null +++ b/tests/runner-e2e/catalog.ts @@ -0,0 +1,857 @@ +import { createHash } from "node:crypto"; +import { createAgentSchema } from "../../packages/shared/src/validators/agent.js"; +import { createEnvironmentSchema } from "../../packages/shared/src/validators/environment.js"; +import { DEFAULT_CODEX_LOCAL_MODEL } from "../../packages/adapters/codex-local/src/index.js"; +import { models as claudeModels } from "../../packages/adapters/claude-local/src/index.js"; +import { QUALIFIED_ACPX_PROFILES } from "../../packages/paperclip-runner/src/drivers/acpx/qualified-profiles.js"; +import { QUALIFIED_OPENCODE_MODEL } from "../../packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.js"; +import { CREDENTIAL_NAMES } from "./types.js"; +import { + openRouterProfileId, + openRouterRankingSnapshot, +} from "./openrouter-ranking.js"; +import type { + AgentFixtureBuildInput, + EnvironmentFixture, + EnvironmentFixtureBuildInput, + MatrixExecution, + RunnerProfileFixture, + RunnerTaskFixture, + RunnerSuiteFixture, + SecretReference, +} from "./types.js"; + +const ENVIRONMENT_IDS = ["local", "daytona"] as const; +const SELECTABLE_GROUPS = [ + "legacy", + "native", + "local", + "daytona", + "core", + "breadth", +] as const; +const SAMPLE_UUID = "11111111-1111-4111-8111-111111111111"; + +export function isImmutableDaytonaImage(value: string | undefined) { + return /^.+@sha256:[0-9a-f]{64}$/i.test(value ?? ""); +} + +function requiredSecret( + input: AgentFixtureBuildInput, + name: RunnerProfileFixture["credential"], +): SecretReference { + const value = input.secretRefs[name]; + if (!value) throw new Error(`Missing fixture secret reference ${name}`); + return value; +} + +function commonAgent( + input: AgentFixtureBuildInput, + fixtureId: string, + adapterType: string, + adapterConfig: Record, +) { + return { + name: `Runner E2E ${fixtureId} ${input.executionId}`, + role: "qa", + title: "Paid full-stack runner acceptance fixture", + capabilities: + "Completes deterministic standard, planning, and ask-mode runner acceptance tasks.", + adapterType, + adapterConfig, + defaultEnvironmentId: input.environmentId, + budgetMonthlyCents: 0, + instructionsBundle: { + entryFile: "AGENTS.md", + files: { + "AGENTS.md": [ + "You are running a paid Paperclip end-to-end acceptance fixture.", + "Follow the assigned task and its Paperclip work mode literally.", + "For standard and ask tasks, publish the requested visible answer and mark the task done.", + "For planning tasks, publish or revise the canonical Plan document and its revision-bound request_confirmation, then wait. Only implement after that exact plan is accepted.", + "Invoke assigned tools only through the runtime's real tool-call channel. Never print XML, DSML, JSON, or other tool-call markup as assistant text.", + "Legacy adapters must use the public Paperclip API and the injected PAPERCLIP_API_URL, PAPERCLIP_API_KEY, PAPERCLIP_TASK_ID, and PAPERCLIP_RUN_ID values for comments, documents, interactions, and status changes.", + ...(adapterType === "paperclip_runner" + ? [] + : [ + 'For a planning task, do not inspect the OpenAPI schema. PUT /api/issues/$PAPERCLIP_TASK_ID/documents/plan with {title:"Plan",format:"markdown",body,changeSummary}; read latestRevisionId and latestRevisionNumber from that response. Then POST /api/issues/$PAPERCLIP_TASK_ID/interactions with {kind:"request_confirmation",continuationPolicy:"wake_assignee",payload:{version:1,prompt,acceptLabel:"Approve",rejectLabel:"Reject",rejectRequiresReason:true,target:{type:"issue_document",key:"plan",revisionId,revisionNumber}}}, and PATCH the issue to {status:"in_review"}. Include Authorization and X-Paperclip-Run-Id on every write.', + ]), + "Never print, persist, or expose credential values, and never create unrelated work.", + ].join("\n"), + }, + }, + runtimeConfig: {}, + }; +} + +function legacyProfile(input: { + id: string; + label: string; + adapterType: "codex_local" | "claude_local" | "opencode_local"; + provider: string; + model: string; + credential: RunnerProfileFixture["credential"]; + extraConfig?: Record; +}): RunnerProfileFixture { + return { + ...input, + modelQualification: { + source: "adapter_constant", + qualificationId: `${input.adapterType}:default-model`, + }, + generation: "legacy", + groups: ["legacy"], + supportedEnvironments: ENVIRONMENT_IDS, + expectedRuntimeMode: "legacy", + expectedRuntimeMetadata: { + adapterType: input.adapterType, + provider: input.provider, + }, + buildAgent(buildInput) { + return commonAgent(buildInput, input.id, input.adapterType, { + // Remote adapters must inherit the lease's provider-owned remoteCwd. + // A host path here would override that mapping inside the sandbox. + ...(buildInput.environmentFixtureId === "local" + ? { cwd: buildInput.workspacePath } + : {}), + model: input.model, + timeoutSec: buildInput.environmentFixtureId === "daytona" ? 780 : 360, + dangerouslySkipPermissions: true, + ...input.extraConfig, + env: { + [input.credential]: requiredSecret(buildInput, input.credential), + }, + }); + }, + }; +} + +function nativeProfile(input: { + id: string; + label: string; + provider: "codex" | "opencode" | "acpx"; + model: string; + credential: RunnerProfileFixture["credential"]; + acpxAgent?: "claude" | "codex"; + supportedEnvironments?: readonly (typeof ENVIRONMENT_IDS)[number][]; + modelQualification?: RunnerProfileFixture["modelQualification"]; + ranking?: RunnerProfileFixture["ranking"]; +}): RunnerProfileFixture { + return { + ...input, + adapterType: "paperclip_runner", + generation: "native", + groups: ["native"], + supportedEnvironments: input.supportedEnvironments ?? ENVIRONMENT_IDS, + expectedRuntimeMode: "native", + modelQualification: input.modelQualification ?? { + source: + input.provider === "acpx" + ? "qualified_runner_profile" + : "adapter_constant", + qualificationId: + input.provider === "acpx" + ? `acpx:${input.acpxAgent}` + : `${input.provider}:qualified-model`, + }, + ...(input.ranking ? { ranking: input.ranking } : {}), + expectedRuntimeMetadata: { + adapterType: "paperclip_runner", + provider: input.provider, + }, + buildAgent(buildInput) { + const credentialRef = requiredSecret(buildInput, input.credential); + const permissionConfig = + input.provider === "codex" + ? { codexPermissionMode: "never" } + : input.provider === "opencode" + ? { opencodePermissionMode: "allow" } + : { acpxPermissionMode: "approve-all", acpxAgent: input.acpxAgent }; + return commonAgent(buildInput, input.id, "paperclip_runner", { + provider: input.provider, + model: input.model, + lifecycleMode: "per_turn", + idleTimeoutMs: 300_000, + ...permissionConfig, + env: { + [input.credential]: credentialRef, + // Codex's supported automation credential is CODEX_API_KEY. Keep + // OPENAI_API_KEY as the operator-facing fixture secret name and bind + // the same encrypted reference to the runtime-specific alias. + ...(input.provider === "codex" + ? { CODEX_API_KEY: credentialRef } + : {}), + }, + }); + }, + }; +} + +const claudeLegacyModel = "claude-sonnet-4-6"; +if (!claudeModels.some((model) => model.id === claudeLegacyModel)) { + throw new Error( + `Claude adapter does not expose the qualified ${claudeLegacyModel} model`, + ); +} + +export const runnerProfiles: readonly RunnerProfileFixture[] = [ + legacyProfile({ + id: "legacy-codex", + label: "Legacy Codex", + adapterType: "codex_local", + provider: "codex", + model: DEFAULT_CODEX_LOCAL_MODEL, + credential: "OPENAI_API_KEY", + // Keep this fixture on the classic adapter/CLI lane. ACP execution is + // covered independently by the native runner ACPX profiles below. + extraConfig: { engine: "cli" }, + }), + legacyProfile({ + id: "legacy-claude", + label: "Legacy Claude", + adapterType: "claude_local", + provider: "claude", + model: claudeLegacyModel, + credential: "ANTHROPIC_API_KEY", + extraConfig: { + engine: "cli", + // Plan fixtures need enough tool turns to read the issue, write the + // canonical Plan document, and request confirmation. Four turns caused + // the Claude CLI to terminate correctly but prematurely with + // `max_turns_exhausted` during the revision flow. + maxTurnsPerRun: 24, + }, + }), + legacyProfile({ + id: "legacy-opencode", + label: "Legacy OpenCode", + adapterType: "opencode_local", + provider: "opencode", + model: QUALIFIED_OPENCODE_MODEL, + credential: "OPENROUTER_API_KEY", + }), + nativeProfile({ + id: "runner-codex", + label: "Runner Codex", + provider: "codex", + model: DEFAULT_CODEX_LOCAL_MODEL, + credential: "OPENAI_API_KEY", + }), + nativeProfile({ + id: "runner-opencode", + label: "Runner OpenCode", + provider: "opencode", + model: QUALIFIED_OPENCODE_MODEL, + credential: "OPENROUTER_API_KEY", + }), + nativeProfile({ + id: "runner-acpx-claude", + label: "Runner ACPX Claude", + provider: "acpx", + acpxAgent: "claude", + model: QUALIFIED_ACPX_PROFILES.claude.qualificationModel, + credential: "ANTHROPIC_API_KEY", + }), + nativeProfile({ + id: "runner-acpx-codex", + label: "Runner ACPX Codex", + provider: "acpx", + acpxAgent: "codex", + model: QUALIFIED_ACPX_PROFILES.codex.qualificationModel, + credential: "OPENAI_API_KEY", + }), +] as const; + +export const openRouterBreadthProfiles: readonly RunnerProfileFixture[] = + openRouterRankingSnapshot.models.map((rankedModel) => + nativeProfile({ + id: openRouterProfileId(rankedModel.id), + label: `#${rankedModel.rank} ${rankedModel.name}`, + provider: "opencode", + model: `openrouter/${rankedModel.id}`, + credential: "OPENROUTER_API_KEY", + supportedEnvironments: ["local"], + modelQualification: { + source: "openrouter_rankings_snapshot", + qualificationId: `${openRouterRankingSnapshot.snapshotId}:${rankedModel.rank}`, + }, + ranking: { + rank: rankedModel.rank, + canonicalModelId: rankedModel.id, + snapshotId: openRouterRankingSnapshot.snapshotId, + capturedAt: openRouterRankingSnapshot.capturedAt, + sourceUrl: openRouterRankingSnapshot.sourceUrl, + }, + }), + ); + +function requiredDaytonaSecret(input: EnvironmentFixtureBuildInput) { + const apiKey = input.secretRefs.DAYTONA_API_KEY; + if (!apiKey) + throw new Error("Missing fixture secret reference DAYTONA_API_KEY"); + return apiKey; +} + +export const runnerEnvironments: readonly EnvironmentFixture[] = [ + { + id: "local", + label: "Isolated local", + groups: ["local"], + driver: "local", + provider: "local", + lifecycle: { + setup: "instance_managed", + probe: "run_context_via_api", + cleanup: "instance_shutdown", + }, + expectedExecutionTarget: { kind: "local" }, + buildEnvironment(input) { + return { + name: `Runner E2E local ${input.executionId}`, + description: "Ephemeral local runner E2E environment", + driver: "local", + config: {}, + envVars: {}, + }; + }, + }, + { + id: "daytona", + label: "Daytona sandbox", + groups: ["daytona"], + driver: "sandbox", + provider: "daytona", + credential: "DAYTONA_API_KEY", + lifecycle: { + setup: "create_via_api", + probe: "run_context_via_api", + cleanup: "delete_via_api_and_destroy_leases", + }, + expectedExecutionTarget: { kind: "remote", transport: "sandbox" }, + buildEnvironment(input) { + if (!isImmutableDaytonaImage(input.daytonaImage)) { + throw new Error( + "PAPERCLIP_E2E_DAYTONA_IMAGE must be an immutable image digest", + ); + } + return { + name: `Runner E2E Daytona ${input.executionId}`, + description: "Ephemeral Daytona runner E2E environment", + driver: "sandbox", + config: { + provider: "daytona", + apiKey: requiredDaytonaSecret(input), + image: input.daytonaImage, + // Pin the billable resource shape so per-test runtime list-price + // estimates remain reproducible when provider defaults change. + cpu: 4, + memory: 4, + disk: 10, + reuseLease: false, + runnerLifecycleMode: "per_turn", + autoStopInterval: 5, + autoArchiveInterval: 15, + autoDeleteInterval: 60, + timeoutMs: 300_000, + livenessTimeoutMs: 30_000, + }, + envVars: {}, + }; + }, + }, +] as const; + +export const runnerTasks: readonly RunnerTaskFixture[] = [ + { + id: "message-marker", + label: "Basic response", + groups: [], + workMode: "standard", + flow: "single_turn", + expectedRunCount: 1, + attemptTimeoutMs: { + local: 8 * 60_000, + daytona: 15 * 60_000, + }, + expectedTerminalState: { issue: "done", run: "succeeded" }, + buildTitle: (nonce) => `Runner E2E PAPERCLIP_E2E_OK_${nonce}`, + buildVisibleMarker: (nonce) => `PAPERCLIP_E2E_OK_${nonce}`, + buildPrompt: (nonce) => + [ + "Complete this task in a single run.", + `The exact marker also appears unescaped in the task title: PAPERCLIP_E2E_OK_${nonce}`, + `Your final visible task-thread response must be exactly this marker: PAPERCLIP_E2E_OK_${nonce}`, + `In a native runner, first emit that exact marker as the complete user-facing final response, then use paperclip_finish with the same marker as its summary and an objective-satisfied claim for the supplied contract revision.`, + `In a legacy runner, post a task comment containing exactly that marker and mark the task Done through the public API.`, + "The visible task-thread response is asserted; hidden reasoning or provider terminal output alone does not count.", + "Use underscore characters exactly as shown and do not insert backslashes.", + "Do not create files, ask questions, start additional tasks, or include any credentials.", + ].join("\n"), + buildMatchers(nonce, execution) { + return [ + { kind: "message_contains", expected: `PAPERCLIP_E2E_OK_${nonce}` }, + { + kind: "issue_status", + expected: execution.task.expectedTerminalState.issue, + }, + { + kind: "run_status", + expected: execution.task.expectedTerminalState.run, + }, + { + kind: "runtime_mode", + expected: execution.profile.expectedRuntimeMode, + }, + { kind: "environment", expected: execution.environment.id }, + ]; + }, + }, + { + id: "plan-revise-accept", + label: "Plan, revise, accept, implement", + groups: [], + workMode: "planning", + flow: "plan_revision_acceptance", + expectedRunCount: 3, + attemptTimeoutMs: { + local: 20 * 60_000, + daytona: 35 * 60_000, + }, + expectedTerminalState: { issue: "done", run: "succeeded" }, + buildTitle: (nonce) => `Runner E2E plan lifecycle ${nonce}`, + buildVisibleMarker: (nonce) => `PAPERCLIP_E2E_PLAN_DONE_${nonce}`, + buildPlanMarkers: (nonce) => ({ + draft: `PAPERCLIP_E2E_PLAN_DRAFT_${nonce}`, + revised: `PAPERCLIP_E2E_PLAN_REVISED_${nonce}`, + }), + buildRevisionRequest: (nonce) => + [ + "Revise this same plan; do not implement it yet.", + `Remove PAPERCLIP_E2E_PLAN_DRAFT_${nonce} and include PAPERCLIP_E2E_PLAN_REVISED_${nonce}.`, + "Change the plan from two steps to exactly three numbered steps, with verification as step 3.", + "Publish the revised canonical Plan revision and request confirmation for that new revision.", + "In a native runner, call write_document for key `plan`, then call request_human_input exactly once with interactionKind `confirmation`, targetRevisionId set to the returned latest Plan revision, and continuationPolicy `wake_assignee`; do not call paperclip_finish while waiting.", + "In a legacy runner, first GET the current `plan` issue document, then PUT the revised Plan with `baseRevisionId` set to that response's `latestRevisionId`; after the update succeeds, create the equivalent request_confirmation targeting the newly returned `latestRevisionId` through the public Paperclip API.", + ].join(" "), + buildPrompt: (nonce) => + [ + "This is a planning-mode lifecycle acceptance task.", + "First, create a small canonical Plan with exactly two numbered steps and request approval; do not implement it.", + `The initial Plan body must contain PAPERCLIP_E2E_PLAN_DRAFT_${nonce}.`, + "If the plan is sent back, revise that same Plan document according to the revision note, publish a new revision-bound confirmation, and still do not implement.", + "Only after the revised plan is accepted, implement it by posting one final visible task-thread response containing exactly " + + `PAPERCLIP_E2E_PLAN_DONE_${nonce}` + + " and mark the task Done.", + "For a native runner, remain in the requested planning collaboration mode. Call write_document for key `plan`, then call request_human_input exactly once with interactionKind `confirmation`, targetRevisionId set to the returned latest Plan revision, and continuationPolicy `wake_assignee`. Do not call paperclip_finish while waiting for either Plan confirmation.", + "For a legacy runner, use the public Paperclip API. The first PUT of the `plan` issue document creates it. For every later PUT, first GET the current document and set `baseRevisionId` to its `latestRevisionId`; a 409 means you must GET again and retry with the new latest revision. Create a `request_confirmation` targeting the successful PUT response's `latestRevisionId` with `continuationPolicy: wake_assignee`, and move the issue to `in_review` while waiting.", + "Do not create files, child tasks, or unrelated work, and do not expose credentials.", + ].join("\n"), + buildMatchers(nonce, execution) { + return [ + { + kind: "message_contains", + expected: `PAPERCLIP_E2E_PLAN_DONE_${nonce}`, + }, + { + kind: "issue_status", + expected: execution.task.expectedTerminalState.issue, + }, + { + kind: "run_status", + expected: execution.task.expectedTerminalState.run, + }, + { + kind: "runtime_mode", + expected: execution.profile.expectedRuntimeMode, + }, + { kind: "environment", expected: execution.environment.id }, + ]; + }, + }, + { + id: "ask-question", + label: "Ask mode question", + groups: [], + workMode: "ask", + flow: "single_turn", + expectedRunCount: 1, + attemptTimeoutMs: { + local: 8 * 60_000, + daytona: 15 * 60_000, + }, + expectedTerminalState: { issue: "done", run: "succeeded" }, + buildTitle: (nonce) => `Runner E2E ask mode ${nonce}`, + buildVisibleMarker: (nonce) => `E2E_ASK_12_${nonce}`, + buildPrompt: (nonce) => + [ + "Answer this question directly in Ask mode: what is seven plus five?", + `Your final visible task-thread answer must contain this complete marker, including its final suffix: E2E_ASK_12_${nonce}.`, + "Do not create or modify files, do not create a plan or additional work, and do not expose credentials.", + "After posting the direct answer, mark this task Done.", + `In a native runner, emit the answer and finish with E2E_ASK_12_${nonce} as the summary.`, + `In a legacy runner, post a task comment containing E2E_ASK_12_${nonce} and mark the task Done through the public API.`, + ].join("\n"), + buildMatchers(nonce, execution) { + return [ + { + kind: "message_contains", + expected: `E2E_ASK_12_${nonce}`, + }, + { + kind: "issue_status", + expected: execution.task.expectedTerminalState.issue, + }, + { + kind: "run_status", + expected: execution.task.expectedTerminalState.run, + }, + { + kind: "runtime_mode", + expected: execution.profile.expectedRuntimeMode, + }, + { kind: "environment", expected: execution.environment.id }, + ]; + }, + }, +] as const; + +function terminalMatchers( + nonceMarker: string, + execution: MatrixExecution, +): readonly ReturnType[number][] { + return [ + { kind: "message_contains", expected: nonceMarker }, + { + kind: "issue_status", + expected: execution.task.expectedTerminalState.issue, + }, + { kind: "run_status", expected: execution.task.expectedTerminalState.run }, + { kind: "runtime_mode", expected: execution.profile.expectedRuntimeMode }, + { kind: "environment", expected: execution.environment.id }, + ]; +} + +function breadthMarker(phase: "H" | "Q_C" | "P_READY" | "P_OK", nonce: string) { + // Keep the complete visible marker comfortably below the shortest output + // fragments observed across ranked models while retaining the attempt nonce. + return `PC_${phase}_${nonce}`; +} + +export const openRouterBreadthTasks: readonly RunnerTaskFixture[] = [ + { + id: "hello-complete", + label: "Hello and complete", + groups: [], + workMode: "standard", + flow: "single_turn", + expectedRunCount: 1, + attemptTimeoutMs: { local: 8 * 60_000, daytona: 8 * 60_000 }, + expectedTerminalState: { issue: "done", run: "succeeded" }, + buildTitle: (nonce) => `OpenRouter breadth hello ${nonce}`, + buildVisibleMarker: (nonce) => breadthMarker("H", nonce), + buildPrompt: (nonce) => + [ + "Complete this deterministic hello task in one turn.", + `Return ${breadthMarker("H", nonce)} as the complete visible response.`, + `Call paperclip_finish with ${breadthMarker("H", nonce)} as its summary and mark the task Done.`, + "Do not create files, plans, interactions, or additional work.", + ].join("\n"), + buildMatchers: (nonce, execution) => + terminalMatchers(breadthMarker("H", nonce), execution), + }, + { + id: "question-resume-complete", + label: "Ask, answer, resume", + groups: [], + workMode: "standard", + flow: "question_resume_completion", + expectedRunCount: 2, + attemptTimeoutMs: { local: 12 * 60_000, daytona: 12 * 60_000 }, + expectedTerminalState: { issue: "done", run: "succeeded" }, + buildTitle: (nonce) => `OpenRouter breadth question ${nonce}`, + buildVisibleMarker: (nonce) => breadthMarker("Q_C", nonce), + buildQuestionAnswer: (nonce) => ({ + optionLabel: "Cobalt", + expectedMarker: breadthMarker("Q_C", nonce), + }), + buildPrompt: (nonce) => + [ + "Ask the user one structured question before completing this task.", + "Call request_human_input exactly once with interactionKind `questions`, title `Verification word`, prompt `Choose the verification word`, continuationPolicy `wake_assignee`, and payload {version:1,questions:[{id:`verification-word`,prompt:`Choose the verification word.`,selectionMode:`single`,required:true,options:[{id:`cobalt`,label:`Cobalt`},{id:`amber`,label:`Amber`}]}]}.", + "Do not call paperclip_finish while the question is pending.", + `After the answer arrives, if it is Cobalt, return ${breadthMarker("Q_C", nonce)} visibly and call paperclip_finish with that marker as the summary.`, + "Do not create files, plans, or additional work.", + ].join("\n"), + buildMatchers: (nonce, execution) => + terminalMatchers(breadthMarker("Q_C", nonce), execution), + }, + { + id: "plan-approve-complete", + label: "Plan, approve, complete", + groups: [], + workMode: "planning", + flow: "plan_approval_completion", + expectedRunCount: 2, + attemptTimeoutMs: { local: 15 * 60_000, daytona: 15 * 60_000 }, + expectedTerminalState: { issue: "done", run: "succeeded" }, + buildTitle: (nonce) => `OpenRouter breadth plan ${nonce}`, + buildVisibleMarker: (nonce) => breadthMarker("P_OK", nonce), + buildPlanMarkers: (nonce) => ({ + draft: breadthMarker("P_READY", nonce), + revised: breadthMarker("P_OK", nonce), + }), + buildPrompt: (nonce) => + [ + "Create a canonical Plan with exactly two numbered steps and request approval; do not implement before approval.", + `The Plan body must contain ${breadthMarker("P_READY", nonce)}.`, + "Call write_document for key `plan`, then call request_human_input exactly once with interactionKind `confirmation`, targetRevisionId set to the returned latest Plan revision, and continuationPolicy `wake_assignee`.", + "Do not call paperclip_finish while confirmation is pending.", + `After that exact Plan revision is accepted, return ${breadthMarker("P_OK", nonce)} visibly and call paperclip_finish with that marker as the summary.`, + "Do not create files, child tasks, or unrelated work.", + ].join("\n"), + buildMatchers: (nonce, execution) => + terminalMatchers(breadthMarker("P_OK", nonce), execution), + }, +] as const; + +const localEnvironment = runnerEnvironments.find( + (environment) => environment.id === "local", +)!; + +export const runnerSuites: readonly RunnerSuiteFixture[] = [ + { + id: "core-compatibility", + label: "Core Runner Compatibility", + description: + "Major provider, runtime generation, and execution-environment compatibility.", + groups: ["core"], + profiles: runnerProfiles, + environments: runnerEnvironments, + tasks: runnerTasks, + expectedMatrixSize: 42, + }, + { + id: "openrouter-model-breadth", + label: "OpenRouter Model Breadth", + description: + "Weekly-ranked tool-capable OpenRouter models through native OpenCode on isolated local workspaces.", + groups: ["breadth"], + profiles: openRouterBreadthProfiles, + environments: [localEnvironment], + tasks: openRouterBreadthTasks, + expectedMatrixSize: 15, + definitionMetadata: { + rankingSnapshotId: openRouterRankingSnapshot.snapshotId, + rankingContentHash: openRouterRankingSnapshot.contentHash, + rankingCapturedAt: openRouterRankingSnapshot.capturedAt, + rankingSourceUrl: openRouterRankingSnapshot.sourceUrl, + }, + }, +] as const; + +export function suiteDefinitionHash(suite: RunnerSuiteFixture) { + return createHash("sha256") + .update( + JSON.stringify({ + id: suite.id, + profiles: suite.profiles.map((profile) => ({ + id: profile.id, + model: profile.model, + qualification: profile.modelQualification, + })), + environments: suite.environments.map((environment) => environment.id), + tasks: suite.tasks.map((task) => ({ + id: task.id, + flow: task.flow, + expectedRunCount: task.expectedRunCount, + })), + metadata: suite.definitionMetadata ?? null, + }), + ) + .digest("hex"); +} + +export function buildRunnerMatrix( + suites: readonly RunnerSuiteFixture[] = runnerSuites, +): MatrixExecution[] { + return suites.flatMap((suite) => + suite.profiles.flatMap((profile) => + suite.environments + .filter((environment) => + profile.supportedEnvironments.includes(environment.id), + ) + .flatMap((environment) => + suite.tasks.map((task) => ({ + id: `${suite.id}.${profile.id}.${environment.id}.${task.id}`, + suite, + suiteDefinitionHash: suiteDefinitionHash(suite), + profile, + environment, + task, + groups: [ + ...new Set([ + ...suite.groups, + ...profile.groups, + ...environment.groups, + ...task.groups, + ]), + ], + requiredCredentials: [ + profile.credential, + ...(environment.credential ? [environment.credential] : []), + ], + })), + ), + ), + ); +} + +function duplicateIds(values: readonly { id: string }[]) { + const seen = new Set(); + return values + .map((value) => value.id) + .filter((id) => { + if (seen.has(id)) return true; + seen.add(id); + return false; + }); +} + +function assertNoRawSecretValues(value: unknown, label: string) { + if (typeof value === "string") { + if (/\b(?:sk-(?:proj-)?|sk-ant-)[A-Za-z0-9_-]{12,}\b/.test(value)) { + throw new Error(`${label} contains a raw secret-looking value`); + } + return; + } + if (Array.isArray(value)) { + value.forEach((entry) => assertNoRawSecretValues(entry, label)); + return; + } + if (value && typeof value === "object") { + Object.entries(value).forEach(([key, entry]) => { + if ( + typeof entry === "string" && + /(?:api.?key|access.?token|credential|secret)$/i.test(key) && + entry.trim() + ) { + throw new Error(`${label} contains a raw credential at ${key}`); + } + assertNoRawSecretValues(entry, label); + }); + } +} + +export function validateRunnerCatalog(): MatrixExecution[] { + const allProfiles = [...runnerProfiles, ...openRouterBreadthProfiles]; + const allTasks = [...runnerTasks, ...openRouterBreadthTasks]; + for (const [label, values] of [ + ["suite", runnerSuites], + ["profile", allProfiles], + ["environment", runnerEnvironments], + ["task", allTasks], + ] as const) { + const duplicates = duplicateIds(values); + if (duplicates.length > 0) + throw new Error( + `Duplicate ${label} fixture ids: ${duplicates.join(", ")}`, + ); + } + + const selectableGroups = new Set(SELECTABLE_GROUPS); + for (const fixture of [ + ...runnerSuites, + ...allProfiles, + ...runnerEnvironments, + ...allTasks, + ]) { + const unknownGroups = fixture.groups.filter( + (group) => !selectableGroups.has(group), + ); + if (unknownGroups.length > 0) { + throw new Error( + `Fixture ${fixture.id} declares unknown groups: ${unknownGroups.join(", ")}`, + ); + } + } + + const sampleRefs = Object.fromEntries( + [ + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "OPENROUTER_API_KEY", + "DAYTONA_API_KEY", + ].map((name, index) => [ + name, + { + type: "secret_ref" as const, + secretId: `${String(index + 1).padStart(8, "0")}-1111-4111-8111-111111111111`, + version: "latest" as const, + }, + ]), + ); + + for (const environment of runnerEnvironments) { + const payload = environment.buildEnvironment({ + secretRefs: sampleRefs, + daytonaImage: + "ghcr.io/paperclipai/paperclip-daytona-runner@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + executionId: "schema-validation", + }); + createEnvironmentSchema.parse(payload); + assertNoRawSecretValues(payload, `environment ${environment.id}`); + } + for (const profile of allProfiles) { + if (!CREDENTIAL_NAMES.includes(profile.credential)) { + throw new Error( + `Profile ${profile.id} declares unknown credential ${profile.credential}`, + ); + } + const unsupportedEnvironmentIds = profile.supportedEnvironments.filter( + (environmentId) => !ENVIRONMENT_IDS.includes(environmentId), + ); + if (unsupportedEnvironmentIds.length > 0) { + throw new Error( + `Profile ${profile.id} declares unknown environments: ${unsupportedEnvironmentIds.join(", ")}`, + ); + } + const payload = profile.buildAgent({ + environmentId: SAMPLE_UUID, + environmentFixtureId: "local", + workspacePath: "/tmp/paperclip-runner-e2e-schema", + secretRefs: sampleRefs, + executionId: "schema-validation", + }); + createAgentSchema.parse(payload); + assertNoRawSecretValues(payload, `profile ${profile.id}`); + } + + const matrix = buildRunnerMatrix(); + const duplicateMatrixIds = duplicateIds(matrix); + if (duplicateMatrixIds.length > 0) { + throw new Error( + `Duplicate matrix execution ids: ${duplicateMatrixIds.join(", ")}`, + ); + } + for (const suite of runnerSuites) { + const suiteSize = matrix.filter( + (execution) => execution.suite.id === suite.id, + ).length; + if (suiteSize !== suite.expectedMatrixSize) { + throw new Error( + `Expected ${suite.expectedMatrixSize} ${suite.id} executions; received ${suiteSize}`, + ); + } + } + if (matrix.length !== 57) + throw new Error(`Expected 57 runner executions; received ${matrix.length}`); + return matrix; +} + +export const runnerMatrix = validateRunnerCatalog(); + +export function runnerExecutionById(id: string): MatrixExecution { + const execution = runnerMatrix.find((candidate) => candidate.id === id); + if (!execution) throw new Error(`Unknown runner E2E execution id: ${id}`); + return execution; +} diff --git a/tests/runner-e2e/dashboard-regenerate.ts b/tests/runner-e2e/dashboard-regenerate.ts new file mode 100644 index 0000000000..2a66ed3a39 --- /dev/null +++ b/tests/runner-e2e/dashboard-regenerate.ts @@ -0,0 +1,203 @@ +import path from "node:path"; +import { mkdir, readFile, readdir, writeFile } from "node:fs/promises"; +import { runnerMatrix } from "./catalog.js"; +import { renderRunnerE2EDashboard } from "./dashboard.js"; +import { + buildRunnerCampaign, + canonicalExecutionId, + upgradeRunnerResult, +} from "./history.js"; +import type { + RunnerE2ECampaign, + RunnerE2EHistoryIndex, + RunnerE2EResult, +} from "./types.js"; + +interface PublishedResult extends RunnerE2EResult { + evidenceValid?: boolean; + evidenceErrors?: string[]; +} + +interface PublishedCampaign { + schema?: string; + campaignId?: string; + generatedAt: string; + expected: string[]; + results: PublishedResult[]; +} + +async function relativeFiles(root: string, current = root): Promise { + const entries = await readdir(current, { withFileTypes: true }).catch( + () => [], + ); + const files: string[] = []; + for (const entry of entries) { + const absolute = path.join(current, entry.name); + if (entry.isDirectory()) + files.push(...(await relativeFiles(root, absolute))); + if (entry.isFile()) { + files.push(path.relative(root, absolute).split(path.sep).join("/")); + } + } + return files; +} + +async function readOptionalHistory(file: string) { + return readFile(file, "utf8") + .then((value) => JSON.parse(value) as RunnerE2EHistoryIndex) + .then((value) => + value.schema === "paperclip.runner-e2e.history/v1" ? value : undefined, + ) + .catch(() => undefined); +} + +export async function regenerateRunnerDashboard(input: { + bundle: string; + historyFile?: string | null; + outputDirectory?: string; + evidenceHrefPrefix?: string; +}) { + const bundle = path.resolve(input.bundle); + const outputDirectory = path.resolve(input.outputDirectory ?? bundle); + const evidenceHrefPrefix = input.evidenceHrefPrefix?.replace( + /^\/+|\/+$/g, + "", + ); + if ( + evidenceHrefPrefix && + (evidenceHrefPrefix.includes("\\") || + evidenceHrefPrefix + .split("/") + .some((segment) => !segment || segment === "." || segment === "..")) + ) { + throw new Error("evidenceHrefPrefix must be a safe relative URL path"); + } + const normalized = JSON.parse( + await readFile(path.join(bundle, "normalized-results.json"), "utf8"), + ) as PublishedCampaign; + if ( + !Array.isArray(normalized.expected) || + !Array.isArray(normalized.results) || + typeof normalized.generatedAt !== "string" + ) { + throw new Error("Published bundle has an invalid normalized-results.json"); + } + + const results = normalized.results.map(upgradeRunnerResult); + const expected = normalized.expected.map(canonicalExecutionId); + const campaign: RunnerE2ECampaign = buildRunnerCampaign({ + campaignId: + normalized.campaignId ?? + `legacy-${normalized.generatedAt.replace(/[:.]/g, "-")}`, + generatedAt: normalized.generatedAt, + expected, + results, + }); + const history = + input.historyFile === null + ? undefined + : await readOptionalHistory( + input.historyFile ?? path.join(bundle, "history.json"), + ); + const entries = await Promise.all( + normalized.results.map(async (publishedResult, index) => { + const result = results[index]!; + const originalExecutionId = publishedResult.executionId; + const evidenceIds = [ + originalExecutionId, + ...(originalExecutionId.startsWith("core-compatibility.") + ? [originalExecutionId.slice("core-compatibility.".length)] + : []), + ]; + let evidenceBaseHref = ""; + let evidenceFiles: string[] = []; + for (const evidenceId of evidenceIds) { + const candidate = [ + "evidence", + evidenceId, + `attempt-${result.attempt}`, + ].join("/"); + const files = await relativeFiles( + path.join(bundle, ...candidate.split("/")), + ); + if (files.length === 0) continue; + evidenceBaseHref = [evidenceHrefPrefix, candidate] + .filter(Boolean) + .join("/"); + evidenceFiles = files; + break; + } + return { + result, + valid: + publishedResult.evidenceValid ?? + (result.status === "passed" && result.cleanup === "passed"), + errors: publishedResult.evidenceErrors ?? [], + evidenceBaseHref, + evidenceFiles, + }; + }), + ); + const dashboard = renderRunnerE2EDashboard({ + title: "Runner Full-Stack E2E", + generatedAt: normalized.generatedAt, + expected, + catalog: runnerMatrix, + entries, + campaign, + history, + }); + const upgraded = { + ...campaign, + results: campaign.results.map((result, index) => ({ + ...result, + evidenceValid: + normalized.results[index]?.evidenceValid ?? + (result.status === "passed" && result.cleanup === "passed"), + evidenceErrors: normalized.results[index]?.evidenceErrors ?? [], + })), + }; + await mkdir(outputDirectory, { recursive: true }); + await Promise.all([ + writeFile(path.join(outputDirectory, "index.html"), dashboard, "utf8"), + writeFile(path.join(outputDirectory, "dashboard.html"), dashboard, "utf8"), + writeFile( + path.join(outputDirectory, "normalized-results.json"), + `${JSON.stringify(upgraded, null, 2)}\n`, + "utf8", + ), + ]); + console.log( + `Regenerated dashboard from ${normalized.results.length} retained result(s) in ${outputDirectory}`, + ); +} + +async function main() { + const arguments_ = process.argv + .slice(2) + .filter((argument) => argument !== "--"); + const bundleArgument = arguments_.find( + (argument) => !argument.startsWith("--"), + ); + if (!bundleArgument) { + throw new Error( + "Usage: pnpm test:e2e:runner:dashboard -- [--history ]", + ); + } + const historyIndex = arguments_.indexOf("--history"); + const historyFile = + historyIndex >= 0 ? arguments_[historyIndex + 1] : undefined; + if (historyIndex >= 0 && !historyFile) + throw new Error("--history requires a path"); + await regenerateRunnerDashboard({ bundle: bundleArgument, historyFile }); +} + +if ( + process.argv[1] && + path.resolve(process.argv[1]) === path.resolve(import.meta.filename) +) { + await main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/tests/runner-e2e/dashboard.ts b/tests/runner-e2e/dashboard.ts new file mode 100644 index 0000000000..23f80352f7 --- /dev/null +++ b/tests/runner-e2e/dashboard.ts @@ -0,0 +1,1059 @@ +import type { + MatrixExecution, + RunnerE2ECampaign, + RunnerE2EHistoryIndex, + RunnerE2EResult, + RunnerE2ESuiteSummary, +} from "./types.js"; +import { + aggregateCampaignBilling, + summarizeExecutionBilling, +} from "./billing.js"; + +export interface RunnerDashboardEntry { + result: RunnerE2EResult; + valid: boolean; + errors: readonly string[]; + evidenceBaseHref?: string; + evidenceFiles?: readonly string[]; +} + +export interface RunnerDashboardInput { + title: string; + generatedAt: string; + expected: readonly string[]; + catalog: readonly MatrixExecution[]; + entries: readonly RunnerDashboardEntry[]; + campaign?: RunnerE2ECampaign; + history?: RunnerE2EHistoryIndex; +} + +interface ResolvedScreenshot { + id: string; + label: string; + file: string; + href: string; +} + +function html(value: unknown) { + return String(value ?? "") + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """) + .replaceAll("'", "'"); +} + +function durationLabel(durationMs: number) { + if (durationMs < 1_000) return `${durationMs}ms`; + const seconds = Math.round(durationMs / 1_000); + if (seconds < 60) return `${seconds}s`; + return `${Math.floor(seconds / 60)}m ${seconds % 60}s`; +} + +function tokenLabel(value: number) { + return new Intl.NumberFormat("en-US").format(value); +} + +function usdLabel(value: number) { + return `$${value.toFixed(value < 0.01 ? 6 : 4)}`; +} + +function safeEvidenceHref(base: string | undefined, relative: string) { + if (!base || /^(?:[a-z]+:|\/\/)/i.test(base)) return null; + const cleanBase = base + .split("/") + .filter((segment) => segment && segment !== "." && segment !== "..") + .map(encodeURIComponent) + .join("/"); + const cleanRelative = relative + .split("/") + .filter((segment) => segment && segment !== "." && segment !== "..") + .map(encodeURIComponent) + .join("/"); + return `${cleanBase}/${cleanRelative}`; +} + +function compactJson(value: unknown) { + const serialized = JSON.stringify(value); + return serialized && serialized.length > 1_200 + ? `${serialized.slice(0, 1_200)}…` + : serialized; +} + +function resolveScreenshots( + entry: RunnerDashboardEntry | undefined, +): ResolvedScreenshot[] { + const declaredScreenshots = entry?.result.screenshots?.length + ? entry.result.screenshots + : entry + ? [ + { + id: "final-state", + label: "Final visible task state", + file: "final-state.png", + }, + ] + : []; + const availableFiles = entry?.evidenceFiles + ? new Set(entry.evidenceFiles) + : null; + const screenshots = declaredScreenshots + .filter((item) => !availableFiles || availableFiles.has(item.file)) + .flatMap((item) => { + const href = safeEvidenceHref(entry?.evidenceBaseHref, item.file); + return href ? [{ ...item, href }] : []; + }); + if ( + entry?.result.status === "failed" && + !screenshots.some((item) => item.file === "failure.png") && + (!availableFiles || availableFiles.has("failure.png")) + ) { + const href = safeEvidenceHref(entry.evidenceBaseHref, "failure.png"); + if (href) { + screenshots.push({ + id: "failure", + label: "Failure state", + file: "failure.png", + href, + }); + } + } + return screenshots; +} + +function renderCase( + execution: MatrixExecution, + expected: ReadonlySet, + entryById: ReadonlyMap, +) { + const selected = expected.has(execution.id); + const entry = entryById.get(execution.id); + const state = !selected + ? "not-selected" + : !entry + ? "missing" + : entry.valid + ? "passed" + : "failed"; + const label = state.replace("-", " "); + const detail = + entry?.errors.join("; ") || + (entry?.valid ? "All invariants passed" : "Not selected"); + const screenshots = resolveScreenshots(entry); + const billing = entry ? summarizeExecutionBilling(entry.result) : null; + const availableFiles = entry?.evidenceFiles + ? new Set(entry.evidenceFiles) + : null; + const playwright = + !availableFiles || availableFiles.has("html-report/index.html") + ? safeEvidenceHref(entry?.evidenceBaseHref, "html-report/index.html") + : null; + const links = + screenshots.length > 0 || playwright + ? `` + : ""; + const matcherRows = (entry?.result.matcherResults ?? []) + .map( + (result) => ` + ${result.passed ? "Pass" : "Fail"} + ${html(result.matcher.kind)} + ${html(compactJson(result.matcher))} + ${html(result.detail)} + `, + ) + .join(""); + const gallery = screenshots.length + ? `` + : ""; + const billingStrip = billing + ? `
+
Tokens${html(tokenLabel(billing.llm.inputTokens))} in · ${html(tokenLabel(billing.llm.outputTokens))} out${html(tokenLabel(billing.llm.cachedInputTokens))} cached · ${billing.llm.runsWithTokenUsage}/${billing.llm.runCount} runs covered
+
LLM spend${billing.llm.runsWithReportedCost > 0 ? html(usdLabel(billing.reportedCostUsd)) : html(billing.llm.costStatus)}${billing.llm.runsWithReportedCost}/${billing.llm.runCount} runs provider-priced
+
Execution${billing.runtime.estimatedListCostUsd === undefined ? html(billing.runtime.costStatus === "not_metered" ? "Local · not metered" : "Cost unavailable") : `${html(usdLabel(billing.runtime.estimatedListCostUsd))} est.`}${html(durationLabel(billing.runtime.agentRunDurationMs))} agent${billing.runtime.leaseDurationMs === null ? "" : ` · ${html(durationLabel(billing.runtime.leaseDurationMs))} lease`}
+
` + : ""; + return `
+
+ ${html(execution.task.label)} + ${html(label)} +
+ ${gallery} + ${billingStrip} + ${html(execution.id)} +
+ Matchers and test context + ${ + entry + ? `
+
Attempt
${entry.result.attempt}
+
Duration
${html(durationLabel(entry.result.durationMs))}
+
Agent runtime
${html(durationLabel(billing!.runtime.agentRunDurationMs))}
+ ${billing!.runtime.leaseDurationMs === null ? "" : `
Environment lease
${html(durationLabel(billing!.runtime.leaseDurationMs))}
`} +
Runtime
${html(entry.result.runtimeMode)}
+
Provider
${html(entry.result.provider)}
+
Model
${html(entry.result.model)}
+ ${entry.result.issueIdentifier ? `
Issue
${html(entry.result.issueIdentifier)}
` : ""} +
` + : "" + } +

${html(detail)}

+ ${matcherRows ? `
${matcherRows}
ResultMatcherExpectationDetail
` : `

No matcher result was recorded.

`} + ${entry ? `
Usage and billing metadata
${html(JSON.stringify({ billing, rawUsage: entry.result.usage ?? null }, null, 2))}
` : ""} + ${links} +
+
`; +} + +function renderTrendChart(input: { + history: RunnerE2EHistoryIndex; + label: string; + value(campaign: RunnerE2EHistoryIndex["campaigns"][number]): number; + format(value: number): string; + include?(campaign: RunnerE2EHistoryIndex["campaigns"][number]): boolean; + fingerprint?(campaign: RunnerE2EHistoryIndex["campaigns"][number]): string; +}) { + const campaigns = input.history.campaigns + .filter(input.include ?? ((campaign) => campaign.complete)) + .slice(0, 20) + .reverse(); + if (campaigns.length === 0) { + return `
${html(input.label)}No complete campaigns
`; + } + const values = campaigns.map(input.value); + const maximum = Math.max(...values, 1); + const pointRows = values.map((value, index) => { + const x = + campaigns.length === 1 ? 50 : (index / (campaigns.length - 1)) * 100; + const y = 96 - (value / maximum) * 88; + return { + point: `${x.toFixed(2)},${y.toFixed(2)}`, + x, + y, + fingerprint: input.fingerprint?.(campaigns[index]!) ?? "stable", + }; + }); + const segments = pointRows.reduce>( + (groups, point) => { + const current = groups.at(-1); + if (!current || current.at(-1)?.fingerprint !== point.fingerprint) { + groups.push([point]); + } else { + current.push(point); + } + return groups; + }, + [], + ); + const definitionCount = new Set(pointRows.map((point) => point.fingerprint)) + .size; + const latest = values.at(-1) ?? 0; + return `
+ ${html(input.label)} + ${html(input.format(latest))} + + ${segments.map((segment) => ``).join("")} + ${pointRows.map((point) => ``).join("")} + + ${campaigns.length} complete campaign${campaigns.length === 1 ? "" : "s"} · ${definitionCount} definition${definitionCount === 1 ? "" : "s"} +
`; +} + +function suiteSummaryFor( + campaign: RunnerE2EHistoryIndex["campaigns"][number], + suiteId: string, +) { + return campaign.suites.find((suite) => suite.suiteId === suiteId); +} + +function renderHistory(history: RunnerE2EHistoryIndex | undefined) { + if (!history || history.campaigns.length === 0) { + return `

History

Campaign trends

No historical campaigns have been published yet.

`; + } + const suiteIds = [ + ...new Set( + history.campaigns.flatMap((campaign) => + campaign.suites.map((suite) => suite.suiteId), + ), + ), + ]; + const charts = [ + renderTrendChart({ + history, + label: "Observed + estimated cost", + value: (campaign) => campaign.billing.observedAndEstimatedCostUsd, + format: usdLabel, + fingerprint: (campaign) => + campaign.suites + .map((suite) => `${suite.suiteId}:${suite.suiteDefinitionHash}`) + .sort() + .join("|"), + }), + renderTrendChart({ + history, + label: "Total tokens", + value: (campaign) => campaign.billing.llm.totalTokens, + format: tokenLabel, + fingerprint: (campaign) => + campaign.suites + .map((suite) => `${suite.suiteId}:${suite.suiteDefinitionHash}`) + .sort() + .join("|"), + }), + renderTrendChart({ + history, + label: "Agent execution time", + value: (campaign) => campaign.billing.agentRunDurationMs, + format: durationLabel, + fingerprint: (campaign) => + campaign.suites + .map((suite) => `${suite.suiteId}:${suite.suiteDefinitionHash}`) + .sort() + .join("|"), + }), + renderTrendChart({ + history, + label: "Daytona lease time", + value: (campaign) => campaign.billing.leaseDurationMs, + format: durationLabel, + fingerprint: (campaign) => + campaign.suites + .map((suite) => `${suite.suiteId}:${suite.suiteDefinitionHash}`) + .sort() + .join("|"), + }), + renderTrendChart({ + history, + label: "Pass rate", + value: (campaign) => + campaign.selected > 0 ? (campaign.passed / campaign.selected) * 100 : 0, + format: (value) => `${value.toFixed(1)}%`, + fingerprint: (campaign) => + campaign.suites + .map((suite) => `${suite.suiteId}:${suite.suiteDefinitionHash}`) + .sort() + .join("|"), + }), + ].join(""); + const suiteCharts = suiteIds + .map((suiteId) => { + const include = (campaign: RunnerE2EHistoryIndex["campaigns"][number]) => + suiteSummaryFor(campaign, suiteId)?.complete === true; + const value = ( + campaign: RunnerE2EHistoryIndex["campaigns"][number], + metric: "cost" | "tokens" | "agent" | "lease" | "passRate", + ) => { + const suite = suiteSummaryFor(campaign, suiteId); + if (!suite) return 0; + if (metric === "cost") return suite.billing.observedAndEstimatedCostUsd; + if (metric === "tokens") return suite.billing.llm.totalTokens; + if (metric === "agent") return suite.billing.agentRunDurationMs; + if (metric === "lease") return suite.billing.leaseDurationMs; + return suite.selected > 0 ? (suite.passed / suite.selected) * 100 : 0; + }; + const fingerprint = ( + campaign: RunnerE2EHistoryIndex["campaigns"][number], + ) => suiteSummaryFor(campaign, suiteId)?.suiteDefinitionHash ?? "unknown"; + return ``; + }) + .join(""); + const rows = history.campaigns + .map((campaign) => { + const status = campaign.failed === 0 ? "passed" : "failed"; + const sha = campaign.source.sha; + const searchable = [ + campaign.campaignId, + sha, + campaign.source.ref, + ...campaign.executions.flatMap((execution) => [ + execution.suiteId, + execution.profileId, + execution.model, + execution.environmentId, + execution.caseId, + execution.status, + ]), + ] + .filter(Boolean) + .join(" ") + .toLowerCase(); + return ` + ${html(campaign.campaignId)}${html(new Date(campaign.generatedAt).toLocaleString("en-US", { timeZone: "UTC" }))} UTC + ${sha ? `${html(sha.slice(0, 10))}` : "Unknown"}${html(campaign.source.ref ?? "unknown ref")} + ${status}${campaign.passed}/${campaign.passed + campaign.failed} passed · ${campaign.complete ? "complete" : "partial"} + ${html(tokenLabel(campaign.billing.llm.inputTokens))} / ${html(tokenLabel(campaign.billing.llm.outputTokens))}input / output · ${html(tokenLabel(campaign.billing.llm.cachedInputTokens))} cached + ${html(usdLabel(campaign.billing.reportedLlmCostUsd))}${html(usdLabel(campaign.billing.estimatedRuntimeCostUsd))} runtime estimate + ${html(durationLabel(campaign.billing.agentRunDurationMs))}${html(durationLabel(campaign.billing.leaseDurationMs))} lease + `; + }) + .join(""); + const latest = history.campaigns.find( + (campaign) => campaign.campaignId === history.latestCampaignId, + ); + const latestGreen = history.campaigns.find( + (campaign) => campaign.campaignId === history.latestGreenCampaignId, + ); + return `
+

History

Campaign trends

Complete campaigns are compared by default. Partial smoke runs remain searchable and are labeled explicitly.

+ +
${charts}
+ ${suiteCharts} +
+ + + + + + +
+
${rows}
CampaignPaperclip SHAResultTokensCostExecution
+ +
`; +} + +function renderSuiteMatrix(input: { + suiteCatalog: readonly MatrixExecution[]; + expected: ReadonlySet; + entryById: ReadonlyMap; + summary?: RunnerE2ESuiteSummary; +}) { + const suite = input.suiteCatalog[0]?.suite; + if (!suite) return ""; + const profiles = [ + ...new Map( + input.suiteCatalog.map((execution) => [ + execution.profile.id, + execution.profile, + ]), + ).values(), + ]; + const environments = [ + ...new Map( + input.suiteCatalog.map((execution) => [ + execution.environment.id, + execution.environment, + ]), + ).values(), + ]; + const rows = profiles + .map((profile, profileIndex) => { + const columns = environments + .map((environment) => { + const executions = input.suiteCatalog.filter( + (execution) => + execution.profile.id === profile.id && + execution.environment.id === environment.id, + ); + return ` +
${html(environment.label)}${html(environment.provider)} · ${html(environment.expectedExecutionTarget.kind)}
+
${executions.map((execution) => renderCase(execution, input.expected, input.entryById)).join("")}
+ `; + }) + .join(""); + return `
+ + ${html(profile.label)}${html(profile.generation)}${html(profile.provider)} · ${html(profile.model)} +
${columns}`; + }) + .join(""); + const environmentHeaders = environments + .map( + (environment) => + `${html(environment.label)}${html(environment.provider)} · ${html(environment.expectedExecutionTarget.kind)}`, + ) + .join(""); + const selected = input.suiteCatalog.filter((execution) => + input.expected.has(execution.id), + ).length; + const summary = input.summary; + const summaryHtml = summary + ? `
+
Pass rate${summary.selected > 0 ? ((summary.passed / summary.selected) * 100).toFixed(1) : "0.0"}%${summary.passed}/${summary.selected} passed
+
Tokens${html(tokenLabel(summary.billing.llm.totalTokens))}${html(tokenLabel(summary.billing.llm.inputTokens))} input · ${html(tokenLabel(summary.billing.llm.outputTokens))} output
+
Cost${html(usdLabel(summary.billing.observedAndEstimatedCostUsd))}reported LLM + runtime estimate
+
Agent time${html(durationLabel(summary.billing.agentRunDurationMs))}${html(durationLabel(summary.billing.leaseDurationMs))} lease
+
Execution${summary.executed}/${summary.selected}${summary.retries} retries · cleanup ${summary.cleanupPassed ? "passed" : "failed"}
+
` + : ""; + return `
+

Test suite

${html(suite.label)}

${html(suite.description)}

+ ${summaryHtml} +
Configuration matrix${profiles.length} profiles · ${environments.length} environments · ${selected} selected
+
${environmentHeaders}${rows}
Agent profile
+
`; +} + +export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { + const expected = new Set(input.expected); + const entryById = new Map( + input.entries.map((entry) => [entry.result.executionId, entry]), + ); + const selectedEntries = input.entries.filter((entry) => + expected.has(entry.result.executionId), + ); + const passed = selectedEntries.filter((entry) => entry.valid).length; + const failed = input.expected.length - passed; + const totalDuration = selectedEntries.reduce( + (total, entry) => total + entry.result.durationMs, + 0, + ); + const screenshotCount = selectedEntries.reduce( + (total, entry) => total + resolveScreenshots(entry).length, + 0, + ); + const campaignBilling = aggregateCampaignBilling( + selectedEntries.map((entry) => entry.result), + ); + const suites = [ + ...new Map( + input.catalog.map((execution) => [execution.suite.id, execution.suite]), + ).values(), + ]; + const suiteSections = suites + .map((suite) => + renderSuiteMatrix({ + suiteCatalog: input.catalog.filter( + (execution) => execution.suite.id === suite.id, + ), + expected, + entryById, + summary: input.campaign?.suites.find( + (summary) => summary.suiteId === suite.id, + ), + }), + ) + .join(""); + const historySection = renderHistory(input.history); + + return ` + + + + + + + + + ${html(input.title)} · Paperclip + + + +
+ + + Paperclip + + Quality engineering · Runner acceptance +
+
+
+
+

Full-stack acceptance campaign

+

${html(input.title)}

+

A browser-verified matrix of runner profiles, execution environments, and deterministic task contracts. Visual evidence is retained in the access-controlled workflow artifact; public history contains inert structured evidence only.

+
+
+
+
${passed}/${input.expected.length}Passed
+
${failed}Failed
+
${html(durationLabel(totalDuration))}Test time
+
+ +
+
+
+
${html(tokenLabel(campaignBilling.llm.inputTokens))}Input tokens
+
${html(tokenLabel(campaignBilling.llm.outputTokens))}Output tokens
+
${html(tokenLabel(campaignBilling.llm.cachedInputTokens))}Cached tokens
+
${html(usdLabel(campaignBilling.reportedLlmCostUsd))}LLM reported subtotal
+
${html(usdLabel(campaignBilling.estimatedRuntimeCostUsd))}Daytona list estimate
+
${html(durationLabel(campaignBilling.agentRunDurationMs))}Agent execution time
+
${html(durationLabel(campaignBilling.leaseDurationMs))}Daytona lease time
+
${campaignBilling.llm.runsWithReportedCost}/${campaignBilling.llm.runCount}Runs provider-priced
+

Model spend is the provider-reported subtotal; unpriced or unavailable runs are excluded, never counted as free. Daytona runtime is a public-list-price estimate from captured lease time and pinned resources, before credits, discounts, storage allowance, or invoice adjustments. Local execution has no external runtime meter.

+
+ + ${suiteSections} + ${historySection} +
Generated ${html(input.generatedAt)}${input.catalog.length} catalog executions · Public history excludes visual evidence
+
+ + + + + + + +`; +} diff --git a/tests/runner-e2e/daytona-image-content.ts b/tests/runner-e2e/daytona-image-content.ts new file mode 100644 index 0000000000..ab566d7bd7 --- /dev/null +++ b/tests/runner-e2e/daytona-image-content.ts @@ -0,0 +1,242 @@ +import { createHash } from "node:crypto"; +import { lstat, readdir, readFile, readlink } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const repositoryRoot = path.resolve(import.meta.dirname, "../.."); + +export const DAYTONA_IMAGE_CONTENT_SCHEMA = + "paperclip-daytona-runner-image-content/v3"; +export const DAYTONA_IMAGE_PLATFORM = "linux/amd64"; +export const DAYTONA_IMAGE_DOCKERFILE_PATH = "docker/daytona-runner/Dockerfile"; + +// This is the audited dependency closure of docker/daytona-runner/Dockerfile. +// Keep it conservative: a false positive only rebuilds the image, while a +// missing input could incorrectly reuse an incompatible paid-test image. +export const DAYTONA_IMAGE_INPUT_PATHS = [ + ".dockerignore", + ".npmrc", + "docker/daytona-runner/Dockerfile", + "package.json", + "patches", + "pnpm-lock.yaml", + "pnpm-workspace.yaml", + "scripts/link-plugin-dev-sdk.mjs", + "tsconfig.base.json", + "packages/paperclip-eval-kernel", + "packages/paperclip-runner", +] as const; + +const ignoredDirectoryPaths = new Set([ + "packages/paperclip-runner/dist", + "packages/paperclip-runner/runner/target", +]); + +export interface DaytonaImageContentOptions { + repositoryRoot?: string; + inputPaths?: readonly string[]; + platform?: string; + baseImages?: readonly string[]; + frontendDigest?: string; +} + +function compareNames(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +function normalizedRelativePath(value: string): string { + return value.split(path.sep).join("/"); +} + +function shouldIgnore(relativePath: string): boolean { + if (ignoredDirectoryPaths.has(relativePath)) return true; + return relativePath.split("/").includes("node_modules"); +} + +function updateRecord( + hash: ReturnType, + kind: string, + relativePath: string, + payload = "", +): void { + hash.update(kind); + hash.update("\0"); + hash.update(relativePath); + hash.update("\0"); + hash.update(payload); + hash.update("\0"); +} + +function assertPinnedBaseImage(reference: string): void { + if (!/@sha256:[0-9a-f]{64}$/.test(reference)) { + throw new Error( + `Daytona image base must use an immutable sha256 digest: ${reference}`, + ); + } +} + +function assertPinnedFrontendDigest(digest: string): void { + if (!/^sha256:[0-9a-f]{64}$/.test(digest)) { + throw new Error( + `Daytona Dockerfile frontend must use an immutable sha256 digest: ${digest}`, + ); + } +} + +export function extractDaytonaDockerfileFrontendDigest( + dockerfile: string, +): string { + const firstLine = dockerfile.split(/\r?\n/, 1)[0] ?? ""; + const match = /^#\s*syntax=\S+@(sha256:[0-9a-f]{64})\s*$/.exec(firstLine); + if (!match) { + throw new Error( + "Daytona Dockerfile first line must pin its syntax frontend to an immutable sha256 digest", + ); + } + const digest = match[1]!; + assertPinnedFrontendDigest(digest); + return digest; +} + +export function extractDaytonaBaseImages(dockerfile: string): string[] { + const baseImages: string[] = []; + const stageAliases = new Set(); + + for (const line of dockerfile.split(/\r?\n/)) { + if (!/^\s*FROM\s/i.test(line)) continue; + const match = /^\s*FROM(?:\s+--\S+)*\s+(\S+)(?:\s+AS\s+(\S+))?\s*$/i.exec( + line, + ); + if (!match) { + throw new Error(`Cannot parse Daytona Dockerfile FROM line: ${line}`); + } + + const reference = match[1]!; + if (!stageAliases.has(reference)) { + assertPinnedBaseImage(reference); + baseImages.push(reference); + } + if (match[2]) stageAliases.add(match[2]); + } + + if (baseImages.length === 0) { + throw new Error("Daytona Dockerfile does not declare a base image"); + } + return baseImages; +} + +async function resolveBaseImages( + root: string, + explicitBaseImages?: readonly string[], +): Promise { + const baseImages = explicitBaseImages + ? [...explicitBaseImages] + : extractDaytonaBaseImages( + await readFile(path.join(root, DAYTONA_IMAGE_DOCKERFILE_PATH), "utf8"), + ); + if (baseImages.length === 0) { + throw new Error("Daytona image content identity requires a base image"); + } + for (const reference of baseImages) assertPinnedBaseImage(reference); + return baseImages.sort(compareNames); +} + +async function resolveFrontendDigest( + root: string, + explicitFrontendDigest?: string, +): Promise { + const digest = + explicitFrontendDigest ?? + extractDaytonaDockerfileFrontendDigest( + await readFile(path.join(root, DAYTONA_IMAGE_DOCKERFILE_PATH), "utf8"), + ); + assertPinnedFrontendDigest(digest); + return digest; +} + +async function hashEntry( + hash: ReturnType, + root: string, + relativePath: string, +): Promise { + const normalizedPath = normalizedRelativePath(relativePath); + if (shouldIgnore(normalizedPath)) return; + + const absolutePath = path.resolve(root, relativePath); + const relativeFromRoot = path.relative(root, absolutePath); + if ( + relativeFromRoot.startsWith(`..${path.sep}`) || + relativeFromRoot === ".." || + path.isAbsolute(relativeFromRoot) + ) { + throw new Error( + `Daytona image input escapes repository root: ${relativePath}`, + ); + } + + const stats = await lstat(absolutePath); + if (stats.isDirectory()) { + updateRecord(hash, "directory", normalizedPath); + const entries = await readdir(absolutePath, { withFileTypes: true }); + entries.sort((left, right) => compareNames(left.name, right.name)); + for (const entry of entries) { + await hashEntry(hash, root, path.join(relativePath, entry.name)); + } + return; + } + if (stats.isSymbolicLink()) { + updateRecord(hash, "symlink", normalizedPath, await readlink(absolutePath)); + return; + } + if (stats.isFile()) { + const executable = + (stats.mode & 0o111) === 0 ? "non-executable" : "executable"; + updateRecord(hash, "file", normalizedPath, executable); + hash.update(await readFile(absolutePath)); + hash.update("\0"); + return; + } + throw new Error(`Unsupported Daytona image input type: ${relativePath}`); +} + +export async function computeDaytonaImageContentId( + options: DaytonaImageContentOptions = {}, +): Promise { + const root = path.resolve(options.repositoryRoot ?? repositoryRoot); + const inputPaths = [ + ...(options.inputPaths ?? DAYTONA_IMAGE_INPUT_PATHS), + ].sort(compareNames); + const hash = createHash("sha256"); + updateRecord( + hash, + "contract", + DAYTONA_IMAGE_CONTENT_SCHEMA, + options.platform ?? DAYTONA_IMAGE_PLATFORM, + ); + updateRecord( + hash, + "dockerfile-frontend", + await resolveFrontendDigest(root, options.frontendDigest), + ); + for (const baseImage of await resolveBaseImages(root, options.baseImages)) { + updateRecord(hash, "base-image", baseImage); + } + for (const inputPath of inputPaths) { + await hashEntry(hash, root, inputPath); + } + return hash.digest("hex"); +} + +const invokedPath = process.argv[1] ? path.resolve(process.argv[1]) : null; +if (invokedPath && import.meta.url === pathToFileURL(invokedPath).href) { + computeDaytonaImageContentId() + .then((contentId) => process.stdout.write(`${contentId}\n`)) + .catch((error: unknown) => { + process.stderr.write( + `${error instanceof Error ? error.message : String(error)}\n`, + ); + process.exitCode = 1; + }); +} + +export const DAYTONA_IMAGE_CONTENT_SCRIPT = fileURLToPath(import.meta.url); diff --git a/tests/runner-e2e/daytona-image.test.ts b/tests/runner-e2e/daytona-image.test.ts new file mode 100644 index 0000000000..9aefe4163d --- /dev/null +++ b/tests/runner-e2e/daytona-image.test.ts @@ -0,0 +1,203 @@ +import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { + computeDaytonaImageContentId, + DAYTONA_IMAGE_DOCKERFILE_PATH, + DAYTONA_IMAGE_INPUT_PATHS, + extractDaytonaBaseImages, + extractDaytonaDockerfileFrontendDigest, +} from "./daytona-image-content.js"; + +const repositoryRoot = path.resolve(import.meta.dirname, "../.."); + +describe("runner E2E Daytona image contract", () => { + it("builds runnerd and the provider pack and verifies every required transport", async () => { + const [dockerfile, dockerignore, workflow] = await Promise.all([ + readFile( + path.join(repositoryRoot, "docker/daytona-runner/Dockerfile"), + "utf8", + ), + readFile(path.join(repositoryRoot, ".dockerignore"), "utf8"), + readFile( + path.join( + repositoryRoot, + ".github/workflows/runner-full-stack-e2e.yml", + ), + "utf8", + ), + ]); + expect(dockerfile).toContain("--bin paperclip-runnerd"); + expect(dockerfile).toContain("build-provider-pack.mjs /provider-pack"); + expect(dockerfile).toContain( + "/opt/paperclip-runner/provider-pack/provider-pack.json", + ); + expect(dockerfile).toContain( + "${PAPERCLIP_RUNNER_PROVIDER_PACK_ROOT}/node_modules/.bin", + ); + for (const command of ["acpx", "claude-agent-acp", "codex-acp"]) { + expect(dockerfile).toContain(command); + } + for (const transport of ["dial_ws_loopback", "dial_wss", "listen_ws"]) { + expect(dockerfile).toContain(transport); + } + expect(dockerfile).toContain( + 'metadata="$(paperclip-runnerd --build-metadata)"', + ); + expect(dockerfile).toContain("provider-pack.json"); + expect(dockerfile).toContain("io.paperclip.runner.content-id"); + expect(dockerfile).toContain("org.opencontainers.image.revision"); + expect(extractDaytonaDockerfileFrontendDigest(dockerfile)).toBe( + "sha256:a57df69d0ea827fb7266491f2813635de6f17269be881f696fbfdf2d83dda33e", + ); + expect(extractDaytonaBaseImages(dockerfile)).toEqual([ + "rust:1.97-bookworm@sha256:408fe88047cef61a2087653b0c5255fa51c0f2d6d94ddedd7a2562a9b91a46f6", + "node:24-bookworm@sha256:9137a20e25879e0b557227b57e3ee4e9af4bde29eb3db66134cd1723e84f830b", + "daytonaio/sandbox:0.8.0@sha256:eadf88e4391072b7ad4bed27d9cadfc9fe9d8ed375d9219d34c2ccb518f213e3", + ]); + expect(dockerignore).toContain("**/node_modules"); + expect(dockerignore).toContain("packages/paperclip-runner/dist"); + expect(dockerignore).toContain("packages/paperclip-runner/runner/target"); + expect(workflow).toContain("--platform linux/amd64"); + expect(workflow).toContain( + "Compute Daytona image content ID with pinned bases", + ); + expect(workflow).toContain( + "e2e-content-${{ needs.catalog.outputs.daytona_image_content_id }}", + ); + expect(workflow).toContain( + '--build-arg "PAPERCLIP_RUNNER_CONTENT_ID=${IMAGE_CONTENT_ID}"', + ); + expect(workflow).not.toContain("e2e-git-${{ github.sha }}"); + expect(workflow).toContain("cosign sign --yes"); + expect(workflow).toContain("docker image inspect"); + expect(workflow).toContain('.Config.User == "daytona"'); + expect(workflow).toContain("PAPERCLIP_RUNNER_PROVIDER_PACK_ROOT="); + expect(workflow).toContain( + "pnpm --filter @paperclipai/paperclip-runner build:provider-pack", + ); + expect(workflow).toContain( + "PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH: ${{ github.workspace }}/packages/paperclip-runner/provider-pack", + ); + expect(workflow).toContain( + "PAPERCLIP_RUNNER_SOURCE_REVISION: ${{ needs.daytona_image.outputs.source_revision }}", + ); + expect(workflow).toContain("anonymous_config"); + }); + + it("hashes the audited image dependency closure rather than the repository revision", async () => { + for (const requiredPath of [ + ".dockerignore", + "docker/daytona-runner/Dockerfile", + "pnpm-lock.yaml", + "patches", + "packages/paperclip-eval-kernel", + "packages/paperclip-runner", + ]) { + expect(DAYTONA_IMAGE_INPUT_PATHS).toContain(requiredPath); + } + expect(DAYTONA_IMAGE_DOCKERFILE_PATH).toBe( + "docker/daytona-runner/Dockerfile", + ); + + const contentId = await computeDaytonaImageContentId(); + expect(contentId).toMatch(/^[0-9a-f]{64}$/); + }); + + it("changes only when an image input, frontend, base, or platform changes", async () => { + const root = await mkdtemp( + path.join(tmpdir(), "paperclip-daytona-image-id-"), + ); + try { + await mkdir(path.join(root, "image-input")); + await writeFile( + path.join(root, "image-input", "runner.ts"), + "version one\n", + ); + const baseline = await computeDaytonaImageContentId({ + repositoryRoot: root, + inputPaths: ["image-input"], + baseImages: [`example.test/base:1@sha256:${"a".repeat(64)}`], + frontendDigest: `sha256:${"c".repeat(64)}`, + }); + expect( + await computeDaytonaImageContentId({ + repositoryRoot: root, + inputPaths: ["image-input"], + baseImages: [`example.test/base:1@sha256:${"b".repeat(64)}`], + frontendDigest: `sha256:${"c".repeat(64)}`, + }), + ).not.toBe(baseline); + expect( + await computeDaytonaImageContentId({ + repositoryRoot: root, + inputPaths: ["image-input"], + baseImages: [`example.test/base:1@sha256:${"a".repeat(64)}`], + frontendDigest: `sha256:${"d".repeat(64)}`, + }), + ).not.toBe(baseline); + + await writeFile( + path.join(root, "unrelated.txt"), + "does not enter the image\n", + ); + expect( + await computeDaytonaImageContentId({ + repositoryRoot: root, + inputPaths: ["image-input"], + baseImages: [`example.test/base:1@sha256:${"a".repeat(64)}`], + frontendDigest: `sha256:${"c".repeat(64)}`, + }), + ).toBe(baseline); + + await writeFile( + path.join(root, "image-input", "runner.ts"), + "version two\n", + ); + expect( + await computeDaytonaImageContentId({ + repositoryRoot: root, + inputPaths: ["image-input"], + baseImages: [`example.test/base:1@sha256:${"a".repeat(64)}`], + frontendDigest: `sha256:${"c".repeat(64)}`, + }), + ).not.toBe(baseline); + expect( + await computeDaytonaImageContentId({ + repositoryRoot: root, + inputPaths: ["image-input"], + platform: "linux/arm64", + baseImages: [`example.test/base:1@sha256:${"a".repeat(64)}`], + frontendDigest: `sha256:${"c".repeat(64)}`, + }), + ).not.toBe(baseline); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + it("rejects mutable Docker base references", () => { + expect(() => extractDaytonaBaseImages("FROM node:24-bookworm\n")).toThrow( + "must use an immutable sha256 digest", + ); + }); + + it("rejects a mutable or missing Dockerfile syntax frontend", async () => { + expect(() => + extractDaytonaDockerfileFrontendDigest( + "# syntax=docker/dockerfile:1.7\nFROM scratch\n", + ), + ).toThrow("must pin its syntax frontend"); + expect(() => + extractDaytonaDockerfileFrontendDigest("FROM scratch\n"), + ).toThrow("must pin its syntax frontend"); + await expect( + computeDaytonaImageContentId({ + inputPaths: [], + baseImages: [`example.test/base:1@sha256:${"a".repeat(64)}`], + frontendDigest: "sha256:mutable", + }), + ).rejects.toThrow("must use an immutable sha256 digest"); + }); +}); diff --git a/tests/runner-e2e/evidence.ts b/tests/runner-e2e/evidence.ts new file mode 100644 index 0000000000..a6d304f01d --- /dev/null +++ b/tests/runner-e2e/evidence.ts @@ -0,0 +1,232 @@ +import { spawn } from "node:child_process"; +import { + mkdir, + readFile, + readdir, + rm, + writeFile, + copyFile, +} from "node:fs/promises"; +import path from "node:path"; +import { + assertSecretFree, + findSecretLeak, + findSecretLeakInJsonValues, + redactText, + sanitizeJson, +} from "./redaction.js"; + +const TEXT_EXTENSIONS = new Set([ + ".json", + ".xml", + ".html", + ".css", + ".js", + ".txt", + ".log", + ".md", +]); +const BINARY_EXTENSIONS = new Set([".png", ".webm"]); +const ACTIVE_CONTENT_EXTENSIONS = new Set([".svg"]); +const ALLOWED_DIRECTORIES = new Set([ + "snapshots", + "playwright-output", + "blob-report", + "html-report", +]); +const ALLOWED_ROOT_FILES = new Set([ + "result.json", + "final-state.png", + "failure.png", + "server.log", + "playwright.log", + "junit.xml", +]); +const REQUIRED_PASS_FILES = [ + "final-state.png", + "result.json", + "junit.xml", + path.join("html-report", "index.html"), + path.join("snapshots", "fixtures.json"), + path.join("snapshots", "api-state.json"), +] as const; + +export interface EvidencePackageResult { + files: string[]; + leaks: Array<{ file: string; reason: string }>; + missing: string[]; +} + +async function walk(root: string, relative = ""): Promise { + const directory = path.join(root, relative); + const entries = await readdir(directory, { withFileTypes: true }).catch( + () => [], + ); + const files: string[] = []; + for (const entry of entries) { + const next = path.join(relative, entry.name); + if (entry.isDirectory()) files.push(...(await walk(root, next))); + else if (entry.isFile()) files.push(next); + } + return files; +} + +function isAllowed(relative: string) { + const segments = relative.split(path.sep); + if (segments.length === 1) + return ( + ALLOWED_ROOT_FILES.has(relative) || + /^(?:plan|question)-[a-z0-9-]+\.png$/.test(relative) + ); + return ALLOWED_DIRECTORIES.has(segments[0]); +} + +async function inspectZip(source: string, secrets: readonly string[]) { + // Failure traces can exceed Node's child-process output buffer. Stream the + // expanded archive through the exact-value scanner with enough overlap to + // detect a credential split across stdout chunks, without retaining the + // archive in memory or weakening fail-closed evidence publication. + const overlap = Math.max( + 256, + ...secrets.map((secret) => Buffer.byteLength(secret, "utf8") + 16), + ); + return new Promise((resolve) => { + const unzip = spawn("unzip", ["-p", source], { + stdio: ["ignore", "pipe", "pipe"], + }); + let carry = Buffer.alloc(0); + let leak: string | null = null; + let spawnError: Error | null = null; + + unzip.stdout.on("data", (chunk: Buffer) => { + if (leak) return; + const data = Buffer.concat([carry, chunk]); + leak = findSecretLeak(data, secrets, { includeShapes: false }); + carry = data.subarray(Math.max(0, data.length - overlap)); + if (leak) unzip.kill(); + }); + // Drain diagnostics so a noisy unzip cannot block. Error text is withheld + // because archive paths and contents belong to private attempt evidence. + unzip.stderr.resume(); + unzip.on("error", (error) => { + spawnError = error; + }); + unzip.on("close", (code) => { + if (leak) return resolve(leak); + if (spawnError) { + return resolve(`zip could not be inspected: ${spawnError.message}`); + } + return resolve( + code === 0 + ? null + : `zip could not be inspected: unzip exited with code ${String(code)}`, + ); + }); + }); +} + +export async function packageEvidence(input: { + privateDir: string; + uploadDir: string; + secrets: readonly string[]; + expectPassScreenshot: boolean; +}): Promise { + await rm(input.uploadDir, { recursive: true, force: true }); + await mkdir(input.uploadDir, { recursive: true }); + const files: string[] = []; + const leaks: EvidencePackageResult["leaks"] = []; + const available = await walk(input.privateDir); + + for (const relative of available.filter(isAllowed)) { + const source = path.join(input.privateDir, relative); + const extension = path.extname(relative).toLowerCase(); + const destination = path.join(input.uploadDir, relative); + await mkdir(path.dirname(destination), { recursive: true }); + if (ACTIVE_CONTENT_EXTENSIONS.has(extension)) { + // SVG can execute script when opened directly from an artifact. Keep the + // source in the disposable private attempt directory, but never admit it + // to the sanitized CI artifact. + continue; + } else if (TEXT_EXTENSIONS.has(extension)) { + const raw = await readFile(source, "utf8"); + const parsed = extension === ".json" ? JSON.parse(raw) : null; + const leak = + extension === ".json" + ? findSecretLeakInJsonValues(parsed, input.secrets) + : findSecretLeak(raw, input.secrets); + if (leak) leaks.push({ file: relative, reason: leak }); + // Redacting an already serialized JSON string can change escape + // boundaries around shell commands. Sanitize parsed values instead so + // uploaded snapshots stay valid JSON. + const safe = + extension === ".json" + ? `${JSON.stringify(sanitizeJson(parsed, input.secrets), null, 2)}\n` + : redactText(raw, input.secrets); + if (extension === ".json") { + const safeLeak = findSecretLeakInJsonValues( + JSON.parse(safe), + input.secrets, + ); + if (safeLeak) + throw new Error(`Secret leak in ${relative}: ${safeLeak}`); + } else { + assertSecretFree(safe, input.secrets, relative); + } + await writeFile(destination, safe, "utf8"); + files.push(relative); + } else if (extension === ".zip") { + const leak = await inspectZip(source, input.secrets); + if (leak) { + leaks.push({ file: relative, reason: leak }); + continue; + } + await copyFile(source, destination); + files.push(relative); + } else if (BINARY_EXTENSIONS.has(extension)) { + // This raw-byte scan catches embedded plaintext credentials, but cannot + // inspect rendered pixels. Raster/video files are retained in the + // access-controlled CI artifact and stripped at the public-history + // boundary by history-publish.ts. + const raw = await readFile(source); + const leak = findSecretLeak(raw, input.secrets); + if (leak) { + leaks.push({ file: relative, reason: leak }); + continue; + } + await copyFile(source, destination); + files.push(relative); + } + } + + const missing = input.expectPassScreenshot + ? [ + ...REQUIRED_PASS_FILES.filter((required) => !files.includes(required)), + ...(files.some( + (file) => + file.startsWith(`blob-report${path.sep}`) && file.endsWith(".zip"), + ) + ? [] + : [path.join("blob-report", "*.zip")]), + ] + : []; + const manifest = { + schema: "paperclip.runner-e2e.evidence/v1", + files: [...files].sort(), + leaks, + missing, + }; + const manifestText = `${JSON.stringify(sanitizeJson(manifest, input.secrets), null, 2)}\n`; + const manifestLeak = findSecretLeakInJsonValues( + JSON.parse(manifestText), + input.secrets, + ); + if (manifestLeak) + throw new Error(`Secret leak in evidence-manifest.json: ${manifestLeak}`); + await writeFile( + path.join(input.uploadDir, "evidence-manifest.json"), + manifestText, + "utf8", + ); + files.push("evidence-manifest.json"); + return { files, leaks, missing }; +} diff --git a/tests/runner-e2e/failure-classifier.ts b/tests/runner-e2e/failure-classifier.ts new file mode 100644 index 0000000000..c70a9fa26b --- /dev/null +++ b/tests/runner-e2e/failure-classifier.ts @@ -0,0 +1,30 @@ +import type { FailureClass } from "./types.js"; + +const TRANSIENT = + /(?:\b429\b|\b5\d\d\b|rate.?limit|ECONN(?:RESET|REFUSED)|socket hang up|network (?:error|interruption|timeout)|service unavailable|(?:provider|server|bootstrap|browser|webserver|health|daytona|sandbox|ingress|preview|connection|harness).*(?:temporar|timed? out|timeout|closed|failed|unavailable|interrupt|reset|refused|create|start|connect)|(?:timed? out|timeout).*(?:provider|server|bootstrap|browser|webserver|health|daytona|sandbox|ingress|preview|connection|harness))/i; +const PERMANENT = + /(?:missing (?:credential|fixture secret)|invalid.*(?:credential|api key)|unauthorized|forbidden|qualification|model.*(?:unsupported|incompatible)|artifact.*incompatible|runner_remote_.*(?:incompatible|unavailable)|immutable image digest)/i; +const CANDIDATE = + /(?:matcher|expected.*observed|marker|issue status|run status|runtime mode|wrong output|missing output)/i; + +export function classifyFailure(error: unknown): FailureClass { + const message = + error instanceof Error ? `${error.name}: ${error.message}` : String(error); + if (/browser bootstrap failed before task creation/i.test(message)) + return "transient_infrastructure"; + if (/secret.*(?:leak|plaintext|redaction)/i.test(message)) + return "secret_leak"; + if (/cleanup|teardown|lease.*release/i.test(message)) { + return TRANSIENT.test(message) + ? "transient_infrastructure" + : "cleanup_failure"; + } + if (PERMANENT.test(message)) return "permanent_infrastructure"; + if (TRANSIENT.test(message)) return "transient_infrastructure"; + if (CANDIDATE.test(message)) return "candidate_failure"; + return "candidate_failure"; +} + +export function shouldRetryFailure(failureClass: FailureClass) { + return failureClass === "transient_infrastructure"; +} diff --git a/tests/runner-e2e/fixture-registry.ts b/tests/runner-e2e/fixture-registry.ts new file mode 100644 index 0000000000..b9b994fb56 --- /dev/null +++ b/tests/runner-e2e/fixture-registry.ts @@ -0,0 +1,74 @@ +export interface FixtureDefinition { + id: string; + dependencies?: readonly string[]; + setup(resolved: ReadonlyMap): Promise; + teardown?(value: T, resolved: ReadonlyMap): Promise; +} + +export class FixtureRegistry { + readonly #definitions = new Map(); + + register(definition: FixtureDefinition) { + if (this.#definitions.has(definition.id)) + throw new Error(`Duplicate fixture id: ${definition.id}`); + this.#definitions.set(definition.id, definition as FixtureDefinition); + return this; + } + + async setupAll() { + const resolved = new Map(); + const completed: FixtureDefinition[] = []; + const visiting = new Set(); + + const setup = async (id: string): Promise => { + if (resolved.has(id)) return; + if (visiting.has(id)) + throw new Error(`Fixture dependency cycle at ${id}`); + const definition = this.#definitions.get(id); + if (!definition) throw new Error(`Unknown fixture dependency: ${id}`); + visiting.add(id); + for (const dependency of definition.dependencies ?? []) + await setup(dependency); + visiting.delete(id); + const value = await definition.setup(resolved); + resolved.set(id, value); + completed.push(definition); + }; + + try { + for (const id of this.#definitions.keys()) await setup(id); + } catch (setupError) { + try { + await this.#teardown(completed, resolved); + } catch (teardownError) { + throw new AggregateError( + [setupError, teardownError], + `Fixture setup failed and cleanup failed: ${teardownError instanceof Error ? teardownError.message : String(teardownError)}`, + ); + } + throw setupError; + } + + return { + values: resolved, + teardown: () => this.#teardown(completed, resolved), + }; + } + + async #teardown( + completed: FixtureDefinition[], + resolved: Map, + ) { + const errors: Error[] = []; + for (const definition of [...completed].reverse()) { + if (!definition.teardown) continue; + try { + await definition.teardown(resolved.get(definition.id), resolved); + } catch (error) { + errors.push(error instanceof Error ? error : new Error(String(error))); + } + } + if (errors.length > 0) + throw new AggregateError(errors, "Fixture teardown failed"); + } +} diff --git a/tests/runner-e2e/harness-env.ts b/tests/runner-e2e/harness-env.ts new file mode 100644 index 0000000000..05e9722666 --- /dev/null +++ b/tests/runner-e2e/harness-env.ts @@ -0,0 +1,91 @@ +import { CREDENTIAL_NAMES } from "./types.js"; + +const DATABASE_KEYS = ["DATABASE_URL", "DATABASE_MIGRATION_URL"] as const; +const AMBIENT_PAPERCLIP_CREDENTIAL_KEYS = [ + "PAPERCLIP_API_KEY", + "PAPERCLIP_AGENT_API_KEY", + "PAPERCLIP_TASK_BRIDGE_TOKEN", + "PAPERCLIP_SETUP_TOKEN", + "PAPERCLIP_SECRETS_MASTER_KEY", + "PAPERCLIP_SECRETS_MASTER_KEY_FILE", +] as const; +const GENERATED_SERVER_SECRET_KEYS = [ + "PAPERCLIP_AGENT_JWT_SECRET", + "PAPERCLIP_DECISION_SIGNING_SECRET", + "PAPERCLIP_TOOL_ACTION_SIGNING_SECRET", + "BETTER_AUTH_SECRET", +] as const; +const AMBIENT_EXTERNAL_STATE_KEYS = [ + "PAPERCLIP_STORAGE_S3_BUCKET", + "PAPERCLIP_STORAGE_S3_REGION", + "PAPERCLIP_STORAGE_S3_ENDPOINT", + "PAPERCLIP_STORAGE_S3_PREFIX", + "PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE", +] as const; +const PROVIDER_SECRET_KEY = /^(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA)(?:_|$)/; + +/** + * Build the environment inherited by the Paperclip server. Paid credentials + * deliberately stay in the launcher/Playwright process and cross the server + * boundary only once, in the encrypted company-secrets API request. + */ +export function buildPaperclipServerEnvironment( + source: NodeJS.ProcessEnv, + overrides: NodeJS.ProcessEnv = {}, +): NodeJS.ProcessEnv { + const result = { ...source }; + for (const key of Object.keys(result)) { + if (PROVIDER_SECRET_KEY.test(key)) delete result[key]; + } + for (const key of [ + ...CREDENTIAL_NAMES, + ...DATABASE_KEYS, + ...AMBIENT_PAPERCLIP_CREDENTIAL_KEYS, + ...AMBIENT_EXTERNAL_STATE_KEYS, + ]) { + delete result[key]; + } + for (const key of GENERATED_SERVER_SECRET_KEYS) delete result[key]; + Object.assign(result, overrides); + return result; +} + +export function assertIsolatedServerEnvironment( + env: NodeJS.ProcessEnv, + expected: { + temporaryRoot: string; + paperclipHome: string; + configPath: string; + }, +) { + const home = env.PAPERCLIP_HOME; + const config = env.PAPERCLIP_CONFIG; + if (home !== expected.paperclipHome || config !== expected.configPath) { + throw new Error( + "Paperclip server environment does not use the allocated home/config paths", + ); + } + if ( + !home.startsWith(`${expected.temporaryRoot}/`) || + !config.startsWith(`${expected.temporaryRoot}/`) + ) { + throw new Error( + "Paperclip server paths escape the isolated temporary root", + ); + } + for (const key of [ + ...CREDENTIAL_NAMES, + ...DATABASE_KEYS, + ...AMBIENT_PAPERCLIP_CREDENTIAL_KEYS, + ...AMBIENT_EXTERNAL_STATE_KEYS, + ]) { + if (env[key]) + throw new Error( + `Paperclip server environment unexpectedly contains ${key}`, + ); + } + for (const key of GENERATED_SERVER_SECRET_KEYS) { + if (!env[key]) + throw new Error(`Paperclip server environment is missing ${key}`); + } +} diff --git a/tests/runner-e2e/history-index.ts b/tests/runner-e2e/history-index.ts new file mode 100644 index 0000000000..3f93ed5cfd --- /dev/null +++ b/tests/runner-e2e/history-index.ts @@ -0,0 +1,221 @@ +import type { + RunnerE2EHistoryCampaign, + RunnerE2EHistoryIndex, +} from "./types.js"; + +function html(value: unknown) { + return String(value ?? "") + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """) + .replaceAll("'", "'"); +} + +function number(value: number) { + return new Intl.NumberFormat("en-US").format(value); +} + +function usd(value: number) { + return `$${value.toFixed(value < 0.01 ? 6 : 2)}`; +} + +function duration(durationMs: number) { + const seconds = Math.round(durationMs / 1_000); + if (seconds < 60) return `${seconds}s`; + const minutes = Math.floor(seconds / 60); + if (minutes < 60) return `${minutes}m ${seconds % 60}s`; + return `${Math.floor(minutes / 60)}h ${minutes % 60}m`; +} + +function date(value: string) { + return new Intl.DateTimeFormat("en-US", { + dateStyle: "medium", + timeStyle: "short", + timeZone: "UTC", + }).format(new Date(value)); +} + +function campaignStatus(campaign: RunnerE2EHistoryCampaign) { + return campaign.failed === 0 && + campaign.passed === campaign.selected && + campaign.executed === campaign.selected && + campaign.cleanupPassed + ? "passed" + : "failed"; +} + +function sourceCell(campaign: RunnerE2EHistoryCampaign) { + const sha = campaign.source.sha; + const shortSha = sha?.slice(0, 8) ?? "Unknown"; + const shaLabel = /^[0-9a-f]{40}$/i.test(sha ?? "") + ? `${html(shortSha)}` + : html(shortSha); + const workflow = campaign.source.workflowRunUrl + ? `Workflow run` + : ""; + return `${shaLabel}${html(campaign.source.ref ?? "Unknown ref")}${workflow}`; +} + +function campaignRow(campaign: RunnerE2EHistoryCampaign) { + const status = campaignStatus(campaign); + const suites = campaign.suites + .map( + (suite) => + `${html(suite.suiteId)} · ${suite.passed}/${suite.selected}`, + ) + .join(""); + const billing = campaign.billing; + return ` + + ${html(campaign.campaignId)} + ${html(date(campaign.generatedAt))} UTC + + + ${status} + ${campaign.complete ? "Complete campaign" : "Partial campaign"}${campaign.retries > 0 ? ` · ${campaign.retries} retries` : ""} + + ${sourceCell(campaign)} +
${suites}
+ + ${campaign.passed}/${campaign.selected} passed + ${campaign.executed} executed · ${campaign.failed} failed + + + ${html(number(billing.llm.totalTokens))} + ${html(number(billing.llm.inputTokens))} in · ${html(number(billing.llm.outputTokens))} out · ${html(number(billing.llm.cachedInputTokens))} cached + + + ${html(usd(billing.observedAndEstimatedCostUsd))} + ${html(usd(billing.reportedLlmCostUsd))} LLM · ${html(usd(billing.estimatedRuntimeCostUsd))} runtime + + + ${html(duration(billing.agentRunDurationMs))} + ${html(duration(billing.leaseDurationMs))} Daytona lease + + Open report → + `; +} + +export function renderRunnerHistoryIndex(history: RunnerE2EHistoryIndex) { + const campaigns = [...history.campaigns].sort((left, right) => + right.generatedAt.localeCompare(left.generatedAt), + ); + const passed = campaigns.filter( + (campaign) => campaignStatus(campaign) === "passed", + ).length; + const latest = campaigns.find( + (campaign) => campaign.campaignId === history.latestCampaignId, + ); + const latestGreen = campaigns.find( + (campaign) => campaign.campaignId === history.latestGreenCampaignId, + ); + const totalCost = campaigns.reduce( + (sum, campaign) => sum + campaign.billing.observedAndEstimatedCostUsd, + 0, + ); + const rows = + campaigns.length > 0 + ? campaigns.map(campaignRow).join("") + : `No campaigns have been published yet.`; + + return ` + + + + + + + + + Runner E2E Campaigns · Paperclip + + + +
+ + + Paperclip + + Quality engineering · Runner acceptance +
+
+
+
+

Historical test reporting

+

Runner E2E campaigns

+

Each row is one workflow campaign against a Paperclip revision. Open a report for its configuration matrices, matchers, per-test billing, and sanitized structured evidence. Visual evidence remains in access-controlled workflow artifacts.

+
+
+
${campaigns.length}Campaigns
+
${passed}Passed
+
${html(usd(totalCost))}Recorded cost
+
+
+ +
+ + + ${rows} +
CampaignStatusSourceSuitesTestsTokensCostAgent time
+
+
Updated ${html(date(history.updatedAt))} UTCImmutable campaign reports · Inert structured public evidence
+
+ +`; +} diff --git a/tests/runner-e2e/history-publish.ts b/tests/runner-e2e/history-publish.ts new file mode 100644 index 0000000000..2e404ab136 --- /dev/null +++ b/tests/runner-e2e/history-publish.ts @@ -0,0 +1,499 @@ +import { createHash } from "node:crypto"; +import { execFile } from "node:child_process"; +import path from "node:path"; +import { promisify } from "node:util"; +import { + mkdtemp, + lstat, + readFile, + readdir, + rm, + stat, + writeFile, +} from "node:fs/promises"; +import os from "node:os"; +import { regenerateRunnerDashboard } from "./dashboard-regenerate.js"; +import { renderRunnerHistoryIndex } from "./history-index.js"; +import { + campaignHistoryRecord, + emptyRunnerHistory, + mergeRunnerHistory, +} from "./history.js"; +import type { RunnerE2ECampaign, RunnerE2EHistoryIndex } from "./types.js"; + +const execFileAsync = promisify(execFile); +const MUTABLE_HISTORY_FILES = new Set([ + "history.json", + "latest.json", + "latest-green.json", +]); +const PUBLISH_ROOT_FILES = new Set([ + "dashboard.html", + "index.html", + "junit.xml", + "normalized-results.json", + "summary.md", +]); +const PUBLIC_EVIDENCE_EXTENSIONS = new Set([".json", ".log", ".md", ".txt"]); +const PRIVATE_EVIDENCE_DIRECTORIES = new Set([ + "blob-report", + "html-report", + "playwright-output", +]); + +function publicEvidencePath(relative: string) { + const match = relative.match( + /^evidence\/[A-Za-z0-9._-]+\/attempt-[1-9][0-9]*\/(.+)$/, + ); + if (!match) return false; + const evidencePath = match[1]!; + const segments = evidencePath.split("/"); + if ( + segments.some( + (segment) => + !segment || + segment === "." || + segment === ".." || + PRIVATE_EVIDENCE_DIRECTORIES.has(segment), + ) + ) { + return false; + } + return PUBLIC_EVIDENCE_EXTENSIONS.has( + path.posix.extname(evidencePath).toLowerCase(), + ); +} + +export function isHistoricalBundlePathAllowed(relative: string) { + if ( + relative.includes("\\") || + relative.startsWith("/") || + relative.includes("..") + ) { + return false; + } + if (PUBLISH_ROOT_FILES.has(relative)) return true; + if ( + relative === "assets/favicon.svg" || + relative === "assets/InterVariable.woff2" + ) { + return true; + } + return publicEvidencePath(relative); +} + +async function pruneEvidenceDirectory(root: string, current: string) { + const entries = await readdir(current, { withFileTypes: true }); + for (const entry of entries) { + const absolute = path.join(current, entry.name); + if (entry.isDirectory()) { + await pruneEvidenceDirectory(root, absolute); + if ((await readdir(absolute)).length === 0) { + await rm(absolute, { recursive: true }); + } + continue; + } + const relative = path.relative(root, absolute).split(path.sep).join("/"); + if (!entry.isFile() || !publicEvidencePath(relative)) { + await rm(absolute, { force: true }); + } + } +} + +export async function prunePrivateHistoryEvidence(root: string) { + const evidenceRoot = path.join(root, "evidence"); + const metadata = await lstat(evidenceRoot).catch(() => null); + if (!metadata) return; + if (!metadata.isDirectory()) { + throw new Error("Historical evidence root must be a directory"); + } + await pruneEvidenceDirectory(root, evidenceRoot); +} + +interface BundleManifest { + schema: "paperclip.runner-e2e.bundle/v1"; + campaignId: string; + bundleDigest: string; + files: Array<{ path: string; sha256: string; bytes: number }>; +} + +function json(value: unknown) { + return `${JSON.stringify(value, null, 2)}\n`; +} + +export function validateHistoryDestination(input: { + bucket: string; + prefix: string; + publicBaseUrl: string; +}) { + if (!/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(input.bucket)) { + throw new Error("RUNNER_E2E_HISTORY_S3_BUCKET is not a valid bucket name"); + } + const prefix = input.prefix.replace(/^\/+|\/+$/g, ""); + if ( + !prefix || + prefix + .split("/") + .some((segment) => !segment || segment === "." || segment === "..") + ) { + throw new Error( + "RUNNER_E2E_HISTORY_PREFIX must be a safe non-empty key prefix", + ); + } + const publicUrl = new URL(input.publicBaseUrl); + if ( + publicUrl.protocol !== "https:" || + publicUrl.username || + publicUrl.password || + publicUrl.search || + publicUrl.hash + ) { + throw new Error( + "RUNNER_E2E_HISTORY_PUBLIC_BASE_URL must be a credential-free HTTPS URL", + ); + } + return { prefix, publicBaseUrl: publicUrl.href.replace(/\/$/, "") }; +} + +async function relativeFiles(root: string, current = root): Promise { + const entries = await readdir(current, { withFileTypes: true }); + const files: string[] = []; + for (const entry of entries) { + const absolute = path.join(current, entry.name); + if (entry.isSymbolicLink()) { + throw new Error(`Refusing to publish symbolic link ${entry.name}`); + } + if (entry.isDirectory()) { + files.push(...(await relativeFiles(root, absolute))); + } else if (entry.isFile()) { + const relative = path.relative(root, absolute).split(path.sep).join("/"); + if (MUTABLE_HISTORY_FILES.has(relative)) continue; + if (!isHistoricalBundlePathAllowed(relative)) { + throw new Error( + `Refusing non-allowlisted historical bundle path ${relative}`, + ); + } + files.push(relative); + } + } + return files; +} + +export async function createBundleManifest( + root: string, + campaignId: string, +): Promise { + if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$/.test(campaignId)) { + throw new Error("Campaign ID is unsafe for immutable object storage"); + } + const files = await Promise.all( + (await relativeFiles(root)).sort().map(async (relative) => { + const absolute = path.join(root, ...relative.split("/")); + const [content, metadata] = await Promise.all([ + readFile(absolute), + stat(absolute), + ]); + return { + path: relative, + sha256: createHash("sha256").update(content).digest("hex"), + bytes: metadata.size, + }; + }), + ); + const bundleDigest = createHash("sha256") + .update(JSON.stringify(files)) + .digest("hex"); + return { + schema: "paperclip.runner-e2e.bundle/v1", + campaignId, + bundleDigest, + files, + }; +} + +export function buildHistoryPointers(history: RunnerE2EHistoryIndex) { + const byCampaign = new Map( + history.campaigns.map((campaign) => [campaign.campaignId, campaign]), + ); + const pointer = (campaignId: string | null | undefined) => { + const campaign = campaignId ? byCampaign.get(campaignId) : undefined; + return campaign + ? { + campaignId: campaign.campaignId, + generatedAt: campaign.generatedAt, + publicUrl: campaign.publicUrl, + sha: campaign.source.sha, + } + : null; + }; + return { + latest: { + schema: "paperclip.runner-e2e.pointer/v1", + updatedAt: history.updatedAt, + overall: pointer(history.latestCampaignId), + suites: Object.fromEntries( + Object.entries(history.latestBySuite).map(([suiteId, campaignId]) => [ + suiteId, + pointer(campaignId), + ]), + ), + }, + latestGreen: { + schema: "paperclip.runner-e2e.pointer/v1", + updatedAt: history.updatedAt, + overall: pointer(history.latestGreenCampaignId), + suites: Object.fromEntries( + Object.entries(history.latestGreenBySuite).map( + ([suiteId, campaignId]) => [suiteId, pointer(campaignId)], + ), + ), + }, + }; +} + +function awsObject(bucket: string, key: string) { + return `s3://${bucket}/${key}`; +} + +async function objectExists(bucket: string, key: string) { + try { + await execFileAsync("aws", [ + "s3api", + "head-object", + "--bucket", + bucket, + "--key", + key, + ]); + return true; + } catch (error) { + const detail = String( + (error as { stderr?: string }).stderr ?? + (error instanceof Error ? error.message : error), + ); + if (/\b(?:404|Not Found|NoSuchKey)\b/i.test(detail)) return false; + throw new Error( + `Unable to inspect historical object: ${detail.slice(0, 400)}`, + ); + } +} + +async function downloadJson( + bucket: string, + key: string, + destination: string, +) { + if (!(await objectExists(bucket, key))) return null; + await execFileAsync("aws", [ + "s3", + "cp", + awsObject(bucket, key), + destination, + "--only-show-errors", + ]); + return JSON.parse(await readFile(destination, "utf8")) as T; +} + +async function uploadJson( + bucket: string, + key: string, + file: string, + cacheControl: string, +) { + await uploadFile(bucket, key, file, "application/json", cacheControl); +} + +async function uploadFile( + bucket: string, + key: string, + file: string, + contentType: string, + cacheControl: string, +) { + await execFileAsync("aws", [ + "s3", + "cp", + file, + awsObject(bucket, key), + "--only-show-errors", + "--content-type", + contentType, + "--cache-control", + cacheControl, + ]); +} + +async function main() { + const reportRoot = path.resolve( + process.env.PAPERCLIP_RUNNER_E2E_REPORT_DIR ?? + "runner-e2e-merged-report/normalized", + ); + const bucket = process.env.RUNNER_E2E_HISTORY_S3_BUCKET ?? ""; + const destination = validateHistoryDestination({ + bucket, + prefix: process.env.RUNNER_E2E_HISTORY_PREFIX ?? "runner-e2e", + publicBaseUrl: process.env.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL ?? "", + }); + const campaign = JSON.parse( + await readFile(path.join(reportRoot, "normalized-results.json"), "utf8"), + ) as RunnerE2ECampaign; + if (campaign.schema !== "paperclip.runner-e2e.campaign/v2") { + throw new Error("Historical publishing requires a v2 normalized campaign"); + } + + const temporary = await mkdtemp( + path.join(os.tmpdir(), "runner-e2e-history-"), + ); + const historyKey = `${destination.prefix}/history.json`; + const current = + (await downloadJson( + bucket, + historyKey, + path.join(temporary, "current-history.json"), + )) ?? emptyRunnerHistory(); + const history = mergeRunnerHistory( + current, + campaignHistoryRecord( + campaign, + `${destination.publicBaseUrl}/${destination.prefix}`, + ), + ); + + // Campaign bundles are immutable and must not capture a mutable history + // file left in a reused local directory. The root landing page below is the + // only dashboard that embeds navigation across campaigns. + // Raster/video pixels are not OCR-scanned for secrets, and generated HTML, + // archives, and SVG may contain or execute active/private content. Preserve + // those in the access-controlled workflow artifact but remove them from the + // directory shared by public S3 and Pages publication. + await prunePrivateHistoryEvidence(reportRoot); + await regenerateRunnerDashboard({ bundle: reportRoot, historyFile: null }); + const manifest = await createBundleManifest(reportRoot, campaign.campaignId); + const campaignPrefix = `${destination.prefix}/campaigns/${campaign.campaignId}`; + const manifestKey = `${campaignPrefix}/bundle-manifest.json`; + const existingManifest = await downloadJson( + bucket, + manifestKey, + path.join(temporary, "existing-manifest.json"), + ); + if ( + existingManifest && + existingManifest.bundleDigest !== manifest.bundleDigest + ) { + throw new Error( + `Immutable campaign ${campaign.campaignId} already exists with a different digest`, + ); + } + if (!existingManifest) { + await execFileAsync("aws", [ + "s3", + "cp", + reportRoot, + awsObject(bucket, campaignPrefix), + "--recursive", + "--only-show-errors", + "--exclude", + "history.json", + "--exclude", + "latest.json", + "--exclude", + "latest-green.json", + "--cache-control", + "public,max-age=31536000,immutable", + ]); + const manifestFile = path.join(temporary, "bundle-manifest.json"); + await writeFile(manifestFile, json(manifest), "utf8"); + await uploadJson( + bucket, + manifestKey, + manifestFile, + "public,max-age=31536000,immutable", + ); + } + + const pointers = buildHistoryPointers(history); + const historyFile = path.join(reportRoot, "history.json"); + const latestFile = path.join(reportRoot, "latest.json"); + const latestGreenFile = path.join(reportRoot, "latest-green.json"); + await Promise.all([ + writeFile(historyFile, json(history), "utf8"), + writeFile(latestFile, json(pointers.latest), "utf8"), + writeFile(latestGreenFile, json(pointers.latestGreen), "utf8"), + ]); + await regenerateRunnerDashboard({ bundle: reportRoot, historyFile }); + const landingDirectory = path.join(temporary, "landing"); + await regenerateRunnerDashboard({ + bundle: reportRoot, + historyFile, + outputDirectory: landingDirectory, + evidenceHrefPrefix: `campaigns/${campaign.campaignId}`, + }); + await writeFile( + path.join(landingDirectory, "index.html"), + renderRunnerHistoryIndex(history), + "utf8", + ); + await Promise.all([ + uploadJson(bucket, historyKey, historyFile, "no-cache"), + uploadJson( + bucket, + `${destination.prefix}/latest.json`, + latestFile, + "no-cache", + ), + uploadJson( + bucket, + `${destination.prefix}/latest-green.json`, + latestGreenFile, + "no-cache", + ), + uploadFile( + bucket, + `${destination.prefix}/index.html`, + path.join(landingDirectory, "index.html"), + "text/html; charset=utf-8", + "no-cache", + ), + uploadFile( + bucket, + `${destination.prefix}/dashboard.html`, + path.join(landingDirectory, "dashboard.html"), + "text/html; charset=utf-8", + "no-cache", + ), + uploadFile( + bucket, + `${destination.prefix}/normalized-results.json`, + path.join(landingDirectory, "normalized-results.json"), + "application/json", + "no-cache", + ), + uploadFile( + bucket, + `${destination.prefix}/assets/favicon.svg`, + path.join(reportRoot, "assets", "favicon.svg"), + "image/svg+xml", + "public,max-age=86400", + ), + uploadFile( + bucket, + `${destination.prefix}/assets/InterVariable.woff2`, + path.join(reportRoot, "assets", "InterVariable.woff2"), + "font/woff2", + "public,max-age=86400", + ), + ]); + console.log( + `Published immutable campaign ${campaign.campaignId} (${manifest.bundleDigest}) and ${history.campaigns.length} history record(s)`, + ); +} + +if ( + process.argv[1] && + path.resolve(process.argv[1]) === path.resolve(import.meta.filename) +) { + await main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/tests/runner-e2e/history.test.ts b/tests/runner-e2e/history.test.ts new file mode 100644 index 0000000000..7fc5116184 --- /dev/null +++ b/tests/runner-e2e/history.test.ts @@ -0,0 +1,363 @@ +import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { runnerMatrix } from "./catalog.js"; +import { regenerateRunnerDashboard } from "./dashboard-regenerate.js"; +import { renderRunnerE2EDashboard } from "./dashboard.js"; +import { + buildHistoryPointers, + createBundleManifest, + isHistoricalBundlePathAllowed, + prunePrivateHistoryEvidence, + validateHistoryDestination, +} from "./history-publish.js"; +import { + buildRunnerCampaign, + campaignHistoryRecord, + canonicalExecutionId, + emptyRunnerHistory, + mergeRunnerHistory, +} from "./history.js"; +import { renderRunnerHistoryIndex } from "./history-index.js"; +import type { MatrixExecution, RunnerE2EResult } from "./types.js"; + +const temporaryDirectories: string[] = []; +afterEach(async () => { + await Promise.all( + temporaryDirectories + .splice(0) + .map((directory) => rm(directory, { recursive: true, force: true })), + ); +}); + +function result(execution: MatrixExecution, status: "passed" | "failed") { + return { + schema: "paperclip.runner-e2e.result/v2", + executionId: execution.id, + suiteId: execution.suite.id, + suiteDefinitionHash: execution.suiteDefinitionHash, + attempt: 1, + status, + profileId: execution.profile.id, + environmentId: execution.environment.id, + caseId: execution.task.id, + provider: execution.profile.provider, + model: execution.profile.model, + runtimeMode: execution.profile.expectedRuntimeMode, + runIds: ["run-1"], + usage: { inputTokens: 100, outputTokens: 25, costUsd: 0.01 }, + startedAt: "2026-08-28T00:00:00.000Z", + finishedAt: "2026-08-28T00:00:01.000Z", + durationMs: 1_000, + cleanup: "passed", + } satisfies RunnerE2EResult; +} + +describe("runner E2E campaign history", () => { + it("migrates v1 execution IDs and keeps partial suite runs out of overall trends", () => { + expect(canonicalExecutionId("legacy-codex.local.message-marker")).toBe( + "core-compatibility.legacy-codex.local.message-marker", + ); + const breadth = runnerMatrix.filter( + (execution) => execution.suite.id === "openrouter-model-breadth", + ); + const campaign = buildRunnerCampaign({ + campaignId: "breadth-smoke", + generatedAt: "2026-08-28T00:01:00.000Z", + expected: breadth.map((execution) => execution.id), + results: breadth.map((execution) => result(execution, "passed")), + }); + expect(campaign).toMatchObject({ complete: false, passed: 15, failed: 0 }); + expect(campaign.suites[0]).toMatchObject({ + suiteId: "openrouter-model-breadth", + complete: true, + selected: 15, + }); + expect(campaign.billing).toMatchObject({ + reportedLlmCostUsd: 0.15, + llm: { inputTokens: 1_500, outputTokens: 375 }, + }); + const history = mergeRunnerHistory( + emptyRunnerHistory(), + campaignHistoryRecord(campaign, "https://history.example/runner-e2e"), + ); + expect(history.latestGreenCampaignId).toBeNull(); + expect(history.latestGreenBySuite).toEqual({ + "openrouter-model-breadth": "breadth-smoke", + }); + }); + + it("retains latest and latest-green pointers independently", () => { + const green = buildRunnerCampaign({ + campaignId: "complete-green", + generatedAt: "2026-08-28T00:01:00.000Z", + expected: runnerMatrix.map((execution) => execution.id), + results: runnerMatrix.map((execution) => result(execution, "passed")), + }); + const red = buildRunnerCampaign({ + campaignId: "complete-red", + generatedAt: "2026-08-28T01:01:00.000Z", + expected: runnerMatrix.map((execution) => execution.id), + results: runnerMatrix.map((execution, index) => + result(execution, index === 0 ? "failed" : "passed"), + ), + }); + let history = mergeRunnerHistory( + emptyRunnerHistory(), + campaignHistoryRecord(green, "https://history.example/runner-e2e"), + ); + history = mergeRunnerHistory( + history, + campaignHistoryRecord(red, "https://history.example/runner-e2e"), + ); + const pointers = buildHistoryPointers(history); + expect(pointers.latest.overall).toMatchObject({ + campaignId: "complete-red", + }); + expect(pointers.latestGreen.overall).toMatchObject({ + campaignId: "complete-green", + }); + expect(history.campaigns).toHaveLength(2); + const dashboard = renderRunnerE2EDashboard({ + title: "Runner Full-Stack E2E", + generatedAt: red.generatedAt, + expected: red.expected, + catalog: runnerMatrix, + campaign: red, + history, + entries: red.results.map((campaignResult) => ({ + result: campaignResult, + valid: campaignResult.status === "passed", + errors: campaignResult.status === "passed" ? [] : ["failed"], + })), + }); + expect(dashboard).toContain("Campaign trends"); + expect(dashboard).toContain("data-history-from"); + expect(dashboard).toContain("data-history-through"); + expect(dashboard).toContain( + 'data-history-suite-trends="core-compatibility"', + ); + expect(dashboard).toContain( + 'data-history-suite-trends="openrouter-model-breadth"', + ); + expect(dashboard).toContain("Suite pass rate"); + expect(dashboard).toContain("lines break at definition changes"); + expect(dashboard).toContain("cleanup passed"); + const index = renderRunnerHistoryIndex(history); + expect(index).toContain("Runner E2E campaigns"); + expect(index).toContain("complete-green"); + expect(index).toContain("complete-red"); + expect(index).toContain("57/57 passed"); + expect(index).toContain("56/57 passed"); + expect(index).toContain("Open report →"); + expect(index).toContain( + "Visual evidence remains in access-controlled workflow artifacts", + ); + expect(index).toContain("Inert structured public evidence"); + expect(index).not.toContain("data-gallery-dialog"); + expect(index).not.toContain("Configuration matrix"); + }); +}); + +describe("historical publication security", () => { + it("keeps visual and active evidence private when building the public dashboard", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "runner-landing-test-")); + const output = path.join(root, "landing"); + temporaryDirectories.push(root); + const execution = runnerMatrix[0]!; + const campaignResult = { + ...result(execution, "passed"), + screenshots: [ + { + id: "final-state", + label: "Final state", + file: "final-state.png", + }, + ], + } satisfies RunnerE2EResult; + const campaign = buildRunnerCampaign({ + campaignId: "campaign-1", + generatedAt: "2026-08-28T00:01:00.000Z", + expected: [execution.id], + results: [campaignResult], + }); + const evidenceDirectory = path.join( + root, + "evidence", + execution.id, + "attempt-1", + ); + await mkdir(evidenceDirectory, { recursive: true }); + await writeFile( + path.join(root, "normalized-results.json"), + JSON.stringify(campaign), + ); + await writeFile(path.join(evidenceDirectory, "final-state.png"), "png"); + await writeFile(path.join(evidenceDirectory, "failure.webm"), "webm"); + await writeFile(path.join(evidenceDirectory, "unsafe.svg"), ""); + await writeFile( + path.join(evidenceDirectory, "junit.xml"), + "", + ); + await writeFile(path.join(evidenceDirectory, "result.json"), "{}\n"); + await mkdir(path.join(evidenceDirectory, "snapshots")); + await writeFile( + path.join(evidenceDirectory, "snapshots", "api-state.json"), + "{}\n", + ); + await mkdir(path.join(evidenceDirectory, "html-report")); + await writeFile( + path.join(evidenceDirectory, "html-report", "index.html"), + "", + ); + await mkdir(path.join(evidenceDirectory, "blob-report")); + await writeFile( + path.join(evidenceDirectory, "blob-report", "report.zip"), + "private archive", + ); + + await prunePrivateHistoryEvidence(root); + await regenerateRunnerDashboard({ + bundle: root, + outputDirectory: output, + evidenceHrefPrefix: "campaigns/campaign-1", + }); + const dashboard = await readFile(path.join(output, "index.html"), "utf8"); + expect(dashboard).not.toContain( + `campaigns/campaign-1/evidence/${execution.id}/attempt-1/final-state.png`, + ); + expect(dashboard).toContain("Visual evidence · workflow artifact only"); + expect(dashboard).toContain( + "public history contains inert structured evidence only", + ); + expect(dashboard).toContain("Public history excludes visual evidence"); + await expect( + readFile(path.join(evidenceDirectory, "final-state.png")), + ).rejects.toThrow(); + await expect( + readFile(path.join(evidenceDirectory, "failure.webm")), + ).rejects.toThrow(); + await expect( + readFile(path.join(evidenceDirectory, "unsafe.svg")), + ).rejects.toThrow(); + await expect( + readFile(path.join(evidenceDirectory, "junit.xml")), + ).rejects.toThrow(); + await expect( + readFile(path.join(evidenceDirectory, "html-report", "index.html")), + ).rejects.toThrow(); + await expect( + readFile(path.join(evidenceDirectory, "blob-report", "report.zip")), + ).rejects.toThrow(); + await expect( + readFile(path.join(evidenceDirectory, "result.json"), "utf8"), + ).resolves.toBe("{}\n"); + await expect( + readFile( + path.join(evidenceDirectory, "snapshots", "api-state.json"), + "utf8", + ), + ).resolves.toBe("{}\n"); + expect( + JSON.parse( + await readFile(path.join(output, "normalized-results.json"), "utf8"), + ).schema, + ).toBe("paperclip.runner-e2e.campaign/v2"); + await expect( + regenerateRunnerDashboard({ + bundle: root, + outputDirectory: output, + evidenceHrefPrefix: "../unsafe", + }), + ).rejects.toThrow("safe relative URL path"); + }); + + it("requires a private-origin-compatible destination shape", () => { + expect( + validateHistoryDestination({ + bucket: "paperclip-runner-e2e-history", + prefix: "/runner-e2e/", + publicBaseUrl: "https://history.paperclip.ai/", + }), + ).toEqual({ + prefix: "runner-e2e", + publicBaseUrl: "https://history.paperclip.ai", + }); + expect(() => + validateHistoryDestination({ + bucket: "paperclip-runner-e2e-history", + prefix: "../unsafe", + publicBaseUrl: "https://history.paperclip.ai/", + }), + ).toThrow("safe non-empty key prefix"); + expect(() => + validateHistoryDestination({ + bucket: "paperclip-runner-e2e-history", + prefix: "runner-e2e", + publicBaseUrl: "http://history.paperclip.ai/", + }), + ).toThrow("credential-free HTTPS"); + }); + + it("rejects non-allowlisted files and fingerprints an immutable bundle", async () => { + expect(isHistoricalBundlePathAllowed("normalized-results.json")).toBe(true); + expect(isHistoricalBundlePathAllowed("junit.xml")).toBe(true); + expect( + isHistoricalBundlePathAllowed( + "evidence/core-compatibility.profile.local.case/attempt-1/final-state.png", + ), + ).toBe(false); + expect( + isHistoricalBundlePathAllowed( + "evidence/core-compatibility.profile.local.case/attempt-1/failure.webm", + ), + ).toBe(false); + expect( + isHistoricalBundlePathAllowed( + "evidence/core-compatibility.profile.local.case/attempt-1/unsafe.svg", + ), + ).toBe(false); + expect( + isHistoricalBundlePathAllowed( + "evidence/core-compatibility.profile.local.case/attempt-1/junit.xml", + ), + ).toBe(false); + expect( + isHistoricalBundlePathAllowed( + "evidence/core-compatibility.profile.local.case/attempt-1/blob-report/report.zip", + ), + ).toBe(false); + expect( + isHistoricalBundlePathAllowed( + "evidence/core-compatibility.profile.local.case/attempt-1/html-report/index.html", + ), + ).toBe(false); + expect( + isHistoricalBundlePathAllowed( + "evidence/core-compatibility.profile.local.case/attempt-1/result.json", + ), + ).toBe(true); + expect( + isHistoricalBundlePathAllowed( + "evidence/core-compatibility.profile.local.case/attempt-1/snapshots/api-state.json", + ), + ).toBe(true); + expect(isHistoricalBundlePathAllowed("paperclip-home/database")).toBe( + false, + ); + + const root = await mkdtemp(path.join(os.tmpdir(), "runner-history-test-")); + temporaryDirectories.push(root); + await mkdir(path.join(root, "assets")); + await writeFile(path.join(root, "index.html"), "safe"); + await writeFile(path.join(root, "assets", "favicon.svg"), "safe"); + const first = await createBundleManifest(root, "campaign-1"); + const second = await createBundleManifest(root, "campaign-1"); + expect(first.bundleDigest).toBe(second.bundleDigest); + await writeFile(path.join(root, "database.sqlite"), "unsafe"); + await expect(createBundleManifest(root, "campaign-1")).rejects.toThrow( + "non-allowlisted", + ); + }); +}); diff --git a/tests/runner-e2e/history.ts b/tests/runner-e2e/history.ts new file mode 100644 index 0000000000..9101836ca7 --- /dev/null +++ b/tests/runner-e2e/history.ts @@ -0,0 +1,245 @@ +import { runnerMatrix, runnerSuites } from "./catalog.js"; +import { + aggregateCampaignBilling, + summarizeExecutionBilling, +} from "./billing.js"; +import type { + RunnerE2ECampaign, + RunnerE2EHistoryCampaign, + RunnerE2EHistoryIndex, + RunnerE2EResult, +} from "./types.js"; + +export function canonicalExecutionId(id: string) { + if (runnerMatrix.some((execution) => execution.id === id)) return id; + const coreId = `core-compatibility.${id}`; + return runnerMatrix.some((execution) => execution.id === coreId) + ? coreId + : id; +} + +export function upgradeRunnerResult(result: RunnerE2EResult): RunnerE2EResult { + const executionId = canonicalExecutionId(result.executionId); + const execution = runnerMatrix.find( + (candidate) => candidate.id === executionId, + ); + if (!execution) return result; + return { + ...result, + executionId, + suiteId: result.suiteId ?? execution.suite.id, + suiteDefinitionHash: + result.suiteDefinitionHash ?? execution.suiteDefinitionHash, + ...(result.schema === "paperclip.runner-e2e.result/v1" + ? { + source: result.source ?? { + sha: null, + ref: null, + workflowRunUrl: null, + }, + } + : {}), + }; +} + +export function buildRunnerCampaign(input: { + campaignId: string; + generatedAt: string; + expected: readonly string[]; + results: readonly RunnerE2EResult[]; + eventName?: string | null; +}): RunnerE2ECampaign { + const expected = input.expected.map(canonicalExecutionId); + const results = input.results.map((result) => ({ + ...upgradeRunnerResult(result), + billing: result.billing ?? summarizeExecutionBilling(result), + })); + const resultSource = results.find((result) => result.source)?.source; + const source = { + sha: resultSource?.sha ?? process.env.GITHUB_SHA ?? null, + ref: resultSource?.ref ?? process.env.GITHUB_REF ?? null, + workflowRunUrl: + resultSource?.workflowRunUrl ?? + (process.env.GITHUB_SERVER_URL && + process.env.GITHUB_REPOSITORY && + process.env.GITHUB_RUN_ID + ? `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}` + : null), + eventName: input.eventName ?? process.env.GITHUB_EVENT_NAME ?? null, + }; + const suites = runnerSuites + .map((suite) => { + const suiteExpected = expected.filter((id) => + id.startsWith(`${suite.id}.`), + ); + if (suiteExpected.length === 0) return null; + const suiteResults = results.filter( + (result) => result.suiteId === suite.id, + ); + const passed = suiteResults.filter( + (result) => result.status === "passed" && result.cleanup === "passed", + ).length; + const executed = suiteResults.filter( + (result) => result.attempt > 0, + ).length; + return { + suiteId: suite.id, + suiteDefinitionHash: + suiteResults[0]?.suiteDefinitionHash ?? + runnerMatrix.find((execution) => execution.suite.id === suite.id)! + .suiteDefinitionHash, + expected: suite.expectedMatrixSize, + selected: suiteExpected.length, + executed, + passed, + failed: suiteExpected.length - passed, + retries: suiteResults.reduce( + (total, result) => total + Math.max(0, result.attempt - 1), + 0, + ), + cleanupPassed: suiteResults.every( + (result) => result.cleanup === "passed", + ), + complete: suiteExpected.length === suite.expectedMatrixSize, + durationMs: suiteResults.reduce( + (total, result) => total + result.durationMs, + 0, + ), + billing: aggregateCampaignBilling(suiteResults), + }; + }) + .filter((suite): suite is NonNullable => Boolean(suite)); + const passed = results.filter( + (result) => result.status === "passed" && result.cleanup === "passed", + ).length; + const rankingSnapshots = [ + ...new Map( + results.flatMap((result) => + result.rankingSnapshot + ? [ + [ + result.rankingSnapshot.snapshotId, + { + snapshotId: result.rankingSnapshot.snapshotId, + capturedAt: result.rankingSnapshot.capturedAt, + sourceUrl: result.rankingSnapshot.sourceUrl, + }, + ] as const, + ] + : [], + ), + ).values(), + ]; + return { + schema: "paperclip.runner-e2e.campaign/v2", + campaignId: input.campaignId, + generatedAt: input.generatedAt, + source, + expected, + complete: + suites.length === runnerSuites.length && + suites.every((suite) => suite.complete), + selected: expected.length, + executed: results.filter((result) => result.attempt > 0).length, + passed, + failed: expected.length - passed, + retries: results.reduce( + (total, result) => total + Math.max(0, result.attempt - 1), + 0, + ), + cleanupPassed: results.every((result) => result.cleanup === "passed"), + rankingSnapshots, + billing: aggregateCampaignBilling(results), + suites, + results, + }; +} + +export function campaignHistoryRecord( + campaign: RunnerE2ECampaign, + publicBaseUrl: string, +): RunnerE2EHistoryCampaign { + const publicUrl = `${publicBaseUrl.replace(/\/$/, "")}/campaigns/${encodeURIComponent(campaign.campaignId)}/`; + return { + campaignId: campaign.campaignId, + generatedAt: campaign.generatedAt, + source: campaign.source, + complete: campaign.complete, + selected: campaign.selected, + executed: campaign.executed, + passed: campaign.passed, + failed: campaign.failed, + retries: campaign.retries, + cleanupPassed: campaign.cleanupPassed, + publicUrl, + billing: campaign.billing, + suites: campaign.suites, + executions: campaign.results.map((result) => ({ + executionId: result.executionId, + suiteId: result.suiteId ?? "core-compatibility", + profileId: result.profileId, + environmentId: result.environmentId, + caseId: result.caseId, + provider: result.provider, + model: result.model, + status: result.status, + durationMs: result.durationMs, + attempt: result.attempt, + cleanup: result.cleanup, + billing: result.billing ?? summarizeExecutionBilling(result), + })), + }; +} + +export function emptyRunnerHistory(): RunnerE2EHistoryIndex { + return { + schema: "paperclip.runner-e2e.history/v1", + updatedAt: new Date(0).toISOString(), + latestCampaignId: null, + latestGreenCampaignId: null, + latestBySuite: {}, + latestGreenBySuite: {}, + campaigns: [], + }; +} + +export function mergeRunnerHistory( + current: RunnerE2EHistoryIndex | null | undefined, + campaign: RunnerE2EHistoryCampaign, +): RunnerE2EHistoryIndex { + const base = + current?.schema === "paperclip.runner-e2e.history/v1" + ? current + : emptyRunnerHistory(); + const campaigns = [ + campaign, + ...base.campaigns.filter( + (candidate) => candidate.campaignId !== campaign.campaignId, + ), + ].sort( + (left, right) => + Date.parse(right.generatedAt) - Date.parse(left.generatedAt), + ); + const latestBySuite: Record = {}; + const latestGreenBySuite: Record = {}; + for (const candidate of campaigns) { + for (const suite of candidate.suites) { + latestBySuite[suite.suiteId] ??= candidate.campaignId; + if (suite.complete && suite.failed === 0) { + latestGreenBySuite[suite.suiteId] ??= candidate.campaignId; + } + } + } + return { + schema: "paperclip.runner-e2e.history/v1", + updatedAt: new Date().toISOString(), + latestCampaignId: campaigns[0]?.campaignId ?? null, + latestGreenCampaignId: + campaigns.find( + (candidate) => candidate.complete && candidate.failed === 0, + )?.campaignId ?? null, + latestBySuite, + latestGreenBySuite, + campaigns, + }; +} diff --git a/tests/runner-e2e/instance-isolation.ts b/tests/runner-e2e/instance-isolation.ts new file mode 100644 index 0000000000..46540f5a9e --- /dev/null +++ b/tests/runner-e2e/instance-isolation.ts @@ -0,0 +1,69 @@ +import path from "node:path"; +import { readFile, stat } from "node:fs/promises"; + +export async function assertEmbeddedDatabaseIsolation( + configPath: string, + temporaryRoot: string, +) { + const configText = await readFile(configPath, "utf8"); + const config = JSON.parse(configText) as { + database?: { + mode?: string; + connectionString?: string; + embeddedPostgresDataDir?: string; + backup?: { dir?: string }; + }; + logging?: { logDir?: string }; + storage?: { + provider?: string; + localDisk?: { baseDir?: string }; + }; + secrets?: { + provider?: string; + strictMode?: boolean; + localEncrypted?: { keyFilePath?: string }; + }; + }; + if ( + config.database?.mode !== "embedded-postgres" || + config.database.connectionString + ) { + throw new Error( + "Runner E2E Paperclip instance did not use its embedded database", + ); + } + if ( + config.storage?.provider !== "local_disk" || + config.secrets?.provider !== "local_encrypted" || + config.secrets.strictMode !== true + ) { + throw new Error( + "Runner E2E instance did not use isolated local storage and strict encrypted secrets", + ); + } + const isolatedPaths = { + database: config.database.embeddedPostgresDataDir, + backup: config.database.backup?.dir, + logs: config.logging?.logDir, + storage: config.storage.localDisk?.baseDir, + secretsKey: config.secrets.localEncrypted?.keyFilePath, + }; + for (const [label, configuredPath] of Object.entries(isolatedPaths)) { + if (!configuredPath) { + throw new Error(`Runner E2E config omitted its ${label} path`); + } + const resolved = path.resolve(configuredPath); + if (!resolved.startsWith(`${temporaryRoot}${path.sep}`)) { + throw new Error( + `Runner E2E ${label} path escaped the isolated root: ${resolved}`, + ); + } + } + const databasePath = path.resolve(isolatedPaths.database!); + const databaseStat = await stat(databasePath); + if (!databaseStat.isDirectory()) + throw new Error("Embedded database path is not a directory"); + const keyStat = await stat(path.resolve(isolatedPaths.secretsKey!)); + if (!keyStat.isFile()) + throw new Error("Encrypted-secrets master key path is not a file"); +} diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts new file mode 100644 index 0000000000..26b8de7301 --- /dev/null +++ b/tests/runner-e2e/launch.ts @@ -0,0 +1,921 @@ +import { randomBytes } from "node:crypto"; +import { spawn } from "node:child_process"; +import { createWriteStream } from "node:fs"; +import { createRequire } from "node:module"; +import { createServer } from "node:net"; +import os from "node:os"; +import path from "node:path"; +import { + chmod, + cp, + lstat, + mkdir, + mkdtemp, + readFile, + readdir, + rm, + symlink, + writeFile, +} from "node:fs/promises"; +import { isImmutableDaytonaImage, runnerMatrix } from "./catalog.js"; +import { renderRunnerE2EDashboard } from "./dashboard.js"; +import { packageEvidence } from "./evidence.js"; +import { classifyFailure, shouldRetryFailure } from "./failure-classifier.js"; +import { buildRunnerCampaign } from "./history.js"; +import { assertEmbeddedDatabaseIsolation } from "./instance-isolation.js"; +import { + assertSecretFree, + findSecretLeakInDirectory, + isEphemeralCodexRuntimeAuthFile, + normalizedSecrets, + sanitizeJson, +} from "./redaction.js"; +import { + buildMatrixJobs, + parseRunnerSelectors, + RunnerSelectorError, + selectRunnerExecutions, +} from "./selectors.js"; +import { + CREDENTIAL_NAMES, + type MatrixExecution, + type RunnerE2EResult, +} from "./types.js"; +import { + reapNewDetachedDarwinSharedMemory, + snapshotDarwinSharedMemory, +} from "./shared-memory.js"; + +const repositoryRoot = path.resolve(import.meta.dirname, "../.."); +const localEnvPath = path.join(repositoryRoot, ".env.runner-e2e.local"); +const resultsRoot = path.join(repositoryRoot, "tests/runner-e2e/results"); +const activeProcessGroups = new Set(); +const activeProcessCleanup = new Map>(); +let cancelled = false; + +function cleanId(value: string) { + const result = value + .replace(/[^A-Za-z0-9_.-]+/g, "-") + .replace(/^-+|-+$/g, ""); + if (!result) + throw new Error( + `Invalid empty identifier derived from ${JSON.stringify(value)}`, + ); + return result; +} + +function secret(bytes = 32) { + return randomBytes(bytes).toString("base64url"); +} + +async function makeDirectoryTreeRemovable(directory: string): Promise { + await chmod(directory, 0o700); + const entries = await readdir(directory, { withFileTypes: true }); + await Promise.all( + entries + .filter((entry) => entry.isDirectory() && !entry.isSymbolicLink()) + .map((entry) => + makeDirectoryTreeRemovable(path.join(directory, entry.name)), + ), + ); +} + +function stopProcessGroup(pid: number, signal: NodeJS.Signals = "SIGTERM") { + try { + if (process.platform === "win32") process.kill(pid, signal); + else process.kill(-pid, signal); + } catch { + // The process tree already exited. + } +} + +function processGroupIsAlive(pid: number) { + try { + process.kill(process.platform === "win32" ? pid : -pid, 0); + return true; + } catch { + return false; + } +} + +async function terminateProcessGroup(pid: number) { + stopProcessGroup(pid, "SIGTERM"); + const gracefulDeadline = Date.now() + 5_000; + while (processGroupIsAlive(pid) && Date.now() < gracefulDeadline) { + await new Promise((resolve) => setTimeout(resolve, 50)); + } + if (!processGroupIsAlive(pid)) return null; + stopProcessGroup(pid, "SIGKILL"); + const forcedDeadline = Date.now() + 5_000; + while (processGroupIsAlive(pid) && Date.now() < forcedDeadline) { + await new Promise((resolve) => setTimeout(resolve, 50)); + } + return processGroupIsAlive(pid) + ? `Paperclip/Playwright process group ${pid} survived SIGKILL` + : null; +} + +for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"] as const) { + process.on(signal, () => { + cancelled = true; + for (const pid of activeProcessGroups) { + if (activeProcessCleanup.has(pid)) { + stopProcessGroup(pid, "SIGKILL"); + continue; + } + activeProcessCleanup.set(pid, terminateProcessGroup(pid)); + } + }); +} + +async function loadLocalEnvironment(target: NodeJS.ProcessEnv) { + const contents = await readFile(localEnvPath, "utf8").catch( + (error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return null; + throw error; + }, + ); + if (contents === null) return; + for (const [index, rawLine] of contents.split(/\r?\n/).entries()) { + const line = rawLine.trim(); + if (!line || line.startsWith("#")) continue; + const match = /^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)=(.*)$/.exec(line); + if (!match) + throw new Error( + `Invalid ${path.basename(localEnvPath)} line ${index + 1}`, + ); + if (target[match[1]] !== undefined) continue; + let value = match[2].trim(); + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + value = value.slice(1, -1); + } + target[match[1]] = value; + } +} + +async function reservePort() { + const server = createServer(); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + if (!address || typeof address === "string") + throw new Error("Failed to reserve a loopback port"); + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + return address.port; +} + +async function prepareProviderPath( + temporaryRoot: string, + inheritedPath: string | undefined, +) { + const toolBin = path.join(temporaryRoot, "provider-bin"); + await mkdir(toolBin, { recursive: true }); + const runnerRequire = createRequire( + path.join(repositoryRoot, "packages/paperclip-runner/package.json"), + ); + const codexAcpPackage = runnerRequire.resolve( + "@agentclientprotocol/codex-acp/package.json", + ); + const codexRequire = createRequire(codexAcpPackage); + const codexPackage = codexRequire.resolve("@openai/codex/package.json"); + const codexManifest = JSON.parse(await readFile(codexPackage, "utf8")) as { + bin?: string | Record; + }; + const codexBin = + typeof codexManifest.bin === "string" + ? codexManifest.bin + : codexManifest.bin?.codex; + if (!codexBin) + throw new Error( + "Production Codex ACP dependency does not expose its pinned Codex executable", + ); + await symlink( + path.resolve(path.dirname(codexPackage), codexBin), + path.join(toolBin, "codex"), + ); + const packageBin = path.join( + repositoryRoot, + "packages/paperclip-runner/node_modules/.bin", + ); + return [toolBin, packageBin, inheritedPath] + .filter(Boolean) + .join(path.delimiter); +} + +async function runProcess( + args: string[], + env: NodeJS.ProcessEnv, + timeoutMs: number | null, + logPath: string, +) { + const log = createWriteStream(logPath, { flags: "a", mode: 0o600 }); + const child = spawn("pnpm", args, { + cwd: repositoryRoot, + env, + stdio: ["inherit", "pipe", "pipe"], + detached: process.platform !== "win32", + }); + if (!child.pid) throw new Error("Failed to start Playwright"); + activeProcessGroups.add(child.pid); + let outputTail = ""; + const recordOutput = (chunk: Buffer, destination: NodeJS.WriteStream) => { + destination.write(chunk); + log.write(chunk); + outputTail += chunk.toString("utf8"); + if (outputTail.length > 1024 * 1024) + outputTail = outputTail.slice(-1024 * 1024); + }; + child.stdout?.on("data", (chunk: Buffer) => + recordOutput(chunk, process.stdout), + ); + child.stderr?.on("data", (chunk: Buffer) => + recordOutput(chunk, process.stderr), + ); + let timedOut = false; + const timer = + timeoutMs === null + ? undefined + : setTimeout(() => { + timedOut = true; + stopProcessGroup(child.pid!, "SIGTERM"); + setTimeout( + () => stopProcessGroup(child.pid!, "SIGKILL"), + 10_000, + ).unref(); + }, timeoutMs); + timer?.unref(); + let spawnError: string | null = null; + const exitCode = await new Promise((resolve, reject) => { + child.once("error", reject); + child.once("exit", (code) => resolve(code ?? 1)); + }).catch((error) => { + spawnError = error instanceof Error ? error.message : String(error); + return 1; + }); + if (timer) clearTimeout(timer); + const processCleanupError = child.pid + ? await (activeProcessCleanup.get(child.pid) ?? + terminateProcessGroup(child.pid)) + : null; + if (child.pid) { + activeProcessGroups.delete(child.pid); + activeProcessCleanup.delete(child.pid); + } + await new Promise((resolve) => log.end(resolve)); + return { + exitCode, + timedOut, + processCleanupError, + spawnError, + outputTail, + }; +} + +function syntheticResult( + execution: MatrixExecution, + attempt: number, + startedAtMs: number, + error: string, + failureClass: RunnerE2EResult["failureClass"] = "transient_infrastructure", +): RunnerE2EResult { + const finishedAtMs = Date.now(); + return { + schema: "paperclip.runner-e2e.result/v2", + executionId: execution.id, + suiteId: execution.suite.id, + suiteDefinitionHash: execution.suiteDefinitionHash, + source: { + sha: process.env.GITHUB_SHA ?? null, + ref: process.env.GITHUB_REF ?? null, + workflowRunUrl: + process.env.GITHUB_SERVER_URL && + process.env.GITHUB_REPOSITORY && + process.env.GITHUB_RUN_ID + ? `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}` + : null, + }, + ...(execution.profile.ranking + ? { rankingSnapshot: execution.profile.ranking } + : {}), + attempt, + status: "failed", + failureClass, + error, + profileId: execution.profile.id, + environmentId: execution.environment.id, + caseId: execution.task.id, + provider: execution.profile.provider, + model: execution.profile.model, + runtimeMode: execution.profile.expectedRuntimeMode, + startedAt: new Date(startedAtMs).toISOString(), + finishedAt: new Date(finishedAtMs).toISOString(), + durationMs: finishedAtMs - startedAtMs, + cleanup: "not_started", + }; +} + +async function readResult(resultPath: string, fallback: RunnerE2EResult) { + try { + return JSON.parse(await readFile(resultPath, "utf8")) as RunnerE2EResult; + } catch { + return fallback; + } +} + +async function copySharedEvidence(privateRoot: string, casePrivateDir: string) { + for (const relative of [ + "server.log", + "playwright.log", + "junit.xml", + "html-report", + "blob-report", + "playwright-output", + ]) { + await cp( + path.join(privateRoot, relative), + path.join(casePrivateDir, relative), + { recursive: true, force: true }, + ).catch((error: NodeJS.ErrnoException) => { + if (error.code !== "ENOENT") throw error; + }); + } +} + +async function runAttempt(input: { + executions: readonly MatrixExecution[]; + attempt: number; + campaignId: string; + options: ReturnType; +}): Promise { + const { executions, attempt, campaignId, options } = input; + const execution = executions[0]; + if (!execution) throw new Error("A runner E2E cell must contain a task case"); + if ( + executions.some( + (candidate) => + candidate.profile.id !== execution.profile.id || + candidate.environment.id !== execution.environment.id, + ) + ) { + throw new Error( + "One isolated runner E2E harness cannot mix profiles or environments", + ); + } + const startedAtMs = Date.now(); + const sharedMemoryBaseline = snapshotDarwinSharedMemory(); + const temporaryRoot = await mkdtemp( + path.join(os.tmpdir(), "paperclip-runner-e2e-"), + ); + const publishedResults: RunnerE2EResult[] = []; + const publishedResultPaths = new Map(); + let attemptSecrets: string[] = []; + try { + const paperclipHome = path.join(temporaryRoot, "paperclip-home"); + const workspace = path.join(temporaryRoot, "workspace"); + const privateDir = path.join(temporaryRoot, "artifacts-private"); + const instanceId = `runner-e2e-${randomBytes(8).toString("hex")}`; + const configPath = path.join( + paperclipHome, + "instances", + instanceId, + "config.json", + ); + const port = await reservePort(); + await Promise.all([ + mkdir(paperclipHome, { recursive: true }), + mkdir(workspace, { recursive: true }), + mkdir(privateDir, { recursive: true }), + ]); + const providerPath = await prepareProviderPath( + temporaryRoot, + process.env.PATH, + ); + const agentJwtSecret = secret(48); + const decisionSigningSecret = secret(48); + const toolActionSigningSecret = secret(48); + const betterAuthSecret = secret(48); + const credentials = normalizedSecrets([ + ...CREDENTIAL_NAMES.map((name) => process.env[name]), + agentJwtSecret, + decisionSigningSecret, + toolActionSigningSecret, + betterAuthSecret, + ]); + attemptSecrets = credentials; + const childEnv: NodeJS.ProcessEnv = { + ...process.env, + PATH: providerPath, + PAPERCLIP_RUNNER_E2E_EXECUTION_IDS: JSON.stringify( + executions.map((candidate) => candidate.id), + ), + PAPERCLIP_RUNNER_E2E_ATTEMPT: String(attempt), + PAPERCLIP_RUNNER_E2E_PORT: String(port), + PAPERCLIP_RUNNER_E2E_TEMP_ROOT: temporaryRoot, + PAPERCLIP_RUNNER_E2E_PRIVATE_DIR: privateDir, + PAPERCLIP_RUNNER_E2E_WORKSPACE: workspace, + PAPERCLIP_RUNNER_E2E_SERVER_LOG: path.join(privateDir, "server.log"), + // Vite's optimized dependency cache embeds revision query strings. A + // private per-attempt cache prevents an earlier cell or local rebuild + // from producing `504 Outdated Optimize Dep` during browser bootstrap. + PAPERCLIP_VITE_CACHE_DIR: path.join(temporaryRoot, "vite-cache"), + PAPERCLIP_RUNNER_E2E_TEST_TIMEOUT_MS: String( + Math.max( + ...executions.map( + (candidate) => + candidate.task.attemptTimeoutMs[candidate.environment.id], + ), + ) + 90_000, + ), + PAPERCLIP_HOME: paperclipHome, + PAPERCLIP_INSTANCE_ID: instanceId, + PAPERCLIP_CONFIG: configPath, + PAPERCLIP_AGENT_JWT_SECRET: agentJwtSecret, + PAPERCLIP_DECISION_SIGNING_SECRET: decisionSigningSecret, + PAPERCLIP_TOOL_ACTION_SIGNING_SECRET: toolActionSigningSecret, + BETTER_AUTH_SECRET: betterAuthSecret, + }; + // The database URLs are stripped here and again at the Playwright web-server + // boundary so a developer's shell can never redirect this paid test. + delete childEnv.DATABASE_URL; + delete childEnv.DATABASE_MIGRATION_URL; + + const playwrightArgs = [ + "exec", + "playwright", + "test", + "--config", + "tests/runner-e2e/playwright.config.ts", + ...(options.headed ? ["--headed"] : []), + ...(options.ui ? ["--ui"] : []), + ...(options.debug ? ["--debug"] : []), + ]; + const watchdog = + options.ui || options.debug + ? null + : executions.reduce( + (total, candidate) => + total + + candidate.task.attemptTimeoutMs[candidate.environment.id] + + 90_000, + 0, + ) + + 5 * 60_000; + const processResult = await runProcess( + playwrightArgs, + childEnv, + watchdog, + path.join(privateDir, "playwright.log"), + ); + const processFailure = processResult.spawnError + ? `Playwright failed to start: ${processResult.spawnError}` + : processResult.timedOut + ? `Harness process exceeded ${Math.round((watchdog ?? 0) / 1000)} seconds` + : `Playwright exited ${processResult.exitCode} before writing a result`; + const processFailureClass = + processResult.spawnError || processResult.timedOut + ? "transient_infrastructure" + : classifyFailure( + new Error( + processResult.outputTail + ? `${processFailure}\n${processResult.outputTail}` + : processFailure, + ), + ); + const results = await Promise.all( + executions.map(async (candidate) => { + const resultPath = path.join( + privateDir, + "cases", + candidate.task.id, + "result.json", + ); + const fallback = syntheticResult( + candidate, + attempt, + startedAtMs, + processFailure, + processFailureClass, + ); + const result = await readResult(resultPath, fallback); + return processResult.processCleanupError + ? { + ...result, + status: "failed" as const, + failureClass: "cleanup_failure" as const, + error: processResult.processCleanupError, + cleanup: "failed" as const, + } + : result; + }), + ); + let isolationError: unknown; + try { + await assertEmbeddedDatabaseIsolation(configPath, temporaryRoot); + } catch (error) { + isolationError = error; + } + let persistedStateError: unknown; + try { + const expectedEphemeralCredentials = new Set(); + for (const [label, directory] of [ + ["Paperclip home", paperclipHome], + ["workspace", workspace], + ] as const) { + while (true) { + // The managed Codex home may legitimately contain upstream source-code + // fixtures with fake `sk-*` strings. Reject exact campaign credentials. + const leak = await findSecretLeakInDirectory(directory, credentials, { + includeShapes: false, + ignoreFile: (file) => expectedEphemeralCredentials.has(file), + }); + if (!leak) break; + const isManagedCodexRuntimeAuth = + label === "Paperclip home" && + isEphemeralCodexRuntimeAuthFile(paperclipHome, leak.file); + if (isManagedCodexRuntimeAuth) { + const metadata = await lstat(leak.file); + if (metadata.isFile() && (metadata.mode & 0o777) === 0o600) { + // Codex CLI API-key mode requires this one runtime auth file. It + // lives only in the disposable cell root, is never published, + // must be owner-only, and is removed with the root below. + expectedEphemeralCredentials.add(leak.file); + continue; + } + } + throw new Error( + `Secret leak in persisted ${label} state at ${path.relative(temporaryRoot, leak.file)}: ${leak.reason}`, + ); + } + } + } catch (error) { + persistedStateError = error; + } + for (const [index, candidate] of executions.entries()) { + let result = results[index]; + if ( + isolationError && + result.failureClass !== "cleanup_failure" && + result.failureClass !== "secret_leak" + ) { + const isolationMessage = + isolationError instanceof Error + ? isolationError.message + : String(isolationError); + const configWasUnavailableDuringTransientBootstrap = + result.failureClass === "transient_infrastructure" && + typeof isolationError === "object" && + isolationError !== null && + "code" in isolationError && + isolationError.code === "ENOENT"; + result = { + ...result, + status: "failed", + failureClass: configWasUnavailableDuringTransientBootstrap + ? result.failureClass + : "permanent_infrastructure", + error: configWasUnavailableDuringTransientBootstrap + ? `${result.error}; isolated config could not be inspected after bootstrap failure: ${isolationMessage}` + : isolationMessage, + }; + } + if (persistedStateError) { + const persistedStateMessage = + persistedStateError instanceof Error + ? persistedStateError.message + : String(persistedStateError); + const persistedStateClass = classifyFailure(persistedStateError); + if ( + persistedStateClass === "secret_leak" || + (result.failureClass !== "secret_leak" && + result.failureClass !== "cleanup_failure") + ) { + result = { + ...result, + status: "failed", + failureClass: + persistedStateClass === "secret_leak" + ? "secret_leak" + : "permanent_infrastructure", + error: persistedStateMessage, + }; + } else { + result = { + ...result, + error: `${result.error}; persisted-state scan also failed: ${persistedStateMessage}`, + }; + } + } + const casePrivateDir = path.join(privateDir, "cases", candidate.task.id); + const resultPath = path.join(casePrivateDir, "result.json"); + const uploadDir = path.join( + resultsRoot, + campaignId, + candidate.suite.id, + candidate.profile.id, + candidate.environment.id, + candidate.task.id, + `attempt-${attempt}`, + ); + await mkdir(casePrivateDir, { recursive: true }); + await copySharedEvidence(privateDir, casePrivateDir); + await writeFile( + resultPath, + `${JSON.stringify(sanitizeJson(result, credentials), null, 2)}\n`, + "utf8", + ); + let evidence = await packageEvidence({ + privateDir: casePrivateDir, + uploadDir, + secrets: credentials, + expectPassScreenshot: result.status === "passed", + }); + if (evidence.leaks.length > 0 || evidence.missing.length > 0) { + result = { + ...result, + status: "failed", + failureClass: + evidence.leaks.length > 0 + ? "secret_leak" + : "permanent_infrastructure", + error: + evidence.leaks.length > 0 + ? `Secret leak rejected from evidence: ${evidence.leaks.map((leak) => leak.file).join(", ")}` + : `Required evidence missing: ${evidence.missing.join(", ")}`, + }; + await writeFile( + resultPath, + `${JSON.stringify(sanitizeJson(result, credentials), null, 2)}\n`, + "utf8", + ); + evidence = await packageEvidence({ + privateDir: casePrivateDir, + uploadDir, + secrets: credentials, + expectPassScreenshot: false, + }); + } + console.log( + `${result.status === "passed" ? "PASS" : "FAIL"} ${candidate.id} attempt ${attempt} -> ${uploadDir}`, + ); + publishedResults.push(result); + publishedResultPaths.set( + candidate.id, + path.join(uploadDir, "result.json"), + ); + } + return [...publishedResults]; + } finally { + reapNewDetachedDarwinSharedMemory(sharedMemoryBaseline); + let cleanupError: unknown; + if ( + temporaryRoot.startsWith(`${os.tmpdir()}${path.sep}paperclip-runner-e2e-`) + ) { + for (let cleanupAttempt = 1; cleanupAttempt <= 3; cleanupAttempt += 1) { + try { + await rm(temporaryRoot, { recursive: true, force: true }); + cleanupError = undefined; + break; + } catch (error) { + cleanupError = error; + if (cleanupAttempt < 3) { + await makeDirectoryTreeRemovable(temporaryRoot).catch(() => {}); + await new Promise((resolve) => + setTimeout(resolve, cleanupAttempt * 250), + ); + } + } + } + } else { + cleanupError = new Error( + `Refusing to remove unexpected temporary path ${temporaryRoot}`, + ); + } + if (cleanupError) { + const message = `Temporary runner E2E state cleanup failed at ${temporaryRoot}: ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}`; + for (const publishedResult of publishedResults) { + const publishedResultPath = publishedResultPaths.get( + publishedResult.executionId, + ); + if (!publishedResultPath) continue; + Object.assign(publishedResult, { + status: "failed", + failureClass: "cleanup_failure", + error: message, + cleanup: "failed", + } satisfies Partial); + const safeResult = `${JSON.stringify( + sanitizeJson(publishedResult, attemptSecrets), + null, + 2, + )}\n`; + assertSecretFree(safeResult, attemptSecrets, publishedResultPath); + await writeFile(publishedResultPath, safeResult, "utf8"); + } + // Cleanup failure is a failed cell, but must not suppress later cells in + // the same campaign. The result above carries the terminal failure. + console.error(message); + } + } +} + +function printList(executions: readonly MatrixExecution[]) { + console.log("ID\tSUITE\tGENERATION\tPROVIDER\tMODEL\tCREDENTIALS"); + for (const execution of executions) { + console.log( + [ + execution.id, + execution.suite.id, + execution.profile.generation, + execution.profile.provider, + execution.profile.model, + execution.requiredCredentials.join(","), + ].join("\t"), + ); + } +} + +async function runExecutionWithRetry(input: { + execution: MatrixExecution; + campaignId: string; + options: ReturnType; +}): Promise { + const { execution, campaignId, options } = input; + const [firstResult] = await runAttempt({ + executions: [execution], + attempt: 1, + campaignId, + options, + }); + if (!firstResult) throw new Error(`No result produced for ${execution.id}`); + if ( + options.ui || + options.debug || + firstResult.status !== "failed" || + !firstResult.failureClass || + !shouldRetryFailure(firstResult.failureClass) + ) { + return firstResult; + } + if (cancelled) throw new Error("Runner E2E campaign cancelled"); + console.warn( + `Retrying ${execution.id} in a fresh isolated harness after transient infrastructure failure`, + ); + const [retryResult] = await runAttempt({ + executions: [execution], + attempt: 2, + campaignId, + options, + }); + if (!retryResult) + throw new Error(`No retry result produced for ${execution.id}`); + return retryResult; +} + +async function runWithConcurrency( + values: readonly T[], + concurrency: number, + worker: (value: T) => Promise, +): Promise { + const results = new Array(values.length); + let cursor = 0; + const workers = Array.from( + { length: Math.min(concurrency, values.length) }, + async () => { + while (true) { + const index = cursor; + cursor += 1; + if (index >= values.length) return; + if (cancelled) throw new Error("Runner E2E campaign cancelled"); + results[index] = await worker(values[index]!); + } + }, + ); + await Promise.all(workers); + return results; +} + +async function main() { + let options: ReturnType; + try { + options = parseRunnerSelectors(process.argv.slice(2)); + } catch (error) { + if (error instanceof RunnerSelectorError) + throw new Error(`${error.message}\nUse --list to inspect valid cells.`); + throw error; + } + const executions = selectRunnerExecutions(options, runnerMatrix); + if (options.list) { + printList(executions); + return; + } + if (options.matrixJson) { + const jobs = buildMatrixJobs(executions); + console.log( + JSON.stringify({ + include: jobs, + needsDaytona: jobs.some((job) => job.needsDaytona), + executionIds: executions.map((execution) => execution.id), + }), + ); + return; + } + + await loadLocalEnvironment(process.env); + const missingCredentials = [ + ...new Set( + executions.flatMap((execution) => execution.requiredCredentials), + ), + ].filter((name) => !process.env[name]?.trim()); + if (missingCredentials.length > 0) { + throw new Error( + `Missing runner E2E credentials: ${missingCredentials.join(", ")}`, + ); + } + if ( + executions.some((execution) => execution.environment.id === "daytona") && + !isImmutableDaytonaImage(process.env.PAPERCLIP_E2E_DAYTONA_IMAGE) + ) { + throw new Error( + "PAPERCLIP_E2E_DAYTONA_IMAGE must be an immutable image@sha256 digest for Daytona cells", + ); + } + + const campaignId = cleanId( + process.env.PAPERCLIP_E2E_CAMPAIGN_ID ?? + `local-${new Date().toISOString().replace(/[:.]/g, "-")}`, + ); + const requestedParallelism = + options.headed || options.ui || options.debug ? 1 : options.maxParallel; + console.log( + `Running ${executions.length} isolated execution(s) with max parallelism ${requestedParallelism}`, + ); + const finalResults = await runWithConcurrency( + executions, + requestedParallelism, + (execution) => runExecutionWithRetry({ execution, campaignId, options }), + ); + + const generatedAt = new Date().toISOString(); + const campaign = buildRunnerCampaign({ + campaignId, + generatedAt, + expected: executions.map((execution) => execution.id), + results: finalResults, + }); + const summaryDir = path.join(resultsRoot, campaignId); + await mkdir(summaryDir, { recursive: true }); + const campaignSecrets = normalizedSecrets( + CREDENTIAL_NAMES.map((name) => process.env[name]), + ); + const safeCampaign = sanitizeJson( + campaign, + campaignSecrets, + ) as typeof campaign; + const campaignText = `${JSON.stringify(safeCampaign, null, 2)}\n`; + assertSecretFree(campaignText, campaignSecrets, "campaign.json"); + await writeFile(path.join(summaryDir, "campaign.json"), campaignText, "utf8"); + const dashboard = renderRunnerE2EDashboard({ + title: `Runner E2E · ${campaignId}`, + generatedAt, + expected: executions.map((execution) => execution.id), + catalog: runnerMatrix, + campaign: safeCampaign, + entries: safeCampaign.results.map((result) => ({ + result, + valid: result.status === "passed" && result.cleanup === "passed", + errors: + result.status === "passed" && result.cleanup === "passed" + ? [] + : [ + result.error ?? + result.failureClass ?? + `cleanup=${result.cleanup}`, + ], + evidenceBaseHref: [ + result.suiteId ?? "core-compatibility", + result.profileId, + result.environmentId, + result.caseId, + `attempt-${result.attempt}`, + ].join("/"), + })), + }); + assertSecretFree(dashboard, campaignSecrets, "dashboard.html"); + await writeFile(path.join(summaryDir, "dashboard.html"), dashboard, "utf8"); + console.log( + `Campaign ${campaignId}: ${safeCampaign.passed}/${safeCampaign.selected} passed`, + ); + if (safeCampaign.failed > 0) process.exitCode = 1; +} + +await main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +}); diff --git a/tests/runner-e2e/live-fixtures.test.ts b/tests/runner-e2e/live-fixtures.test.ts new file mode 100644 index 0000000000..1fe34be8e0 --- /dev/null +++ b/tests/runner-e2e/live-fixtures.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from "vitest"; +import type { RunnerApi } from "./api.js"; +import { runnerMatrix } from "./catalog.js"; +import { setupLiveFixtures } from "./live-fixtures.js"; + +describe("live runner fixtures", () => { + it("installs the Daytona provider through the public API before creating its environment", async () => { + const calls: string[] = []; + const api = { + async post(path: string, data?: Record) { + calls.push(`POST ${path}`); + if (path === "/api/plugins/install") { + expect(data).toMatchObject({ isLocalPath: true }); + expect(data?.packageName).toEqual( + expect.stringContaining( + "packages/plugins/sandbox-providers/daytona", + ), + ); + return { + id: "plugin-daytona", + pluginKey: "paperclip.daytona-sandbox-provider", + status: "ready", + }; + } + if (path === "/api/companies") { + return { id: "company-1", name: "Runner E2E" }; + } + if (path.endsWith("/environments")) { + expect(calls).toContain("POST /api/plugins/install"); + return { id: "environment-1", driver: "sandbox" }; + } + if (path.endsWith("/agents")) { + return { id: "agent-1", name: "Agent", companyId: "company-1" }; + } + throw new Error(`Unexpected POST ${path}`); + }, + async postSensitive(path: string, data?: Record) { + calls.push(`POST ${path}`); + return { id: `secret-${String(data?.key).toLowerCase()}` }; + }, + async delete(path: string) { + calls.push(`DELETE ${path}`); + }, + } as unknown as RunnerApi; + const execution = runnerMatrix.find( + (candidate) => + candidate.id === + "core-compatibility.legacy-codex.daytona.message-marker", + ); + expect(execution).toBeDefined(); + + const fixtures = await setupLiveFixtures({ + api, + execution: execution!, + executionNonce: "nonce", + workspacePath: "/tmp/workspace", + credentials: { + OPENAI_API_KEY: "openai-test-value", + DAYTONA_API_KEY: "daytona-test-value", + }, + daytonaImage: + "ghcr.io/paperclip/image@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + }); + + expect(calls.indexOf("POST /api/plugins/install")).toBeLessThan( + calls.indexOf("POST /api/companies/company-1/environments"), + ); + await fixtures.teardown(); + expect(calls).toContain( + "DELETE /api/environments/environment-1?destroyReusableSandboxLeases=true", + ); + }); +}); diff --git a/tests/runner-e2e/live-fixtures.ts b/tests/runner-e2e/live-fixtures.ts new file mode 100644 index 0000000000..4dba281fb7 --- /dev/null +++ b/tests/runner-e2e/live-fixtures.ts @@ -0,0 +1,221 @@ +import path from "node:path"; +import { FixtureRegistry } from "./fixture-registry.js"; +import type { RunnerApi } from "./api.js"; +import type { + CredentialName, + MatrixExecution, + SecretReference, + SecretReferenceMap, +} from "./types.js"; + +interface CompanyRecord { + id: string; + issuePrefix?: string | null; + name: string; +} + +interface SecretRecord { + id: string; +} +interface PluginRecord { + id: string; + pluginKey: string; + status: string; +} +interface EnvironmentRecord { + id: string; + driver: string; + config?: Record; +} +interface AgentRecord { + id: string; + name: string; + companyId: string; +} + +export interface LiveFixtureValues { + company: CompanyRecord; + secretRefs: SecretReferenceMap; + environment: EnvironmentRecord; + agent: AgentRecord; + teardown(): Promise; +} + +function value(resolved: ReadonlyMap, id: string): T { + const result = resolved.get(id); + if (!result) throw new Error(`Missing resolved fixture ${id}`); + return result as T; +} + +async function deleteDaytonaEnvironment(api: RunnerApi, environmentId: string) { + const deadlineAt = Date.now() + 120_000; + let lastError: unknown; + while (Date.now() < deadlineAt) { + try { + await api.delete( + `/api/environments/${environmentId}?destroyReusableSandboxLeases=true`, + { + allowNotFound: true, + }, + ); + return; + } catch (error) { + lastError = error; + await new Promise((resolve) => setTimeout(resolve, 3_000)); + } + } + throw new Error( + `Daytona lease cleanup failed: ${lastError instanceof Error ? lastError.message : String(lastError)}`, + ); +} + +export async function setupLiveFixtures(input: { + api: RunnerApi; + execution: MatrixExecution; + executionNonce: string; + workspacePath: string; + credentials: Partial>; + daytonaImage?: string; +}): Promise { + const { api, execution } = input; + const registry = new FixtureRegistry(); + + if (execution.environment.id === "daytona") { + registry.register({ + id: "sandbox-provider", + async setup() { + return api.post("/api/plugins/install", { + packageName: path.resolve( + import.meta.dirname, + "../../packages/plugins/sandbox-providers/daytona", + ), + isLocalPath: true, + }); + }, + async teardown() { + // The plugin is installed only in the isolated instance/database. The + // launcher removes that complete instance after the environment lease + // has been destroyed, so no global uninstall mutation is necessary. + }, + }); + } + + registry.register({ + id: "company", + async setup() { + return api.post("/api/companies", { + name: `Runner E2E ${execution.id} ${input.executionNonce}`, + description: "Ephemeral paid full-stack runner acceptance fixture", + budgetMonthlyCents: 0, + }); + }, + async teardown() { + // The isolated instance/database is removed by the launcher. Do not call + // company deletion here: metered runs intentionally retain cost-event + // references until that instance-wide teardown. + }, + }); + + registry.register({ + id: "secrets", + dependencies: ["company"], + async setup(resolved) { + const company = value(resolved, "company"); + const refs: SecretReferenceMap = {}; + for (const credentialName of execution.requiredCredentials) { + const rawValue = input.credentials[credentialName]; + if (!rawValue) throw new Error(`Missing credential ${credentialName}`); + const secret = await api.postSensitive( + `/api/companies/${company.id}/secrets`, + { + name: `Runner E2E ${credentialName} ${input.executionNonce}`, + key: credentialName, + value: rawValue, + description: `Ephemeral credential for ${execution.id}`, + }, + ); + refs[credentialName] = { + type: "secret_ref", + secretId: secret.id, + version: "latest", + } satisfies SecretReference; + } + return refs; + }, + }); + + registry.register({ + id: "environment", + dependencies: [ + "company", + "secrets", + ...(execution.environment.id === "daytona" ? ["sandbox-provider"] : []), + ], + async setup(resolved) { + const company = value(resolved, "company"); + const secretRefs = value(resolved, "secrets"); + if (execution.environment.id === "local") { + // Paperclip has one instance-managed local environment. The company + // creation API ensures it exists; creating a second local environment + // is intentionally rejected by the public API. + const environments = await api.get( + `/api/companies/${company.id}/environments?driver=local`, + ); + const local = environments.find( + (candidate) => candidate.driver === "local", + ); + if (!local) + throw new Error( + "Isolated Paperclip instance did not create its managed local environment", + ); + return local; + } + return api.post( + `/api/companies/${company.id}/environments`, + execution.environment.buildEnvironment({ + secretRefs, + daytonaImage: input.daytonaImage, + executionId: input.executionNonce, + }), + ); + }, + async teardown(environment) { + if (execution.environment.id === "daytona") { + await deleteDaytonaEnvironment(api, environment.id); + } + }, + }); + + registry.register({ + id: "agent", + dependencies: ["company", "secrets", "environment"], + async setup(resolved) { + const company = value(resolved, "company"); + const environment = value(resolved, "environment"); + const secretRefs = value(resolved, "secrets"); + return api.post( + `/api/companies/${company.id}/agents`, + execution.profile.buildAgent({ + environmentId: environment.id, + environmentFixtureId: execution.environment.id, + workspacePath: input.workspacePath, + secretRefs, + executionId: input.executionNonce, + }), + ); + }, + async teardown() { + // Agent state is instance-local. Daytona environment teardown below is + // the only fixture cleanup that must reach an external provider. + }, + }); + + const setup = await registry.setupAll(); + return { + company: value(setup.values, "company"), + secretRefs: value(setup.values, "secrets"), + environment: value(setup.values, "environment"), + agent: value(setup.values, "agent"), + teardown: setup.teardown, + }; +} diff --git a/tests/runner-e2e/matchers.ts b/tests/runner-e2e/matchers.ts new file mode 100644 index 0000000000..c36eb676d1 --- /dev/null +++ b/tests/runner-e2e/matchers.ts @@ -0,0 +1,149 @@ +import { createRequire } from "node:module"; +import { readFile } from "node:fs/promises"; +import type { Matcher } from "./types.js"; + +interface JsonSchemaValidator { + (value: unknown): boolean; + errors?: unknown; +} + +interface Ajv2020Instance { + compile(schema: Record): JsonSchemaValidator; +} + +const runnerRequire = createRequire( + new URL("../../packages/paperclip-runner/package.json", import.meta.url), +); +const Ajv2020 = runnerRequire("ajv/dist/2020.js").default as new (options: { + allErrors: boolean; + strict: boolean; +}) => Ajv2020Instance; +const jsonSchemaCompiler = new Ajv2020({ allErrors: true, strict: false }); + +export interface MatcherObservation { + message?: string; + issueStatus?: string; + runStatus?: string; + runtimeMode?: string; + environment?: string; + files?: Record; + artifacts?: Array<{ name: string; mimeType?: string }>; + json?: unknown; +} + +export interface MatcherResult { + matcher: Matcher; + passed: boolean; + detail: string; +} + +function normalizeMessage(value: string | undefined) { + return ( + (value ?? "") + .replace(/\r\n/g, "\n") + // Some provider renderers escape underscores in plain-text identifiers + // before persisting Markdown. Treat that presentation-only escape as the + // same visible marker for exact/contains/ordered message assertions. + .replace(/\\_/g, "_") + .replace(/[ \t]+/g, " ") + .trim() + ); +} + +function readJsonPath(value: unknown, path: string): unknown { + return path + .split(".") + .filter(Boolean) + .reduce((current, segment) => { + if (!current || typeof current !== "object") return undefined; + return (current as Record)[segment]; + }, value); +} + +export async function evaluateMatcher( + matcher: Matcher, + observation: MatcherObservation, +): Promise { + const message = normalizeMessage(observation.message); + let passed = false; + let actual: unknown; + if (matcher.kind === "message_exact") { + actual = message; + passed = message === normalizeMessage(matcher.expected); + } else if (matcher.kind === "message_contains") { + actual = message; + passed = message.includes(normalizeMessage(matcher.expected)); + } else if (matcher.kind === "message_regex") { + actual = message; + passed = new RegExp(matcher.pattern, matcher.flags).test(message); + } else if (matcher.kind === "message_ordered") { + actual = message; + let cursor = 0; + passed = matcher.expected.every((expected) => { + const normalizedExpected = normalizeMessage(expected); + const index = message.indexOf(normalizedExpected, cursor); + if (index < 0) return false; + cursor = index + normalizedExpected.length; + return true; + }); + } else if (matcher.kind === "issue_status") { + actual = observation.issueStatus; + passed = actual === matcher.expected; + } else if (matcher.kind === "run_status") { + actual = observation.runStatus; + passed = actual === matcher.expected; + } else if (matcher.kind === "runtime_mode") { + actual = observation.runtimeMode; + passed = actual === matcher.expected; + } else if (matcher.kind === "environment") { + actual = observation.environment; + passed = actual === matcher.expected; + } else if ( + matcher.kind === "file_exists" || + matcher.kind === "file_contains" + ) { + try { + actual = + observation.files?.[matcher.path] ?? + (await readFile(matcher.path, "utf8")); + passed = + matcher.kind === "file_exists" || + String(actual).includes(matcher.expected); + } catch { + actual = undefined; + passed = false; + } + } else if (matcher.kind === "artifact_exists") { + actual = observation.artifacts ?? []; + passed = (observation.artifacts ?? []).some( + (artifact) => + artifact.name === matcher.name && + (!matcher.mimeType || artifact.mimeType === matcher.mimeType), + ); + } else if (matcher.kind === "json_path") { + actual = readJsonPath(observation.json, matcher.path); + passed = JSON.stringify(actual) === JSON.stringify(matcher.expected); + } else { + const validate = jsonSchemaCompiler.compile(matcher.schema); + passed = validate(observation.json); + actual = passed + ? observation.json + : { value: observation.json, errors: validate.errors ?? [] }; + } + return { + matcher, + passed, + detail: passed + ? "matched" + : `expected ${JSON.stringify(matcher)}; observed ${JSON.stringify(actual)}`, + }; +} + +export async function evaluateMatchers( + matchers: readonly Matcher[], + observation: MatcherObservation, +): Promise { + return Promise.all( + matchers.map((matcher) => evaluateMatcher(matcher, observation)), + ); +} diff --git a/tests/runner-e2e/merge.config.ts b/tests/runner-e2e/merge.config.ts new file mode 100644 index 0000000000..906482cb44 --- /dev/null +++ b/tests/runner-e2e/merge.config.ts @@ -0,0 +1,14 @@ +import path from "node:path"; + +const output = path.resolve( + process.env.PAPERCLIP_RUNNER_E2E_MERGED_REPORT_DIR ?? + "runner-e2e-merged-report", +); + +export default { + testDir: ".", + reporter: [ + ["html", { open: "never", outputFolder: path.join(output, "html") }], + ["junit", { outputFile: path.join(output, "playwright-junit.xml") }], + ], +}; diff --git a/tests/runner-e2e/openrouter-models-update.ts b/tests/runner-e2e/openrouter-models-update.ts new file mode 100644 index 0000000000..e0359da814 --- /dev/null +++ b/tests/runner-e2e/openrouter-models-update.ts @@ -0,0 +1,59 @@ +import { writeFile } from "node:fs/promises"; +import { + rankingContentHash, + validateOpenRouterRankingSnapshot, + type OpenRouterRankedModel, +} from "./openrouter-ranking.js"; + +const sourceUrl = + "https://openrouter.ai/api/v1/models?sort=top-weekly&supported_parameters=tools"; + +interface OpenRouterModelResponse { + data?: Array<{ + id?: string; + name?: string; + supported_parameters?: string[]; + }>; +} + +const response = await fetch(sourceUrl, { + headers: { Accept: "application/json" }, +}); +if (!response.ok) { + throw new Error(`OpenRouter ranking request failed with ${response.status}`); +} +const payload = (await response.json()) as OpenRouterModelResponse; +const models: OpenRouterRankedModel[] = (payload.data ?? []) + .filter( + ( + model, + ): model is Required< + NonNullable[number] + > => + Boolean( + model.id && model.name && model.supported_parameters?.includes("tools"), + ), + ) + .slice(0, 5) + .map((model, index) => ({ + rank: index + 1, + id: model.id, + name: model.name, + supportedParameters: [...new Set(model.supported_parameters)].sort(), + })); +const capturedAt = new Date().toISOString(); +const snapshot = validateOpenRouterRankingSnapshot({ + schema: "paperclip.runner-e2e.openrouter-ranking/v1", + snapshotId: `top-weekly-tools-${capturedAt.slice(0, 10)}`, + ranking: "top-weekly", + requiredParameter: "tools", + sourceUrl, + capturedAt, + contentHash: rankingContentHash(models), + models, +}); +const output = new URL("./openrouter-models.json", import.meta.url); +await writeFile(output, `${JSON.stringify(snapshot, null, 2)}\n`, "utf8"); +console.log( + `Pinned ${snapshot.models.length} OpenRouter model(s) in ${output.pathname}`, +); diff --git a/tests/runner-e2e/openrouter-models.json b/tests/runner-e2e/openrouter-models.json new file mode 100644 index 0000000000..fa1f00af05 --- /dev/null +++ b/tests/runner-e2e/openrouter-models.json @@ -0,0 +1,41 @@ +{ + "schema": "paperclip.runner-e2e.openrouter-ranking/v1", + "snapshotId": "top-weekly-tools-2026-08-28", + "ranking": "top-weekly", + "requiredParameter": "tools", + "sourceUrl": "https://openrouter.ai/api/v1/models?sort=top-weekly&supported_parameters=tools", + "capturedAt": "2026-08-28T00:00:00.000Z", + "contentHash": "c1b564582712dd2769e4855a79bc5604e860bf7a85fb98445fff6b3c4c100a3c", + "models": [ + { + "rank": 1, + "id": "deepseek/deepseek-v4-flash-0731", + "name": "DeepSeek V4 Flash 0731", + "supportedParameters": ["tools"] + }, + { + "rank": 2, + "id": "xiaomi/mimo-v2.5", + "name": "MiMo V2.5", + "supportedParameters": ["tools"] + }, + { + "rank": 3, + "id": "tencent/hy3", + "name": "Tencent HY 3", + "supportedParameters": ["tools"] + }, + { + "rank": 4, + "id": "nvidia/nemotron-3-ultra-550b-a55b:free", + "name": "Nemotron 3 Ultra 550B A55B (free)", + "supportedParameters": ["tools"] + }, + { + "rank": 5, + "id": "openai/gpt-5.6-luna", + "name": "GPT-5.6 Luna", + "supportedParameters": ["tools"] + } + ] +} diff --git a/tests/runner-e2e/openrouter-ranking.ts b/tests/runner-e2e/openrouter-ranking.ts new file mode 100644 index 0000000000..9010274e86 --- /dev/null +++ b/tests/runner-e2e/openrouter-ranking.ts @@ -0,0 +1,78 @@ +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; + +export interface OpenRouterRankedModel { + rank: number; + id: string; + name: string; + supportedParameters: string[]; +} + +export interface OpenRouterRankingSnapshot { + schema: "paperclip.runner-e2e.openrouter-ranking/v1"; + snapshotId: string; + ranking: "top-weekly"; + requiredParameter: "tools"; + sourceUrl: string; + capturedAt: string; + contentHash: string; + models: OpenRouterRankedModel[]; +} + +export function rankingContentHash(models: readonly OpenRouterRankedModel[]) { + return createHash("sha256").update(JSON.stringify(models)).digest("hex"); +} + +export function validateOpenRouterRankingSnapshot( + value: unknown, +): OpenRouterRankingSnapshot { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("OpenRouter ranking snapshot must be an object"); + } + const snapshot = value as Partial; + if (snapshot.schema !== "paperclip.runner-e2e.openrouter-ranking/v1") { + throw new Error("OpenRouter ranking snapshot has an unknown schema"); + } + if ( + snapshot.ranking !== "top-weekly" || + snapshot.requiredParameter !== "tools" || + !snapshot.snapshotId?.trim() || + !snapshot.sourceUrl?.startsWith("https://openrouter.ai/") || + !snapshot.capturedAt || + Number.isNaN(Date.parse(snapshot.capturedAt)) || + !Array.isArray(snapshot.models) || + snapshot.models.length !== 5 + ) { + throw new Error("OpenRouter ranking snapshot metadata is invalid"); + } + const ids = new Set(); + for (const [index, model] of snapshot.models.entries()) { + if ( + model.rank !== index + 1 || + !/^[a-z0-9][a-z0-9._-]*\/[a-z0-9][a-z0-9._:-]*$/i.test(model.id) || + !model.name?.trim() || + !model.supportedParameters?.includes("tools") || + ids.has(model.id) + ) { + throw new Error(`OpenRouter ranked model ${index + 1} is invalid`); + } + ids.add(model.id); + } + if (snapshot.contentHash !== rankingContentHash(snapshot.models)) { + throw new Error("OpenRouter ranking snapshot content hash is invalid"); + } + return snapshot as OpenRouterRankingSnapshot; +} + +export const openRouterRankingSnapshot = validateOpenRouterRankingSnapshot( + JSON.parse( + readFileSync(new URL("./openrouter-models.json", import.meta.url), "utf8"), + ), +); + +export function openRouterProfileId(modelId: string) { + return `openrouter-${modelId}` + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, ""); +} diff --git a/tests/runner-e2e/playwright.config.ts b/tests/runner-e2e/playwright.config.ts new file mode 100644 index 0000000000..0bbc899847 --- /dev/null +++ b/tests/runner-e2e/playwright.config.ts @@ -0,0 +1,69 @@ +import path from "node:path"; +import { defineConfig } from "@playwright/test"; +import { runnerE2EWebServerCommand } from "./web-server-command.js"; + +function required(name: string) { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`${name} is required`); + return value; +} + +const port = Number(required("PAPERCLIP_RUNNER_E2E_PORT")); +const temporaryRoot = required("PAPERCLIP_RUNNER_E2E_TEMP_ROOT"); +const privateDir = required("PAPERCLIP_RUNNER_E2E_PRIVATE_DIR"); +const paperclipHome = required("PAPERCLIP_HOME"); +const configPath = required("PAPERCLIP_CONFIG"); +const baseURL = `http://127.0.0.1:${port}`; +required("PAPERCLIP_INSTANCE_ID"); +required("PAPERCLIP_AGENT_JWT_SECRET"); +required("PAPERCLIP_DECISION_SIGNING_SECRET"); +required("PAPERCLIP_TOOL_ACTION_SIGNING_SECRET"); +required("BETTER_AUTH_SECRET"); +if ( + !paperclipHome.startsWith(`${temporaryRoot}${path.sep}`) || + !configPath.startsWith(`${temporaryRoot}${path.sep}`) +) { + throw new Error("Paperclip server paths escape the isolated temporary root"); +} +const repositoryRoot = path.resolve(import.meta.dirname, "../.."); + +export default defineConfig({ + testDir: ".", + testMatch: "runner.spec.ts", + timeout: Number(process.env.PAPERCLIP_RUNNER_E2E_TEST_TIMEOUT_MS ?? 600_000), + expect: { timeout: 30_000 }, + fullyParallel: false, + workers: 1, + retries: 0, + use: { + baseURL, + browserName: "chromium", + headless: true, + actionTimeout: 30_000, + navigationTimeout: 30_000, + screenshot: "only-on-failure", + trace: "retain-on-failure", + video: "retain-on-failure", + }, + webServer: { + // Do not put an env object here: Playwright serializes webServer config in + // blob reports. The wrapper inherits the test process and strips provider + // keys before spawning the real Paperclip process. + command: runnerE2EWebServerCommand(repositoryRoot), + url: `${baseURL}/api/health`, + reuseExistingServer: false, + timeout: 180_000, + stdout: "pipe", + stderr: "pipe", + }, + outputDir: path.join(privateDir, "playwright-output"), + reporter: [ + ["list"], + ["blob", { outputDir: path.join(privateDir, "blob-report") }], + ["junit", { outputFile: path.join(privateDir, "junit.xml") }], + [ + "html", + { open: "never", outputFolder: path.join(privateDir, "html-report") }, + ], + ], +}); diff --git a/tests/runner-e2e/redaction.ts b/tests/runner-e2e/redaction.ts new file mode 100644 index 0000000000..5c42e529c6 --- /dev/null +++ b/tests/runner-e2e/redaction.ts @@ -0,0 +1,188 @@ +import { createReadStream } from "node:fs"; +import { readdir } from "node:fs/promises"; +import path from "node:path"; +import { redactDiagnosticText } from "../../packages/adapter-utils/src/command-redaction.js"; + +const SECRET_SHAPES = [ + /\bsk-ant-[A-Za-z0-9_-]{16,}\b/g, + /\bsk-(?:proj-)?[A-Za-z0-9_-]{16,}\b/g, + /\b(?:openrouter|daytona)[-_]?(?:api)?[-_]?key["'=:\s]+[A-Za-z0-9._-]{12,}\b/gi, +] as const; + +export function normalizedSecrets(values: readonly (string | undefined)[]) { + return [ + ...new Set( + values + .map((value) => value?.trim()) + .filter((value): value is string => Boolean(value)), + ), + ].sort((left, right) => right.length - left.length); +} + +export function isEphemeralCodexRuntimeAuthFile( + paperclipHome: string, + file: string, +) { + const relative = path.relative(paperclipHome, file).split(path.sep).join("/"); + return ( + /^instances\/[^/]+\/companies\/[^/]+\/agents\/[^/]+\/codex-home\/auth\.json$/.test( + relative, + ) || + /^instances\/[^/]+\/runtime\/paperclip-runner\/durable-sessions\/[^/]+\/codex-home\/auth\.json$/.test( + relative, + ) + ); +} + +export function redactText(value: string, secrets: readonly string[]) { + let redacted = redactDiagnosticText(value, "[REDACTED]"); + return redactKnownSecretsAndShapes(redacted, secrets); +} + +function redactKnownSecretsAndShapes( + value: string, + secrets: readonly string[], +) { + let redacted = value; + for (const secret of normalizedSecrets(secrets)) + redacted = redacted.split(secret).join("[REDACTED]"); + for (const pattern of SECRET_SHAPES) + redacted = redacted.replace(pattern, "[REDACTED_SECRET_SHAPE]"); + return redacted; +} + +function redactStructuredText(value: string, secrets: readonly string[]) { + const knownSafe = redactKnownSecretsAndShapes(value, secrets); + if ( + !/(?:Authorization\s*:\s*Bearer|(?:api[-_]?key|token|secret|password)\s*=)/i.test( + knownSafe, + ) + ) { + return knownSafe; + } + return redactKnownSecretsAndShapes( + redactDiagnosticText(knownSafe, "[REDACTED]"), + secrets, + ); +} + +export function findSecretLeak( + value: string | Buffer, + secrets: readonly string[], + options: { includeShapes?: boolean } = {}, +): string | null { + const text = Buffer.isBuffer(value) ? value.toString("utf8") : value; + for (const secret of normalizedSecrets(secrets)) { + if (text.includes(secret)) return "exact secret value"; + } + if (options.includeShapes !== false) { + for (const pattern of SECRET_SHAPES) { + pattern.lastIndex = 0; + if (pattern.test(text)) return "secret-shaped value"; + } + } + return null; +} + +/** + * Scan structured JSON one key/value string at a time. Scanning the serialized + * document for provider key shapes can join an object key, punctuation, and an + * unrelated value into a false positive that never existed in the payload. + */ +export function findSecretLeakInJsonValues( + value: unknown, + secrets: readonly string[], + options: { includeShapes?: boolean } = {}, +): string | null { + if (typeof value === "string") return findSecretLeak(value, secrets, options); + if (Array.isArray(value)) { + for (const entry of value) { + const leak = findSecretLeakInJsonValues(entry, secrets, options); + if (leak) return leak; + } + return null; + } + if (value && typeof value === "object") { + for (const [key, entry] of Object.entries(value)) { + const keyLeak = findSecretLeak(key, secrets, options); + if (keyLeak) return keyLeak; + const valueLeak = findSecretLeakInJsonValues(entry, secrets, options); + if (valueLeak) return valueLeak; + } + } + return null; +} + +export function sanitizeJson( + value: unknown, + secrets: readonly string[], +): unknown { + // Structured values are not shell diagnostics. Applying the command/JWT + // heuristic here would redact stable dotted identifiers such as our schema + // name. Exact loaded secrets and well-known provider key shapes are enough. + if (typeof value === "string") return redactStructuredText(value, secrets); + if (Array.isArray(value)) + return value.map((entry) => sanitizeJson(entry, secrets)); + if (value && typeof value === "object") { + return Object.fromEntries( + Object.entries(value).map(([key, entry]) => [ + key, + sanitizeJson(entry, secrets), + ]), + ); + } + return value; +} + +export function assertSecretFree( + value: string | Buffer, + secrets: readonly string[], + label: string, + options: { includeShapes?: boolean } = {}, +) { + const leak = findSecretLeak(value, secrets, options); + if (leak) throw new Error(`Secret leak in ${label}: ${leak}`); +} + +export async function findSecretLeakInDirectory( + root: string, + secrets: readonly string[], + options: { + includeShapes?: boolean; + ignoreFile?: (file: string) => boolean; + } = {}, +): Promise<{ file: string; reason: string } | null> { + const overlap = Math.max( + 256, + ...normalizedSecrets(secrets).map((secret) => secret.length + 16), + ); + const scan = async ( + directory: string, + ): Promise<{ + file: string; + reason: string; + } | null> => { + const entries = await readdir(directory, { withFileTypes: true }); + for (const entry of entries) { + const file = path.join(directory, entry.name); + if (entry.isDirectory()) { + const leak = await scan(file); + if (leak) return leak; + } else if (entry.isFile()) { + if (options.ignoreFile?.(file)) continue; + let carry = Buffer.alloc(0); + for await (const chunk of createReadStream(file)) { + const data = Buffer.concat([ + carry, + Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk), + ]); + const reason = findSecretLeak(data, secrets, options); + if (reason) return { file, reason }; + carry = data.subarray(Math.max(0, data.length - overlap)); + } + } + } + return null; + }; + return scan(root); +} diff --git a/tests/runner-e2e/report.test.ts b/tests/runner-e2e/report.test.ts new file mode 100644 index 0000000000..0aef99e8e3 --- /dev/null +++ b/tests/runner-e2e/report.test.ts @@ -0,0 +1,358 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import type { RunnerE2EResult } from "./types.js"; + +const execFileAsync = promisify(execFile); +const repositoryRoot = path.resolve(import.meta.dirname, "../.."); +const cleanupDirectories: string[] = []; +afterEach(async () => { + await Promise.all( + cleanupDirectories + .splice(0) + .map((directory) => rm(directory, { recursive: true, force: true })), + ); +}); + +describe("runner E2E report aggregation", () => { + it("selects the latest retry and enforces cleanup and pass evidence", async () => { + const root = await mkdtemp( + path.join(os.tmpdir(), "runner-e2e-report-test-"), + ); + cleanupDirectories.push(root); + const executionId = "legacy-codex.local.message-marker"; + const base: RunnerE2EResult = { + schema: "paperclip.runner-e2e.result/v1", + executionId, + attempt: 2, + status: "passed", + profileId: "legacy-codex", + environmentId: "local", + caseId: "message-marker", + provider: "codex", + model: "fixture-model", + runtimeMode: "legacy", + startedAt: "2026-08-26T00:00:00.000Z", + finishedAt: "2026-08-26T00:00:01.000Z", + durationMs: 1_000, + runIds: ["run-2"], + usage: { + inputTokens: 1_250, + outputTokens: 75, + cachedInputTokens: 500, + costUsd: 0.0125, + }, + cleanup: "passed", + }; + for (const attempt of [1, 2]) { + const directory = path.join(root, `attempt-${attempt}`); + await mkdir(directory, { recursive: true }); + const result = + attempt === 1 + ? { + ...base, + attempt, + status: "failed" as const, + failureClass: "transient_infrastructure" as const, + } + : { + ...base, + matcherResults: [ + { + matcher: { + kind: "message_contains" as const, + expected: "PAPERCLIP_E2E_OK", + }, + passed: true, + detail: "matched", + }, + ], + screenshots: [ + { + id: "final-state", + label: "Final visible task state", + file: "final-state.png", + }, + ], + }; + await writeFile( + path.join(directory, "result.json"), + JSON.stringify(result), + ); + if (attempt === 2) { + await writeFile(path.join(directory, "final-state.png"), "fake-png"); + } + await writeFile( + path.join(directory, "evidence-manifest.json"), + JSON.stringify({ + files: attempt === 2 ? ["final-state.png"] : [], + leaks: [], + missing: [], + }), + ); + } + const staleDuplicate = path.join(root, "attempt-2-stale-duplicate"); + await mkdir(staleDuplicate, { recursive: true }); + await writeFile( + path.join(staleDuplicate, "result.json"), + JSON.stringify({ + ...base, + status: "failed", + failureClass: "candidate_failure", + finishedAt: "2026-08-26T00:00:00.500Z", + }), + ); + await writeFile( + path.join(staleDuplicate, "evidence-manifest.json"), + JSON.stringify({ files: [], leaks: [], missing: [] }), + ); + const output = path.join(root, "merged"); + await execFileAsync( + process.execPath, + [ + path.join(repositoryRoot, "cli/node_modules/tsx/dist/cli.mjs"), + path.join(repositoryRoot, "tests/runner-e2e/report.ts"), + ], + { + cwd: repositoryRoot, + env: { + ...process.env, + PAPERCLIP_RUNNER_E2E_REPORT_ROOT: root, + PAPERCLIP_RUNNER_E2E_REPORT_OUT: output, + PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: JSON.stringify([executionId]), + }, + }, + ); + const normalized = JSON.parse( + await readFile(path.join(output, "normalized-results.json"), "utf8"), + ); + expect(normalized).toMatchObject({ + schema: "paperclip.runner-e2e.campaign/v2", + selected: 1, + executed: 1, + passed: 1, + failed: 0, + retries: 1, + cleanupPassed: true, + }); + expect(normalized.billing).toMatchObject({ + reportedLlmCostUsd: 0.0125, + llm: { + inputTokens: 1_250, + outputTokens: 75, + runsWithReportedCost: 1, + }, + }); + expect(normalized.results[0]).toMatchObject({ + attempt: 2, + evidenceValid: true, + }); + const dashboard = await readFile( + path.join(output, "dashboard.html"), + "utf8", + ); + expect(dashboard).toContain("Runner Full-Stack E2E"); + expect(dashboard).toContain(executionId); + expect(dashboard).toContain("case-passed"); + expect(dashboard).toContain( + "core-compatibility.runner-acpx-codex.daytona.message-marker", + ); + expect(dashboard).toContain("case-not-selected"); + expect(dashboard).toContain(" { + const root = await mkdtemp( + path.join(os.tmpdir(), "runner-e2e-report-rerun-test-"), + ); + cleanupDirectories.push(root); + const executionId = "runner-opencode.local.ask-question"; + const common: RunnerE2EResult = { + schema: "paperclip.runner-e2e.result/v1", + executionId, + attempt: 2, + status: "failed", + failureClass: "candidate_failure", + profileId: "runner-opencode", + environmentId: "local", + caseId: "ask-question", + provider: "opencode", + model: "fixture-model", + runtimeMode: "native", + startedAt: "2026-08-26T00:00:00.000Z", + finishedAt: "2026-08-26T00:00:01.000Z", + durationMs: 1_000, + cleanup: "not_started", + }; + const failedDirectory = path.join(root, "old-campaign", "attempt-2"); + const passedDirectory = path.join(root, "new-campaign", "attempt-1"); + await mkdir(failedDirectory, { recursive: true }); + await mkdir(passedDirectory, { recursive: true }); + await writeFile( + path.join(failedDirectory, "result.json"), + JSON.stringify(common), + ); + await writeFile( + path.join(failedDirectory, "evidence-manifest.json"), + JSON.stringify({ files: [], leaks: [], missing: [] }), + ); + await writeFile( + path.join(passedDirectory, "result.json"), + JSON.stringify({ + ...common, + attempt: 1, + status: "passed", + failureClass: undefined, + finishedAt: "2026-08-26T00:00:02.000Z", + cleanup: "passed", + }), + ); + await writeFile( + path.join(passedDirectory, "evidence-manifest.json"), + JSON.stringify({ + files: ["final-state.png"], + leaks: [], + missing: [], + }), + ); + await writeFile(path.join(passedDirectory, "final-state.png"), "fake-png"); + const output = path.join(root, "merged"); + await execFileAsync( + process.execPath, + [ + path.join(repositoryRoot, "cli/node_modules/tsx/dist/cli.mjs"), + path.join(repositoryRoot, "tests/runner-e2e/report.ts"), + ], + { + cwd: repositoryRoot, + env: { + ...process.env, + PAPERCLIP_RUNNER_E2E_REPORT_ROOT: root, + PAPERCLIP_RUNNER_E2E_REPORT_OUT: output, + PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: JSON.stringify([executionId]), + }, + }, + ); + const normalized = JSON.parse( + await readFile(path.join(output, "normalized-results.json"), "utf8"), + ); + expect(normalized).toMatchObject({ passed: 1, failed: 0 }); + expect(normalized.results[0]).toMatchObject({ + attempt: 1, + status: "passed", + evidenceValid: true, + }); + }); + + it("constructs the public root JUnit from fixed markup and escaped fields", async () => { + const root = await mkdtemp( + path.join(os.tmpdir(), "runner-e2e-report-junit-test-"), + ); + cleanupDirectories.push(root); + const executionId = "legacy-codex.local.message-marker"; + const directory = path.join(root, "attempt-1"); + await mkdir(directory, { recursive: true }); + await writeFile( + path.join(directory, "result.json"), + JSON.stringify({ + schema: "paperclip.runner-e2e.result/v1", + executionId, + attempt: 1, + status: "failed", + failureClass: "candidate_failure", + error: `provider said \">&`, + profileId: "legacy-codex", + environmentId: "local", + caseId: "message-marker", + provider: "codex", + model: "fixture-model", + runtimeMode: "legacy", + startedAt: "2026-08-26T00:00:00.000Z", + finishedAt: "2026-08-26T00:00:01.000Z", + durationMs: 1_000, + cleanup: "passed", + } satisfies RunnerE2EResult), + ); + await writeFile( + path.join(directory, "evidence-manifest.json"), + JSON.stringify({ files: [], leaks: [], missing: [] }), + ); + const output = path.join(root, "merged"); + + await expect( + execFileAsync( + process.execPath, + [ + path.join(repositoryRoot, "cli/node_modules/tsx/dist/cli.mjs"), + path.join(repositoryRoot, "tests/runner-e2e/report.ts"), + ], + { + cwd: repositoryRoot, + env: { + ...process.env, + PAPERCLIP_RUNNER_E2E_REPORT_ROOT: root, + PAPERCLIP_RUNNER_E2E_REPORT_OUT: output, + PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: JSON.stringify([executionId]), + }, + }, + ), + ).rejects.toBeDefined(); + + const junit = await readFile(path.join(output, "junit.xml"), "utf8"); + expect(junit).toContain( + `message="provider said "><script>alert(1)</script>&"`, + ); + expect(junit).not.toContain("