231 lines
8.7 KiB
YAML
231 lines
8.7 KiB
YAML
name: Unified Tests (Fly Runner)
|
|
|
|
on:
|
|
push:
|
|
branches: [main]
|
|
paths:
|
|
- 'src/**'
|
|
- 'tests/**'
|
|
# Manual trigger for PRs: add the `run-unified-tests` label to run the suite
|
|
# against the PR's merge commit. The label is purged as soon as the run
|
|
# starts so it can be re-added to trigger another run.
|
|
pull_request:
|
|
types: [labeled]
|
|
|
|
# Cap spend: at most one active run per PR (per ref for push). Re-triggering
|
|
# a PR run cancels the in-flight one instead of stacking Fly machines; pushes
|
|
# to main queue instead of cancelling so main CI results aren't lost. The
|
|
# cleanup-machine job runs `if: always()`, which still executes on cancelled
|
|
# runs, so a cancelled run's Fly machine and runner are still torn down.
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
|
cancel-in-progress: ${{ github.event_name != 'push' }}
|
|
|
|
permissions:
|
|
contents: read
|
|
actions: read
|
|
|
|
jobs:
|
|
# Only code owners (.github/CODEOWNERS) may trigger the suite manually via
|
|
# the label; the gate also purges the trigger label so it can be re-added
|
|
# to trigger another run.
|
|
gate:
|
|
name: Gate manual trigger
|
|
permissions:
|
|
contents: read
|
|
pull-requests: write
|
|
uses: ./.github/workflows/manual-trigger-gate.yml
|
|
with:
|
|
label: run-unified-tests
|
|
|
|
start-runner:
|
|
name: Start Fly Runner
|
|
needs: gate
|
|
# always() lets this run on push events, where the gate's jobs are skipped.
|
|
# Label adds other than run-unified-tests trigger the workflow but skip
|
|
# every job here; the run-unified-tests label additionally requires the
|
|
# gate's CODEOWNERS check to have passed.
|
|
if: >-
|
|
always() &&
|
|
(github.event_name == 'push' ||
|
|
(github.event.label.name == 'run-unified-tests' &&
|
|
needs.gate.outputs.authorized == 'true'))
|
|
uses: ./.github/workflows/start-fly-runner.yml
|
|
secrets: inherit
|
|
|
|
unified-tests:
|
|
name: Run Unified Tests
|
|
runs-on: ${{ fromJSON(format('[{0}]', needs.start-runner.outputs.runner-labels)) }}
|
|
needs: start-runner
|
|
if: needs.start-runner.outputs.runner-ready == 'true'
|
|
timeout-minutes: 90
|
|
environment: unified-tests
|
|
permissions:
|
|
id-token: write # Required for OIDC authentication with AWS
|
|
contents: read
|
|
|
|
env:
|
|
PYTHONUNBUFFERED: "1"
|
|
TEST_DISCORD_WEBHOOK_URL: ${{ secrets.TEST_DISCORD_WEBHOOK_URL }}
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
with:
|
|
ref: ${{ github.sha }}
|
|
|
|
- name: Configure AWS credentials
|
|
uses: aws-actions/configure-aws-credentials@v4
|
|
with:
|
|
role-to-assume: ${{ vars.AWS_OIDC_ROLE_ARN }}
|
|
aws-region: us-east-1
|
|
role-duration-seconds: 43200 # 12 hours
|
|
|
|
# Resolves secret ids from the newest release tags, fetches the newest
|
|
# available staging secret into the job env, and fails if none loaded.
|
|
- name: Load staging secrets
|
|
uses: ./.github/actions/load-staging-secrets
|
|
with:
|
|
secret-prefix: ${{ secrets.STAGING_SECRET_PREFIX }}
|
|
|
|
# Layer test-specific overrides on top of the staging secret. The staging
|
|
# dotenv tracks the deployed release and can drift from what main's config
|
|
# expects; the TESTING_SECRET_ID secret holds only the keys (flat JSON,
|
|
# exact env var names) the unified tests need to pin. The get-secrets
|
|
# action refuses to inject an env var that already exists, so the
|
|
# overrides are fetched under a prefix alias here and promoted over the
|
|
# staging values in the next step.
|
|
- name: Fetch testing secret overrides
|
|
uses: aws-actions/aws-secretsmanager-get-secrets@v2
|
|
with:
|
|
secret-ids: |
|
|
HONCHO_TEST_OVERRIDE,${{ secrets.TESTING_SECRET_ID }}
|
|
parse-json-secrets: true
|
|
|
|
# Re-export each HONCHO_TEST_OVERRIDE_* var under its real name; the
|
|
# later $GITHUB_ENV write wins over the value loaded from the staging
|
|
# secret. Values are already masked by the fetch step above.
|
|
- name: Apply testing secret overrides
|
|
run: |
|
|
set -euo pipefail
|
|
applied=0
|
|
while IFS= read -r -d '' entry; do
|
|
name="${entry%%=*}"
|
|
value="${entry#*=}"
|
|
case "$name" in
|
|
HONCHO_TEST_OVERRIDE_*)
|
|
target="${name#HONCHO_TEST_OVERRIDE_}"
|
|
{
|
|
echo "${target}<<__HONCHO_OVERRIDE_EOF__"
|
|
printf '%s\n' "$value"
|
|
echo "__HONCHO_OVERRIDE_EOF__"
|
|
} >> "$GITHUB_ENV"
|
|
echo "Overriding ${target}"
|
|
applied=$((applied + 1))
|
|
;;
|
|
esac
|
|
done < <(env -0)
|
|
echo "Applied ${applied} override(s)"
|
|
|
|
# Configure the test environment. Disables auth/Sentry/CloudEvents telemetry
|
|
# (their endpoints aren't reachable from CI), and points REASONING_TRACES_FILE
|
|
# at a shared path so the API + deriver record full LLM I/O for auditing — the
|
|
# runner uploads it to S3. Written after the fetch steps so these win over the
|
|
# values loaded from Secrets Manager (last $GITHUB_ENV write wins). Stale
|
|
# config keys loaded from the staging secret (e.g. settings that have since
|
|
# been renamed or removed on main) must always be ignored by the app config.
|
|
- name: Configure test environment
|
|
run: |
|
|
{
|
|
echo "AUTH_USE_AUTH=false"
|
|
echo "SENTRY_ENABLED=false"
|
|
echo "TELEMETRY_ENABLED=false"
|
|
echo "REASONING_TRACES_FILE=unified-reasoning-traces.jsonl"
|
|
} >> "$GITHUB_ENV"
|
|
|
|
- name: Verify Docker is available
|
|
run: docker info
|
|
|
|
- name: Verify uv and Python
|
|
run: |
|
|
uv --version
|
|
python3.12 --version
|
|
which python3.12
|
|
|
|
- name: Install the project
|
|
run: uv sync --all-extras
|
|
|
|
- name: Run unified tests
|
|
run: uv run python -m tests.unified.run
|
|
|
|
cleanup-machine:
|
|
name: Cleanup Fly Machine and Runner
|
|
runs-on: ubuntu-latest
|
|
needs: [start-runner, unified-tests]
|
|
if: always() && needs.start-runner.outputs.machine-id != ''
|
|
env:
|
|
FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN_TESTING }}
|
|
GITHUB_TOKEN: ${{ secrets.GH_TOKEN_ACTIONS }}
|
|
FLY_RUNNER_APP: ivysaur
|
|
steps:
|
|
- name: Setup Fly CLI
|
|
uses: superfly/flyctl-actions/setup-flyctl@1.5
|
|
|
|
- name: Cleanup fly machine
|
|
run: |
|
|
set -euo pipefail
|
|
MACHINE_ID="${{ needs.start-runner.outputs.machine-id }}"
|
|
if [ -z "$MACHINE_ID" ]; then
|
|
echo "No machine ID provided, skipping Fly cleanup."
|
|
exit 0
|
|
fi
|
|
|
|
echo "🧹 Cleaning up machine: $MACHINE_ID"
|
|
flyctl machines stop "$MACHINE_ID" -a "$FLY_RUNNER_APP" || echo "Machine may already be stopped"
|
|
flyctl machines destroy "$MACHINE_ID" -a "$FLY_RUNNER_APP" --force || echo "Failed to destroy machine"
|
|
|
|
- name: Cleanup GitHub runner
|
|
run: |
|
|
set -euo pipefail
|
|
RUNNER_NAME="${{ needs.start-runner.outputs.runner-name }}"
|
|
FALLBACK_LABEL="${{ github.run_id }}"
|
|
|
|
echo "🗑️ Cleaning up GitHub runner (name: ${RUNNER_NAME:-unknown}, label: ${FALLBACK_LABEL})"
|
|
|
|
RUNNERS_RESPONSE=$(curl -s \
|
|
-H "Authorization: Bearer $GITHUB_TOKEN" \
|
|
-H "Accept: application/vnd.github+json" \
|
|
"https://api.github.com/repos/${{ github.repository }}/actions/runners")
|
|
|
|
if echo "$RUNNERS_RESPONSE" | grep -q '"message"'; then
|
|
echo "⚠️ Failed to fetch runners: $(echo "$RUNNERS_RESPONSE" | jq -r '.message')"
|
|
exit 0
|
|
fi
|
|
|
|
RUNNER_ID=""
|
|
if [ -n "$RUNNER_NAME" ]; then
|
|
RUNNER_ID=$(echo "$RUNNERS_RESPONSE" | jq -r --arg name "$RUNNER_NAME" '.runners[]? | select(.name == $name) | .id')
|
|
fi
|
|
|
|
if [ -z "$RUNNER_ID" ]; then
|
|
RUNNER_ID=$(echo "$RUNNERS_RESPONSE" | jq -r --arg label "$FALLBACK_LABEL" '.runners[]? | select([.labels[].name] | index($label)) | .id' | head -n 1)
|
|
fi
|
|
|
|
if [ -z "$RUNNER_ID" ] || [ "$RUNNER_ID" = "null" ]; then
|
|
echo "⚠️ Runner not found, nothing to delete."
|
|
exit 0
|
|
fi
|
|
|
|
DELETE_RESPONSE=$(curl -s -w "%{http_code}" \
|
|
-X DELETE \
|
|
-H "Accept: application/vnd.github+json" \
|
|
-H "Authorization: Bearer $GITHUB_TOKEN" \
|
|
"https://api.github.com/repos/${{ github.repository }}/actions/runners/$RUNNER_ID")
|
|
|
|
HTTP_CODE="${DELETE_RESPONSE: -3}"
|
|
if [ "$HTTP_CODE" = "204" ]; then
|
|
echo "✅ Successfully deleted runner."
|
|
else
|
|
echo "⚠️ Failed to delete runner. HTTP code: $HTTP_CODE"
|
|
echo "Response: ${DELETE_RESPONSE%???}"
|
|
fi
|