name: Unified Tests (Fly Runner) on: push: branches: [main] paths: - 'src/**' - 'tests/**' permissions: contents: read actions: read jobs: start-runner: name: Start Fly Runner uses: ./.github/workflows/start-fly-runner.yml secrets: inherit unified-tests: name: Run Unified Tests runs-on: ${{ fromJSON(format('[{0}]', needs.start-runner.outputs.runner-labels)) }} needs: start-runner if: needs.start-runner.outputs.runner-ready == 'true' timeout-minutes: 90 environment: unified-tests permissions: id-token: write # Required for OIDC authentication with AWS contents: read env: PYTHONUNBUFFERED: "1" TEST_DISCORD_WEBHOOK_URL: ${{ secrets.TEST_DISCORD_WEBHOOK_URL }} steps: - name: Checkout repository uses: actions/checkout@v4 with: ref: ${{ github.sha }} - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ vars.AWS_OIDC_ROLE_ARN }} aws-region: us-east-1 role-duration-seconds: 43200 # 12 hours - name: Resolve secret ids from latest git tags id: resolve-secret env: SECRET_PREFIX: ${{ secrets.STAGING_SECRET_PREFIX }} run: | set -euo pipefail : "${SECRET_PREFIX:?STAGING_SECRET_PREFIX secret is not set for this environment}" # Keep the secret-name prefix out of public CI logs. echo "::add-mask::${SECRET_PREFIX}" # Two newest v tags, highest first (tags are public). versions="$(git ls-remote --tags origin 'v*' \ | sed -n 's#.*refs/tags/v\([0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\)$#\1#p' \ | sort -t. -k1,1nr -k2,2nr -k3,3nr -u)" latest="$(printf '%s\n' "$versions" | sed -n '1p')" second="$(printf '%s\n' "$versions" | sed -n '2p')" if [ -z "${latest:-}" ]; then echo "::error::No v git tags found to resolve a secret version" exit 1 fi latest_id="${SECRET_PREFIX}${latest}" echo "::add-mask::${latest_id}" echo "latest-id=${latest_id}" >> "$GITHUB_OUTPUT" echo "Latest version: ${latest}" if [ -n "${second:-}" ]; then second_id="${SECRET_PREFIX}${second}" echo "::add-mask::${second_id}" echo "second-id=${second_id}" >> "$GITHUB_OUTPUT" echo "Fallback version: ${second}" fi # Fetch the latest tag's secret. continue-on-error so a not-yet-published # latest falls through to the second-latest instead of failing the job. - name: Fetch staging secret (latest) id: fetch-latest continue-on-error: true uses: aws-actions/aws-secretsmanager-get-secrets@v2 with: secret-ids: | ,${{ steps.resolve-secret.outputs.latest-id }} parse-json-secrets: true # Runs only if the latest fetch failed; this one is NOT continue-on-error, # so if the fallback also fails the job fails loudly. - name: Fetch staging secret (fallback to second-latest) if: steps.fetch-latest.outcome == 'failure' && steps.resolve-secret.outputs.second-id != '' uses: aws-actions/aws-secretsmanager-get-secrets@v2 with: secret-ids: | ,${{ steps.resolve-secret.outputs.second-id }} parse-json-secrets: true # Configure the test environment. Disables auth/Sentry/CloudEvents telemetry # (their endpoints aren't reachable from CI), and points REASONING_TRACES_FILE # at a shared path so the API + deriver record full LLM I/O for auditing — the # runner uploads it to S3. Written after the fetch steps so these win over the # values loaded from Secrets Manager (last $GITHUB_ENV write wins). - name: Configure test environment run: | echo "AUTH_USE_AUTH=false" >> "$GITHUB_ENV" echo "SENTRY_ENABLED=false" >> "$GITHUB_ENV" echo "TELEMETRY_ENABLED=false" >> "$GITHUB_ENV" echo "REASONING_TRACES_FILE=unified-reasoning-traces.jsonl" >> "$GITHUB_ENV" - name: Verify Docker is available run: docker info - name: Verify uv and Python run: | uv --version python3.12 --version which python3.12 - name: Install the project run: uv sync --all-extras - name: Run unified tests run: uv run python -m tests.unified.run cleanup-machine: name: Cleanup Fly Machine and Runner runs-on: ubuntu-latest needs: [start-runner, unified-tests] if: always() && needs.start-runner.outputs.machine-id != '' env: FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN_TESTING }} GITHUB_TOKEN: ${{ secrets.GH_TOKEN_ACTIONS }} FLY_RUNNER_APP: ivysaur steps: - name: Setup Fly CLI uses: superfly/flyctl-actions/setup-flyctl@1.5 - name: Cleanup fly machine run: | set -euo pipefail MACHINE_ID="${{ needs.start-runner.outputs.machine-id }}" if [ -z "$MACHINE_ID" ]; then echo "No machine ID provided, skipping Fly cleanup." exit 0 fi echo "🧹 Cleaning up machine: $MACHINE_ID" flyctl machines stop "$MACHINE_ID" -a "$FLY_RUNNER_APP" || echo "Machine may already be stopped" flyctl machines destroy "$MACHINE_ID" -a "$FLY_RUNNER_APP" --force || echo "Failed to destroy machine" - name: Cleanup GitHub runner run: | set -euo pipefail RUNNER_NAME="${{ needs.start-runner.outputs.runner-name }}" FALLBACK_LABEL="${{ github.run_id }}" echo "🗑️ Cleaning up GitHub runner (name: ${RUNNER_NAME:-unknown}, label: ${FALLBACK_LABEL})" RUNNERS_RESPONSE=$(curl -s \ -H "Authorization: Bearer $GITHUB_TOKEN" \ -H "Accept: application/vnd.github+json" \ "https://api.github.com/repos/${{ github.repository }}/actions/runners") if echo "$RUNNERS_RESPONSE" | grep -q '"message"'; then echo "⚠️ Failed to fetch runners: $(echo "$RUNNERS_RESPONSE" | jq -r '.message')" exit 0 fi RUNNER_ID=""  if [ -n "$RUNNER_NAME" ]; then RUNNER_ID=$(echo "$RUNNERS_RESPONSE" | jq -r --arg name "$RUNNER_NAME" '.runners[]? | select(.name == $name) | .id') fi if [ -z "$RUNNER_ID" ]; then RUNNER_ID=$(echo "$RUNNERS_RESPONSE" | jq -r --arg label "$FALLBACK_LABEL" '.runners[]? | select([.labels[].name] | index($label)) | .id' | head -n 1) fi if [ -z "$RUNNER_ID" ] || [ "$RUNNER_ID" = "null" ]; then echo "⚠️ Runner not found, nothing to delete." exit 0 fi DELETE_RESPONSE=$(curl -s -w "%{http_code}" \ -X DELETE \ -H "Accept: application/vnd.github+json" \ -H "Authorization: Bearer $GITHUB_TOKEN" \ "https://api.github.com/repos/${{ github.repository }}/actions/runners/$RUNNER_ID") HTTP_CODE="${DELETE_RESPONSE: -3}" if [ "$HTTP_CODE" = "204" ]; then echo "✅ Successfully deleted runner." else echo "⚠️ Failed to delete runner. HTTP code: $HTTP_CODE" echo "Response: ${DELETE_RESPONSE%???}" fi