honcho/.github/workflows/unified-tests.yml

name: Unified Tests (Fly Runner)

on:
  push:
    branches: [main]
    paths:
      - 'src/**'
      - 'tests/**'

permissions:
  contents: read
  actions: read

jobs:
  start-runner:
    name: Start Fly Runner
    uses: ./.github/workflows/start-fly-runner.yml
    secrets: inherit

  unified-tests:
    name: Run Unified Tests
    runs-on: ${{ fromJSON(format('[{0}]', needs.start-runner.outputs.runner-labels)) }}
    needs: start-runner
    if: needs.start-runner.outputs.runner-ready == 'true'
    timeout-minutes: 90
    environment: unified-tests
    permissions:
      id-token: write  # Required for OIDC authentication with AWS
      contents: read

    env:
      PYTHONUNBUFFERED: "1"
      TEST_DISCORD_WEBHOOK_URL: ${{ secrets.TEST_DISCORD_WEBHOOK_URL }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          ref: ${{ github.sha }}

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ vars.AWS_OIDC_ROLE_ARN }}
          aws-region: us-east-1
          role-duration-seconds: 43200 # 12 hours

      - name: Resolve secret ids from latest git tags
        id: resolve-secret
        env:
          SECRET_PREFIX: ${{ secrets.STAGING_SECRET_PREFIX }}
        run: |
          set -euo pipefail
          : "${SECRET_PREFIX:?STAGING_SECRET_PREFIX secret is not set for this environment}"
          # Keep the secret-name prefix out of public CI logs.
          echo "::add-mask::${SECRET_PREFIX}"

          # Two newest v<major.minor.patch> tags, highest first (tags are public).
          versions="$(git ls-remote --tags origin 'v*' \
            | sed -n 's#.*refs/tags/v\([0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\)$#\1#p' \
            | sort -t. -k1,1nr -k2,2nr -k3,3nr -u)"
          latest="$(printf '%s\n' "$versions" | sed -n '1p')"
          second="$(printf '%s\n' "$versions" | sed -n '2p')"
          if [ -z "${latest:-}" ]; then
            echo "::error::No v<semver> git tags found to resolve a secret version"
            exit 1
          fi

          latest_id="${SECRET_PREFIX}${latest}"
          echo "::add-mask::${latest_id}"
          echo "latest-id=${latest_id}" >> "$GITHUB_OUTPUT"
          echo "Latest version: ${latest}"
          if [ -n "${second:-}" ]; then
            second_id="${SECRET_PREFIX}${second}"
            echo "::add-mask::${second_id}"
            echo "second-id=${second_id}" >> "$GITHUB_OUTPUT"
            echo "Fallback version: ${second}"
          fi          

      # Fetch the latest tag's secret. continue-on-error so a not-yet-published
      # latest falls through to the second-latest instead of failing the job.
      - name: Fetch staging secret (latest)
        id: fetch-latest
        continue-on-error: true
        uses: aws-actions/aws-secretsmanager-get-secrets@v2
        with:
          secret-ids: |
            ,${{ steps.resolve-secret.outputs.latest-id }}            
          parse-json-secrets: true

      # Runs only if the latest fetch failed; this one is NOT continue-on-error,
      # so if the fallback also fails the job fails loudly.
      - name: Fetch staging secret (fallback to second-latest)
        if: steps.fetch-latest.outcome == 'failure' && steps.resolve-secret.outputs.second-id != ''
        uses: aws-actions/aws-secretsmanager-get-secrets@v2
        with:
          secret-ids: |
            ,${{ steps.resolve-secret.outputs.second-id }}            
          parse-json-secrets: true

      # Configure the test environment. Disables auth/Sentry/CloudEvents telemetry
      # (their endpoints aren't reachable from CI), and points REASONING_TRACES_FILE
      # at a shared path so the API + deriver record full LLM I/O for auditing — the
      # runner uploads it to S3. Written after the fetch steps so these win over the
      # values loaded from Secrets Manager (last $GITHUB_ENV write wins).
      - name: Configure test environment
        run: |
          echo "AUTH_USE_AUTH=false" >> "$GITHUB_ENV"
          echo "SENTRY_ENABLED=false" >> "$GITHUB_ENV"
          echo "TELEMETRY_ENABLED=false" >> "$GITHUB_ENV"
          echo "REASONING_TRACES_FILE=unified-reasoning-traces.jsonl" >> "$GITHUB_ENV"          

      - name: Verify Docker is available
        run: docker info

      - name: Verify uv and Python
        run: |
          uv --version
          python3.12 --version
          which python3.12          

      - name: Install the project
        run: uv sync --all-extras

      - name: Run unified tests
        run: uv run python -m tests.unified.run

  cleanup-machine:
    name: Cleanup Fly Machine and Runner
    runs-on: ubuntu-latest
    needs: [start-runner, unified-tests]
    if: always() && needs.start-runner.outputs.machine-id != ''
    env:
      FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN_TESTING }}
      GITHUB_TOKEN: ${{ secrets.GH_TOKEN_ACTIONS }}
      FLY_RUNNER_APP: ivysaur
    steps:
      - name: Setup Fly CLI
        uses: superfly/flyctl-actions/setup-flyctl@1.5

      - name: Cleanup fly machine
        run: |
          set -euo pipefail
          MACHINE_ID="${{ needs.start-runner.outputs.machine-id }}"
          if [ -z "$MACHINE_ID" ]; then
            echo "No machine ID provided, skipping Fly cleanup."
            exit 0
          fi

          echo "🧹 Cleaning up machine: $MACHINE_ID"
          flyctl machines stop "$MACHINE_ID" -a "$FLY_RUNNER_APP" || echo "Machine may already be stopped"
          flyctl machines destroy "$MACHINE_ID" -a "$FLY_RUNNER_APP" --force || echo "Failed to destroy machine"          

      - name: Cleanup GitHub runner
        run: |
          set -euo pipefail
          RUNNER_NAME="${{ needs.start-runner.outputs.runner-name }}"
          FALLBACK_LABEL="${{ github.run_id }}"

          echo "🗑️ Cleaning up GitHub runner (name: ${RUNNER_NAME:-unknown}, label: ${FALLBACK_LABEL})"

          RUNNERS_RESPONSE=$(curl -s \
            -H "Authorization: Bearer $GITHUB_TOKEN" \
            -H "Accept: application/vnd.github+json" \
            "https://api.github.com/repos/${{ github.repository }}/actions/runners")

          if echo "$RUNNERS_RESPONSE" | grep -q '"message"'; then
            echo "⚠️ Failed to fetch runners: $(echo "$RUNNERS_RESPONSE" | jq -r '.message')"
            exit 0
          fi

          RUNNER_ID="" 
          if [ -n "$RUNNER_NAME" ]; then
            RUNNER_ID=$(echo "$RUNNERS_RESPONSE" | jq -r --arg name "$RUNNER_NAME" '.runners[]? | select(.name == $name) | .id')
          fi

          if [ -z "$RUNNER_ID" ]; then
            RUNNER_ID=$(echo "$RUNNERS_RESPONSE" | jq -r --arg label "$FALLBACK_LABEL" '.runners[]? | select([.labels[].name] | index($label)) | .id' | head -n 1)
          fi

          if [ -z "$RUNNER_ID" ] || [ "$RUNNER_ID" = "null" ]; then
            echo "⚠️ Runner not found, nothing to delete."
            exit 0
          fi

          DELETE_RESPONSE=$(curl -s -w "%{http_code}" \
            -X DELETE \
            -H "Accept: application/vnd.github+json" \
            -H "Authorization: Bearer $GITHUB_TOKEN" \
            "https://api.github.com/repos/${{ github.repository }}/actions/runners/$RUNNER_ID")

          HTTP_CODE="${DELETE_RESPONSE: -3}"
          if [ "$HTTP_CODE" = "204" ]; then
            echo "✅ Successfully deleted runner."
          else
            echo "⚠️ Failed to delete runner. HTTP code: $HTTP_CODE"
            echo "Response: ${DELETE_RESPONSE%???}"
          fi