chore: use git tags to fetch secrets for unified test (#838)

* chore: use git tags to fetch secrets for unified test

* fix: test failure

* fix: override AUTH_USE_AUTH and SENTRY_ENABLED

* fix: upload traces

* chore: rm run on PR

* fix: rm bucket from logs

.github/workflows/unified-tests.yml

@@ -40,17 +40,75 @@ jobs:
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4
         with:
-          role-to-assume: arn:aws:iam::444554165670:role/GitHubActionsS3Role
+          role-to-assume: ${{ vars.AWS_OIDC_ROLE_ARN }}
           aws-region: us-east-1
           role-duration-seconds: 43200 # 12 hours
 
-      - name: Fetch secrets from AWS Secrets Manager
+      - name: Resolve secret ids from latest git tags
+        id: resolve-secret
+        env:
+          SECRET_PREFIX: ${{ secrets.STAGING_SECRET_PREFIX }}
+        run: |
+          set -euo pipefail
+          : "${SECRET_PREFIX:?STAGING_SECRET_PREFIX secret is not set for this environment}"
+          # Keep the secret-name prefix out of public CI logs.
+          echo "::add-mask::${SECRET_PREFIX}"
+
+          # Two newest v<major.minor.patch> tags, highest first (tags are public).
+          versions="$(git ls-remote --tags origin 'v*' \
+            | sed -n 's#.*refs/tags/v\([0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\)$#\1#p' \
+            | sort -t. -k1,1nr -k2,2nr -k3,3nr -u)"
+          latest="$(printf '%s\n' "$versions" | sed -n '1p')"
+          second="$(printf '%s\n' "$versions" | sed -n '2p')"
+          if [ -z "${latest:-}" ]; then
+            echo "::error::No v<semver> git tags found to resolve a secret version"
+            exit 1
+          fi
+
+          latest_id="${SECRET_PREFIX}${latest}"
+          echo "::add-mask::${latest_id}"
+          echo "latest-id=${latest_id}" >> "$GITHUB_OUTPUT"
+          echo "Latest version: ${latest}"
+          if [ -n "${second:-}" ]; then
+            second_id="${SECRET_PREFIX}${second}"
+            echo "::add-mask::${second_id}"
+            echo "second-id=${second_id}" >> "$GITHUB_OUTPUT"
+            echo "Fallback version: ${second}"
+          fi
+
+      # Fetch the latest tag's secret. continue-on-error so a not-yet-published
+      # latest falls through to the second-latest instead of failing the job.
+      - name: Fetch staging secret (latest)
+        id: fetch-latest
+        continue-on-error: true
         uses: aws-actions/aws-secretsmanager-get-secrets@v2
         with:
           secret-ids: |
-            ,testing/unified/tests
+            ,${{ steps.resolve-secret.outputs.latest-id }}
           parse-json-secrets: true
 
+      # Runs only if the latest fetch failed; this one is NOT continue-on-error,
+      # so if the fallback also fails the job fails loudly.
+      - name: Fetch staging secret (fallback to second-latest)
+        if: steps.fetch-latest.outcome == 'failure' && steps.resolve-secret.outputs.second-id != ''
+        uses: aws-actions/aws-secretsmanager-get-secrets@v2
+        with:
+          secret-ids: |
+            ,${{ steps.resolve-secret.outputs.second-id }}
+          parse-json-secrets: true
+
+      # Configure the test environment. Disables auth/Sentry/CloudEvents telemetry
+      # (their endpoints aren't reachable from CI), and points REASONING_TRACES_FILE
+      # at a shared path so the API + deriver record full LLM I/O for auditing — the
+      # runner uploads it to S3. Written after the fetch steps so these win over the
+      # values loaded from Secrets Manager (last $GITHUB_ENV write wins).
+      - name: Configure test environment
+        run: |
+          echo "AUTH_USE_AUTH=false" >> "$GITHUB_ENV"
+          echo "SENTRY_ENABLED=false" >> "$GITHUB_ENV"
+          echo "TELEMETRY_ENABLED=false" >> "$GITHUB_ENV"
+          echo "REASONING_TRACES_FILE=unified-reasoning-traces.jsonl" >> "$GITHUB_ENV"
+
       - name: Verify Docker is available
         run: docker info
 

tests/unified/runner.py

@@ -119,6 +119,7 @@ async def save_results_to_s3(
         # Create comprehensive results object
         timestamp = datetime.now(timezone.utc).isoformat()
         github_run_id = os.getenv("GITHUB_RUN_ID", "local")
+        github_run_attempt = os.getenv("GITHUB_RUN_ATTEMPT", "1")
         github_sha = os.getenv("GITHUB_SHA", "unknown")
         github_ref = os.getenv("GITHUB_REF_NAME", "unknown")
 
@@ -132,6 +133,7 @@ async def save_results_to_s3(
             },
             "metadata": {
                 "github_run_id": github_run_id,
+                "github_run_attempt": github_run_attempt,
                 "github_sha": github_sha,
                 "github_ref": github_ref,
             },
@@ -145,31 +147,59 @@ async def save_results_to_s3(
             ],
         }
 
+        # One "folder" per run: <prefix>/<date>/<run>/ holding results.json plus
+        # the reasoning-trace file(s), so a run's summary and full LLM I/O live together.
         date_str = datetime.now(timezone.utc).strftime("%Y-%m-%d")
         sha_short = github_sha[:7] if github_sha != "unknown" else "unknown"
-        ref_name = github_ref if github_ref != "unknown" else "unknown"
-        key = f"{s3_prefix}/{date_str}-{ref_name}-{sha_short}.json"
+        ref_slug = github_ref.replace("/", "-")  # branch names may contain "/"
+        run_slug = f"{ref_slug}-{sha_short}-{github_run_id}-{github_run_attempt}"
+        run_prefix = f"{s3_prefix}/{date_str}/{run_slug}"
+        results_key = f"{run_prefix}/results.json"
 
         s3_client = boto3.client("s3", region_name=aws_region)  # pyright: ignore
         s3_client.put_object(  # pyright: ignore
             Bucket=s3_bucket,
-            Key=key,
+            Key=results_key,
             Body=json.dumps(comprehensive_results, indent=2).encode("utf-8"),
             ContentType="application/json",
         )
+        logger.info(f"Saved test results to S3 key {results_key}")
+
+        # Upload the reasoning traces (full LLM/deriver I/O) captured this run. The
+        # API and deriver both append to REASONING_TRACES_FILE (file-locked). Use
+        # upload_file so large trace files stream via multipart instead of buffering.
+        traces_path_str = os.getenv("REASONING_TRACES_FILE")
+        if traces_path_str:
+            traces_path = Path(traces_path_str)
+            if traces_path.is_file() and traces_path.stat().st_size > 0:
+                traces_key = f"{run_prefix}/{traces_path.name}"
+                try:
+                    s3_client.upload_file(  # pyright: ignore
+                        str(traces_path),
+                        s3_bucket,
+                        traces_key,
+                        ExtraArgs={"ContentType": "application/x-ndjson"},
+                    )
+                    logger.info(f"Saved reasoning traces to S3 key {traces_key}")
+                except Exception as e:
+                    logger.error(
+                        f"Failed to upload reasoning traces: {e}", exc_info=True
+                    )
+            else:
+                logger.warning(
+                    f"REASONING_TRACES_FILE={traces_path} is missing or empty; no traces uploaded"
+                )
 
         try:
             url: str = s3_client.generate_presigned_url(  # pyright: ignore
                 "get_object",
-                Params={"Bucket": s3_bucket, "Key": key},
+                Params={"Bucket": s3_bucket, "Key": results_key},
                 ExpiresIn=259200,  # 3 days
             )
-            logger.info(f"Saved test results to s3://{s3_bucket}/{key}")
-            return url, key  # pyright: ignore
+            return url, results_key  # pyright: ignore
         except Exception as e:
             logger.warning(f"Could not generate S3 presigned URL: {e}")
-            logger.info(f"Saved test results to s3://{s3_bucket}/{key}")
-            return None, key
+            return None, results_key
 
     except Exception as e:
         logger.error(f"Failed to save results to S3: {e}", exc_info=True)