From 1c23a2e24ac4784077745a8d9d6fd8d961feaef0 Mon Sep 17 00:00:00 2001 From: Rajat Ahuja Date: Wed, 24 Jun 2026 10:29:52 -0400 Subject: [PATCH] chore: use git tags to fetch secrets for unified test (#838) * chore: use git tags to fetch secrets for unified test * fix: test failure * fix: override AUTH_USE_AUTH and SENTRY_ENABLED * fix: upload traces * chore: rm run on PR * fix: rm bucket from logs --- .github/workflows/unified-tests.yml | 64 +++++++++++++++++++++++++++-- tests/unified/runner.py | 46 +++++++++++++++++---- 2 files changed, 99 insertions(+), 11 deletions(-) diff --git a/.github/workflows/unified-tests.yml b/.github/workflows/unified-tests.yml index bfc27018..55b938e7 100644 --- a/.github/workflows/unified-tests.yml +++ b/.github/workflows/unified-tests.yml @@ -40,17 +40,75 @@ jobs: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: - role-to-assume: arn:aws:iam::444554165670:role/GitHubActionsS3Role + role-to-assume: ${{ vars.AWS_OIDC_ROLE_ARN }} aws-region: us-east-1 role-duration-seconds: 43200 # 12 hours - - name: Fetch secrets from AWS Secrets Manager + - name: Resolve secret ids from latest git tags + id: resolve-secret + env: + SECRET_PREFIX: ${{ secrets.STAGING_SECRET_PREFIX }} + run: | + set -euo pipefail + : "${SECRET_PREFIX:?STAGING_SECRET_PREFIX secret is not set for this environment}" + # Keep the secret-name prefix out of public CI logs. + echo "::add-mask::${SECRET_PREFIX}" + + # Two newest v tags, highest first (tags are public). + versions="$(git ls-remote --tags origin 'v*' \ + | sed -n 's#.*refs/tags/v\([0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\)$#\1#p' \ + | sort -t. -k1,1nr -k2,2nr -k3,3nr -u)" + latest="$(printf '%s\n' "$versions" | sed -n '1p')" + second="$(printf '%s\n' "$versions" | sed -n '2p')" + if [ -z "${latest:-}" ]; then + echo "::error::No v git tags found to resolve a secret version" + exit 1 + fi + + latest_id="${SECRET_PREFIX}${latest}" + echo "::add-mask::${latest_id}" + echo "latest-id=${latest_id}" >> "$GITHUB_OUTPUT" + echo "Latest version: ${latest}" + if [ -n "${second:-}" ]; then + second_id="${SECRET_PREFIX}${second}" + echo "::add-mask::${second_id}" + echo "second-id=${second_id}" >> "$GITHUB_OUTPUT" + echo "Fallback version: ${second}" + fi + + # Fetch the latest tag's secret. continue-on-error so a not-yet-published + # latest falls through to the second-latest instead of failing the job. + - name: Fetch staging secret (latest) + id: fetch-latest + continue-on-error: true uses: aws-actions/aws-secretsmanager-get-secrets@v2 with: secret-ids: | - ,testing/unified/tests + ,${{ steps.resolve-secret.outputs.latest-id }} parse-json-secrets: true + # Runs only if the latest fetch failed; this one is NOT continue-on-error, + # so if the fallback also fails the job fails loudly. + - name: Fetch staging secret (fallback to second-latest) + if: steps.fetch-latest.outcome == 'failure' && steps.resolve-secret.outputs.second-id != '' + uses: aws-actions/aws-secretsmanager-get-secrets@v2 + with: + secret-ids: | + ,${{ steps.resolve-secret.outputs.second-id }} + parse-json-secrets: true + + # Configure the test environment. Disables auth/Sentry/CloudEvents telemetry + # (their endpoints aren't reachable from CI), and points REASONING_TRACES_FILE + # at a shared path so the API + deriver record full LLM I/O for auditing — the + # runner uploads it to S3. Written after the fetch steps so these win over the + # values loaded from Secrets Manager (last $GITHUB_ENV write wins). + - name: Configure test environment + run: | + echo "AUTH_USE_AUTH=false" >> "$GITHUB_ENV" + echo "SENTRY_ENABLED=false" >> "$GITHUB_ENV" + echo "TELEMETRY_ENABLED=false" >> "$GITHUB_ENV" + echo "REASONING_TRACES_FILE=unified-reasoning-traces.jsonl" >> "$GITHUB_ENV" + - name: Verify Docker is available run: docker info diff --git a/tests/unified/runner.py b/tests/unified/runner.py index 297b7100..eaa78409 100644 --- a/tests/unified/runner.py +++ b/tests/unified/runner.py @@ -119,6 +119,7 @@ async def save_results_to_s3( # Create comprehensive results object timestamp = datetime.now(timezone.utc).isoformat() github_run_id = os.getenv("GITHUB_RUN_ID", "local") + github_run_attempt = os.getenv("GITHUB_RUN_ATTEMPT", "1") github_sha = os.getenv("GITHUB_SHA", "unknown") github_ref = os.getenv("GITHUB_REF_NAME", "unknown") @@ -132,6 +133,7 @@ async def save_results_to_s3( }, "metadata": { "github_run_id": github_run_id, + "github_run_attempt": github_run_attempt, "github_sha": github_sha, "github_ref": github_ref, }, @@ -145,31 +147,59 @@ async def save_results_to_s3( ], } + # One "folder" per run: /// holding results.json plus + # the reasoning-trace file(s), so a run's summary and full LLM I/O live together. date_str = datetime.now(timezone.utc).strftime("%Y-%m-%d") sha_short = github_sha[:7] if github_sha != "unknown" else "unknown" - ref_name = github_ref if github_ref != "unknown" else "unknown" - key = f"{s3_prefix}/{date_str}-{ref_name}-{sha_short}.json" + ref_slug = github_ref.replace("/", "-") # branch names may contain "/" + run_slug = f"{ref_slug}-{sha_short}-{github_run_id}-{github_run_attempt}" + run_prefix = f"{s3_prefix}/{date_str}/{run_slug}" + results_key = f"{run_prefix}/results.json" s3_client = boto3.client("s3", region_name=aws_region) # pyright: ignore s3_client.put_object( # pyright: ignore Bucket=s3_bucket, - Key=key, + Key=results_key, Body=json.dumps(comprehensive_results, indent=2).encode("utf-8"), ContentType="application/json", ) + logger.info(f"Saved test results to S3 key {results_key}") + + # Upload the reasoning traces (full LLM/deriver I/O) captured this run. The + # API and deriver both append to REASONING_TRACES_FILE (file-locked). Use + # upload_file so large trace files stream via multipart instead of buffering. + traces_path_str = os.getenv("REASONING_TRACES_FILE") + if traces_path_str: + traces_path = Path(traces_path_str) + if traces_path.is_file() and traces_path.stat().st_size > 0: + traces_key = f"{run_prefix}/{traces_path.name}" + try: + s3_client.upload_file( # pyright: ignore + str(traces_path), + s3_bucket, + traces_key, + ExtraArgs={"ContentType": "application/x-ndjson"}, + ) + logger.info(f"Saved reasoning traces to S3 key {traces_key}") + except Exception as e: + logger.error( + f"Failed to upload reasoning traces: {e}", exc_info=True + ) + else: + logger.warning( + f"REASONING_TRACES_FILE={traces_path} is missing or empty; no traces uploaded" + ) try: url: str = s3_client.generate_presigned_url( # pyright: ignore "get_object", - Params={"Bucket": s3_bucket, "Key": key}, + Params={"Bucket": s3_bucket, "Key": results_key}, ExpiresIn=259200, # 3 days ) - logger.info(f"Saved test results to s3://{s3_bucket}/{key}") - return url, key # pyright: ignore + return url, results_key # pyright: ignore except Exception as e: logger.warning(f"Could not generate S3 presigned URL: {e}") - logger.info(f"Saved test results to s3://{s3_bucket}/{key}") - return None, key + return None, results_key except Exception as e: logger.error(f"Failed to save results to S3: {e}", exc_info=True)