From 2aec845c1e38ebd61b502cbcbd3b7a65be722b23 Mon Sep 17 00:00:00 2001
From: Garry Tan <garrytan@gmail.com>
Date: Fri, 14 Aug 2026 15:57:25 -0700
Subject: [PATCH] ci(evals): skip eval jobs deterministically on fork PRs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Fork PRs never receive repository secrets, so every API-calling eval failed
at SDK auth — but only when Docker-cache luck let the jobs start at all,
making fork PRs randomly red or grey. Skip the eval and report jobs
explicitly for fork-origin PRs, keep the image BUILD (validates
Dockerfile.ci changes) without the push a fork token can't perform, and
leave full coverage for same-repo PRs, pushes, and dispatches.

Contributed by @andrey-esipov (PR #2345).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 .github/workflows/evals.yml | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml
index fa911176d..3b30271e6 100644
--- a/.github/workflows/evals.yml
+++ b/.github/workflows/evals.yml
@@ -45,19 +45,30 @@ jobs:
       - if: steps.check.outputs.exists == 'false'
         run: cp package.json bun.lock .github/docker/
 
+      # A fork PR's GITHUB_TOKEN only has `packages: read`, so pushing fails.
+      # Still BUILD (validates Dockerfile.ci changes), just don't publish. This
+      # job intentionally keeps no `if:` so fork PRs still get one real, honest
+      # green check here instead of a run where every job is grey.
       - if: steps.check.outputs.exists == 'false'
         uses: docker/build-push-action@v6
         with:
           context: .github/docker
           file: .github/docker/Dockerfile.ci
-          push: true
+          push: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
           tags: |
             ${{ steps.meta.outputs.tag }}
             ${{ env.IMAGE }}:latest
 
+  # Fork PRs never receive repository secrets (ANTHROPIC_API_KEY et al), so every
+  # API-calling eval fails at SDK auth before a model runs. Skip deterministically
+  # rather than leaving the outcome to Docker-cache luck: a warm cache let these
+  # run and fail, a cold one made build-image fail its push and the shards skip.
+  # Same-repo PRs, pushes, and workflow_dispatch keep full coverage. Fork work
+  # gets real coverage via a trusted base-repo branch.
   evals:
     runs-on: ${{ matrix.suite.runner || 'ubicloud-standard-8' }}
     needs: build-image
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
     container:
       image: ${{ needs.build-image.outputs.image-tag }}
       credentials:
@@ -276,7 +287,7 @@ jobs:
   report:
     runs-on: ubicloud-standard-8
     needs: evals
-    if: always() && github.event_name == 'pull_request'
+    if: always() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
     timeout-minutes: 5
     permissions:
       contents: read