From 9a9ead29e4d7d252ba04ab7d77e184b3ff225128 Mon Sep 17 00:00:00 2001
From: Garry Tan <garrytan@gmail.com>
Date: Sat, 15 Aug 2026 18:00:45 -0700
Subject: [PATCH] =?UTF-8?q?ci(evals):=20per-row=20retry=20override=20?=
 =?UTF-8?q?=E2=80=94=20two=20receipted=20rows=20keep=20the=20third=20attem?=
 =?UTF-8?q?pt?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Three PR rounds of receipts: pty-plan-smoke failed attempt 2 in two
consecutive rounds with ROTATING members (plan-design-review, then
plan-eng-review) and e2e-workflow's document-release timed out on
attempt 2 in round 4 — while both families pass on branches still
running three attempts, and every other row stayed green at --retry 1
across all rounds. Matrix rows gain an optional retries field
(default 1); only these two rows set 2, keeping the measured
retry-amplification win everywhere else.
---
 .github/workflows/evals.yml | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml
index 56253b223..5400b19d4 100644
--- a/.github/workflows/evals.yml
+++ b/.github/workflows/evals.yml
@@ -123,6 +123,10 @@ jobs:
             file: test/skill-e2e-review-attribution.test.ts
           - name: e2e-workflow
             file: test/skill-e2e-workflow.test.ts
+            # Earned its extra attempt with receipts: document-release is a
+            # long multi-step E2E that timed out on attempt 2 under in-shard
+            # concurrency (PR #2593 round 4) while passing other rounds.
+            retries: 2
           # Rehomed from the deleted pre-split monolith (its filename never
           # matched the skill-e2e-* glob, so these gate tests silently never
           # ran). Both files hold gate-tier tests: review/plan-eng coverage
@@ -147,6 +151,12 @@ jobs:
           - name: e2e-pty-plan-smoke
             file: test/skill-e2e-office-hours-auto-mode.test.ts test/skill-e2e-plan-mode-no-op.test.ts
             timeout: 35
+            # The documented contention-heavy PTY family: ROTATING members
+            # failed attempt 2 in consecutive PR #2593 rounds
+            # (plan-design-review, then plan-eng-review) while the family
+            # passes on branches still running three attempts. Every other
+            # row keeps --retry 1.
+            retries: 2
     steps:
       - uses: actions/checkout@v4
         with:
@@ -306,7 +316,7 @@ jobs:
           GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
           EVALS_CONCURRENCY: "40"
           PLAYWRIGHT_BROWSERS_PATH: /opt/playwright-browsers
-        run: EVALS=1 bun test --retry 1 --concurrent --max-concurrency 40 ${{ matrix.suite.file }}
+        run: EVALS=1 bun test --retry ${{ matrix.suite.retries || 1 }} --concurrent --max-concurrency 40 ${{ matrix.suite.file }}
 
       - name: Upload eval results
         if: always()