From f6a9cd841f0c0863eaea007687a929840666c506 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Fri, 14 Aug 2026 21:22:37 -0700 Subject: [PATCH] fix(eval-harness): eng-review phase boundary fires on qid-tagged questions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Step 0 boundary only matched two prose phrases, but plan-eng-review may legitimately reach the review phase without either — every per-finding AskUserQuestion then counted as pre-review and the batching regression test read 0 questions while watching the agent ask them one by one. The boundary now also fires on the first answered question carrying a gstack-qid:eng-review- marker. Additive only; 119 runner unit tests green. --- test/helpers/claude-pty-runner.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/helpers/claude-pty-runner.ts b/test/helpers/claude-pty-runner.ts index e0600430c..4f39570f7 100644 --- a/test/helpers/claude-pty-runner.ts +++ b/test/helpers/claude-pty-runner.ts @@ -1254,7 +1254,13 @@ export const ceoStep0Boundary: Step0BoundaryPredicate = (fp) => export const engStep0Boundary: Step0BoundaryPredicate = (fp) => /scope reduction recommendation|cross[\s-]?project learnings/i.test( fp.promptSnippet, - ); + ) || + // plan-eng-review's Step 0 may legitimately end with NO scope-reduction / + // learnings AUQ. When it does, the first answered review-phase question — + // tagged by the question-tuning resolver — + // must fire the boundary, or every per-finding AUQ stays classified + // preReview and the multi-finding batching counter reads 0. + /gstack-qid:\s*eng-review-/i.test(fp.promptSnippet); export const designStep0Boundary: Step0BoundaryPredicate = (fp) => /design system|design posture|design score|first dimension/i.test(