From 7ef96ed0bdf787afef8e6509ac42a35101c8422a Mon Sep 17 00:00:00 2001
From: Garry Tan <garrytan@gmail.com>
Date: Fri, 14 Aug 2026 19:24:42 -0700
Subject: [PATCH] fix(test-infra): judge uses resolveClaudeBinary; eval:watch
 reads the real partials dir
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

judgePtyState spawned the bare string 'claude' three definitions below the
resolveClaudeBinary() helper this same file exports — broken under hermetic
PATHs where every other launch in the file resolves correctly.

eval:watch read _partial-e2e.json from the legacy global ~/.gstack-dev/evals/
while EvalCollector writes it into the per-project eval dir (or
GSTACK_EVAL_DIR) — so the dashboard's completed-tests panel was empty
whenever slug detection succeeded, i.e. the normal case. The heartbeat and
per-run progress logs stay global by design (session-runner.ts: 'heartbeat
stays global'). The three eval-CLI docstrings stop claiming the legacy dir
is the primary location.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 scripts/eval-compare.ts           |  3 ++-
 scripts/eval-list.ts              |  3 ++-
 scripts/eval-summary.ts           |  3 ++-
 scripts/eval-watch.ts             | 11 ++++++++++-
 test/helpers/claude-pty-runner.ts |  5 ++++-
 5 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/scripts/eval-compare.ts b/scripts/eval-compare.ts
index a7b0dba03..30bc3864e 100644
--- a/scripts/eval-compare.ts
+++ b/scripts/eval-compare.ts
@@ -1,6 +1,7 @@
 #!/usr/bin/env bun
 /**
- * Compare two eval runs from ~/.gstack-dev/evals/
+ * Compare two eval runs from the project eval dir (~/.gstack/projects/<slug>/evals;
+ * legacy fallback ~/.gstack-dev/evals)
  *
  * Usage:
  *   bun run eval:compare                    # compare two most recent of same tier
diff --git a/scripts/eval-list.ts b/scripts/eval-list.ts
index bb4ee9a83..38c14e7f2 100644
--- a/scripts/eval-list.ts
+++ b/scripts/eval-list.ts
@@ -1,6 +1,7 @@
 #!/usr/bin/env bun
 /**
- * List eval runs from ~/.gstack-dev/evals/
+ * List eval runs from the project eval dir (~/.gstack/projects/<slug>/evals;
+ * legacy fallback ~/.gstack-dev/evals)
  *
  * Usage: bun run eval:list [--branch <name>] [--tier e2e|llm-judge] [--limit N]
  */
diff --git a/scripts/eval-summary.ts b/scripts/eval-summary.ts
index a3b8d8bc6..5122b165d 100644
--- a/scripts/eval-summary.ts
+++ b/scripts/eval-summary.ts
@@ -1,6 +1,7 @@
 #!/usr/bin/env bun
 /**
- * Aggregate summary of all eval runs from ~/.gstack-dev/evals/
+ * Aggregate summary of eval runs from the project eval dir
+ * (~/.gstack/projects/<slug>/evals; legacy fallback ~/.gstack-dev/evals)
  *
  * Usage: bun run eval:summary
  */
diff --git a/scripts/eval-watch.ts b/scripts/eval-watch.ts
index ba96faf4b..1b5a0d2ad 100644
--- a/scripts/eval-watch.ts
+++ b/scripts/eval-watch.ts
@@ -11,10 +11,19 @@
 import * as fs from 'fs';
 import * as path from 'path';
 import * as os from 'os';
+import { getProjectEvalDir } from '../test/helpers/eval-store';
 
 const GSTACK_DEV_DIR = path.join(os.homedir(), '.gstack-dev');
+// Heartbeat + per-run progress logs are GLOBAL by design — session-runner.ts
+// writes ~/.gstack-dev/e2e-live.json regardless of project ("heartbeat stays
+// global"). The PARTIAL file is per-project: EvalCollector writes it into
+// getProjectEvalDir() (or GSTACK_EVAL_DIR), so watching the legacy global
+// path missed it whenever slug detection succeeded — i.e. the normal case.
 const HEARTBEAT_PATH = path.join(GSTACK_DEV_DIR, 'e2e-live.json');
-const PARTIAL_PATH = path.join(GSTACK_DEV_DIR, 'evals', '_partial-e2e.json');
+const PARTIAL_PATH = path.join(
+  process.env.GSTACK_EVAL_DIR || getProjectEvalDir(),
+  '_partial-e2e.json',
+);
 const STALE_THRESHOLD_SEC = 600; // 10 minutes
 
 export interface HeartbeatData {
diff --git a/test/helpers/claude-pty-runner.ts b/test/helpers/claude-pty-runner.ts
index 669dacb48..ea8a46ea1 100644
--- a/test/helpers/claude-pty-runner.ts
+++ b/test/helpers/claude-pty-runner.ts
@@ -454,8 +454,11 @@ ${tail}
   };
 
   try {
+    // Use the same binary resolution as every PTY launch in this file —
+    // judgePtyState previously hardcoded bare 'claude' three definitions
+    // below resolveClaudeBinary(), breaking under hermetic PATHs.
     const result = nodeSpawnSync(
-      'claude',
+      resolveClaudeBinary() ?? 'claude',
       ['-p', '--model', 'claude-haiku-4-5', '--max-turns', '1'],
       {
         input: prompt,