From 1e35386c540339d7857b1b4a8c7e96f08f33b4f3 Mon Sep 17 00:00:00 2001
From: Garry Tan <garrytan@gmail.com>
Date: Sat, 15 Aug 2026 17:01:41 -0700
Subject: [PATCH] fix(evals): preflight fails fast on spawn error, timeout, and
 exit 127
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The ping only grepped stdout for two connection strings — a missing
claude binary, a 30s timeout kill, or command-not-found all returned
'ok', and the fleet then burned ~30 shard timeouts discovering the
outage one child at a time. Cross-model finding (testing specialist +
codex adversarial). Other non-zero exits stay deliberately fail-open:
a flaky preflight must not block a runnable suite; pinned both ways.
---
 test/anthropic-preflight.test.ts    | 29 +++++++++++++++++++++++++++++
 test/helpers/anthropic-preflight.ts | 14 ++++++++++++++
 2 files changed, 43 insertions(+)

diff --git a/test/anthropic-preflight.test.ts b/test/anthropic-preflight.test.ts
index a207bee96..a04add123 100644
--- a/test/anthropic-preflight.test.ts
+++ b/test/anthropic-preflight.test.ts
@@ -50,4 +50,33 @@ describe('preflightAnthropicApi', () => {
     expect(result).toBe('ok');
     expect(calls.length).toBe(1);
   });
+
+  // Failure modes that guarantee every shard fails too must fail the
+  // preflight — previously a missing binary, a timeout kill, and exit 127
+  // all returned 'ok' and the fleet burned its own discovery of the outage.
+  const shapedSpawn = (shape: Partial<ReturnType<typeof import('child_process').spawnSync>>) =>
+    ((() => ({ stdout: Buffer.from(''), stderr: Buffer.from(''), status: 0, ...shape })) as unknown as
+      typeof import('child_process').spawnSync);
+
+  test('spawn error (unlaunchable shell/binary) throws', () => {
+    expect(() =>
+      preflightAnthropicApi({}, shapedSpawn({ error: new Error('spawn sh ENOENT') })),
+    ).toThrow(/could not run/);
+  });
+
+  test('timeout kill (signal set) throws', () => {
+    expect(() =>
+      preflightAnthropicApi({}, shapedSpawn({ signal: 'SIGTERM' })),
+    ).toThrow(/timed out/);
+  });
+
+  test('exit 127 (claude not found) throws', () => {
+    expect(() =>
+      preflightAnthropicApi({}, shapedSpawn({ status: 127 })),
+    ).toThrow(/not found on PATH/);
+  });
+
+  test('other non-zero exits stay fail-open (a flaky preflight must not block a runnable suite)', () => {
+    expect(preflightAnthropicApi({}, shapedSpawn({ status: 1, stdout: Buffer.from('transient') }))).toBe('ok');
+  });
 });
diff --git a/test/helpers/anthropic-preflight.ts b/test/helpers/anthropic-preflight.ts
index 718e6d8f5..9ca9f1b76 100644
--- a/test/helpers/anthropic-preflight.ts
+++ b/test/helpers/anthropic-preflight.ts
@@ -27,6 +27,20 @@ export function preflightAnthropicApi(
     ['-c', 'echo "ping" | claude -p --max-turns 1 --output-format stream-json --verbose --dangerously-skip-permissions'],
     { stdio: 'pipe', timeout: 30_000 },
   );
+  // Fail fast on the failure modes that guarantee EVERY shard fails too:
+  // spawn error (sh/claude unlaunchable), a timeout kill (signal set), or
+  // exit 127 (command not found). Anything else stays fail-open — a flaky
+  // preflight must not block a run the shards could complete (auth prompts
+  // and transient non-zero exits are the shards' problem to report).
+  if (check.error) {
+    throw new Error(`Anthropic preflight could not run (${check.error.message}) — aborting E2E suite before spawning shards.`);
+  }
+  if (check.signal) {
+    throw new Error(`Anthropic preflight timed out (killed with ${check.signal}) — aborting E2E suite. Check connectivity/auth and retry.`);
+  }
+  if (check.status === 127) {
+    throw new Error('Anthropic preflight: `claude` not found on PATH (exit 127) — aborting E2E suite before spawning shards.');
+  }
   const output = check.stdout?.toString() || '';
   if (output.includes('ConnectionRefused') || output.includes('Unable to connect')) {
     throw new Error('Anthropic API unreachable — aborting E2E suite. Fix connectivity and retry.');