From ea1c2851e52b489bdf5cda857ae9cd1c60a7b436 Mon Sep 17 00:00:00 2001
From: Garry Tan <garrytan@gmail.com>
Date: Wed, 12 Aug 2026 11:09:07 -0700
Subject: [PATCH] refactor(evals): single shared paid-test-set module

test/helpers/paid-test-set.ts is now the one definition of which test
files are paid (the exact globs package.json's test:gate expands).
scripts/test-free-shards.ts derives its free/paid exclusion from it
instead of a private regex list, dropping the dead
browse/test/security-review-fullstack.test.ts pattern (file no longer
exists). The sharded paid runner derives its enumeration from the same
module, so a file added to one list can no longer silently miss the
other.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
(cherry picked from commit a7f36479a6a1f3656452370f5883371f3cb65623)
---
 scripts/test-free-shards.ts   | 14 ++------------
 test/helpers/paid-test-set.ts | 25 +++++++++++++++++++++++++
 2 files changed, 27 insertions(+), 12 deletions(-)
 create mode 100644 test/helpers/paid-test-set.ts

diff --git a/scripts/test-free-shards.ts b/scripts/test-free-shards.ts
index 5be84a1f7..8bf98c066 100755
--- a/scripts/test-free-shards.ts
+++ b/scripts/test-free-shards.ts
@@ -27,22 +27,12 @@
 import * as fs from 'fs';
 import * as path from 'path';
 import { spawnSync } from 'child_process';
+import { isPaidTestFile } from '../test/helpers/paid-test-set';
 
 const ROOT = path.resolve(import.meta.dir, '..');
 const TEST_ROOTS = ['browse/test', 'test', 'make-pdf/test'] as const;
 const TEST_FILE_REGEX = /\.test\.(?:[cm]?[jt]s|tsx|jsx)$/;
 
-// Tests that require API spend, external services, or e2e harnesses.
-// These are filtered out before any sharding or curation.
-const PAID_EVAL_TESTS = [
-  /^browse\/test\/security-review-fullstack\.test\.ts$/,
-  /^test\/skill-e2e-.*\.test\.ts$/,
-  /^test\/skill-llm-eval\.test\.ts$/,
-  /^test\/skill-routing-e2e\.test\.ts$/,
-  /^test\/codex-e2e\.test\.ts$/,
-  /^test\/gemini-e2e\.test\.ts$/,
-] as const;
-
 // POSIX-only patterns that indicate a test will fail on windows-latest no
 // matter how the runner shards. Codex's v1.18.0.0 review flagged the first
 // three as concrete examples in the existing free suite (test/ship-version-sync.test.ts:72,
@@ -118,7 +108,7 @@ export function normalizeRelativePath(filePath: string): string {
 export function isFreeTestFile(relativePath: string): boolean {
   const normalized = normalizeRelativePath(relativePath);
   if (!TEST_FILE_REGEX.test(normalized)) return false;
-  return !PAID_EVAL_TESTS.some(pattern => pattern.test(normalized));
+  return !isPaidTestFile(normalized);
 }
 
 /**
diff --git a/test/helpers/paid-test-set.ts b/test/helpers/paid-test-set.ts
new file mode 100644
index 000000000..a4faf0b75
--- /dev/null
+++ b/test/helpers/paid-test-set.ts
@@ -0,0 +1,25 @@
+/**
+ * The ONE definition of which test files are paid (API spend, external
+ * services, e2e harnesses). package.json's test:gate/test:evals globs, the
+ * free-suite exclusion in scripts/test-free-shards.ts, and the sharded paid
+ * runner in scripts/test-paid-shards.ts all derive from this list — a file
+ * added to one and not the others either burns money in the free suite or
+ * silently never runs in the paid tier.
+ */
+
+import { matchGlob } from './touchfiles';
+
+/** The exact globs package.json's `test:gate` passes to `bun test`. */
+export const PAID_TEST_GLOBS = [
+  'test/skill-llm-eval.test.ts',
+  'test/skill-e2e-*.test.ts',
+  'test/skill-routing-e2e.test.ts',
+  'test/codex-e2e.test.ts',
+  'test/gemini-e2e.test.ts',
+] as const;
+
+/** True when a repo-relative path (either slash style) is a paid test file. */
+export function isPaidTestFile(relativePath: string): boolean {
+  const normalized = relativePath.replace(/\\/g, '/');
+  return PAID_TEST_GLOBS.some((glob) => matchGlob(normalized, glob));
+}