RomanNum3ral
/
gstack
mirror of https://github.com/garrytan/gstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor(evals): single shared paid-test-set module 

test/helpers/paid-test-set.ts is now the one definition of which test
files are paid (the exact globs package.json's test:gate expands).
scripts/test-free-shards.ts derives its free/paid exclusion from it
instead of a private regex list, dropping the dead
browse/test/security-review-fullstack.test.ts pattern (file no longer
exists). The sharded paid runner derives its enumeration from the same
module, so a file added to one list can no longer silently miss the
other.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
(cherry picked from commit a7f36479a6a1f3656452370f5883371f3cb65623)
This commit is contained in:
Garry Tan 2026-08-12 11:09:07 -07:00
parent 281227262d
commit ea1c2851e5
No known key found for this signature in database
GPG Key ID: C1F69E85C74EFE1D
2 changed files with 27 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
scripts/test-free-shards.ts
View File

@ -27,22 +27,12 @@
import * as fs from 'fs';
import * as path from 'path';
import { spawnSync } from 'child_process';
import { isPaidTestFile } from '../test/helpers/paid-test-set';
const ROOT = path.resolve(import.meta.dir, '..');
const TEST_ROOTS = ['browse/test', 'test', 'make-pdf/test'] as const;
const TEST_FILE_REGEX = /\.test\.(?:[cm]?[jt]s|tsx|jsx)$/;
// Tests that require API spend, external services, or e2e harnesses.
// These are filtered out before any sharding or curation.
const PAID_EVAL_TESTS = [
/^browse\/test\/security-review-fullstack\.test\.ts$/,
/^test\/skill-e2e-.*\.test\.ts$/,
/^test\/skill-llm-eval\.test\.ts$/,
/^test\/skill-routing-e2e\.test\.ts$/,
/^test\/codex-e2e\.test\.ts$/,
/^test\/gemini-e2e\.test\.ts$/,
] as const;
// POSIX-only patterns that indicate a test will fail on windows-latest no
// matter how the runner shards. Codex's v1.18.0.0 review flagged the first
// three as concrete examples in the existing free suite (test/ship-version-sync.test.ts:72,
@ -118,7 +108,7 @@ export function normalizeRelativePath(filePath: string): string {
export function isFreeTestFile(relativePath: string): boolean {
const normalized = normalizeRelativePath(relativePath);
if (!TEST_FILE_REGEX.test(normalized)) return false;
return !PAID_EVAL_TESTS.some(pattern => pattern.test(normalized));
return !isPaidTestFile(normalized);
}
/**

25
test/helpers/paid-test-set.ts Normal file
View File

@ -0,0 +1,25 @@
/**
* The ONE definition of which test files are paid (API spend, external
* services, e2e harnesses). package.json's test:gate/test:evals globs, the
* free-suite exclusion in scripts/test-free-shards.ts, and the sharded paid
* runner in scripts/test-paid-shards.ts all derive from this list — a file
* added to one and not the others either burns money in the free suite or
* silently never runs in the paid tier.
*/
import { matchGlob } from './touchfiles';
/** The exact globs package.json's `test:gate` passes to `bun test`. */
export const PAID_TEST_GLOBS = [
'test/skill-llm-eval.test.ts',
'test/skill-e2e-*.test.ts',
'test/skill-routing-e2e.test.ts',
'test/codex-e2e.test.ts',
'test/gemini-e2e.test.ts',
] as const;
/** True when a repo-relative path (either slash style) is a paid test file. */
export function isPaidTestFile(relativePath: string): boolean {
const normalized = relativePath.replace(/\\/g, '/');
return PAID_TEST_GLOBS.some((glob) => matchGlob(normalized, glob));
}