diff --git a/test/helpers/llm-judge.ts b/test/helpers/llm-judge.ts index 6788d4c2f..a85e540f0 100644 --- a/test/helpers/llm-judge.ts +++ b/test/helpers/llm-judge.ts @@ -56,11 +56,17 @@ export interface RecommendationScore { * existing callers; pass a model id (e.g. claude-haiku-4-5-20251001) * for cheaper bounded judgments like judgeRecommendation. */ -// Default judge model is Haiku 4.5 (D1a, 2026-08): these are rubric-scoring -// calls, a duty Haiku is already proven at in this repo (claude-pty-runner's -// hung/working classifier, first-task-scaffold, hermetic-canary). Tests that -// need a stronger judge pass a model explicitly. -export async function callJudge(prompt: string, model: string = 'claude-haiku-4-5-20251001'): Promise { +// Default judge model: Sonnet. D1a tried Haiku 4.5 here and the first live +// run regressed the doc-rubric family — a controlled A/B on the identical +// health-rubric prompt scored 2/2/2 under Haiku vs 4/3/4 under Sonnet (both +// with coherent reasoning; Haiku is simply a harsher grader on long-document +// rubrics, and every >=4 threshold in skill-llm-eval was calibrated against +// months of Sonnet baselines). Per D1a's pin-on-regressors protocol the +// default stays Sonnet; recalibrating the 25 rubrics for Haiku is separately +// scoped work. Override per run with GSTACK_EVAL_MODEL_JUDGE; Haiku remains +// the right default for classifier-grade duties (pty hung/working, warmup, +// distill — see lib/eval-model.ts). +export async function callJudge(prompt: string, model: string = process.env.GSTACK_EVAL_MODEL_JUDGE || 'claude-sonnet-4-6'): Promise { const client = new Anthropic(); const makeRequest = () => client.messages.create({