From 5c67b11978be939d069fcccecc641b7aa51dcaec Mon Sep 17 00:00:00 2001
From: Garry Tan <garrytan@gmail.com>
Date: Sat, 15 Aug 2026 20:59:27 -0700
Subject: [PATCH] =?UTF-8?q?evals:=20Opus=20pin=20on=20the=20spec=20AUQ-mat?=
 =?UTF-8?q?rix=20entry=20=E2=80=94=20D1a=20regressor,=20receipts=20in-file?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The periodic re-baseline for the capture default (Opus -> Sonnet)
found exactly one regressor across the seven-entry AUQ behavioral
matrix: spec failed twice under Sonnet ('never reached a question in
budget', 242s) while its six siblings passed; the controlled Opus
re-run passed cleanly (7/7 format, substance 5, 160s), and a second
run through the new per-entry model plumbing confirms. MatrixSkill
gains an optional model field wired into captureFirstAuq; only spec
sets it. TODOS gains the re-baseline receipts for the never-baselined
periodic tail (three setup-gbrain files + ship-idempotency, all
local-only).
---
 TODOS.md                          |  8 ++++++++
 test/skill-e2e-auq-matrix.test.ts | 10 ++++++++++
 2 files changed, 18 insertions(+)

diff --git a/TODOS.md b/TODOS.md
index a62e2bac6..553a9724e 100644
--- a/TODOS.md
+++ b/TODOS.md
@@ -197,6 +197,14 @@ claim true.
 some orphans are deliberately manual (ios-device, opus-47 overlay harness), so a plain
 glob is wrong — needs a curated exclude list.
 
+**Fresh receipts (2026-08-16, v1.66.0.0 re-baseline):** the first full local
+periodic run in this store gave the never-baselined tail its first results:
+`skill-e2e-setup-gbrain-{bad-token,path4-local-pglite,remote}` all failed
+(spawned-process exit 1 — likely live-gbrain interference on a dev box) and
+`skill-e2e-ship-idempotency` timed out at the 1800s shard wall. None are in
+the weekly matrix, so these failures are invisible to CI — exactly this
+item's thesis. Start the burn-down with those four.
+
 **Context / where to start:** `.github/workflows/evals-periodic.yml:71` (matrix),
 `test/helpers/touchfiles.ts` E2E_TIERS (tier labels already exist per test), orphan
 list generated via `comm -23` between `ls test/skill-e2e-*.test.ts` and the file lists
diff --git a/test/skill-e2e-auq-matrix.test.ts b/test/skill-e2e-auq-matrix.test.ts
index 80a3da7ff..e8c5eef93 100644
--- a/test/skill-e2e-auq-matrix.test.ts
+++ b/test/skill-e2e-auq-matrix.test.ts
@@ -67,6 +67,8 @@ interface MatrixSkill {
   skill: string;
   fixtures: Record<string, string>;
   scenario: string;
+  /** D1a regressor pin: explicit capture model when the Sonnet default measurably fails this entry. */
+  model?: string;
 }
 
 const MATRIX: MatrixSkill[] = [
@@ -99,6 +101,13 @@ const MATRIX: MatrixSkill[] = [
     skill: 'spec',
     fixtures: {},
     scenario: 'Turn this vague intent into a precise spec: "add email notifications when a task is assigned to someone." Walk the spec workflow until the first AskUserQuestion.',
+    // D1a pin-on-regressors, with receipts (2026-08-16 re-baseline): under
+    // the Sonnet capture default this entry failed twice ("never reached a
+    // question in budget", 242s) while the six sibling entries passed; the
+    // controlled Opus re-run passed cleanly (7/7 format, substance 5, 160s).
+    // The spec workflow's long pre-question phase needs the stronger model
+    // to reach its first AskUserQuestion inside the turn budget.
+    model: 'claude-opus-4-7',
   },
   {
     skill: 'design-consultation',
@@ -130,6 +139,7 @@ describeE2E('AUQ behavioral matrix (periodic)', () => {
             scenario: m.scenario,
             testName: `auq-matrix-${m.skill}`,
             runId,
+            model: m.model,
           });
         } finally {
           fs.rmSync(dir, { recursive: true, force: true });