mirror of https://github.com/garrytan/gstack.git
evals: Opus pin on the spec AUQ-matrix entry — D1a regressor, receipts in-file
The periodic re-baseline for the capture default (Opus -> Sonnet)
found exactly one regressor across the seven-entry AUQ behavioral
matrix: spec failed twice under Sonnet ('never reached a question in
budget', 242s) while its six siblings passed; the controlled Opus
re-run passed cleanly (7/7 format, substance 5, 160s), and a second
run through the new per-entry model plumbing confirms. MatrixSkill
gains an optional model field wired into captureFirstAuq; only spec
sets it. TODOS gains the re-baseline receipts for the never-baselined
periodic tail (three setup-gbrain files + ship-idempotency, all
local-only).
This commit is contained in:
parent
9c2d44253d
commit
5c67b11978
8
TODOS.md
8
TODOS.md
|
|
@ -197,6 +197,14 @@ claim true.
|
||||||
some orphans are deliberately manual (ios-device, opus-47 overlay harness), so a plain
|
some orphans are deliberately manual (ios-device, opus-47 overlay harness), so a plain
|
||||||
glob is wrong — needs a curated exclude list.
|
glob is wrong — needs a curated exclude list.
|
||||||
|
|
||||||
|
**Fresh receipts (2026-08-16, v1.66.0.0 re-baseline):** the first full local
|
||||||
|
periodic run in this store gave the never-baselined tail its first results:
|
||||||
|
`skill-e2e-setup-gbrain-{bad-token,path4-local-pglite,remote}` all failed
|
||||||
|
(spawned-process exit 1 — likely live-gbrain interference on a dev box) and
|
||||||
|
`skill-e2e-ship-idempotency` timed out at the 1800s shard wall. None are in
|
||||||
|
the weekly matrix, so these failures are invisible to CI — exactly this
|
||||||
|
item's thesis. Start the burn-down with those four.
|
||||||
|
|
||||||
**Context / where to start:** `.github/workflows/evals-periodic.yml:71` (matrix),
|
**Context / where to start:** `.github/workflows/evals-periodic.yml:71` (matrix),
|
||||||
`test/helpers/touchfiles.ts` E2E_TIERS (tier labels already exist per test), orphan
|
`test/helpers/touchfiles.ts` E2E_TIERS (tier labels already exist per test), orphan
|
||||||
list generated via `comm -23` between `ls test/skill-e2e-*.test.ts` and the file lists
|
list generated via `comm -23` between `ls test/skill-e2e-*.test.ts` and the file lists
|
||||||
|
|
|
||||||
|
|
@ -67,6 +67,8 @@ interface MatrixSkill {
|
||||||
skill: string;
|
skill: string;
|
||||||
fixtures: Record<string, string>;
|
fixtures: Record<string, string>;
|
||||||
scenario: string;
|
scenario: string;
|
||||||
|
/** D1a regressor pin: explicit capture model when the Sonnet default measurably fails this entry. */
|
||||||
|
model?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
const MATRIX: MatrixSkill[] = [
|
const MATRIX: MatrixSkill[] = [
|
||||||
|
|
@ -99,6 +101,13 @@ const MATRIX: MatrixSkill[] = [
|
||||||
skill: 'spec',
|
skill: 'spec',
|
||||||
fixtures: {},
|
fixtures: {},
|
||||||
scenario: 'Turn this vague intent into a precise spec: "add email notifications when a task is assigned to someone." Walk the spec workflow until the first AskUserQuestion.',
|
scenario: 'Turn this vague intent into a precise spec: "add email notifications when a task is assigned to someone." Walk the spec workflow until the first AskUserQuestion.',
|
||||||
|
// D1a pin-on-regressors, with receipts (2026-08-16 re-baseline): under
|
||||||
|
// the Sonnet capture default this entry failed twice ("never reached a
|
||||||
|
// question in budget", 242s) while the six sibling entries passed; the
|
||||||
|
// controlled Opus re-run passed cleanly (7/7 format, substance 5, 160s).
|
||||||
|
// The spec workflow's long pre-question phase needs the stronger model
|
||||||
|
// to reach its first AskUserQuestion inside the turn budget.
|
||||||
|
model: 'claude-opus-4-7',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
skill: 'design-consultation',
|
skill: 'design-consultation',
|
||||||
|
|
@ -130,6 +139,7 @@ describeE2E('AUQ behavioral matrix (periodic)', () => {
|
||||||
scenario: m.scenario,
|
scenario: m.scenario,
|
||||||
testName: `auq-matrix-${m.skill}`,
|
testName: `auq-matrix-${m.skill}`,
|
||||||
runId,
|
runId,
|
||||||
|
model: m.model,
|
||||||
});
|
});
|
||||||
} finally {
|
} finally {
|
||||||
fs.rmSync(dir, { recursive: true, force: true });
|
fs.rmSync(dir, { recursive: true, force: true });
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue