mirror of https://github.com/garrytan/gstack.git
test: cover benchmark alias routing
This commit is contained in:
parent
db3aeb1ac9
commit
49e6b7e48c
|
|
@ -368,6 +368,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||||
'journey-design-system': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
'journey-design-system': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||||
'journey-visual-qa': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
'journey-visual-qa': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||||
'journey-web-performance-benchmark': ['benchmark/**', 'scripts/gen-skill-docs.ts'],
|
'journey-web-performance-benchmark': ['benchmark/**', 'scripts/gen-skill-docs.ts'],
|
||||||
|
'journey-web-performance-benchmark-legacy-alias': ['benchmark/**', 'scripts/gen-skill-docs.ts', 'setup'],
|
||||||
|
|
||||||
// Opus 4.7 behavior evals — keys match testName: values in the test file.
|
// Opus 4.7 behavior evals — keys match testName: values in the test file.
|
||||||
// Routing sub-tests use template literal `routing-${c.name}` testNames,
|
// Routing sub-tests use template literal `routing-${c.name}` testNames,
|
||||||
|
|
@ -737,6 +738,7 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
|
||||||
'journey-design-system': 'periodic',
|
'journey-design-system': 'periodic',
|
||||||
'journey-visual-qa': 'periodic',
|
'journey-visual-qa': 'periodic',
|
||||||
'journey-web-performance-benchmark': 'periodic',
|
'journey-web-performance-benchmark': 'periodic',
|
||||||
|
'journey-web-performance-benchmark-legacy-alias': 'periodic',
|
||||||
|
|
||||||
// Opus 4.7 overlay evals — periodic (non-deterministic LLM behavior + Opus cost)
|
// Opus 4.7 overlay evals — periodic (non-deterministic LLM behavior + Opus cost)
|
||||||
'fanout-arm-overlay-on': 'periodic',
|
'fanout-arm-overlay-on': 'periodic',
|
||||||
|
|
|
||||||
|
|
@ -71,7 +71,10 @@ function installSkills(tmpDir: string) {
|
||||||
{ source: 'plan-design-review' }, { source: 'design-review' },
|
{ source: 'plan-design-review' }, { source: 'design-review' },
|
||||||
{ source: 'design-consultation' }, { source: 'retro' },
|
{ source: 'design-consultation' }, { source: 'retro' },
|
||||||
{ source: 'document-release' }, { source: 'investigate' }, { source: 'office-hours' },
|
{ source: 'document-release' }, { source: 'investigate' }, { source: 'office-hours' },
|
||||||
{ source: 'browse' }, { source: 'benchmark', command: 'web-performance-benchmark' },
|
{ source: 'browse' },
|
||||||
|
{ source: 'benchmark', command: 'web-performance-benchmark' },
|
||||||
|
{ source: 'benchmark', command: 'benchmark' },
|
||||||
|
{ source: 'benchmark', command: 'gstack-benchmark' },
|
||||||
{ source: 'setup-browser-cookies' }, { source: 'gstack-upgrade' }, { source: 'humanizer' },
|
{ source: 'setup-browser-cookies' }, { source: 'gstack-upgrade' }, { source: 'humanizer' },
|
||||||
];
|
];
|
||||||
|
|
||||||
|
|
@ -111,6 +114,7 @@ Key routing rules:
|
||||||
- Visual audit, design polish → invoke design-review
|
- Visual audit, design polish → invoke design-review
|
||||||
- Architecture review → invoke plan-eng-review
|
- Architecture review → invoke plan-eng-review
|
||||||
- Web page speed, Core Web Vitals, or performance regression → invoke web-performance-benchmark
|
- Web page speed, Core Web Vitals, or performance regression → invoke web-performance-benchmark
|
||||||
|
- /benchmark and /gstack-benchmark are compatibility aliases for web-performance-benchmark
|
||||||
`);
|
`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -213,6 +217,34 @@ describeE2E('Skill Routing E2E — Developer Journey', () => {
|
||||||
}
|
}
|
||||||
}, 150_000);
|
}, 150_000);
|
||||||
|
|
||||||
|
testIfSelected('journey-web-performance-benchmark-legacy-alias', async () => {
|
||||||
|
const tmpDir = createRoutingWorkDir('web-performance-benchmark-legacy-alias');
|
||||||
|
try {
|
||||||
|
const testName = 'journey-web-performance-benchmark-legacy-alias';
|
||||||
|
const expectedSkill = 'web-performance-benchmark';
|
||||||
|
const result = await runSkillTest({
|
||||||
|
prompt: 'Run /benchmark for the checkout page. Compare the current Core Web Vitals against the baseline and identify any performance regression.',
|
||||||
|
workingDirectory: tmpDir,
|
||||||
|
maxTurns: 5,
|
||||||
|
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||||
|
timeout: 60_000,
|
||||||
|
testName,
|
||||||
|
runId,
|
||||||
|
});
|
||||||
|
|
||||||
|
const skillCalls = result.toolCalls.filter(tc => tc.tool === 'Skill');
|
||||||
|
const actualSkill = skillCalls[0]?.input?.skill;
|
||||||
|
|
||||||
|
logCost(`journey: ${testName}`, result);
|
||||||
|
recordRouting(testName, result, expectedSkill, actualSkill);
|
||||||
|
|
||||||
|
expect(skillCalls.length, `Expected Skill tool to be called but got 0 calls. Claude may have answered directly without invoking a skill. Tool calls: ${result.toolCalls.map(tc => tc.tool).join(', ')}`).toBeGreaterThan(0);
|
||||||
|
expect(actualSkill).toBe(expectedSkill);
|
||||||
|
} finally {
|
||||||
|
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
}, 150_000);
|
||||||
|
|
||||||
testIfSelected('journey-ideation', async () => {
|
testIfSelected('journey-ideation', async () => {
|
||||||
const tmpDir = createRoutingWorkDir('ideation');
|
const tmpDir = createRoutingWorkDir('ideation');
|
||||||
try {
|
try {
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue