RomanNum3ral
/
gstack
mirror of https://github.com/garrytan/gstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

test: cover benchmark alias routing

This commit is contained in:
Nader Helmy 2026-07-15 20:26:37 -05:00
parent db3aeb1ac9
commit 49e6b7e48c
No known key found for this signature in database
2 changed files with 35 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
test/helpers/touchfiles.ts
View File

@ -368,6 +368,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
'journey-design-system': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
'journey-visual-qa': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
'journey-web-performance-benchmark': ['benchmark/**', 'scripts/gen-skill-docs.ts'],
'journey-web-performance-benchmark-legacy-alias': ['benchmark/**', 'scripts/gen-skill-docs.ts', 'setup'],
// Opus 4.7 behavior evals — keys match testName: values in the test file.
// Routing sub-tests use template literal `routing-${c.name}` testNames,
@ -737,6 +738,7 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
'journey-design-system': 'periodic',
'journey-visual-qa': 'periodic',
'journey-web-performance-benchmark': 'periodic',
'journey-web-performance-benchmark-legacy-alias': 'periodic',
// Opus 4.7 overlay evals — periodic (non-deterministic LLM behavior + Opus cost)
'fanout-arm-overlay-on': 'periodic',

34
test/skill-routing-e2e.test.ts
View File

@ -71,7 +71,10 @@ function installSkills(tmpDir: string) {
{ source: 'plan-design-review' }, { source: 'design-review' },
{ source: 'design-consultation' }, { source: 'retro' },
{ source: 'document-release' }, { source: 'investigate' }, { source: 'office-hours' },
{ source: 'browse' }, { source: 'benchmark', command: 'web-performance-benchmark' },
{ source: 'browse' },
{ source: 'benchmark', command: 'web-performance-benchmark' },
{ source: 'benchmark', command: 'benchmark' },
{ source: 'benchmark', command: 'gstack-benchmark' },
{ source: 'setup-browser-cookies' }, { source: 'gstack-upgrade' }, { source: 'humanizer' },
];
@ -111,6 +114,7 @@ Key routing rules:
- Visual audit, design polish → invoke design-review
- Architecture review → invoke plan-eng-review
- Web page speed, Core Web Vitals, or performance regression → invoke web-performance-benchmark
- /benchmark and /gstack-benchmark are compatibility aliases for web-performance-benchmark
`);
}
@ -213,6 +217,34 @@ describeE2E('Skill Routing E2E — Developer Journey', () => {
}
}, 150_000);
testIfSelected('journey-web-performance-benchmark-legacy-alias', async () => {
const tmpDir = createRoutingWorkDir('web-performance-benchmark-legacy-alias');
try {
const testName = 'journey-web-performance-benchmark-legacy-alias';
const expectedSkill = 'web-performance-benchmark';
const result = await runSkillTest({
prompt: 'Run /benchmark for the checkout page. Compare the current Core Web Vitals against the baseline and identify any performance regression.',
workingDirectory: tmpDir,
maxTurns: 5,
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
timeout: 60_000,
testName,
runId,
});
const skillCalls = result.toolCalls.filter(tc => tc.tool === 'Skill');
const actualSkill = skillCalls[0]?.input?.skill;
logCost(`journey: ${testName}`, result);
recordRouting(testName, result, expectedSkill, actualSkill);
expect(skillCalls.length, `Expected Skill tool to be called but got 0 calls. Claude may have answered directly without invoking a skill. Tool calls: ${result.toolCalls.map(tc => tc.tool).join(', ')}`).toBeGreaterThan(0);
expect(actualSkill).toBe(expectedSkill);
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
}, 150_000);
testIfSelected('journey-ideation', async () => {
const tmpDir = createRoutingWorkDir('ideation');
try {