diff --git a/test/helpers/touchfiles.ts b/test/helpers/touchfiles.ts index dce600e2d..7fba3f301 100644 --- a/test/helpers/touchfiles.ts +++ b/test/helpers/touchfiles.ts @@ -368,6 +368,7 @@ export const E2E_TOUCHFILES: Record = { 'journey-design-system': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'], 'journey-visual-qa': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'], 'journey-web-performance-benchmark': ['benchmark/**', 'scripts/gen-skill-docs.ts'], + 'journey-web-performance-benchmark-legacy-alias': ['benchmark/**', 'scripts/gen-skill-docs.ts', 'setup'], // Opus 4.7 behavior evals — keys match testName: values in the test file. // Routing sub-tests use template literal `routing-${c.name}` testNames, @@ -737,6 +738,7 @@ export const E2E_TIERS: Record = { 'journey-design-system': 'periodic', 'journey-visual-qa': 'periodic', 'journey-web-performance-benchmark': 'periodic', + 'journey-web-performance-benchmark-legacy-alias': 'periodic', // Opus 4.7 overlay evals — periodic (non-deterministic LLM behavior + Opus cost) 'fanout-arm-overlay-on': 'periodic', diff --git a/test/skill-routing-e2e.test.ts b/test/skill-routing-e2e.test.ts index 9d510087c..7bc3e3651 100644 --- a/test/skill-routing-e2e.test.ts +++ b/test/skill-routing-e2e.test.ts @@ -71,7 +71,10 @@ function installSkills(tmpDir: string) { { source: 'plan-design-review' }, { source: 'design-review' }, { source: 'design-consultation' }, { source: 'retro' }, { source: 'document-release' }, { source: 'investigate' }, { source: 'office-hours' }, - { source: 'browse' }, { source: 'benchmark', command: 'web-performance-benchmark' }, + { source: 'browse' }, + { source: 'benchmark', command: 'web-performance-benchmark' }, + { source: 'benchmark', command: 'benchmark' }, + { source: 'benchmark', command: 'gstack-benchmark' }, { source: 'setup-browser-cookies' }, { source: 'gstack-upgrade' }, { source: 'humanizer' }, ]; @@ -111,6 +114,7 @@ Key routing rules: - Visual audit, design polish → invoke design-review - Architecture review → invoke plan-eng-review - Web page speed, Core Web Vitals, or performance regression → invoke web-performance-benchmark +- /benchmark and /gstack-benchmark are compatibility aliases for web-performance-benchmark `); } @@ -213,6 +217,34 @@ describeE2E('Skill Routing E2E — Developer Journey', () => { } }, 150_000); + testIfSelected('journey-web-performance-benchmark-legacy-alias', async () => { + const tmpDir = createRoutingWorkDir('web-performance-benchmark-legacy-alias'); + try { + const testName = 'journey-web-performance-benchmark-legacy-alias'; + const expectedSkill = 'web-performance-benchmark'; + const result = await runSkillTest({ + prompt: 'Run /benchmark for the checkout page. Compare the current Core Web Vitals against the baseline and identify any performance regression.', + workingDirectory: tmpDir, + maxTurns: 5, + allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'], + timeout: 60_000, + testName, + runId, + }); + + const skillCalls = result.toolCalls.filter(tc => tc.tool === 'Skill'); + const actualSkill = skillCalls[0]?.input?.skill; + + logCost(`journey: ${testName}`, result); + recordRouting(testName, result, expectedSkill, actualSkill); + + expect(skillCalls.length, `Expected Skill tool to be called but got 0 calls. Claude may have answered directly without invoking a skill. Tool calls: ${result.toolCalls.map(tc => tc.tool).join(', ')}`).toBeGreaterThan(0); + expect(actualSkill).toBe(expectedSkill); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }, 150_000); + testIfSelected('journey-ideation', async () => { const tmpDir = createRoutingWorkDir('ideation'); try {