fix: scope benchmark skill to web performance

This commit is contained in:
Nader Helmy 2026-06-21 22:51:35 -05:00
parent a3259400a3
commit b76a2d9555
No known key found for this signature in database
6 changed files with 91 additions and 36 deletions

View File

@ -1,12 +1,12 @@
--- ---
name: benchmark name: gstack-benchmark
preamble-tier: 1 preamble-tier: 1
version: 1.0.0 version: 1.0.0
description: Performance regression detection using the browse daemon. (gstack) description: Web page performance regression detection using the browse daemon. (gstack)
triggers: triggers:
- performance benchmark - web performance benchmark
- check page speed - check page speed
- detect performance regression - detect page performance regression
allowed-tools: allowed-tools:
- Bash - Bash
- Read - Read
@ -23,8 +23,9 @@ allowed-tools:
Establishes Establishes
baselines for page load times, Core Web Vitals, and resource sizes. baselines for page load times, Core Web Vitals, and resource sizes.
Compares before/after on every PR. Tracks performance trends over time. Compares before/after on every PR. Tracks performance trends over time.
Use when: "performance", "benchmark", "page speed", "lighthouse", "web vitals", Use when: "web performance benchmark", "page speed", "lighthouse",
"bundle size", "load time". "web vitals", "bundle size", "load time". Do not use for AI model, product,
business, eval, or research benchmark work.
Voice triggers (speech-to-text aliases): "speed test", "check performance". Voice triggers (speech-to-text aliases): "speed test", "check performance".
@ -84,7 +85,7 @@ _QUESTION_TUNING=$(~/.claude/skills/gstack/bin/gstack-config get question_tuning
echo "QUESTION_TUNING: $_QUESTION_TUNING" echo "QUESTION_TUNING: $_QUESTION_TUNING"
mkdir -p ~/.gstack/analytics mkdir -p ~/.gstack/analytics
if [ "$_TEL" != "off" ]; then if [ "$_TEL" != "off" ]; then
echo '{"skill":"benchmark","ts":"'$(date -u +%Y-%m-%dT%H:%M:%SZ)'","repo":"'$(_repo=$(basename "$(git rev-parse --show-toplevel 2>/dev/null)" 2>/dev/null | tr -cd 'a-zA-Z0-9._-'); echo "${_repo:-unknown}")'"}' >> ~/.gstack/analytics/skill-usage.jsonl 2>/dev/null || true echo '{"skill":"gstack-benchmark","ts":"'$(date -u +%Y-%m-%dT%H:%M:%SZ)'","repo":"'$(_repo=$(basename "$(git rev-parse --show-toplevel 2>/dev/null)" 2>/dev/null | tr -cd 'a-zA-Z0-9._-'); echo "${_repo:-unknown}")'"}' >> ~/.gstack/analytics/skill-usage.jsonl 2>/dev/null || true
fi fi
for _PF in $(find ~/.gstack/analytics -maxdepth 1 -name '.pending-*' 2>/dev/null); do for _PF in $(find ~/.gstack/analytics -maxdepth 1 -name '.pending-*' 2>/dev/null); do
if [ -f "$_PF" ]; then if [ -f "$_PF" ]; then
@ -106,7 +107,7 @@ if [ -f "$_LEARN_FILE" ]; then
else else
echo "LEARNINGS: 0" echo "LEARNINGS: 0"
fi fi
~/.claude/skills/gstack/bin/gstack-timeline-log '{"skill":"benchmark","event":"started","branch":"'"$_BRANCH"'","session":"'"$_SESSION_ID"'"}' 2>/dev/null & ~/.claude/skills/gstack/bin/gstack-timeline-log '{"skill":"gstack-benchmark","event":"started","branch":"'"$_BRANCH"'","session":"'"$_SESSION_ID"'"}' 2>/dev/null &
_HAS_ROUTING="no" _HAS_ROUTING="no"
if [ -f CLAUDE.md ] && grep -q "## Skill routing" CLAUDE.md 2>/dev/null; then if [ -f CLAUDE.md ] && grep -q "## Skill routing" CLAUDE.md 2>/dev/null; then
_HAS_ROUTING="yes" _HAS_ROUTING="yes"
@ -571,22 +572,29 @@ If `NEEDS_SETUP`:
fi fi
``` ```
# /benchmark — Performance Regression Detection # /gstack-benchmark - Web Page Performance Regression Detection
You are a **Performance Engineer** who has optimized apps serving millions of requests. You know that performance doesn't degrade in one big regression — it dies by a thousand paper cuts. Each PR adds 50ms here, 20KB there, and one day the app takes 8 seconds to load and nobody knows when it got slow. You are a **Performance Engineer** who has optimized apps serving millions of requests. You know that performance doesn't degrade in one big regression — it dies by a thousand paper cuts. Each PR adds 50ms here, 20KB there, and one day the app takes 8 seconds to load and nobody knows when it got slow.
Your job is to measure, baseline, compare, and alert. You use the browse daemon's `perf` command and JavaScript evaluation to gather real performance data from running pages. Your job is to measure, baseline, compare, and alert. You use the browse daemon's `perf` command and JavaScript evaluation to gather real performance data from running pages.
## Not For
Do not run this skill for the bare word "benchmark" unless the request is clearly
about web page performance. Skip this skill for model benchmarks, agent evals,
business benchmark research, product metric design, or atrib behavior-impact
measurement. For cross-model gstack skill comparisons, use `/benchmark-models`.
## User-invocable ## User-invocable
When the user types `/benchmark`, run this skill. When the user types `/gstack-benchmark`, run this skill.
## Arguments ## Arguments
- `/benchmark <url>` — full performance audit with baseline comparison - `/gstack-benchmark <url>` — full performance audit with baseline comparison
- `/benchmark <url> --baseline` — capture baseline (run before making changes) - `/gstack-benchmark <url> --baseline` — capture baseline (run before making changes)
- `/benchmark <url> --quick` — single-pass timing check (no baseline needed) - `/gstack-benchmark <url> --quick` — single-pass timing check (no baseline needed)
- `/benchmark <url> --pages /,/dashboard,/api/health` — specify pages - `/gstack-benchmark <url> --pages /,/dashboard,/api/health` — specify pages
- `/benchmark --diff` — benchmark only pages affected by current branch - `/gstack-benchmark --diff` — benchmark only pages affected by current branch
- `/benchmark --trend` — show performance trends from historical data - `/gstack-benchmark --trend` — show performance trends from historical data
## Instructions ## Instructions

View File

@ -1,20 +1,21 @@
--- ---
name: benchmark name: gstack-benchmark
preamble-tier: 1 preamble-tier: 1
version: 1.0.0 version: 1.0.0
description: | description: |
Performance regression detection using the browse daemon. Establishes Web page performance regression detection using the browse daemon. Establishes
baselines for page load times, Core Web Vitals, and resource sizes. baselines for page load times, Core Web Vitals, and resource sizes.
Compares before/after on every PR. Tracks performance trends over time. Compares before/after on every PR. Tracks performance trends over time.
Use when: "performance", "benchmark", "page speed", "lighthouse", "web vitals", Use when: "web performance benchmark", "page speed", "lighthouse",
"bundle size", "load time". (gstack) "web vitals", "bundle size", "load time". Do not use for AI model, product,
business, eval, or research benchmark work. (gstack)
voice-triggers: voice-triggers:
- "speed test" - "speed test"
- "check performance" - "check performance"
triggers: triggers:
- performance benchmark - web performance benchmark
- check page speed - check page speed
- detect performance regression - detect page performance regression
allowed-tools: allowed-tools:
- Bash - Bash
- Read - Read
@ -27,22 +28,29 @@ allowed-tools:
{{BROWSE_SETUP}} {{BROWSE_SETUP}}
# /benchmark — Performance Regression Detection # /gstack-benchmark - Web Page Performance Regression Detection
You are a **Performance Engineer** who has optimized apps serving millions of requests. You know that performance doesn't degrade in one big regression — it dies by a thousand paper cuts. Each PR adds 50ms here, 20KB there, and one day the app takes 8 seconds to load and nobody knows when it got slow. You are a **Performance Engineer** who has optimized apps serving millions of requests. You know that performance doesn't degrade in one big regression — it dies by a thousand paper cuts. Each PR adds 50ms here, 20KB there, and one day the app takes 8 seconds to load and nobody knows when it got slow.
Your job is to measure, baseline, compare, and alert. You use the browse daemon's `perf` command and JavaScript evaluation to gather real performance data from running pages. Your job is to measure, baseline, compare, and alert. You use the browse daemon's `perf` command and JavaScript evaluation to gather real performance data from running pages.
## Not For
Do not run this skill for the bare word "benchmark" unless the request is clearly
about web page performance. Skip this skill for model benchmarks, agent evals,
business benchmark research, product metric design, or atrib behavior-impact
measurement. For cross-model gstack skill comparisons, use `/benchmark-models`.
## User-invocable ## User-invocable
When the user types `/benchmark`, run this skill. When the user types `/gstack-benchmark`, run this skill.
## Arguments ## Arguments
- `/benchmark <url>` — full performance audit with baseline comparison - `/gstack-benchmark <url>` — full performance audit with baseline comparison
- `/benchmark <url> --baseline` — capture baseline (run before making changes) - `/gstack-benchmark <url> --baseline` — capture baseline (run before making changes)
- `/benchmark <url> --quick` — single-pass timing check (no baseline needed) - `/gstack-benchmark <url> --quick` — single-pass timing check (no baseline needed)
- `/benchmark <url> --pages /,/dashboard,/api/health` — specify pages - `/gstack-benchmark <url> --pages /,/dashboard,/api/health` — specify pages
- `/benchmark --diff` — benchmark only pages affected by current branch - `/gstack-benchmark --diff` — benchmark only pages affected by current branch
- `/benchmark --trend` — show performance trends from historical data - `/gstack-benchmark --trend` — show performance trends from historical data
## Instructions ## Instructions

View File

@ -24,10 +24,12 @@ import { COMMAND_DESCRIPTIONS as BROWSE_COMMANDS } from '../browse/src/commands'
const ROOT = path.resolve(import.meta.dir, '..'); const ROOT = path.resolve(import.meta.dir, '..');
const OUTPUT = path.join(ROOT, 'gstack', 'llms.txt'); const OUTPUT = path.join(ROOT, 'gstack', 'llms.txt');
const CLAUDE_SKIPPED_SKILL_DIRS = new Set(['claude']);
interface SkillEntry { interface SkillEntry {
name: string; name: string;
description: string; description: string;
href: string;
} }
/** /**
@ -113,6 +115,11 @@ function oneLine(text: string): string {
return first.replace(/\s+/g, ' ').trim(); return first.replace(/\s+/g, ' ').trim();
} }
function skillDirForOutput(output: string): string {
const dir = path.dirname(output);
return dir === '.' ? '.' : dir;
}
interface GenerateOptions { interface GenerateOptions {
/** Override repo root (for tests). */ /** Override repo root (for tests). */
root?: string; root?: string;
@ -135,6 +142,9 @@ export async function generateLlmsTxt(opts: GenerateOptions = {}): Promise<Gener
const templates = discoverTemplates(root); const templates = discoverTemplates(root);
const skills: SkillEntry[] = []; const skills: SkillEntry[] = [];
for (const t of templates) { for (const t of templates) {
if (CLAUDE_SKIPPED_SKILL_DIRS.has(skillDirForOutput(t.output))) {
continue;
}
const filePath = path.join(root, t.tmpl); const filePath = path.join(root, t.tmpl);
const entry = parseSkillFrontmatter(filePath); const entry = parseSkillFrontmatter(filePath);
if (!entry) { if (!entry) {
@ -144,7 +154,7 @@ export async function generateLlmsTxt(opts: GenerateOptions = {}): Promise<Gener
} }
continue; continue;
} }
skills.push(entry); skills.push({ ...entry, href: t.output });
} }
skills.sort((a, b) => a.name.localeCompare(b.name)); skills.sort((a, b) => a.name.localeCompare(b.name));
@ -167,7 +177,7 @@ export async function generateLlmsTxt(opts: GenerateOptions = {}): Promise<Gener
lines.push(''); lines.push('');
for (const skill of skills) { for (const skill of skills) {
const summary = oneLine(skill.description); const summary = oneLine(skill.description);
lines.push(`- [/${skill.name}](${skill.name}/SKILL.md): ${summary}`); lines.push(`- [/${skill.name}](${skill.href}): ${summary}`);
} }
lines.push(''); lines.push('');

View File

@ -9,8 +9,8 @@
"voice_line": "Voice triggers (speech-to-text aliases): \"auto plan\", \"automatic review\"." "voice_line": "Voice triggers (speech-to-text aliases): \"auto plan\", \"automatic review\"."
}, },
"benchmark": { "benchmark": {
"lead": "Performance regression detection using the browse daemon.", "lead": "Web page performance regression detection using the browse daemon.",
"routing": "Establishes\nbaselines for page load times, Core Web Vitals, and resource sizes.\nCompares before/after on every PR. Tracks performance trends over time.\nUse when: \"performance\", \"benchmark\", \"page speed\", \"lighthouse\", \"web vitals\",\n\"bundle size\", \"load time\".", "routing": "Establishes\nbaselines for page load times, Core Web Vitals, and resource sizes.\nCompares before/after on every PR. Tracks performance trends over time.\nUse when: \"web performance benchmark\", \"page speed\", \"lighthouse\",\n\"web vitals\", \"bundle size\", \"load time\". Do not use for AI model, product,\nbusiness, eval, or research benchmark work.",
"voice_line": "Voice triggers (speech-to-text aliases): \"speed test\", \"check performance\"." "voice_line": "Voice triggers (speech-to-text aliases): \"speed test\", \"check performance\"."
}, },
"benchmark-models": { "benchmark-models": {

View File

@ -467,6 +467,23 @@ describe('gen-skill-docs', () => {
} }
}); });
test('benchmark skill is scoped to web performance', () => {
const generated = fs.readFileSync(path.join(ROOT, 'benchmark', 'SKILL.md'), 'utf-8');
const template = fs.readFileSync(path.join(ROOT, 'benchmark', 'SKILL.md.tmpl'), 'utf-8');
for (const content of [generated, template]) {
const frontmatter = content.match(/^---\n([\s\S]*?)\n---/)?.[1] ?? '';
expect(frontmatter).toContain('name: gstack-benchmark');
expect(frontmatter).not.toContain('name: benchmark');
expect(frontmatter).toContain('web performance benchmark');
expect(frontmatter).not.toMatch(/["']benchmark["']/);
expect(content).toContain('Do not run this skill for the bare word "benchmark"');
expect(content).toContain('For cross-model gstack skill comparisons, use `/benchmark-models`.');
expect(content).toContain('/gstack-benchmark <url>');
expect(content).not.toContain('When the user types `/benchmark`, run this skill.');
}
});
test('qa and qa-only templates use QA_METHODOLOGY placeholder', () => { test('qa and qa-only templates use QA_METHODOLOGY placeholder', () => {
const qaTmpl = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md.tmpl'), 'utf-8'); const qaTmpl = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md.tmpl'), 'utf-8');
expect(qaTmpl).toContain('{{QA_METHODOLOGY}}'); expect(qaTmpl).toContain('{{QA_METHODOLOGY}}');

View File

@ -25,17 +25,29 @@ describe('gen-llms-txt — shape', () => {
expect(generated.content).toContain('auto-generated'); expect(generated.content).toContain('auto-generated');
}); });
test('every skill .tmpl in the repo appears in the index', () => { test('every generated Claude skill appears in the index', () => {
const templates = discoverTemplates(ROOT); const templates = discoverTemplates(ROOT);
const claudeGeneratedTemplates = templates.filter((t) => path.dirname(t.output) !== 'claude');
// Filter to those that successfully parsed (have name + description). // Filter to those that successfully parsed (have name + description).
expect(generated.skills.length).toBeGreaterThan(0); expect(generated.skills.length).toBeGreaterThan(0);
expect(generated.skills.length).toBeLessThanOrEqual(templates.length); expect(generated.skills.length).toBeLessThanOrEqual(claudeGeneratedTemplates.length);
expect(generated.content).not.toContain('[/claude]');
for (const skill of generated.skills) { for (const skill of generated.skills) {
expect(generated.content).toMatch(new RegExp(`/${skill.name}\\b`)); expect(generated.content).toMatch(new RegExp(`/${skill.name}\\b`));
} }
}); });
test('every skill link points at an existing generated file', () => {
const skillsSection = generated.content.split('## Skills')[1].split('## Browse Commands')[0];
const links = [...skillsSection.matchAll(/- \[\/[^\]]+\]\(([^)]+)\):/g)].map((m) => m[1]);
expect(links.length).toBe(generated.skills.length);
for (const href of links) {
expect(fs.existsSync(path.join(ROOT, href)), href).toBe(true);
}
});
test('every browse command in COMMAND_DESCRIPTIONS appears in the index', () => { test('every browse command in COMMAND_DESCRIPTIONS appears in the index', () => {
expect(generated.browseCommands.length).toBeGreaterThan(0); expect(generated.browseCommands.length).toBeGreaterThan(0);
for (const cmd of generated.browseCommands) { for (const cmd of generated.browseCommands) {