mirror of https://github.com/garrytan/gstack.git
fix: scope benchmark skill to web performance
This commit is contained in:
parent
a3259400a3
commit
b76a2d9555
|
|
@ -1,12 +1,12 @@
|
||||||
---
|
---
|
||||||
name: benchmark
|
name: gstack-benchmark
|
||||||
preamble-tier: 1
|
preamble-tier: 1
|
||||||
version: 1.0.0
|
version: 1.0.0
|
||||||
description: Performance regression detection using the browse daemon. (gstack)
|
description: Web page performance regression detection using the browse daemon. (gstack)
|
||||||
triggers:
|
triggers:
|
||||||
- performance benchmark
|
- web performance benchmark
|
||||||
- check page speed
|
- check page speed
|
||||||
- detect performance regression
|
- detect page performance regression
|
||||||
allowed-tools:
|
allowed-tools:
|
||||||
- Bash
|
- Bash
|
||||||
- Read
|
- Read
|
||||||
|
|
@ -23,8 +23,9 @@ allowed-tools:
|
||||||
Establishes
|
Establishes
|
||||||
baselines for page load times, Core Web Vitals, and resource sizes.
|
baselines for page load times, Core Web Vitals, and resource sizes.
|
||||||
Compares before/after on every PR. Tracks performance trends over time.
|
Compares before/after on every PR. Tracks performance trends over time.
|
||||||
Use when: "performance", "benchmark", "page speed", "lighthouse", "web vitals",
|
Use when: "web performance benchmark", "page speed", "lighthouse",
|
||||||
"bundle size", "load time".
|
"web vitals", "bundle size", "load time". Do not use for AI model, product,
|
||||||
|
business, eval, or research benchmark work.
|
||||||
|
|
||||||
Voice triggers (speech-to-text aliases): "speed test", "check performance".
|
Voice triggers (speech-to-text aliases): "speed test", "check performance".
|
||||||
|
|
||||||
|
|
@ -84,7 +85,7 @@ _QUESTION_TUNING=$(~/.claude/skills/gstack/bin/gstack-config get question_tuning
|
||||||
echo "QUESTION_TUNING: $_QUESTION_TUNING"
|
echo "QUESTION_TUNING: $_QUESTION_TUNING"
|
||||||
mkdir -p ~/.gstack/analytics
|
mkdir -p ~/.gstack/analytics
|
||||||
if [ "$_TEL" != "off" ]; then
|
if [ "$_TEL" != "off" ]; then
|
||||||
echo '{"skill":"benchmark","ts":"'$(date -u +%Y-%m-%dT%H:%M:%SZ)'","repo":"'$(_repo=$(basename "$(git rev-parse --show-toplevel 2>/dev/null)" 2>/dev/null | tr -cd 'a-zA-Z0-9._-'); echo "${_repo:-unknown}")'"}' >> ~/.gstack/analytics/skill-usage.jsonl 2>/dev/null || true
|
echo '{"skill":"gstack-benchmark","ts":"'$(date -u +%Y-%m-%dT%H:%M:%SZ)'","repo":"'$(_repo=$(basename "$(git rev-parse --show-toplevel 2>/dev/null)" 2>/dev/null | tr -cd 'a-zA-Z0-9._-'); echo "${_repo:-unknown}")'"}' >> ~/.gstack/analytics/skill-usage.jsonl 2>/dev/null || true
|
||||||
fi
|
fi
|
||||||
for _PF in $(find ~/.gstack/analytics -maxdepth 1 -name '.pending-*' 2>/dev/null); do
|
for _PF in $(find ~/.gstack/analytics -maxdepth 1 -name '.pending-*' 2>/dev/null); do
|
||||||
if [ -f "$_PF" ]; then
|
if [ -f "$_PF" ]; then
|
||||||
|
|
@ -106,7 +107,7 @@ if [ -f "$_LEARN_FILE" ]; then
|
||||||
else
|
else
|
||||||
echo "LEARNINGS: 0"
|
echo "LEARNINGS: 0"
|
||||||
fi
|
fi
|
||||||
~/.claude/skills/gstack/bin/gstack-timeline-log '{"skill":"benchmark","event":"started","branch":"'"$_BRANCH"'","session":"'"$_SESSION_ID"'"}' 2>/dev/null &
|
~/.claude/skills/gstack/bin/gstack-timeline-log '{"skill":"gstack-benchmark","event":"started","branch":"'"$_BRANCH"'","session":"'"$_SESSION_ID"'"}' 2>/dev/null &
|
||||||
_HAS_ROUTING="no"
|
_HAS_ROUTING="no"
|
||||||
if [ -f CLAUDE.md ] && grep -q "## Skill routing" CLAUDE.md 2>/dev/null; then
|
if [ -f CLAUDE.md ] && grep -q "## Skill routing" CLAUDE.md 2>/dev/null; then
|
||||||
_HAS_ROUTING="yes"
|
_HAS_ROUTING="yes"
|
||||||
|
|
@ -571,22 +572,29 @@ If `NEEDS_SETUP`:
|
||||||
fi
|
fi
|
||||||
```
|
```
|
||||||
|
|
||||||
# /benchmark — Performance Regression Detection
|
# /gstack-benchmark - Web Page Performance Regression Detection
|
||||||
|
|
||||||
You are a **Performance Engineer** who has optimized apps serving millions of requests. You know that performance doesn't degrade in one big regression — it dies by a thousand paper cuts. Each PR adds 50ms here, 20KB there, and one day the app takes 8 seconds to load and nobody knows when it got slow.
|
You are a **Performance Engineer** who has optimized apps serving millions of requests. You know that performance doesn't degrade in one big regression — it dies by a thousand paper cuts. Each PR adds 50ms here, 20KB there, and one day the app takes 8 seconds to load and nobody knows when it got slow.
|
||||||
|
|
||||||
Your job is to measure, baseline, compare, and alert. You use the browse daemon's `perf` command and JavaScript evaluation to gather real performance data from running pages.
|
Your job is to measure, baseline, compare, and alert. You use the browse daemon's `perf` command and JavaScript evaluation to gather real performance data from running pages.
|
||||||
|
|
||||||
|
## Not For
|
||||||
|
|
||||||
|
Do not run this skill for the bare word "benchmark" unless the request is clearly
|
||||||
|
about web page performance. Skip this skill for model benchmarks, agent evals,
|
||||||
|
business benchmark research, product metric design, or atrib behavior-impact
|
||||||
|
measurement. For cross-model gstack skill comparisons, use `/benchmark-models`.
|
||||||
|
|
||||||
## User-invocable
|
## User-invocable
|
||||||
When the user types `/benchmark`, run this skill.
|
When the user types `/gstack-benchmark`, run this skill.
|
||||||
|
|
||||||
## Arguments
|
## Arguments
|
||||||
- `/benchmark <url>` — full performance audit with baseline comparison
|
- `/gstack-benchmark <url>` — full performance audit with baseline comparison
|
||||||
- `/benchmark <url> --baseline` — capture baseline (run before making changes)
|
- `/gstack-benchmark <url> --baseline` — capture baseline (run before making changes)
|
||||||
- `/benchmark <url> --quick` — single-pass timing check (no baseline needed)
|
- `/gstack-benchmark <url> --quick` — single-pass timing check (no baseline needed)
|
||||||
- `/benchmark <url> --pages /,/dashboard,/api/health` — specify pages
|
- `/gstack-benchmark <url> --pages /,/dashboard,/api/health` — specify pages
|
||||||
- `/benchmark --diff` — benchmark only pages affected by current branch
|
- `/gstack-benchmark --diff` — benchmark only pages affected by current branch
|
||||||
- `/benchmark --trend` — show performance trends from historical data
|
- `/gstack-benchmark --trend` — show performance trends from historical data
|
||||||
|
|
||||||
## Instructions
|
## Instructions
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,20 +1,21 @@
|
||||||
---
|
---
|
||||||
name: benchmark
|
name: gstack-benchmark
|
||||||
preamble-tier: 1
|
preamble-tier: 1
|
||||||
version: 1.0.0
|
version: 1.0.0
|
||||||
description: |
|
description: |
|
||||||
Performance regression detection using the browse daemon. Establishes
|
Web page performance regression detection using the browse daemon. Establishes
|
||||||
baselines for page load times, Core Web Vitals, and resource sizes.
|
baselines for page load times, Core Web Vitals, and resource sizes.
|
||||||
Compares before/after on every PR. Tracks performance trends over time.
|
Compares before/after on every PR. Tracks performance trends over time.
|
||||||
Use when: "performance", "benchmark", "page speed", "lighthouse", "web vitals",
|
Use when: "web performance benchmark", "page speed", "lighthouse",
|
||||||
"bundle size", "load time". (gstack)
|
"web vitals", "bundle size", "load time". Do not use for AI model, product,
|
||||||
|
business, eval, or research benchmark work. (gstack)
|
||||||
voice-triggers:
|
voice-triggers:
|
||||||
- "speed test"
|
- "speed test"
|
||||||
- "check performance"
|
- "check performance"
|
||||||
triggers:
|
triggers:
|
||||||
- performance benchmark
|
- web performance benchmark
|
||||||
- check page speed
|
- check page speed
|
||||||
- detect performance regression
|
- detect page performance regression
|
||||||
allowed-tools:
|
allowed-tools:
|
||||||
- Bash
|
- Bash
|
||||||
- Read
|
- Read
|
||||||
|
|
@ -27,22 +28,29 @@ allowed-tools:
|
||||||
|
|
||||||
{{BROWSE_SETUP}}
|
{{BROWSE_SETUP}}
|
||||||
|
|
||||||
# /benchmark — Performance Regression Detection
|
# /gstack-benchmark - Web Page Performance Regression Detection
|
||||||
|
|
||||||
You are a **Performance Engineer** who has optimized apps serving millions of requests. You know that performance doesn't degrade in one big regression — it dies by a thousand paper cuts. Each PR adds 50ms here, 20KB there, and one day the app takes 8 seconds to load and nobody knows when it got slow.
|
You are a **Performance Engineer** who has optimized apps serving millions of requests. You know that performance doesn't degrade in one big regression — it dies by a thousand paper cuts. Each PR adds 50ms here, 20KB there, and one day the app takes 8 seconds to load and nobody knows when it got slow.
|
||||||
|
|
||||||
Your job is to measure, baseline, compare, and alert. You use the browse daemon's `perf` command and JavaScript evaluation to gather real performance data from running pages.
|
Your job is to measure, baseline, compare, and alert. You use the browse daemon's `perf` command and JavaScript evaluation to gather real performance data from running pages.
|
||||||
|
|
||||||
|
## Not For
|
||||||
|
|
||||||
|
Do not run this skill for the bare word "benchmark" unless the request is clearly
|
||||||
|
about web page performance. Skip this skill for model benchmarks, agent evals,
|
||||||
|
business benchmark research, product metric design, or atrib behavior-impact
|
||||||
|
measurement. For cross-model gstack skill comparisons, use `/benchmark-models`.
|
||||||
|
|
||||||
## User-invocable
|
## User-invocable
|
||||||
When the user types `/benchmark`, run this skill.
|
When the user types `/gstack-benchmark`, run this skill.
|
||||||
|
|
||||||
## Arguments
|
## Arguments
|
||||||
- `/benchmark <url>` — full performance audit with baseline comparison
|
- `/gstack-benchmark <url>` — full performance audit with baseline comparison
|
||||||
- `/benchmark <url> --baseline` — capture baseline (run before making changes)
|
- `/gstack-benchmark <url> --baseline` — capture baseline (run before making changes)
|
||||||
- `/benchmark <url> --quick` — single-pass timing check (no baseline needed)
|
- `/gstack-benchmark <url> --quick` — single-pass timing check (no baseline needed)
|
||||||
- `/benchmark <url> --pages /,/dashboard,/api/health` — specify pages
|
- `/gstack-benchmark <url> --pages /,/dashboard,/api/health` — specify pages
|
||||||
- `/benchmark --diff` — benchmark only pages affected by current branch
|
- `/gstack-benchmark --diff` — benchmark only pages affected by current branch
|
||||||
- `/benchmark --trend` — show performance trends from historical data
|
- `/gstack-benchmark --trend` — show performance trends from historical data
|
||||||
|
|
||||||
## Instructions
|
## Instructions
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -24,10 +24,12 @@ import { COMMAND_DESCRIPTIONS as BROWSE_COMMANDS } from '../browse/src/commands'
|
||||||
|
|
||||||
const ROOT = path.resolve(import.meta.dir, '..');
|
const ROOT = path.resolve(import.meta.dir, '..');
|
||||||
const OUTPUT = path.join(ROOT, 'gstack', 'llms.txt');
|
const OUTPUT = path.join(ROOT, 'gstack', 'llms.txt');
|
||||||
|
const CLAUDE_SKIPPED_SKILL_DIRS = new Set(['claude']);
|
||||||
|
|
||||||
interface SkillEntry {
|
interface SkillEntry {
|
||||||
name: string;
|
name: string;
|
||||||
description: string;
|
description: string;
|
||||||
|
href: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -113,6 +115,11 @@ function oneLine(text: string): string {
|
||||||
return first.replace(/\s+/g, ' ').trim();
|
return first.replace(/\s+/g, ' ').trim();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function skillDirForOutput(output: string): string {
|
||||||
|
const dir = path.dirname(output);
|
||||||
|
return dir === '.' ? '.' : dir;
|
||||||
|
}
|
||||||
|
|
||||||
interface GenerateOptions {
|
interface GenerateOptions {
|
||||||
/** Override repo root (for tests). */
|
/** Override repo root (for tests). */
|
||||||
root?: string;
|
root?: string;
|
||||||
|
|
@ -135,6 +142,9 @@ export async function generateLlmsTxt(opts: GenerateOptions = {}): Promise<Gener
|
||||||
const templates = discoverTemplates(root);
|
const templates = discoverTemplates(root);
|
||||||
const skills: SkillEntry[] = [];
|
const skills: SkillEntry[] = [];
|
||||||
for (const t of templates) {
|
for (const t of templates) {
|
||||||
|
if (CLAUDE_SKIPPED_SKILL_DIRS.has(skillDirForOutput(t.output))) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
const filePath = path.join(root, t.tmpl);
|
const filePath = path.join(root, t.tmpl);
|
||||||
const entry = parseSkillFrontmatter(filePath);
|
const entry = parseSkillFrontmatter(filePath);
|
||||||
if (!entry) {
|
if (!entry) {
|
||||||
|
|
@ -144,7 +154,7 @@ export async function generateLlmsTxt(opts: GenerateOptions = {}): Promise<Gener
|
||||||
}
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
skills.push(entry);
|
skills.push({ ...entry, href: t.output });
|
||||||
}
|
}
|
||||||
skills.sort((a, b) => a.name.localeCompare(b.name));
|
skills.sort((a, b) => a.name.localeCompare(b.name));
|
||||||
|
|
||||||
|
|
@ -167,7 +177,7 @@ export async function generateLlmsTxt(opts: GenerateOptions = {}): Promise<Gener
|
||||||
lines.push('');
|
lines.push('');
|
||||||
for (const skill of skills) {
|
for (const skill of skills) {
|
||||||
const summary = oneLine(skill.description);
|
const summary = oneLine(skill.description);
|
||||||
lines.push(`- [/${skill.name}](${skill.name}/SKILL.md): ${summary}`);
|
lines.push(`- [/${skill.name}](${skill.href}): ${summary}`);
|
||||||
}
|
}
|
||||||
lines.push('');
|
lines.push('');
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -9,8 +9,8 @@
|
||||||
"voice_line": "Voice triggers (speech-to-text aliases): \"auto plan\", \"automatic review\"."
|
"voice_line": "Voice triggers (speech-to-text aliases): \"auto plan\", \"automatic review\"."
|
||||||
},
|
},
|
||||||
"benchmark": {
|
"benchmark": {
|
||||||
"lead": "Performance regression detection using the browse daemon.",
|
"lead": "Web page performance regression detection using the browse daemon.",
|
||||||
"routing": "Establishes\nbaselines for page load times, Core Web Vitals, and resource sizes.\nCompares before/after on every PR. Tracks performance trends over time.\nUse when: \"performance\", \"benchmark\", \"page speed\", \"lighthouse\", \"web vitals\",\n\"bundle size\", \"load time\".",
|
"routing": "Establishes\nbaselines for page load times, Core Web Vitals, and resource sizes.\nCompares before/after on every PR. Tracks performance trends over time.\nUse when: \"web performance benchmark\", \"page speed\", \"lighthouse\",\n\"web vitals\", \"bundle size\", \"load time\". Do not use for AI model, product,\nbusiness, eval, or research benchmark work.",
|
||||||
"voice_line": "Voice triggers (speech-to-text aliases): \"speed test\", \"check performance\"."
|
"voice_line": "Voice triggers (speech-to-text aliases): \"speed test\", \"check performance\"."
|
||||||
},
|
},
|
||||||
"benchmark-models": {
|
"benchmark-models": {
|
||||||
|
|
|
||||||
|
|
@ -467,6 +467,23 @@ describe('gen-skill-docs', () => {
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('benchmark skill is scoped to web performance', () => {
|
||||||
|
const generated = fs.readFileSync(path.join(ROOT, 'benchmark', 'SKILL.md'), 'utf-8');
|
||||||
|
const template = fs.readFileSync(path.join(ROOT, 'benchmark', 'SKILL.md.tmpl'), 'utf-8');
|
||||||
|
|
||||||
|
for (const content of [generated, template]) {
|
||||||
|
const frontmatter = content.match(/^---\n([\s\S]*?)\n---/)?.[1] ?? '';
|
||||||
|
expect(frontmatter).toContain('name: gstack-benchmark');
|
||||||
|
expect(frontmatter).not.toContain('name: benchmark');
|
||||||
|
expect(frontmatter).toContain('web performance benchmark');
|
||||||
|
expect(frontmatter).not.toMatch(/["']benchmark["']/);
|
||||||
|
expect(content).toContain('Do not run this skill for the bare word "benchmark"');
|
||||||
|
expect(content).toContain('For cross-model gstack skill comparisons, use `/benchmark-models`.');
|
||||||
|
expect(content).toContain('/gstack-benchmark <url>');
|
||||||
|
expect(content).not.toContain('When the user types `/benchmark`, run this skill.');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test('qa and qa-only templates use QA_METHODOLOGY placeholder', () => {
|
test('qa and qa-only templates use QA_METHODOLOGY placeholder', () => {
|
||||||
const qaTmpl = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md.tmpl'), 'utf-8');
|
const qaTmpl = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md.tmpl'), 'utf-8');
|
||||||
expect(qaTmpl).toContain('{{QA_METHODOLOGY}}');
|
expect(qaTmpl).toContain('{{QA_METHODOLOGY}}');
|
||||||
|
|
|
||||||
|
|
@ -25,17 +25,29 @@ describe('gen-llms-txt — shape', () => {
|
||||||
expect(generated.content).toContain('auto-generated');
|
expect(generated.content).toContain('auto-generated');
|
||||||
});
|
});
|
||||||
|
|
||||||
test('every skill .tmpl in the repo appears in the index', () => {
|
test('every generated Claude skill appears in the index', () => {
|
||||||
const templates = discoverTemplates(ROOT);
|
const templates = discoverTemplates(ROOT);
|
||||||
|
const claudeGeneratedTemplates = templates.filter((t) => path.dirname(t.output) !== 'claude');
|
||||||
// Filter to those that successfully parsed (have name + description).
|
// Filter to those that successfully parsed (have name + description).
|
||||||
expect(generated.skills.length).toBeGreaterThan(0);
|
expect(generated.skills.length).toBeGreaterThan(0);
|
||||||
expect(generated.skills.length).toBeLessThanOrEqual(templates.length);
|
expect(generated.skills.length).toBeLessThanOrEqual(claudeGeneratedTemplates.length);
|
||||||
|
expect(generated.content).not.toContain('[/claude]');
|
||||||
|
|
||||||
for (const skill of generated.skills) {
|
for (const skill of generated.skills) {
|
||||||
expect(generated.content).toMatch(new RegExp(`/${skill.name}\\b`));
|
expect(generated.content).toMatch(new RegExp(`/${skill.name}\\b`));
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('every skill link points at an existing generated file', () => {
|
||||||
|
const skillsSection = generated.content.split('## Skills')[1].split('## Browse Commands')[0];
|
||||||
|
const links = [...skillsSection.matchAll(/- \[\/[^\]]+\]\(([^)]+)\):/g)].map((m) => m[1]);
|
||||||
|
expect(links.length).toBe(generated.skills.length);
|
||||||
|
|
||||||
|
for (const href of links) {
|
||||||
|
expect(fs.existsSync(path.join(ROOT, href)), href).toBe(true);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test('every browse command in COMMAND_DESCRIPTIONS appears in the index', () => {
|
test('every browse command in COMMAND_DESCRIPTIONS appears in the index', () => {
|
||||||
expect(generated.browseCommands.length).toBeGreaterThan(0);
|
expect(generated.browseCommands.length).toBeGreaterThan(0);
|
||||||
for (const cmd of generated.browseCommands) {
|
for (const cmd of generated.browseCommands) {
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue