mirror of https://github.com/garrytan/gstack.git
fix: scope benchmark skill to web performance
This commit is contained in:
Nader Helmy
parent
a3259400a3
commit
b76a2d9555
|
|
@ -1,12 +1,12 @@
|
|||
---
|
||||
name: benchmark
|
||||
name: gstack-benchmark
|
||||
preamble-tier: 1
|
||||
version: 1.0.0
|
||||
description: Performance regression detection using the browse daemon. (gstack)
|
||||
description: Web page performance regression detection using the browse daemon. (gstack)
|
||||
triggers:
|
||||
- performance benchmark
|
||||
- web performance benchmark
|
||||
- check page speed
|
||||
- detect performance regression
|
||||
- detect page performance regression
|
||||
allowed-tools:
|
||||
- Bash
|
||||
- Read
|
||||
|
|
@ -23,8 +23,9 @@ allowed-tools:
|
|||
Establishes
|
||||
baselines for page load times, Core Web Vitals, and resource sizes.
|
||||
Compares before/after on every PR. Tracks performance trends over time.
|
||||
Use when: "performance", "benchmark", "page speed", "lighthouse", "web vitals",
|
||||
"bundle size", "load time".
|
||||
Use when: "web performance benchmark", "page speed", "lighthouse",
|
||||
"web vitals", "bundle size", "load time". Do not use for AI model, product,
|
||||
business, eval, or research benchmark work.
|
||||
|
||||
Voice triggers (speech-to-text aliases): "speed test", "check performance".
|
||||
|
||||
|
|
@ -84,7 +85,7 @@ _QUESTION_TUNING=$(~/.claude/skills/gstack/bin/gstack-config get question_tuning
|
|||
echo "QUESTION_TUNING: $_QUESTION_TUNING"
|
||||
mkdir -p ~/.gstack/analytics
|
||||
if [ "$_TEL" != "off" ]; then
|
||||
echo '{"skill":"benchmark","ts":"'$(date -u +%Y-%m-%dT%H:%M:%SZ)'","repo":"'$(_repo=$(basename "$(git rev-parse --show-toplevel 2>/dev/null)" 2>/dev/null | tr -cd 'a-zA-Z0-9._-'); echo "${_repo:-unknown}")'"}' >> ~/.gstack/analytics/skill-usage.jsonl 2>/dev/null || true
|
||||
echo '{"skill":"gstack-benchmark","ts":"'$(date -u +%Y-%m-%dT%H:%M:%SZ)'","repo":"'$(_repo=$(basename "$(git rev-parse --show-toplevel 2>/dev/null)" 2>/dev/null | tr -cd 'a-zA-Z0-9._-'); echo "${_repo:-unknown}")'"}' >> ~/.gstack/analytics/skill-usage.jsonl 2>/dev/null || true
|
||||
fi
|
||||
for _PF in $(find ~/.gstack/analytics -maxdepth 1 -name '.pending-*' 2>/dev/null); do
|
||||
if [ -f "$_PF" ]; then
|
||||
|
|
@ -106,7 +107,7 @@ if [ -f "$_LEARN_FILE" ]; then
|
|||
else
|
||||
echo "LEARNINGS: 0"
|
||||
fi
|
||||
~/.claude/skills/gstack/bin/gstack-timeline-log '{"skill":"benchmark","event":"started","branch":"'"$_BRANCH"'","session":"'"$_SESSION_ID"'"}' 2>/dev/null &
|
||||
~/.claude/skills/gstack/bin/gstack-timeline-log '{"skill":"gstack-benchmark","event":"started","branch":"'"$_BRANCH"'","session":"'"$_SESSION_ID"'"}' 2>/dev/null &
|
||||
_HAS_ROUTING="no"
|
||||
if [ -f CLAUDE.md ] && grep -q "## Skill routing" CLAUDE.md 2>/dev/null; then
|
||||
_HAS_ROUTING="yes"
|
||||
|
|
@ -571,22 +572,29 @@ If `NEEDS_SETUP`:
|
|||
fi
|
||||
```
|
||||
|
||||
# /benchmark — Performance Regression Detection
|
||||
# /gstack-benchmark - Web Page Performance Regression Detection
|
||||
|
||||
You are a **Performance Engineer** who has optimized apps serving millions of requests. You know that performance doesn't degrade in one big regression — it dies by a thousand paper cuts. Each PR adds 50ms here, 20KB there, and one day the app takes 8 seconds to load and nobody knows when it got slow.
|
||||
|
||||
Your job is to measure, baseline, compare, and alert. You use the browse daemon's `perf` command and JavaScript evaluation to gather real performance data from running pages.
|
||||
|
||||
## Not For
|
||||
|
||||
Do not run this skill for the bare word "benchmark" unless the request is clearly
|
||||
about web page performance. Skip this skill for model benchmarks, agent evals,
|
||||
business benchmark research, product metric design, or atrib behavior-impact
|
||||
measurement. For cross-model gstack skill comparisons, use `/benchmark-models`.
|
||||
|
||||
## User-invocable
|
||||
When the user types `/benchmark`, run this skill.
|
||||
When the user types `/gstack-benchmark`, run this skill.
|
||||
|
||||
## Arguments
|
||||
- `/benchmark <url>` — full performance audit with baseline comparison
|
||||
- `/benchmark <url> --baseline` — capture baseline (run before making changes)
|
||||
- `/benchmark <url> --quick` — single-pass timing check (no baseline needed)
|
||||
- `/benchmark <url> --pages /,/dashboard,/api/health` — specify pages
|
||||
- `/benchmark --diff` — benchmark only pages affected by current branch
|
||||
- `/benchmark --trend` — show performance trends from historical data
|
||||
- `/gstack-benchmark <url>` — full performance audit with baseline comparison
|
||||
- `/gstack-benchmark <url> --baseline` — capture baseline (run before making changes)
|
||||
- `/gstack-benchmark <url> --quick` — single-pass timing check (no baseline needed)
|
||||
- `/gstack-benchmark <url> --pages /,/dashboard,/api/health` — specify pages
|
||||
- `/gstack-benchmark --diff` — benchmark only pages affected by current branch
|
||||
- `/gstack-benchmark --trend` — show performance trends from historical data
|
||||
|
||||
## Instructions
|
||||
|
||||
|
|
|
|||
|
|
@ -1,20 +1,21 @@
|
|||
---
|
||||
name: benchmark
|
||||
name: gstack-benchmark
|
||||
preamble-tier: 1
|
||||
version: 1.0.0
|
||||
description: |
|
||||
Performance regression detection using the browse daemon. Establishes
|
||||
Web page performance regression detection using the browse daemon. Establishes
|
||||
baselines for page load times, Core Web Vitals, and resource sizes.
|
||||
Compares before/after on every PR. Tracks performance trends over time.
|
||||
Use when: "performance", "benchmark", "page speed", "lighthouse", "web vitals",
|
||||
"bundle size", "load time". (gstack)
|
||||
Use when: "web performance benchmark", "page speed", "lighthouse",
|
||||
"web vitals", "bundle size", "load time". Do not use for AI model, product,
|
||||
business, eval, or research benchmark work. (gstack)
|
||||
voice-triggers:
|
||||
- "speed test"
|
||||
- "check performance"
|
||||
triggers:
|
||||
- performance benchmark
|
||||
- web performance benchmark
|
||||
- check page speed
|
||||
- detect performance regression
|
||||
- detect page performance regression
|
||||
allowed-tools:
|
||||
- Bash
|
||||
- Read
|
||||
|
|
@ -27,22 +28,29 @@ allowed-tools:
|
|||
|
||||
{{BROWSE_SETUP}}
|
||||
|
||||
# /benchmark — Performance Regression Detection
|
||||
# /gstack-benchmark - Web Page Performance Regression Detection
|
||||
|
||||
You are a **Performance Engineer** who has optimized apps serving millions of requests. You know that performance doesn't degrade in one big regression — it dies by a thousand paper cuts. Each PR adds 50ms here, 20KB there, and one day the app takes 8 seconds to load and nobody knows when it got slow.
|
||||
|
||||
Your job is to measure, baseline, compare, and alert. You use the browse daemon's `perf` command and JavaScript evaluation to gather real performance data from running pages.
|
||||
|
||||
## Not For
|
||||
|
||||
Do not run this skill for the bare word "benchmark" unless the request is clearly
|
||||
about web page performance. Skip this skill for model benchmarks, agent evals,
|
||||
business benchmark research, product metric design, or atrib behavior-impact
|
||||
measurement. For cross-model gstack skill comparisons, use `/benchmark-models`.
|
||||
|
||||
## User-invocable
|
||||
When the user types `/benchmark`, run this skill.
|
||||
When the user types `/gstack-benchmark`, run this skill.
|
||||
|
||||
## Arguments
|
||||
- `/benchmark <url>` — full performance audit with baseline comparison
|
||||
- `/benchmark <url> --baseline` — capture baseline (run before making changes)
|
||||
- `/benchmark <url> --quick` — single-pass timing check (no baseline needed)
|
||||
- `/benchmark <url> --pages /,/dashboard,/api/health` — specify pages
|
||||
- `/benchmark --diff` — benchmark only pages affected by current branch
|
||||
- `/benchmark --trend` — show performance trends from historical data
|
||||
- `/gstack-benchmark <url>` — full performance audit with baseline comparison
|
||||
- `/gstack-benchmark <url> --baseline` — capture baseline (run before making changes)
|
||||
- `/gstack-benchmark <url> --quick` — single-pass timing check (no baseline needed)
|
||||
- `/gstack-benchmark <url> --pages /,/dashboard,/api/health` — specify pages
|
||||
- `/gstack-benchmark --diff` — benchmark only pages affected by current branch
|
||||
- `/gstack-benchmark --trend` — show performance trends from historical data
|
||||
|
||||
## Instructions
|
||||
|
||||
|
|
|
|||
|
|
@ -24,10 +24,12 @@ import { COMMAND_DESCRIPTIONS as BROWSE_COMMANDS } from '../browse/src/commands'
|
|||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const OUTPUT = path.join(ROOT, 'gstack', 'llms.txt');
|
||||
const CLAUDE_SKIPPED_SKILL_DIRS = new Set(['claude']);
|
||||
|
||||
interface SkillEntry {
|
||||
name: string;
|
||||
description: string;
|
||||
href: string;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -113,6 +115,11 @@ function oneLine(text: string): string {
|
|||
return first.replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
|
||||
function skillDirForOutput(output: string): string {
|
||||
const dir = path.dirname(output);
|
||||
return dir === '.' ? '.' : dir;
|
||||
}
|
||||
|
||||
interface GenerateOptions {
|
||||
/** Override repo root (for tests). */
|
||||
root?: string;
|
||||
|
|
@ -135,6 +142,9 @@ export async function generateLlmsTxt(opts: GenerateOptions = {}): Promise<Gener
|
|||
const templates = discoverTemplates(root);
|
||||
const skills: SkillEntry[] = [];
|
||||
for (const t of templates) {
|
||||
if (CLAUDE_SKIPPED_SKILL_DIRS.has(skillDirForOutput(t.output))) {
|
||||
continue;
|
||||
}
|
||||
const filePath = path.join(root, t.tmpl);
|
||||
const entry = parseSkillFrontmatter(filePath);
|
||||
if (!entry) {
|
||||
|
|
@ -144,7 +154,7 @@ export async function generateLlmsTxt(opts: GenerateOptions = {}): Promise<Gener
|
|||
}
|
||||
continue;
|
||||
}
|
||||
skills.push(entry);
|
||||
skills.push({ ...entry, href: t.output });
|
||||
}
|
||||
skills.sort((a, b) => a.name.localeCompare(b.name));
|
||||
|
||||
|
|
@ -167,7 +177,7 @@ export async function generateLlmsTxt(opts: GenerateOptions = {}): Promise<Gener
|
|||
lines.push('');
|
||||
for (const skill of skills) {
|
||||
const summary = oneLine(skill.description);
|
||||
lines.push(`- [/${skill.name}](${skill.name}/SKILL.md): ${summary}`);
|
||||
lines.push(`- [/${skill.name}](${skill.href}): ${summary}`);
|
||||
}
|
||||
lines.push('');
|
||||
|
||||
|
|
|
|||
|
|
@ -9,8 +9,8 @@
|
|||
"voice_line": "Voice triggers (speech-to-text aliases): \"auto plan\", \"automatic review\"."
|
||||
},
|
||||
"benchmark": {
|
||||
"lead": "Performance regression detection using the browse daemon.",
|
||||
"routing": "Establishes\nbaselines for page load times, Core Web Vitals, and resource sizes.\nCompares before/after on every PR. Tracks performance trends over time.\nUse when: \"performance\", \"benchmark\", \"page speed\", \"lighthouse\", \"web vitals\",\n\"bundle size\", \"load time\".",
|
||||
"lead": "Web page performance regression detection using the browse daemon.",
|
||||
"routing": "Establishes\nbaselines for page load times, Core Web Vitals, and resource sizes.\nCompares before/after on every PR. Tracks performance trends over time.\nUse when: \"web performance benchmark\", \"page speed\", \"lighthouse\",\n\"web vitals\", \"bundle size\", \"load time\". Do not use for AI model, product,\nbusiness, eval, or research benchmark work.",
|
||||
"voice_line": "Voice triggers (speech-to-text aliases): \"speed test\", \"check performance\"."
|
||||
},
|
||||
"benchmark-models": {
|
||||
|
|
|
|||
|
|
@ -467,6 +467,23 @@ describe('gen-skill-docs', () => {
|
|||
}
|
||||
});
|
||||
|
||||
test('benchmark skill is scoped to web performance', () => {
|
||||
const generated = fs.readFileSync(path.join(ROOT, 'benchmark', 'SKILL.md'), 'utf-8');
|
||||
const template = fs.readFileSync(path.join(ROOT, 'benchmark', 'SKILL.md.tmpl'), 'utf-8');
|
||||
|
||||
for (const content of [generated, template]) {
|
||||
const frontmatter = content.match(/^---\n([\s\S]*?)\n---/)?.[1] ?? '';
|
||||
expect(frontmatter).toContain('name: gstack-benchmark');
|
||||
expect(frontmatter).not.toContain('name: benchmark');
|
||||
expect(frontmatter).toContain('web performance benchmark');
|
||||
expect(frontmatter).not.toMatch(/["']benchmark["']/);
|
||||
expect(content).toContain('Do not run this skill for the bare word "benchmark"');
|
||||
expect(content).toContain('For cross-model gstack skill comparisons, use `/benchmark-models`.');
|
||||
expect(content).toContain('/gstack-benchmark <url>');
|
||||
expect(content).not.toContain('When the user types `/benchmark`, run this skill.');
|
||||
}
|
||||
});
|
||||
|
||||
test('qa and qa-only templates use QA_METHODOLOGY placeholder', () => {
|
||||
const qaTmpl = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md.tmpl'), 'utf-8');
|
||||
expect(qaTmpl).toContain('{{QA_METHODOLOGY}}');
|
||||
|
|
|
|||
|
|
@ -25,17 +25,29 @@ describe('gen-llms-txt — shape', () => {
|
|||
expect(generated.content).toContain('auto-generated');
|
||||
});
|
||||
|
||||
test('every skill .tmpl in the repo appears in the index', () => {
|
||||
test('every generated Claude skill appears in the index', () => {
|
||||
const templates = discoverTemplates(ROOT);
|
||||
const claudeGeneratedTemplates = templates.filter((t) => path.dirname(t.output) !== 'claude');
|
||||
// Filter to those that successfully parsed (have name + description).
|
||||
expect(generated.skills.length).toBeGreaterThan(0);
|
||||
expect(generated.skills.length).toBeLessThanOrEqual(templates.length);
|
||||
expect(generated.skills.length).toBeLessThanOrEqual(claudeGeneratedTemplates.length);
|
||||
expect(generated.content).not.toContain('[/claude]');
|
||||
|
||||
for (const skill of generated.skills) {
|
||||
expect(generated.content).toMatch(new RegExp(`/${skill.name}\\b`));
|
||||
}
|
||||
});
|
||||
|
||||
test('every skill link points at an existing generated file', () => {
|
||||
const skillsSection = generated.content.split('## Skills')[1].split('## Browse Commands')[0];
|
||||
const links = [...skillsSection.matchAll(/- \[\/[^\]]+\]\(([^)]+)\):/g)].map((m) => m[1]);
|
||||
expect(links.length).toBe(generated.skills.length);
|
||||
|
||||
for (const href of links) {
|
||||
expect(fs.existsSync(path.join(ROOT, href)), href).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
test('every browse command in COMMAND_DESCRIPTIONS appears in the index', () => {
|
||||
expect(generated.browseCommands.length).toBeGreaterThan(0);
|
||||
for (const cmd of generated.browseCommands) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue