From 866159861d6f8141988b8884e7217b75498cfb42 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sun, 21 Jun 2026 07:16:24 -0700 Subject: [PATCH] test: cover first-run detection + repoint browse-content assertions to /browse - New unit tests for every detection bucket, the eval-safe enum contract, and the first-run gating (test/preamble-first-task-scaffold.test.ts); periodic E2E that runs the detector through the real harness (test/skill-e2e-first-task-scaffold.test.ts). - Repoint browse-content assertions (gen-skill-docs, audit-compliance, skill-validation, LLM-judge eval) from the root skill to browse/SKILL.md following the router split; add a regression pinning that the router carries no browse body. - Register first-task-scaffold touchfiles + periodic tier; bump parity/carve size caps ~1-2KB per skill for the shared first-run-guidance preamble section. - Refresh ship golden fixtures for the preamble addition. --- test/audit-compliance.test.ts | 11 +- test/fixtures/golden/claude-ship-SKILL.md | 29 ++++ test/fixtures/golden/codex-ship-SKILL.md | 29 ++++ test/fixtures/golden/factory-ship-SKILL.md | 29 ++++ test/gen-skill-docs.test.ts | 28 ++-- test/helpers/carve-guards.ts | 23 ++- test/helpers/parity-harness.ts | 10 +- test/helpers/touchfiles.ts | 7 + test/preamble-first-task-scaffold.test.ts | 171 +++++++++++++++++++++ test/skill-e2e-first-task-scaffold.test.ts | 97 ++++++++++++ test/skill-llm-eval.test.ts | 36 +++-- test/skill-validation.test.ts | 19 ++- 12 files changed, 438 insertions(+), 51 deletions(-) create mode 100644 test/preamble-first-task-scaffold.test.ts create mode 100644 test/skill-e2e-first-task-scaffold.test.ts diff --git a/test/audit-compliance.test.ts b/test/audit-compliance.test.ts index d7ab9af29..80ef6dced 100644 --- a/test/audit-compliance.test.ts +++ b/test/audit-compliance.test.ts @@ -23,12 +23,14 @@ function getAllSkillMds(): Array<{ name: string; content: string }> { describe('Audit compliance', () => { // Fix 1: W007 — No hardcoded credentials in documentation test('no hardcoded credential patterns in SKILL.md.tmpl', () => { - const tmpl = readFileSync(join(ROOT, 'SKILL.md.tmpl'), 'utf-8'); + // P2 (v1.2.0): the browse QA examples moved from the root router to + // browse/SKILL.md.tmpl. The security intent is unchanged — the QA form + // examples must not ship real-looking credentials; generic placeholders + // ("user@test.com", "password") are fine. + const tmpl = readFileSync(join(ROOT, 'browse', 'SKILL.md.tmpl'), 'utf-8'); expect(tmpl).not.toContain('"password123"'); expect(tmpl).not.toContain('"test@example.com"'); expect(tmpl).not.toContain('"test@test.com"'); - expect(tmpl).toContain('$TEST_EMAIL'); - expect(tmpl).toContain('$TEST_PASSWORD'); }); // Fix 2: Conditional telemetry — binary calls wrapped with existence check @@ -71,7 +73,8 @@ describe('Audit compliance', () => { // Fix 4: W011 — Untrusted content warning in command reference test('command reference includes untrusted content warning after Navigation', () => { - const rootSkill = readFileSync(join(ROOT, 'SKILL.md'), 'utf-8'); + // P2 (v1.2.0): the command reference moved from the root router to browse/SKILL.md. + const rootSkill = readFileSync(join(ROOT, 'browse', 'SKILL.md'), 'utf-8'); const navIdx = rootSkill.indexOf('### Navigation'); const readingIdx = rootSkill.indexOf('### Reading'); expect(navIdx).toBeGreaterThan(-1); diff --git a/test/fixtures/golden/claude-ship-SKILL.md b/test/fixtures/golden/claude-ship-SKILL.md index a8ebb77d7..92749a7ce 100644 --- a/test/fixtures/golden/claude-ship-SKILL.md +++ b/test/fixtures/golden/claude-ship-SKILL.md @@ -60,6 +60,17 @@ echo "SESSION_KIND: $_SESSION_KIND" if [ "$_SESSION_KIND" != "headless" ] && { [ -n "${CONDUCTOR_WORKSPACE_PATH:-}" ] || [ -n "${CONDUCTOR_PORT:-}" ]; }; then echo "CONDUCTOR_SESSION: true" fi +_ACTIVATED=$([ -f ~/.gstack/.activated ] && echo "yes" || echo "no") +_FIRST_LOOP_SHOWN=$([ -f ~/.gstack/.first-loop-tip-shown ] && echo "yes" || echo "no") +echo "ACTIVATED: $_ACTIVATED" +echo "FIRST_LOOP_SHOWN: $_FIRST_LOOP_SHOWN" +# First-run project detection: run the detector ONLY on the first-ever skill run +# (ACTIVATED=no, interactive) so it stays off the hot path for every run after. +_FIRST_TASK="" +if [ "$_ACTIVATED" = "no" ] && [ "$_SESSION_KIND" != "headless" ]; then + _FIRST_TASK=$(~/.claude/skills/gstack/bin/gstack-first-task-detect 2>/dev/null || true) +fi +echo "FIRST_TASK: $_FIRST_TASK" _LAKE_SEEN=$([ -f ~/.gstack/.completeness-intro-seen ] && echo "yes" || echo "no") echo "LAKE_INTRO: $_LAKE_SEEN" _TEL=$(~/.claude/skills/gstack/bin/gstack-config get telemetry 2>/dev/null || true) @@ -229,6 +240,24 @@ touch ~/.gstack/.proactive-prompted Skip if `PROACTIVE_PROMPTED` is `yes`. +## First-run guidance (one-time) + +If `ACTIVATED` is `no` (first skill run on this machine) AND the preamble printed a non-empty `FIRST_TASK:` value that is NOT `nongit`: show ONE short, project-specific line mapped from the token, as a heads-up, then CONTINUE with whatever the user actually asked — do NOT halt their task. Map the token: `greenfield` → "Fresh repo — shape it first with `/spec` or `/office-hours`." `code_node`/`code_python`/`code_rust`/`code_go`/`code_ruby`/`code_ios` → "There's code here — `/qa` to see it work, or `/investigate` if something's off." `branch_ahead` → "Unshipped work on this branch — `/review` then `/ship`." `dirty_default` → "Uncommitted changes — `/review` before committing." `clean_default` → "Pick one: `/spec`, `/investigate`, or `/qa`." Then substitute the token you saw for TASK_TOKEN and run (best-effort), and mark activated: +```bash +~/.claude/skills/gstack/bin/gstack-telemetry-log --event-type first_task_scaffold_shown --skill "TASK_TOKEN" --outcome shown 2>/dev/null || true +touch ~/.gstack/.activated 2>/dev/null || true +``` + +If `ACTIVATED` is `no` but `FIRST_TASK:` is empty or `nongit` (headless, non-git, or nothing actionable): show nothing, just run `touch ~/.gstack/.activated 2>/dev/null || true`. + +Else if `ACTIVATED` is `yes` AND `FIRST_LOOP_SHOWN` is `no`: say once as a heads-up (then continue): + +> Tip: gstack pays off when you complete one loop — **plan → review → ship**. A common first loop: `/office-hours` or `/spec` to shape it, `/plan-eng-review` to lock it, then `/ship`. + +Then run `touch ~/.gstack/.first-loop-tip-shown 2>/dev/null || true`. + +Skip this section if `ACTIVATED` and `FIRST_LOOP_SHOWN` are both `yes`. + If `HAS_ROUTING` is `no` AND `ROUTING_DECLINED` is `false` AND `PROACTIVE_PROMPTED` is `yes`: Check if a CLAUDE.md file exists in the project root. If it does not exist, create it. diff --git a/test/fixtures/golden/codex-ship-SKILL.md b/test/fixtures/golden/codex-ship-SKILL.md index 347e1ff81..b79fdaee7 100644 --- a/test/fixtures/golden/codex-ship-SKILL.md +++ b/test/fixtures/golden/codex-ship-SKILL.md @@ -46,6 +46,17 @@ echo "SESSION_KIND: $_SESSION_KIND" if [ "$_SESSION_KIND" != "headless" ] && { [ -n "${CONDUCTOR_WORKSPACE_PATH:-}" ] || [ -n "${CONDUCTOR_PORT:-}" ]; }; then echo "CONDUCTOR_SESSION: true" fi +_ACTIVATED=$([ -f ~/.gstack/.activated ] && echo "yes" || echo "no") +_FIRST_LOOP_SHOWN=$([ -f ~/.gstack/.first-loop-tip-shown ] && echo "yes" || echo "no") +echo "ACTIVATED: $_ACTIVATED" +echo "FIRST_LOOP_SHOWN: $_FIRST_LOOP_SHOWN" +# First-run project detection: run the detector ONLY on the first-ever skill run +# (ACTIVATED=no, interactive) so it stays off the hot path for every run after. +_FIRST_TASK="" +if [ "$_ACTIVATED" = "no" ] && [ "$_SESSION_KIND" != "headless" ]; then + _FIRST_TASK=$($GSTACK_BIN/gstack-first-task-detect 2>/dev/null || true) +fi +echo "FIRST_TASK: $_FIRST_TASK" _LAKE_SEEN=$([ -f ~/.gstack/.completeness-intro-seen ] && echo "yes" || echo "no") echo "LAKE_INTRO: $_LAKE_SEEN" _TEL=$($GSTACK_BIN/gstack-config get telemetry 2>/dev/null || true) @@ -215,6 +226,24 @@ touch ~/.gstack/.proactive-prompted Skip if `PROACTIVE_PROMPTED` is `yes`. +## First-run guidance (one-time) + +If `ACTIVATED` is `no` (first skill run on this machine) AND the preamble printed a non-empty `FIRST_TASK:` value that is NOT `nongit`: show ONE short, project-specific line mapped from the token, as a heads-up, then CONTINUE with whatever the user actually asked — do NOT halt their task. Map the token: `greenfield` → "Fresh repo — shape it first with `/spec` or `/office-hours`." `code_node`/`code_python`/`code_rust`/`code_go`/`code_ruby`/`code_ios` → "There's code here — `/qa` to see it work, or `/investigate` if something's off." `branch_ahead` → "Unshipped work on this branch — `/review` then `/ship`." `dirty_default` → "Uncommitted changes — `/review` before committing." `clean_default` → "Pick one: `/spec`, `/investigate`, or `/qa`." Then substitute the token you saw for TASK_TOKEN and run (best-effort), and mark activated: +```bash +$GSTACK_BIN/gstack-telemetry-log --event-type first_task_scaffold_shown --skill "TASK_TOKEN" --outcome shown 2>/dev/null || true +touch ~/.gstack/.activated 2>/dev/null || true +``` + +If `ACTIVATED` is `no` but `FIRST_TASK:` is empty or `nongit` (headless, non-git, or nothing actionable): show nothing, just run `touch ~/.gstack/.activated 2>/dev/null || true`. + +Else if `ACTIVATED` is `yes` AND `FIRST_LOOP_SHOWN` is `no`: say once as a heads-up (then continue): + +> Tip: gstack pays off when you complete one loop — **plan → review → ship**. A common first loop: `/office-hours` or `/spec` to shape it, `/plan-eng-review` to lock it, then `/ship`. + +Then run `touch ~/.gstack/.first-loop-tip-shown 2>/dev/null || true`. + +Skip this section if `ACTIVATED` and `FIRST_LOOP_SHOWN` are both `yes`. + If `HAS_ROUTING` is `no` AND `ROUTING_DECLINED` is `false` AND `PROACTIVE_PROMPTED` is `yes`: Check if a CLAUDE.md file exists in the project root. If it does not exist, create it. diff --git a/test/fixtures/golden/factory-ship-SKILL.md b/test/fixtures/golden/factory-ship-SKILL.md index ee0ee83a2..1ec252c93 100644 --- a/test/fixtures/golden/factory-ship-SKILL.md +++ b/test/fixtures/golden/factory-ship-SKILL.md @@ -48,6 +48,17 @@ echo "SESSION_KIND: $_SESSION_KIND" if [ "$_SESSION_KIND" != "headless" ] && { [ -n "${CONDUCTOR_WORKSPACE_PATH:-}" ] || [ -n "${CONDUCTOR_PORT:-}" ]; }; then echo "CONDUCTOR_SESSION: true" fi +_ACTIVATED=$([ -f ~/.gstack/.activated ] && echo "yes" || echo "no") +_FIRST_LOOP_SHOWN=$([ -f ~/.gstack/.first-loop-tip-shown ] && echo "yes" || echo "no") +echo "ACTIVATED: $_ACTIVATED" +echo "FIRST_LOOP_SHOWN: $_FIRST_LOOP_SHOWN" +# First-run project detection: run the detector ONLY on the first-ever skill run +# (ACTIVATED=no, interactive) so it stays off the hot path for every run after. +_FIRST_TASK="" +if [ "$_ACTIVATED" = "no" ] && [ "$_SESSION_KIND" != "headless" ]; then + _FIRST_TASK=$($GSTACK_BIN/gstack-first-task-detect 2>/dev/null || true) +fi +echo "FIRST_TASK: $_FIRST_TASK" _LAKE_SEEN=$([ -f ~/.gstack/.completeness-intro-seen ] && echo "yes" || echo "no") echo "LAKE_INTRO: $_LAKE_SEEN" _TEL=$($GSTACK_BIN/gstack-config get telemetry 2>/dev/null || true) @@ -217,6 +228,24 @@ touch ~/.gstack/.proactive-prompted Skip if `PROACTIVE_PROMPTED` is `yes`. +## First-run guidance (one-time) + +If `ACTIVATED` is `no` (first skill run on this machine) AND the preamble printed a non-empty `FIRST_TASK:` value that is NOT `nongit`: show ONE short, project-specific line mapped from the token, as a heads-up, then CONTINUE with whatever the user actually asked — do NOT halt their task. Map the token: `greenfield` → "Fresh repo — shape it first with `/spec` or `/office-hours`." `code_node`/`code_python`/`code_rust`/`code_go`/`code_ruby`/`code_ios` → "There's code here — `/qa` to see it work, or `/investigate` if something's off." `branch_ahead` → "Unshipped work on this branch — `/review` then `/ship`." `dirty_default` → "Uncommitted changes — `/review` before committing." `clean_default` → "Pick one: `/spec`, `/investigate`, or `/qa`." Then substitute the token you saw for TASK_TOKEN and run (best-effort), and mark activated: +```bash +$GSTACK_BIN/gstack-telemetry-log --event-type first_task_scaffold_shown --skill "TASK_TOKEN" --outcome shown 2>/dev/null || true +touch ~/.gstack/.activated 2>/dev/null || true +``` + +If `ACTIVATED` is `no` but `FIRST_TASK:` is empty or `nongit` (headless, non-git, or nothing actionable): show nothing, just run `touch ~/.gstack/.activated 2>/dev/null || true`. + +Else if `ACTIVATED` is `yes` AND `FIRST_LOOP_SHOWN` is `no`: say once as a heads-up (then continue): + +> Tip: gstack pays off when you complete one loop — **plan → review → ship**. A common first loop: `/office-hours` or `/spec` to shape it, `/plan-eng-review` to lock it, then `/ship`. + +Then run `touch ~/.gstack/.first-loop-tip-shown 2>/dev/null || true`. + +Skip this section if `ACTIVATED` and `FIRST_LOOP_SHOWN` are both `yes`. + If `HAS_ROUTING` is `no` AND `ROUTING_DECLINED` is `false` AND `PROACTIVE_PROMPTED` is `yes`: Check if a CLAUDE.md file exists in the project root. If it does not exist, create it. diff --git a/test/gen-skill-docs.test.ts b/test/gen-skill-docs.test.ts index 431209a7f..2fb783ffd 100644 --- a/test/gen-skill-docs.test.ts +++ b/test/gen-skill-docs.test.ts @@ -108,7 +108,7 @@ const CLAUDE_GENERATED_SKILLS = ALL_SKILLS.filter(skill => !CLAUDE_SKIPPED_SKILL describe('gen-skill-docs', () => { test('generated SKILL.md contains all command categories', () => { - const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8'); + const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8'); const categories = new Set(Object.values(COMMAND_DESCRIPTIONS).map(d => d.category)); for (const cat of categories) { expect(content).toContain(`### ${cat}`); @@ -116,7 +116,7 @@ describe('gen-skill-docs', () => { }); test('generated SKILL.md contains all commands', () => { - const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8'); + const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8'); for (const [cmd, meta] of Object.entries(COMMAND_DESCRIPTIONS)) { const display = meta.usage || cmd; expect(content).toContain(display); @@ -124,7 +124,7 @@ describe('gen-skill-docs', () => { }); test('command table is sorted alphabetically within categories', () => { - const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8'); + const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8'); // Extract command names from the Navigation section as a test const navSection = content.match(/### Navigation\n\|.*\n\|.*\n([\s\S]*?)(?=\n###|\n## )/); expect(navSection).not.toBeNull(); @@ -149,7 +149,7 @@ describe('gen-skill-docs', () => { }); test('snapshot flags section contains all flags', () => { - const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8'); + const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8'); for (const flag of SNAPSHOT_FLAGS) { expect(content).toContain(flag.short); expect(content).toContain(flag.description); @@ -284,10 +284,12 @@ describe('gen-skill-docs', () => { }); test('templates contain placeholders', () => { + // P2 (v1.2.0): the root template is a pure router — only {{PREAMBLE}}. + // The browse command/snapshot placeholders live in browse/SKILL.md.tmpl now. const rootTmpl = fs.readFileSync(path.join(ROOT, 'SKILL.md.tmpl'), 'utf-8'); - expect(rootTmpl).toContain('{{COMMAND_REFERENCE}}'); - expect(rootTmpl).toContain('{{SNAPSHOT_FLAGS}}'); expect(rootTmpl).toContain('{{PREAMBLE}}'); + expect(rootTmpl).not.toContain('{{COMMAND_REFERENCE}}'); + expect(rootTmpl).not.toContain('{{SNAPSHOT_FLAGS}}'); const browseTmpl = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md.tmpl'), 'utf-8'); expect(browseTmpl).toContain('{{COMMAND_REFERENCE}}'); @@ -592,7 +594,7 @@ describe('GitLab support in generated skills', () => { describe('description quality evals', () => { // Regression: snapshot flags lost value hints (-d , -s , -o ) test('snapshot flags with values include value hints in output', () => { - const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8'); + const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8'); for (const flag of SNAPSHOT_FLAGS) { if (flag.takesValue) { expect(flag.valueHint).toBeDefined(); @@ -659,11 +661,13 @@ describe('description quality evals', () => { // Guard: generated output uses → not -> test('generated SKILL.md uses unicode arrows', () => { - const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8'); - // Check the Tips section specifically (where we regressed -> from →) - const tipsSection = content.slice(content.indexOf('## Tips')); - expect(tipsSection).toContain('→'); - expect(tipsSection).not.toContain('->'); + // P2 (v1.2.0): the browse body moved out of the top-level router into + // browse/SKILL.md. Guard arrow style on the browse body (sliced from its + // H1 so the auto-generated `-->` header comments are excluded). + const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8'); + const body = content.slice(content.indexOf('# browse: QA Testing')); + expect(body).toContain('→'); + expect(body).not.toContain('->'); }); }); diff --git a/test/helpers/carve-guards.ts b/test/helpers/carve-guards.ts index 11e559d8b..010bbed7a 100644 --- a/test/helpers/carve-guards.ts +++ b/test/helpers/carve-guards.ts @@ -161,7 +161,8 @@ export const CARVE_GUARDS: Record = { gateAfterStop: 'EXIT PLAN MODE GATE', }, behavioral: 'plan', - maxSkeletonBytes: 62_000, + // v1.2.0 activation lift: first-run-guidance section in the shared preamble. + maxSkeletonBytes: 65_000, minUnionBytes: 70_000, mustContain: ['Architecture', 'Code Quality', 'Test', 'Performance'], // Cross-cutting preamble growth (v1.57.2.0 AUQ-failure prose fallback + the @@ -185,7 +186,8 @@ export const CARVE_GUARDS: Record = { behavioral: 'plan', // +Conductor AUQ-default-prose rule + one-way/continuation safety in the // always-loaded AskUserQuestion Format section. - maxSkeletonBytes: 84_000, + // v1.2.0 activation lift: first-run-guidance section in the shared preamble. + maxSkeletonBytes: 86_000, minUnionBytes: 70_000, mustContain: ['design', 'visual'], }, @@ -203,7 +205,8 @@ export const CARVE_GUARDS: Record = { behavioral: 'plan', // +Conductor AUQ-default-prose rule + one-way/destructive prose safety + // continuation protocol in the always-loaded AskUserQuestion Format section. - maxSkeletonBytes: 78_000, + // v1.2.0 activation lift: first-run-guidance section in the shared preamble. + maxSkeletonBytes: 80_000, minUnionBytes: 70_000, mustContain: ['developer experience', 'Getting Started'], // Default-on Codex outside-voice (codexPreflight block + CODEX_MODE branch @@ -224,9 +227,12 @@ export const CARVE_GUARDS: Record = { gateAfterStop: undefined, }, behavioral: 'prompt', - maxSkeletonBytes: 96_000, + // v1.2.0 activation lift: first-run-guidance section in the shared preamble, + // plus the P1 office-hours closing handoff (AUQ that launches the next skill). + maxSkeletonBytes: 98_000, minUnionBytes: 70_000, mustContain: ['design doc', 'problem statement'], + maxSizeRatio: 1.07, }, 'document-release': { skill: 'document-release', @@ -243,7 +249,8 @@ export const CARVE_GUARDS: Record = { behavioral: 'prompt', // +Conductor AUQ-default-prose rule + one-way/continuation safety in the // always-loaded AskUserQuestion Format section. - maxSkeletonBytes: 53_000, + // v1.2.0 activation lift: first-run-guidance section in the shared preamble. + maxSkeletonBytes: 56_000, minUnionBytes: 55_000, mustContain: ['CHANGELOG', 'Diataxis', 'coverage'], // Two intentional additions stack on this small skill: the AUQ-failure prose @@ -270,7 +277,8 @@ export const CARVE_GUARDS: Record = { behavioral: 'prompt', // +Conductor AUQ-default-prose rule + one-way/continuation safety in the // always-loaded AskUserQuestion Format section. - maxSkeletonBytes: 67_000, + // v1.2.0 activation lift: first-run-guidance section in the shared preamble. + maxSkeletonBytes: 69_000, minUnionBytes: 72_000, mustContain: ['Typography', 'Color', 'Aesthetic Direction'], // Cross-cutting preamble growth (v1.57.2.0 AUQ-failure prose fallback ~2KB + @@ -308,7 +316,8 @@ export const CARVE_GUARDS: Record = { behavioral: 'prompt', // +Conductor AUQ-default-prose rule + one-way/continuation safety in the // always-loaded AskUserQuestion Format section. - maxSkeletonBytes: 73_000, + // v1.2.0 activation lift: first-run-guidance section in the shared preamble. + maxSkeletonBytes: 75_000, minUnionBytes: 72_000, mustContain: ['OWASP', 'STRIDE', 'daily', 'comprehensive', 'verif'], // cso keeps its mode-dispatch + FP-filtering phases always-loaded, so the diff --git a/test/helpers/parity-harness.ts b/test/helpers/parity-harness.ts index ee668ff05..3d5f08e2a 100644 --- a/test/helpers/parity-harness.ts +++ b/test/helpers/parity-harness.ts @@ -221,7 +221,9 @@ const MONOLITH_INVARIANTS: ParityInvariant[] = [ skill: 'qa', mustContain: ['bug', 'browse', 'fix'], mustHaveHeadings: ['## Preamble', '## When to invoke'], - maxSizeRatio: 1.05, + // v1.2.0 activation lift: the unified first-run-guidance section (P4 scaffold + + // P3 loop tip) is added to every skill's shared preamble — intentional, ~1KB. + maxSizeRatio: 1.07, minBytes: 50_000, }, { @@ -231,14 +233,16 @@ const MONOLITH_INVARIANTS: ParityInvariant[] = [ // Cross-cutting preamble growth (v1.57.2.0 AUQ-failure prose fallback ~2KB + the // cross-session decision-memory nudge) lands this skill just over the strict 1.05; // headroom for the shared preamble additions (matches the carved-skill overrides). - maxSizeRatio: 1.07, + // v1.2.0 activation lift adds the first-run-guidance section on top. + maxSizeRatio: 1.09, minBytes: 30_000, }, { skill: 'autoplan', mustContain: ['ceo', 'eng', 'design'], mustHaveHeadings: ['## Preamble', '## When to invoke'], - maxSizeRatio: 1.05, + // v1.2.0 activation lift: shared first-run-guidance preamble section. + maxSizeRatio: 1.07, minBytes: 70_000, }, ]; diff --git a/test/helpers/touchfiles.ts b/test/helpers/touchfiles.ts index 946a38007..6abd6cf85 100644 --- a/test/helpers/touchfiles.ts +++ b/test/helpers/touchfiles.ts @@ -41,6 +41,10 @@ export const E2E_TOUCHFILES: Record = { 'hermetic-canary': ['test/helpers/hermetic-env.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-hermetic-canary.test.ts', 'lib/conductor-env-shim.ts'], 'hermetic-sentinel': ['test/helpers/hermetic-env.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-hermetic-canary.test.ts', 'lib/conductor-env-shim.ts'], + // P4 first-run scaffold (activation lift) — the detection binary end-to-end + // through the real runner, plus the preamble wiring that gates + maps it. + 'first-task-scaffold': ['bin/gstack-first-task-detect', 'scripts/resolvers/preamble/generate-first-run-guidance.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'test/skill-e2e-first-task-scaffold.test.ts', 'test/helpers/session-runner.ts'], + // SKILL.md setup + preamble (depend on ROOT SKILL.md + gen-skill-docs) 'skillmd-setup-discovery': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'], 'skillmd-no-local-binary': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'], @@ -459,6 +463,9 @@ export const E2E_TIERS: Record = { 'session-awareness': 'gate', 'operational-learning': 'gate', + // P4 first-run scaffold — periodic (onboarding, non-safety, model-touched marker) + 'first-task-scaffold': 'periodic', + // QA — gate for functional, periodic for quality/benchmarks 'qa-quick': 'gate', 'qa-b6-static': 'periodic', diff --git a/test/preamble-first-task-scaffold.test.ts b/test/preamble-first-task-scaffold.test.ts new file mode 100644 index 000000000..d2bd4d68d --- /dev/null +++ b/test/preamble-first-task-scaffold.test.ts @@ -0,0 +1,171 @@ +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { execFileSync, execSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +// P4 first-run scaffold (activation lift). Two surfaces under test: +// 1. bin/gstack-first-task-detect — classifies a repo into ONE enum bucket. +// 2. The unified first-run-guidance preamble wiring (generated into SKILL.md). + +const ROOT = path.join(import.meta.dir, '..'); +const DETECT = path.join(ROOT, 'bin', 'gstack-first-task-detect'); + +// The complete, closed set the detector is ever allowed to emit. The eval-safety +// guarantee is that nothing outside this set ever reaches the preamble. +const ENUM = new Set([ + 'greenfield', 'code_node', 'code_python', 'code_rust', 'code_go', + 'code_ruby', 'code_ios', 'branch_ahead', 'dirty_default', 'clean_default', 'nongit', +]); + +const GIT_ENV = { + ...process.env, + GIT_AUTHOR_NAME: 'T', GIT_AUTHOR_EMAIL: 't@e.x', + GIT_COMMITTER_NAME: 'T', GIT_COMMITTER_EMAIL: 't@e.x', +}; + +function detect(cwd: string): string { + return execFileSync(DETECT, [], { cwd, encoding: 'utf-8', env: GIT_ENV }).trim(); +} +function git(cwd: string, args: string) { + execSync(`git ${args}`, { cwd, env: GIT_ENV, stdio: 'ignore' }); +} + +let tmp: string; +beforeAll(() => { tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ftd-')); }); +afterAll(() => { fs.rmSync(tmp, { recursive: true, force: true }); }); + +function freshRepo(name: string): string { + const d = path.join(tmp, name); + fs.mkdirSync(d, { recursive: true }); + git(d, 'init -q -b main'); + return d; +} + +describe('gstack-first-task-detect — bucket classification', () => { + test('non-git directory → nongit', () => { + const d = path.join(tmp, 'plain'); fs.mkdirSync(d, { recursive: true }); + expect(detect(d)).toBe('nongit'); + }); + + test('git repo, no commits → greenfield', () => { + expect(detect(freshRepo('green'))).toBe('greenfield'); + }); + + test('Node project with a commit → code_node', () => { + const d = freshRepo('node'); + fs.writeFileSync(path.join(d, 'package.json'), '{"name":"x"}'); + git(d, 'add -A'); git(d, 'commit -qm init'); + expect(detect(d)).toBe('code_node'); + }); + + test('Python project with a commit → code_python', () => { + const d = freshRepo('py'); + fs.writeFileSync(path.join(d, 'pyproject.toml'), '[project]\nname="x"'); + git(d, 'add -A'); git(d, 'commit -qm init'); + expect(detect(d)).toBe('code_python'); + }); + + // The remaining language markers (a typo in any would ship undetected). + for (const [name, file, token] of [ + ['Rust', 'Cargo.toml', 'code_rust'], + ['Go', 'go.mod', 'code_go'], + ['Ruby', 'Gemfile', 'code_ruby'], + ] as const) { + test(`${name} project with a commit → ${token}`, () => { + const d = freshRepo(`lang-${token}`); + fs.writeFileSync(path.join(d, file), 'x'); + git(d, 'add -A'); git(d, 'commit -qm init'); + expect(detect(d)).toBe(token); + }); + } + + test('iOS project (.xcodeproj) with a commit → code_ios', () => { + const d = freshRepo('ios'); + fs.mkdirSync(path.join(d, 'App.xcodeproj')); + fs.writeFileSync(path.join(d, 'App.xcodeproj', 'project.pbxproj'), '// x'); + git(d, 'add -A'); git(d, 'commit -qm init'); + expect(detect(d)).toBe('code_ios'); + }); + + // Precedence (the detector's most fragile logic): branch-state buckets must + // win over language markers, so a real repo isn't mislabeled "verify tests". + test('feature branch ahead + package.json → branch_ahead (not code_node)', () => { + const origin = freshRepo('prec-origin'); + git(origin, 'commit -qm base --allow-empty'); + const clone = path.join(tmp, 'prec-clone'); + git(tmp, `clone -q ${origin} prec-clone`); + fs.writeFileSync(path.join(clone, 'package.json'), '{"name":"x"}'); + git(clone, 'checkout -q -b feature'); + git(clone, 'add -A'); git(clone, 'commit -qm work'); + expect(detect(clone)).toBe('branch_ahead'); + }); + + test('dirty default branch + package.json → dirty_default (not code_node)', () => { + const d = freshRepo('prec-dirty'); + fs.writeFileSync(path.join(d, 'package.json'), '{"name":"x"}'); + git(d, 'add -A'); git(d, 'commit -qm init'); + fs.writeFileSync(path.join(d, 'package.json'), '{"name":"x","v":2}'); + expect(detect(d)).toBe('dirty_default'); + }); + + test('feature branch ahead of origin → branch_ahead', () => { + const origin = freshRepo('origin'); + git(origin, 'commit -qm base --allow-empty'); + const clone = path.join(tmp, 'clone'); + git(tmp, `clone -q ${origin} clone`); + git(clone, 'checkout -q -b feature'); + fs.writeFileSync(path.join(clone, 'f.txt'), 'x'); + git(clone, 'add -A'); git(clone, 'commit -qm work'); + expect(detect(clone)).toBe('branch_ahead'); + }); + + test('uncommitted changes on default branch → dirty_default', () => { + const d = freshRepo('dirty'); + fs.writeFileSync(path.join(d, 'a.txt'), 'x'); + git(d, 'add -A'); git(d, 'commit -qm init'); + fs.writeFileSync(path.join(d, 'a.txt'), 'changed'); + // No recognized language marker, so the dirty-default branch must win. + expect(detect(d)).toBe('dirty_default'); + }); + + test('clean default branch, 5+ commits, no language marker → clean_default', () => { + const d = freshRepo('clean'); + for (let i = 0; i < 6; i++) git(d, `commit -qm c${i} --allow-empty`); + expect(detect(d)).toBe('clean_default'); + }); +}); + +describe('gstack-first-task-detect — contract', () => { + test('output is always a whitelisted enum token or empty (eval-safe)', () => { + for (const name of ['plain', 'green', 'node', 'py', 'clone', 'dirty', 'clean']) { + const out = detect(path.join(tmp, name)); + if (out !== '') expect(ENUM.has(out)).toBe(true); + } + }); + + test('detector is executable', () => { + expect(fs.statSync(DETECT).mode & 0o111).toBeGreaterThan(0); + }); +}); + +describe('first-run-guidance preamble wiring (generated)', () => { + const md = fs.readFileSync(path.join(ROOT, 'ship', 'SKILL.md'), 'utf-8'); + + test('detection is gated to the first-ever run only (ACTIVATED=no, not headless)', () => { + expect(md).toContain('if [ "$_ACTIVATED" = "no" ] && [ "$_SESSION_KIND" != "headless" ]'); + expect(md).toContain('gstack-first-task-detect'); + }); + + test('emits the unified first-run guidance section branching on ACTIVATED', () => { + expect(md).toContain('## First-run guidance (one-time)'); + expect(md).toContain('`ACTIVATED` is `no`'); // P4 scaffold branch + expect(md).toContain('`ACTIVATED` is `yes` AND `FIRST_LOOP_SHOWN` is `no`'); // P3 tip branch + }); + + test('marks activated + logs the scaffold telemetry only on the shown path', () => { + expect(md).toContain('first_task_scaffold_shown'); + expect(md).toContain('touch ~/.gstack/.activated'); + expect(md).toContain('touch ~/.gstack/.first-loop-tip-shown'); + }); +}); diff --git a/test/skill-e2e-first-task-scaffold.test.ts b/test/skill-e2e-first-task-scaffold.test.ts new file mode 100644 index 000000000..871781fcb --- /dev/null +++ b/test/skill-e2e-first-task-scaffold.test.ts @@ -0,0 +1,97 @@ +/** + * P4 first-run scaffold — E2E (periodic tier, ~$0.02 each, deterministic). + * + * Exercises bin/gstack-first-task-detect END-TO-END through the real runner + + * hermetic env (path resolution, execution, git-in-cwd), not just the unit + * harness. Deterministic by construction: it asserts the binary's enum token + * from the Bash tool_result in the stream-json transcript (never the model's + * prose), so it pins the detector's integration contract without depending on + * non-deterministic model phrasing. + * + * Periodic (not gate): onboarding behavior is non-safety, and the scaffold + * marker is model-touched (best-effort). The deterministic bucket logic itself + * is fully covered by the unit test (test/preamble-first-task-scaffold.test.ts). + */ + +import { expect, afterAll } from 'bun:test'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import { execSync } from 'node:child_process'; +import { runSkillTest } from './helpers/session-runner'; +import { + describeIfSelected, testIfSelected, createEvalCollector, finalizeEvalCollector, + recordE2E, runId, logCost, +} from './helpers/e2e-helpers'; + +const ROOT = path.join(import.meta.dir, '..'); +const DETECT = path.join(ROOT, 'bin', 'gstack-first-task-detect'); +const evalCollector = createEvalCollector('e2e-first-task-scaffold'); +const MODEL = 'claude-haiku-4-5-20251001'; + +const GIT_ENV = { + ...process.env, + GIT_AUTHOR_NAME: 'T', GIT_AUTHOR_EMAIL: 't@e.x', + GIT_COMMITTER_NAME: 'T', GIT_COMMITTER_EMAIL: 't@e.x', +}; + +/** Concatenated Bash tool_result text from the stream-json transcript. */ +function toolResultText(transcript: any[]): string { + const chunks: string[] = []; + for (const event of transcript) { + if (event.type !== 'user') continue; + for (const item of event.message?.content ?? []) { + if (item.type !== 'tool_result') continue; + if (typeof item.content === 'string') chunks.push(item.content); + else for (const c of item.content ?? []) if (c.type === 'text') chunks.push(c.text); + } + } + return chunks.join('\n'); +} + +async function detectVia(workDir: string, testName: string): Promise { + const result = await runSkillTest({ + prompt: `Run exactly this one bash command and then stop, printing its output verbatim: ${DETECT}`, + workingDirectory: workDir, + maxTurns: 3, + allowedTools: ['Bash'], + timeout: 120_000, + testName, + runId, + model: MODEL, + }); + logCost(testName, result); + recordE2E(evalCollector, testName, 'e2e-first-task-scaffold', result); + expect(result.exitReason).toBe('success'); + return toolResultText(result.transcript); +} + +describeIfSelected('first-run scaffold detection (E2E)', ['first-task-scaffold'], () => { + testIfSelected('first-task-scaffold', async () => { + if (!process.env.ANTHROPIC_API_KEY) { + throw new Error('first-task-scaffold requires ANTHROPIC_API_KEY (source ~/.zshrc); refusing to skip'); + } + + // code_node bucket: package.json + a commit, on the default branch. + const nodeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'fts-node-')); + // greenfield bucket: git repo, zero commits. + const greenDir = fs.mkdtempSync(path.join(os.tmpdir(), 'fts-green-')); + try { + execSync('git init -q -b main', { cwd: nodeDir, env: GIT_ENV }); + fs.writeFileSync(path.join(nodeDir, 'package.json'), '{"name":"x"}'); + execSync('git add -A && git commit -qm init', { cwd: nodeDir, env: GIT_ENV }); + execSync('git init -q -b main', { cwd: greenDir, env: GIT_ENV }); + + const nodeOut = await detectVia(nodeDir, 'first-task-scaffold'); + expect(nodeOut).toContain('code_node'); + + const greenOut = await detectVia(greenDir, 'first-task-scaffold-greenfield'); + expect(greenOut).toContain('greenfield'); + } finally { + fs.rmSync(nodeDir, { recursive: true, force: true }); + fs.rmSync(greenDir, { recursive: true, force: true }); + } + }, 300_000); +}); + +afterAll(() => finalizeEvalCollector(evalCollector)); diff --git a/test/skill-llm-eval.test.ts b/test/skill-llm-eval.test.ts index 3b0d8661c..17b1adf0d 100644 --- a/test/skill-llm-eval.test.ts +++ b/test/skill-llm-eval.test.ts @@ -65,10 +65,10 @@ describeIfSelected('LLM-as-judge quality evals', [ ], () => { testIfSelected('command reference table', async () => { const t0 = Date.now(); - const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8'); - const start = content.indexOf('## Command Reference'); - const end = content.indexOf('## Tips'); - const section = content.slice(start, end); + // P2 (v1.2.0): the command reference moved from the root router to browse/SKILL.md. + const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8'); + const start = content.indexOf('## Full Command List'); + const section = content.slice(start); const scores = await judge('command reference table', section); console.log('Command reference scores:', JSON.stringify(scores, null, 2)); @@ -94,9 +94,10 @@ describeIfSelected('LLM-as-judge quality evals', [ testIfSelected('snapshot flags reference', async () => { const t0 = Date.now(); - const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8'); - const start = content.indexOf('## Snapshot System'); - const end = content.indexOf('## Command Reference'); + // P2 (v1.2.0): snapshot flags moved from the root router to browse/SKILL.md. + const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8'); + const start = content.indexOf('## Snapshot Flags'); + const end = content.indexOf('## CSS Inspector'); const section = content.slice(start, end); const scores = await judge('snapshot flags reference', section); @@ -145,9 +146,10 @@ describeIfSelected('LLM-as-judge quality evals', [ testIfSelected('setup block', async () => { const t0 = Date.now(); - const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8'); + // P2 (v1.2.0): the browse setup block moved from the root router to browse/SKILL.md. + const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8'); const setupStart = content.indexOf('## SETUP'); - const setupEnd = content.indexOf('## IMPORTANT'); + const setupEnd = content.indexOf('## Core QA Patterns'); const section = content.slice(setupStart, setupEnd); const scores = await judge('setup/binary discovery instructions', section); @@ -172,10 +174,10 @@ describeIfSelected('LLM-as-judge quality evals', [ testIfSelected('regression vs baseline', async () => { const t0 = Date.now(); - const generated = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8'); - const genStart = generated.indexOf('## Command Reference'); - const genEnd = generated.indexOf('## Tips'); - const genSection = generated.slice(genStart, genEnd); + // P2 (v1.2.0): the command reference moved from the root router to browse/SKILL.md. + const generated = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8'); + const genStart = generated.indexOf('## Full Command List'); + const genSection = generated.slice(genStart); const baseline = `## Command Reference @@ -480,10 +482,10 @@ describeIfSelected('Baseline score pinning', ['baseline score pinning'], () => { const baselines = JSON.parse(fs.readFileSync(baselinesPath, 'utf-8')); const regressions: string[] = []; - const skillContent = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8'); - const cmdStart = skillContent.indexOf('## Command Reference'); - const cmdEnd = skillContent.indexOf('## Tips'); - const cmdSection = skillContent.slice(cmdStart, cmdEnd); + // P2 (v1.2.0): the command reference moved from the root router to browse/SKILL.md. + const skillContent = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8'); + const cmdStart = skillContent.indexOf('## Full Command List'); + const cmdSection = skillContent.slice(cmdStart); const cmdScores = await judge('command reference table', cmdSection); for (const dim of ['clarity', 'completeness', 'actionability'] as const) { diff --git a/test/skill-validation.test.ts b/test/skill-validation.test.ts index e7def7dfa..99d4eb83b 100644 --- a/test/skill-validation.test.ts +++ b/test/skill-validation.test.ts @@ -26,15 +26,18 @@ function readShipUnion(): string { } describe('SKILL.md command validation', () => { - test('all $B commands in SKILL.md are valid browse commands', () => { + // P2 (v1.2.0): the top-level gstack skill is a pure ROUTER, not the browse + // skill. The browse body lives only in browse/SKILL.md now. This regression + // pins the split: the router carries routing rules and zero browse commands, + // while browse/SKILL.md still advertises the full QA surface (asserted below). + test('top-level SKILL.md is a router with no browse body (P2)', () => { + const md = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8'); + expect(md).not.toContain('gstack browse: QA Testing'); // browse body removed + expect(md).toContain('## Route first'); // router head present + expect(md).toContain('invoke `/investigate`'); // routing rules present const result = validateSkill(path.join(ROOT, 'SKILL.md')); - expect(result.invalid).toHaveLength(0); - expect(result.valid.length).toBeGreaterThan(0); - }); - - test('all snapshot flags in SKILL.md are valid', () => { - const result = validateSkill(path.join(ROOT, 'SKILL.md')); - expect(result.snapshotFlagErrors).toHaveLength(0); + expect(result.invalid).toHaveLength(0); // no INVALID browse commands + expect(result.valid.length).toBe(0); // and no browse commands at all — it routes, not browses }); test('all $B commands in browse/SKILL.md are valid browse commands', () => {