From b70483495bcd37b5369afd4d5fb07dafaaecdbaa Mon Sep 17 00:00:00 2001 From: Tim White Date: Mon, 20 Jul 2026 14:33:28 -0700 Subject: [PATCH] fix(qa,qa-only): define evidence layout edge cases; drop unavailable recording Remove $B record / repro.webm advertising (browse has no recorder yet). Document flat-default vs --evidence-per-finding nested layout, deterministic collision handling (no silent overwrite; sequential NNN and -2/-3 suffixes), and missing/unreadable asset rules (report and continue; never invent files). Add focused contract tests for both /qa and /qa-only. --- qa-only/SKILL.md | 49 ++++++++++++++---- qa-only/SKILL.md.tmpl | 2 +- qa/SKILL.md | 49 ++++++++++++++---- qa/SKILL.md.tmpl | 2 +- scripts/resolvers/utility.ts | 47 +++++++++++++---- test/gen-skill-docs.test.ts | 95 +++++++++++++++++++++++++++++++++++ test/skill-validation.test.ts | 30 +++++++++++ 7 files changed, 243 insertions(+), 31 deletions(-) diff --git a/qa-only/SKILL.md b/qa-only/SKILL.md index 73e80c885..28bf3d405 100644 --- a/qa-only/SKILL.md +++ b/qa-only/SKILL.md @@ -1054,9 +1054,9 @@ $B snapshot -i -a -o "$REPORT_DIR/screenshots/issue-002.png" #### Evidence layout: flat (default) vs per-finding -By default, evidence files share one `screenshots/` directory and the report references them by filename (`issue-001-step-1.png`, etc). This stays compact and works well for 1-5 findings. +**Default is flat.** Evidence files share one `screenshots/` directory and the report references them by filename (`issue-001-step-1.png`, etc). Flat is used unless the user explicitly opts in. This stays compact and works well for 1-5 findings. -When the run is invoked with **`--evidence-per-finding`**, switch to one folder per finding: +**Opt-in nested layout.** When the run is invoked with **`--evidence-per-finding`** (or natural-language variants: `evidence per finding`, `one folder per bug`), switch to one folder per finding: ``` .gstack/qa-reports/ @@ -1065,10 +1065,9 @@ When the run is invoked with **`--evidence-per-finding`**, switch to one folder ├── findings/ │ ├── 001-critical-checkout-500-on-submit/ │ │ ├── finding.md # severity + repro + env + expected/actual - │ │ ├── step-1.png # before action - │ │ ├── step-2.png # after action - │ │ ├── result.png # final state - │ │ └── repro.webm # OPTIONAL — present iff `$B record` was active + │ │ ├── step-1.png # before action (when capture succeeded) + │ │ ├── step-2.png # after action (when capture succeeded) + │ │ └── result.png # final state (when capture succeeded) │ ├── 002-high-search-no-results/ │ │ └── ... │ └── 003-low-cosmetic-spacing/ @@ -1076,6 +1075,36 @@ When the run is invoked with **`--evidence-per-finding`**, switch to one folder └── baseline.json # unchanged ``` +**Directory / file creation and naming:** + +1. **Flat (default):** + - Ensure `.gstack/qa-reports/screenshots/` exists (`mkdir -p`). + - Report file: `qa-report-{domain}-{YYYY-MM-DD}.md` under the output dir. + - Evidence files: `issue-{NNN}-step-{K}.png`, `issue-{NNN}-result.png` (and for /qa fix loop: `issue-{NNN}-before.png` / `issue-{NNN}-after.png`). + - `{NNN}` is a sequential zero-padded index starting at `001`. Assign the next free index by scanning existing `issue-*.png` / report issue IDs and taking max + 1. +2. **Per-finding (opt-in):** + - Create a per-run report directory: `qa-report-{domain}-{YYYY-MM-DD}/` with `findings/` inside (`mkdir -p`). + - Top-level report is `REPORT.md` inside that directory (not a sibling `.md` file). + - Each finding folder: `{NNN}-{severity}-{kebab-slug}/` where `{NNN}` is sequential (`001`, `002`, …), severity is lowercase (`critical`/`high`/`medium`/`low`/`cosmetic`), and slug is a short kebab-case title stem (alphanumeric + hyphens only, max ~40 chars). + - Inside each finding folder write `finding.md` first, then only the screenshot files that capture actually produced (`step-1.png`, `step-2.png`, `result.png`; /qa fix loop may add `before.png` / `after.png`). + - Create a finding folder only when that finding is documented — do not pre-create empty finding dirs. + +**Collisions (must not silently overwrite evidence):** + +- Never overwrite an existing evidence file or finding directory. If a target path already exists, pick a deterministic free name: + - **Sequential IDs first:** prefer the next free `{NNN}` (flat filenames and nested folder prefixes). Scan existing siblings, take max + 1. + - **Same-day report path collision:** if `qa-report-{domain}-{YYYY-MM-DD}.md` (flat) or `qa-report-{domain}-{YYYY-MM-DD}/` (nested) already exists from an earlier run, append `-2`, then `-3`, etc. (`qa-report-{domain}-{YYYY-MM-DD}-2.md` / `…-2/`) until the path is free. Do not clobber the prior run. + - **Nested folder slug collision under the same NNN:** if `findings/{NNN}-{severity}-{slug}/` exists, append `-2`, `-3`, … to the folder name before writing. + - **Flat filename collision:** if `issue-{NNN}-….png` exists for the chosen NNN, bump NNN (auto-increment) rather than overwriting. +- This matches the existing regression-test collision rule: check existing names, take max number + 1 / next free suffix — never silent overwrite. + +**Missing or unreadable evidence assets:** + +- Capture failures are normal (timeout, navigation error, permission). **Do not invent placeholder image files** and do not write empty/dummy PNGs. +- In `finding.md` / the report **Evidence** section, list only files that exist on disk after a successful write. For a failed capture, note it inline, e.g. `_(capture failed: {short reason})_`, and continue the run. +- If a referenced path is unreadable at report time, treat it as missing: omit the file link, note `_(unreadable: {path})_`, continue. Never block the rest of documentation on one bad asset. +- Top-level `REPORT.md` / flat report screenshot counts must reflect files that actually exist. + **finding.md** (per-finding) MUST have this shape: ```markdown @@ -1107,18 +1136,18 @@ When the run is invoked with **`--evidence-per-finding`**, switch to one folder - `step-1.png` — before the action - `step-2.png` — after the action - `result.png` — final state -- `repro.webm` — full interactive repro (if recorded) ``` +(Only list evidence files that exist. If a step capture failed, replace that bullet with `_(capture failed: {reason})_`.) + **When per-finding is the right call:** - Run produces ≥5 findings (the flat layout gets noisy past that). - Any finding is critical or high severity — those tickets travel further, need self-contained evidence. -- An interactive bug needs video evidence — `record start` then `record stop` (Playwright `recordVideo`) writes a `.webm`; move it into the corresponding finding folder during Phase 6. - Findings will be handed off as Linear/Jira tickets — each folder zips into a self-contained attachment. **When per-finding is overkill:** quick smoke runs, 1-2 findings, regression-mode reruns where the baseline is the canonical artifact. Stick with the flat layout. -Top-level `REPORT.md` content is identical between the two layouts; only the on-disk filing differs. +Top-level `REPORT.md` content is identical between the two layouts; only the on-disk filing differs. Screenshots remain the evidence format — video recording is not part of this layout. ### Phase 6: Wrap Up @@ -1259,7 +1288,7 @@ Write to `~/.gstack/projects/{slug}/{user}-{branch}-test-outcome-{datetime}.md` Report filenames use the domain and date: `qa-report-myapp-com-2026-03-12.md` -**With `--evidence-per-finding`:** one folder per finding under a per-run report dir. Each folder is self-contained — `finding.md` plus step screenshots and (if `$B record` was active) a `repro.webm`. See "Evidence layout: flat vs per-finding" in the Document phase above for the full structure and `finding.md` template. +**With `--evidence-per-finding`:** one folder per finding under a per-run report dir. Each folder is self-contained — `finding.md` plus step screenshots when those captures succeeded. Default remains flat (`screenshots/` + single report `.md`). Nested mode must not silently overwrite prior evidence; see "Evidence layout: flat vs per-finding" in the Document phase above for naming, collision, and missing-asset rules. --- diff --git a/qa-only/SKILL.md.tmpl b/qa-only/SKILL.md.tmpl index cec4d9bc3..e23eaec5a 100644 --- a/qa-only/SKILL.md.tmpl +++ b/qa-only/SKILL.md.tmpl @@ -107,7 +107,7 @@ Write to `~/.gstack/projects/{slug}/{user}-{branch}-test-outcome-{datetime}.md` Report filenames use the domain and date: `qa-report-myapp-com-2026-03-12.md` -**With `--evidence-per-finding`:** one folder per finding under a per-run report dir. Each folder is self-contained — `finding.md` plus step screenshots and (if `$B record` was active) a `repro.webm`. See "Evidence layout: flat vs per-finding" in the Document phase above for the full structure and `finding.md` template. +**With `--evidence-per-finding`:** one folder per finding under a per-run report dir. Each folder is self-contained — `finding.md` plus step screenshots when those captures succeeded. Default remains flat (`screenshots/` + single report `.md`). Nested mode must not silently overwrite prior evidence; see "Evidence layout: flat vs per-finding" in the Document phase above for naming, collision, and missing-asset rules. --- diff --git a/qa/SKILL.md b/qa/SKILL.md index 4fed20ec6..d79259fef 100644 --- a/qa/SKILL.md +++ b/qa/SKILL.md @@ -1288,9 +1288,9 @@ $B snapshot -i -a -o "$REPORT_DIR/screenshots/issue-002.png" #### Evidence layout: flat (default) vs per-finding -By default, evidence files share one `screenshots/` directory and the report references them by filename (`issue-001-step-1.png`, etc). This stays compact and works well for 1-5 findings. +**Default is flat.** Evidence files share one `screenshots/` directory and the report references them by filename (`issue-001-step-1.png`, etc). Flat is used unless the user explicitly opts in. This stays compact and works well for 1-5 findings. -When the run is invoked with **`--evidence-per-finding`**, switch to one folder per finding: +**Opt-in nested layout.** When the run is invoked with **`--evidence-per-finding`** (or natural-language variants: `evidence per finding`, `one folder per bug`), switch to one folder per finding: ``` .gstack/qa-reports/ @@ -1299,10 +1299,9 @@ When the run is invoked with **`--evidence-per-finding`**, switch to one folder ├── findings/ │ ├── 001-critical-checkout-500-on-submit/ │ │ ├── finding.md # severity + repro + env + expected/actual - │ │ ├── step-1.png # before action - │ │ ├── step-2.png # after action - │ │ ├── result.png # final state - │ │ └── repro.webm # OPTIONAL — present iff `$B record` was active + │ │ ├── step-1.png # before action (when capture succeeded) + │ │ ├── step-2.png # after action (when capture succeeded) + │ │ └── result.png # final state (when capture succeeded) │ ├── 002-high-search-no-results/ │ │ └── ... │ └── 003-low-cosmetic-spacing/ @@ -1310,6 +1309,36 @@ When the run is invoked with **`--evidence-per-finding`**, switch to one folder └── baseline.json # unchanged ``` +**Directory / file creation and naming:** + +1. **Flat (default):** + - Ensure `.gstack/qa-reports/screenshots/` exists (`mkdir -p`). + - Report file: `qa-report-{domain}-{YYYY-MM-DD}.md` under the output dir. + - Evidence files: `issue-{NNN}-step-{K}.png`, `issue-{NNN}-result.png` (and for /qa fix loop: `issue-{NNN}-before.png` / `issue-{NNN}-after.png`). + - `{NNN}` is a sequential zero-padded index starting at `001`. Assign the next free index by scanning existing `issue-*.png` / report issue IDs and taking max + 1. +2. **Per-finding (opt-in):** + - Create a per-run report directory: `qa-report-{domain}-{YYYY-MM-DD}/` with `findings/` inside (`mkdir -p`). + - Top-level report is `REPORT.md` inside that directory (not a sibling `.md` file). + - Each finding folder: `{NNN}-{severity}-{kebab-slug}/` where `{NNN}` is sequential (`001`, `002`, …), severity is lowercase (`critical`/`high`/`medium`/`low`/`cosmetic`), and slug is a short kebab-case title stem (alphanumeric + hyphens only, max ~40 chars). + - Inside each finding folder write `finding.md` first, then only the screenshot files that capture actually produced (`step-1.png`, `step-2.png`, `result.png`; /qa fix loop may add `before.png` / `after.png`). + - Create a finding folder only when that finding is documented — do not pre-create empty finding dirs. + +**Collisions (must not silently overwrite evidence):** + +- Never overwrite an existing evidence file or finding directory. If a target path already exists, pick a deterministic free name: + - **Sequential IDs first:** prefer the next free `{NNN}` (flat filenames and nested folder prefixes). Scan existing siblings, take max + 1. + - **Same-day report path collision:** if `qa-report-{domain}-{YYYY-MM-DD}.md` (flat) or `qa-report-{domain}-{YYYY-MM-DD}/` (nested) already exists from an earlier run, append `-2`, then `-3`, etc. (`qa-report-{domain}-{YYYY-MM-DD}-2.md` / `…-2/`) until the path is free. Do not clobber the prior run. + - **Nested folder slug collision under the same NNN:** if `findings/{NNN}-{severity}-{slug}/` exists, append `-2`, `-3`, … to the folder name before writing. + - **Flat filename collision:** if `issue-{NNN}-….png` exists for the chosen NNN, bump NNN (auto-increment) rather than overwriting. +- This matches the existing regression-test collision rule: check existing names, take max number + 1 / next free suffix — never silent overwrite. + +**Missing or unreadable evidence assets:** + +- Capture failures are normal (timeout, navigation error, permission). **Do not invent placeholder image files** and do not write empty/dummy PNGs. +- In `finding.md` / the report **Evidence** section, list only files that exist on disk after a successful write. For a failed capture, note it inline, e.g. `_(capture failed: {short reason})_`, and continue the run. +- If a referenced path is unreadable at report time, treat it as missing: omit the file link, note `_(unreadable: {path})_`, continue. Never block the rest of documentation on one bad asset. +- Top-level `REPORT.md` / flat report screenshot counts must reflect files that actually exist. + **finding.md** (per-finding) MUST have this shape: ```markdown @@ -1341,18 +1370,18 @@ When the run is invoked with **`--evidence-per-finding`**, switch to one folder - `step-1.png` — before the action - `step-2.png` — after the action - `result.png` — final state -- `repro.webm` — full interactive repro (if recorded) ``` +(Only list evidence files that exist. If a step capture failed, replace that bullet with `_(capture failed: {reason})_`.) + **When per-finding is the right call:** - Run produces ≥5 findings (the flat layout gets noisy past that). - Any finding is critical or high severity — those tickets travel further, need self-contained evidence. -- An interactive bug needs video evidence — `record start` then `record stop` (Playwright `recordVideo`) writes a `.webm`; move it into the corresponding finding folder during Phase 6. - Findings will be handed off as Linear/Jira tickets — each folder zips into a self-contained attachment. **When per-finding is overkill:** quick smoke runs, 1-2 findings, regression-mode reruns where the baseline is the canonical artifact. Stick with the flat layout. -Top-level `REPORT.md` content is identical between the two layouts; only the on-disk filing differs. +Top-level `REPORT.md` content is identical between the two layouts; only the on-disk filing differs. Screenshots remain the evidence format — video recording is not part of this layout. ### Phase 6: Wrap Up @@ -1485,7 +1514,7 @@ Record baseline health score at end of Phase 6. Report filenames use the domain and date: `qa-report-myapp-com-2026-03-12.md` -**With `--evidence-per-finding`:** one folder per finding under a per-run report dir. Each folder is self-contained — `finding.md` plus step screenshots, fix before/after, and (if `$B record` was active) a `repro.webm`. See "Evidence layout: flat vs per-finding" in the Document phase below for the full structure and `finding.md` template. +**With `--evidence-per-finding`:** one folder per finding under a per-run report dir. Each folder is self-contained — `finding.md` plus step screenshots and fix before/after when those captures succeeded. Default remains flat (`screenshots/` + single report `.md`). Nested mode must not silently overwrite prior evidence; see "Evidence layout: flat vs per-finding" in the Document phase (Phases 1-6) for naming, collision, and missing-asset rules. --- diff --git a/qa/SKILL.md.tmpl b/qa/SKILL.md.tmpl index d076e6230..c7bece91a 100644 --- a/qa/SKILL.md.tmpl +++ b/qa/SKILL.md.tmpl @@ -145,7 +145,7 @@ Record baseline health score at end of Phase 6. Report filenames use the domain and date: `qa-report-myapp-com-2026-03-12.md` -**With `--evidence-per-finding`:** one folder per finding under a per-run report dir. Each folder is self-contained — `finding.md` plus step screenshots, fix before/after, and (if `$B record` was active) a `repro.webm`. See "Evidence layout: flat vs per-finding" in the Document phase below for the full structure and `finding.md` template. +**With `--evidence-per-finding`:** one folder per finding under a per-run report dir. Each folder is self-contained — `finding.md` plus step screenshots and fix before/after when those captures succeeded. Default remains flat (`screenshots/` + single report `.md`). Nested mode must not silently overwrite prior evidence; see "Evidence layout: flat vs per-finding" in the Document phase (Phases 1-6) for naming, collision, and missing-asset rules. --- diff --git a/scripts/resolvers/utility.ts b/scripts/resolvers/utility.ts index 4be68c843..088285548 100644 --- a/scripts/resolvers/utility.ts +++ b/scripts/resolvers/utility.ts @@ -259,9 +259,9 @@ $B snapshot -i -a -o "$REPORT_DIR/screenshots/issue-002.png" #### Evidence layout: flat (default) vs per-finding -By default, evidence files share one \`screenshots/\` directory and the report references them by filename (\`issue-001-step-1.png\`, etc). This stays compact and works well for 1-5 findings. +**Default is flat.** Evidence files share one \`screenshots/\` directory and the report references them by filename (\`issue-001-step-1.png\`, etc). Flat is used unless the user explicitly opts in. This stays compact and works well for 1-5 findings. -When the run is invoked with **\`--evidence-per-finding\`**, switch to one folder per finding: +**Opt-in nested layout.** When the run is invoked with **\`--evidence-per-finding\`** (or natural-language variants: \`evidence per finding\`, \`one folder per bug\`), switch to one folder per finding: \`\`\` .gstack/qa-reports/ @@ -270,10 +270,9 @@ When the run is invoked with **\`--evidence-per-finding\`**, switch to one folde ├── findings/ │ ├── 001-critical-checkout-500-on-submit/ │ │ ├── finding.md # severity + repro + env + expected/actual - │ │ ├── step-1.png # before action - │ │ ├── step-2.png # after action - │ │ ├── result.png # final state - │ │ └── repro.webm # OPTIONAL — present iff \`$B record\` was active + │ │ ├── step-1.png # before action (when capture succeeded) + │ │ ├── step-2.png # after action (when capture succeeded) + │ │ └── result.png # final state (when capture succeeded) │ ├── 002-high-search-no-results/ │ │ └── ... │ └── 003-low-cosmetic-spacing/ @@ -281,6 +280,36 @@ When the run is invoked with **\`--evidence-per-finding\`**, switch to one folde └── baseline.json # unchanged \`\`\` +**Directory / file creation and naming:** + +1. **Flat (default):** + - Ensure \`.gstack/qa-reports/screenshots/\` exists (\`mkdir -p\`). + - Report file: \`qa-report-{domain}-{YYYY-MM-DD}.md\` under the output dir. + - Evidence files: \`issue-{NNN}-step-{K}.png\`, \`issue-{NNN}-result.png\` (and for /qa fix loop: \`issue-{NNN}-before.png\` / \`issue-{NNN}-after.png\`). + - \`{NNN}\` is a sequential zero-padded index starting at \`001\`. Assign the next free index by scanning existing \`issue-*.png\` / report issue IDs and taking max + 1. +2. **Per-finding (opt-in):** + - Create a per-run report directory: \`qa-report-{domain}-{YYYY-MM-DD}/\` with \`findings/\` inside (\`mkdir -p\`). + - Top-level report is \`REPORT.md\` inside that directory (not a sibling \`.md\` file). + - Each finding folder: \`{NNN}-{severity}-{kebab-slug}/\` where \`{NNN}\` is sequential (\`001\`, \`002\`, …), severity is lowercase (\`critical\`/\`high\`/\`medium\`/\`low\`/\`cosmetic\`), and slug is a short kebab-case title stem (alphanumeric + hyphens only, max ~40 chars). + - Inside each finding folder write \`finding.md\` first, then only the screenshot files that capture actually produced (\`step-1.png\`, \`step-2.png\`, \`result.png\`; /qa fix loop may add \`before.png\` / \`after.png\`). + - Create a finding folder only when that finding is documented — do not pre-create empty finding dirs. + +**Collisions (must not silently overwrite evidence):** + +- Never overwrite an existing evidence file or finding directory. If a target path already exists, pick a deterministic free name: + - **Sequential IDs first:** prefer the next free \`{NNN}\` (flat filenames and nested folder prefixes). Scan existing siblings, take max + 1. + - **Same-day report path collision:** if \`qa-report-{domain}-{YYYY-MM-DD}.md\` (flat) or \`qa-report-{domain}-{YYYY-MM-DD}/\` (nested) already exists from an earlier run, append \`-2\`, then \`-3\`, etc. (\`qa-report-{domain}-{YYYY-MM-DD}-2.md\` / \`…-2/\`) until the path is free. Do not clobber the prior run. + - **Nested folder slug collision under the same NNN:** if \`findings/{NNN}-{severity}-{slug}/\` exists, append \`-2\`, \`-3\`, … to the folder name before writing. + - **Flat filename collision:** if \`issue-{NNN}-….png\` exists for the chosen NNN, bump NNN (auto-increment) rather than overwriting. +- This matches the existing regression-test collision rule: check existing names, take max number + 1 / next free suffix — never silent overwrite. + +**Missing or unreadable evidence assets:** + +- Capture failures are normal (timeout, navigation error, permission). **Do not invent placeholder image files** and do not write empty/dummy PNGs. +- In \`finding.md\` / the report **Evidence** section, list only files that exist on disk after a successful write. For a failed capture, note it inline, e.g. \`_(capture failed: {short reason})_\`, and continue the run. +- If a referenced path is unreadable at report time, treat it as missing: omit the file link, note \`_(unreadable: {path})_\`, continue. Never block the rest of documentation on one bad asset. +- Top-level \`REPORT.md\` / flat report screenshot counts must reflect files that actually exist. + **finding.md** (per-finding) MUST have this shape: \`\`\`markdown @@ -312,18 +341,18 @@ When the run is invoked with **\`--evidence-per-finding\`**, switch to one folde - \`step-1.png\` — before the action - \`step-2.png\` — after the action - \`result.png\` — final state -- \`repro.webm\` — full interactive repro (if recorded) \`\`\` +(Only list evidence files that exist. If a step capture failed, replace that bullet with \`_(capture failed: {reason})_\`.) + **When per-finding is the right call:** - Run produces ≥5 findings (the flat layout gets noisy past that). - Any finding is critical or high severity — those tickets travel further, need self-contained evidence. -- An interactive bug needs video evidence — \`record start\` then \`record stop\` (Playwright \`recordVideo\`) writes a \`.webm\`; move it into the corresponding finding folder during Phase 6. - Findings will be handed off as Linear/Jira tickets — each folder zips into a self-contained attachment. **When per-finding is overkill:** quick smoke runs, 1-2 findings, regression-mode reruns where the baseline is the canonical artifact. Stick with the flat layout. -Top-level \`REPORT.md\` content is identical between the two layouts; only the on-disk filing differs. +Top-level \`REPORT.md\` content is identical between the two layouts; only the on-disk filing differs. Screenshots remain the evidence format — video recording is not part of this layout. ### Phase 6: Wrap Up diff --git a/test/gen-skill-docs.test.ts b/test/gen-skill-docs.test.ts index 4bf8abeee..1e523f107 100644 --- a/test/gen-skill-docs.test.ts +++ b/test/gen-skill-docs.test.ts @@ -476,6 +476,101 @@ describe('gen-skill-docs', () => { }); }); +/** + * Evidence layout contract for /qa and /qa-only. + * + * Shared methodology (via {{QA_METHODOLOGY}}) plus each leaf's Setup row and + * Output Structure pointer must stay aligned: flat default, opt-in nested + * layout, no unavailable recording claims, explicit collision and missing-asset + * rules. + */ +describe('QA evidence layout contract (qa + qa-only)', () => { + const qaContent = () => fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md'), 'utf-8'); + const qaOnlyContent = () => fs.readFileSync(path.join(ROOT, 'qa-only', 'SKILL.md'), 'utf-8'); + const both = () => [ + { name: 'qa', content: qaContent() }, + { name: 'qa-only', content: qaOnlyContent() }, + ]; + + test('both modes document flat default and --evidence-per-finding opt-in', () => { + for (const { name, content } of both()) { + expect(content, name).toContain('--evidence-per-finding'); + expect(content, name).toMatch(/Evidence layout.*flat \(default\)/s); + expect(content, name).toContain('Evidence layout: flat (default) vs per-finding'); + expect(content, name).toContain('findings/'); + expect(content, name).toContain('finding.md'); + // Flat remains the default on-disk shape + expect(content, name).toContain('screenshots/'); + expect(content, name).toMatch(/Default \(flat layout\)/); + } + }); + + test('both modes define finding folder naming and nested tree shape', () => { + for (const { name, content } of both()) { + // Canonical nested paths from the shared methodology section + expect(content, name).toContain('001-critical-checkout-500-on-submit/'); + expect(content, name).toContain('step-1.png'); + expect(content, name).toContain('result.png'); + expect(content, name).toContain('REPORT.md'); + // finding.md shape fields + expect(content, name).toContain('**Severity:**'); + expect(content, name).toContain('## Repro steps'); + expect(content, name).toContain('## Evidence'); + } + }); + + test('both modes ban unavailable video recording claims', () => { + for (const { name, content } of both()) { + // $B record / recordVideo / repro.webm are not implemented in browse — + // skills must not advertise them (reviewer feedback on PR #1484). + expect(content, name).not.toMatch(/\$B record\b/); + expect(content, name).not.toMatch(/\brepro\.webm\b/); + expect(content, name).not.toMatch(/\brecordVideo\b/); + expect(content, name).not.toMatch(/`record start`/); + expect(content, name).not.toMatch(/`record stop`/); + } + }); + + test('both modes specify collision handling (no silent overwrite)', () => { + for (const { name, content } of both()) { + expect(content, name).toMatch(/collision|must not silently overwrite|never overwrite/i); + // Deterministic conflict resolution: sequential NNN and/or -2, -3 suffix + expect(content, name).toMatch(/-2|-3|auto-increment|sequential/i); + } + }); + + test('both modes specify missing/unreadable evidence asset behavior', () => { + for (const { name, content } of both()) { + expect(content, name).toMatch(/missing|unreadable|capture failed/i); + // Do not invent placeholder files; report and continue + expect(content, name).toMatch(/do not invent|must not invent|never invent|omit/i); + } + }); + + test('leaf templates carry Setup row and Output Structure pointer without recording', () => { + const qaTmpl = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md.tmpl'), 'utf-8'); + const qaOnlyTmpl = fs.readFileSync(path.join(ROOT, 'qa-only', 'SKILL.md.tmpl'), 'utf-8'); + for (const { name, tmpl } of [ + { name: 'qa', tmpl: qaTmpl }, + { name: 'qa-only', tmpl: qaOnlyTmpl }, + ]) { + expect(tmpl, name).toContain('| Evidence layout | flat (default) |'); + expect(tmpl, name).toContain('`--evidence-per-finding`'); + expect(tmpl, name).toMatch(/With `--evidence-per-finding`:/); + expect(tmpl, name).not.toMatch(/\$B record\b/); + expect(tmpl, name).not.toMatch(/\brepro\.webm\b/); + } + }); + + test('/qa nested pointer keeps fix before/after; /qa-only does not claim fix assets', () => { + const qaTmpl = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md.tmpl'), 'utf-8'); + const qaOnlyTmpl = fs.readFileSync(path.join(ROOT, 'qa-only', 'SKILL.md.tmpl'), 'utf-8'); + expect(qaTmpl).toMatch(/fix before\/after|before\/after/); + // qa-only is report-only — no fix-loop evidence language required + expect(qaOnlyTmpl).toContain('finding.md'); + }); +}); + describe('BASE_BRANCH_DETECT resolver', () => { // Find a generated SKILL.md that uses the placeholder (ship is guaranteed to) const shipContent = fs.readFileSync(path.join(ROOT, 'ship', 'SKILL.md'), 'utf-8'); diff --git a/test/skill-validation.test.ts b/test/skill-validation.test.ts index 53c7c33aa..63a26d447 100644 --- a/test/skill-validation.test.ts +++ b/test/skill-validation.test.ts @@ -421,6 +421,36 @@ describe('QA skill structure validation', () => { expect(qaContent).toContain('screenshots/'); expect(qaContent).toContain('.gstack/qa-reports/'); }); + + test('evidence layout: flat default, opt-in nested, no recording claims', () => { + expect(qaContent).toContain('--evidence-per-finding'); + expect(qaContent).toContain('Evidence layout: flat (default) vs per-finding'); + expect(qaContent).toContain('findings/'); + expect(qaContent).toContain('finding.md'); + // Unavailable browse capability — must not advertise + expect(qaContent).not.toMatch(/\brepro\.webm\b/); + expect(qaContent).not.toMatch(/\$B record\b/); + expect(qaContent).not.toMatch(/\brecordVideo\b/); + // Edge cases required by the layout contract + expect(qaContent).toMatch(/must not silently overwrite|never overwrite/i); + expect(qaContent).toMatch(/missing|unreadable|capture failed/i); + }); +}); + +describe('QA-only evidence layout contract', () => { + const qaOnlyContent = fs.readFileSync(path.join(ROOT, 'qa-only', 'SKILL.md'), 'utf-8'); + + test('mirrors nested evidence contract without recording claims', () => { + expect(qaOnlyContent).toContain('--evidence-per-finding'); + expect(qaOnlyContent).toContain('Evidence layout: flat (default) vs per-finding'); + expect(qaOnlyContent).toContain('findings/'); + expect(qaOnlyContent).toContain('finding.md'); + expect(qaOnlyContent).not.toMatch(/\brepro\.webm\b/); + expect(qaOnlyContent).not.toMatch(/\$B record\b/); + expect(qaOnlyContent).not.toMatch(/\brecordVideo\b/); + expect(qaOnlyContent).toMatch(/must not silently overwrite|never overwrite/i); + expect(qaOnlyContent).toMatch(/missing|unreadable|capture failed/i); + }); }); // --- Part 7: Greptile history format consistency (A3) ---