From 202d3d84ebb5e77a1f7c3ba610257da62bafed35 Mon Sep 17 00:00:00 2001 From: Tim White Date: Wed, 13 May 2026 20:16:05 -0700 Subject: [PATCH 1/3] feat(qa,qa-only): add --evidence-per-finding evidence layout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an opt-in per-finding evidence layout to /qa and /qa-only. When the user passes --evidence-per-finding (or natural-language variants like "evidence per finding" / "one folder per bug"), the run writes one self-contained folder per finding instead of the flat shared-screenshots layout: .gstack/qa-reports/qa-report-{domain}-{date}/ ├── REPORT.md ├── findings/ │ ├── 001-critical-checkout-500-on-submit/ │ │ ├── finding.md (severity, repro, env, expected/actual) │ │ ├── step-1.png │ │ ├── step-2.png │ │ ├── result.png │ │ └── repro.webm (optional — present iff $B record was active) │ └── 002-high-search-no-results/ │ └── ... └── baseline.json The default flat layout is unchanged. When per-finding is the right call (now in the shared methodology): - Run produces ≥5 findings — the flat layout gets noisy past that. - Any finding is critical or high — those tickets travel further and need self-contained evidence. - An interactive bug needs video evidence — pairs with $B record (a separate PR adds the recording primitive at the browse layer). - Findings will be handed off as Linear/Jira tickets — each folder zips into a single attachment. Skip per-finding for quick smoke runs, 1-2 findings, or regression-mode reruns where baseline.json is the canonical artifact. Why a shared resolver: the structure and finding.md template are identical for /qa and /qa-only. Per gstack's "no copy-paste across leaves" prompt-size guidance, the shared content goes through generateQAMethodology() (loaded into both via {{QA_METHODOLOGY}}). Each leaf SKILL.md.tmpl only gets one new Setup-table row and a one-line Output-Structure pointer to the shared section. 712 existing tests in test/gen-skill-docs.test.ts and test/skill-validation.test.ts still pass. --- qa-only/SKILL.md.tmpl | 5 +++ qa/SKILL.md.tmpl | 5 +++ scripts/resolvers/utility.ts | 68 ++++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+) diff --git a/qa-only/SKILL.md.tmpl b/qa-only/SKILL.md.tmpl index 75c4123cc..cec4d9bc3 100644 --- a/qa-only/SKILL.md.tmpl +++ b/qa-only/SKILL.md.tmpl @@ -38,6 +38,7 @@ You are a QA engineer. Test web applications like a real user — click everythi | Target URL | (auto-detect or required) | `https://myapp.com`, `http://localhost:3000` | | Mode | full | `--quick`, `--regression .gstack/qa-reports/baseline.json` | | Output dir | `.gstack/qa-reports/` | `Output to /tmp/qa` | +| Evidence layout | flat (default) | `--evidence-per-finding`, `evidence per finding`, `one folder per bug` | | Scope | Full app (or diff-scoped) | `Focus on the billing page` | | Auth | None | `Sign in to user@example.com`, `Import cookies from cookies.json` | @@ -91,6 +92,8 @@ Write to `~/.gstack/projects/{slug}/{user}-{branch}-test-outcome-{datetime}.md` ### Output Structure +**Default (flat layout):** + ``` .gstack/qa-reports/ ├── qa-report-{domain}-{YYYY-MM-DD}.md # Structured report @@ -104,6 +107,8 @@ Write to `~/.gstack/projects/{slug}/{user}-{branch}-test-outcome-{datetime}.md` Report filenames use the domain and date: `qa-report-myapp-com-2026-03-12.md` +**With `--evidence-per-finding`:** one folder per finding under a per-run report dir. Each folder is self-contained — `finding.md` plus step screenshots and (if `$B record` was active) a `repro.webm`. See "Evidence layout: flat vs per-finding" in the Document phase above for the full structure and `finding.md` template. + --- {{LEARNINGS_LOG}} diff --git a/qa/SKILL.md.tmpl b/qa/SKILL.md.tmpl index 11997f7b8..d076e6230 100644 --- a/qa/SKILL.md.tmpl +++ b/qa/SKILL.md.tmpl @@ -50,6 +50,7 @@ You are a QA engineer AND a bug-fix engineer. Test web applications like a real | Tier | Standard | `--quick`, `--exhaustive` | | Mode | full | `--regression .gstack/qa-reports/baseline.json` | | Output dir | `.gstack/qa-reports/` | `Output to /tmp/qa` | +| Evidence layout | flat (default) | `--evidence-per-finding`, `evidence per finding`, `one folder per bug` | | Scope | Full app (or diff-scoped) | `Focus on the billing page` | | Auth | None | `Sign in to user@example.com`, `Import cookies from cookies.json` | @@ -127,6 +128,8 @@ Record baseline health score at end of Phase 6. ## Output Structure +**Default (flat layout):** + ``` .gstack/qa-reports/ ├── qa-report-{domain}-{YYYY-MM-DD}.md # Structured report @@ -142,6 +145,8 @@ Record baseline health score at end of Phase 6. Report filenames use the domain and date: `qa-report-myapp-com-2026-03-12.md` +**With `--evidence-per-finding`:** one folder per finding under a per-run report dir. Each folder is self-contained — `finding.md` plus step screenshots, fix before/after, and (if `$B record` was active) a `repro.webm`. See "Evidence layout: flat vs per-finding" in the Document phase below for the full structure and `finding.md` template. + --- ## Phase 7: Triage diff --git a/scripts/resolvers/utility.ts b/scripts/resolvers/utility.ts index 3d2e368a2..4be68c843 100644 --- a/scripts/resolvers/utility.ts +++ b/scripts/resolvers/utility.ts @@ -257,6 +257,74 @@ $B snapshot -i -a -o "$REPORT_DIR/screenshots/issue-002.png" **Write each issue to the report immediately** using the template format from \`qa/templates/qa-report-template.md\`. +#### Evidence layout: flat (default) vs per-finding + +By default, evidence files share one \`screenshots/\` directory and the report references them by filename (\`issue-001-step-1.png\`, etc). This stays compact and works well for 1-5 findings. + +When the run is invoked with **\`--evidence-per-finding\`**, switch to one folder per finding: + +\`\`\` +.gstack/qa-reports/ +└── qa-report-{domain}-{YYYY-MM-DD}/ + ├── REPORT.md # top-level report (same content as flat mode) + ├── findings/ + │ ├── 001-critical-checkout-500-on-submit/ + │ │ ├── finding.md # severity + repro + env + expected/actual + │ │ ├── step-1.png # before action + │ │ ├── step-2.png # after action + │ │ ├── result.png # final state + │ │ └── repro.webm # OPTIONAL — present iff \`$B record\` was active + │ ├── 002-high-search-no-results/ + │ │ └── ... + │ └── 003-low-cosmetic-spacing/ + │ └── ... + └── baseline.json # unchanged +\`\`\` + +**finding.md** (per-finding) MUST have this shape: + +\`\`\`markdown +# {NNN}: {Title} + +**Severity:** critical / high / medium / low / cosmetic +**Category:** functional / visual / ux / accessibility / performance / content / console / links +**Page:** \`\` +**Detected:** {YYYY-MM-DD HH:mm} ({tier} mode) + +## What's wrong +{one paragraph plain-language description} + +## Repro steps +1. {step} +2. {step} — see \`step-1.png\` +3. {step} — see \`step-2.png\` + +## Expected vs actual +- **Expected:** {what should happen} +- **Actual:** {what does happen} + +## Environment +- Browser: Chromium {version} +- Viewport: {WxH} +- Auth: {persona/cookie name or "none"} + +## Evidence +- \`step-1.png\` — before the action +- \`step-2.png\` — after the action +- \`result.png\` — final state +- \`repro.webm\` — full interactive repro (if recorded) +\`\`\` + +**When per-finding is the right call:** +- Run produces ≥5 findings (the flat layout gets noisy past that). +- Any finding is critical or high severity — those tickets travel further, need self-contained evidence. +- An interactive bug needs video evidence — \`record start\` then \`record stop\` (Playwright \`recordVideo\`) writes a \`.webm\`; move it into the corresponding finding folder during Phase 6. +- Findings will be handed off as Linear/Jira tickets — each folder zips into a self-contained attachment. + +**When per-finding is overkill:** quick smoke runs, 1-2 findings, regression-mode reruns where the baseline is the canonical artifact. Stick with the flat layout. + +Top-level \`REPORT.md\` content is identical between the two layouts; only the on-disk filing differs. + ### Phase 6: Wrap Up 1. **Compute health score** using the rubric below From 07ca46b9e0d2ae8fbbd98b2fef944d5f568ebbec Mon Sep 17 00:00:00 2001 From: Tim White Date: Wed, 13 May 2026 20:16:11 -0700 Subject: [PATCH 2/3] docs(qa,qa-only): regenerate SKILL.md for --evidence-per-finding Output of `bun run gen:skill-docs --host all` after the prior commit. Picks up the new Setup-table row + the shared Document-phase Evidence Layout section in both qa/SKILL.md and qa-only/SKILL.md. --- qa-only/SKILL.md | 73 ++++++++++++++++++++++++++++++++++++++++++++++++ qa/SKILL.md | 73 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 146 insertions(+) diff --git a/qa-only/SKILL.md b/qa-only/SKILL.md index 77dcc4d23..73e80c885 100644 --- a/qa-only/SKILL.md +++ b/qa-only/SKILL.md @@ -776,6 +776,7 @@ You are a QA engineer. Test web applications like a real user — click everythi | Target URL | (auto-detect or required) | `https://myapp.com`, `http://localhost:3000` | | Mode | full | `--quick`, `--regression .gstack/qa-reports/baseline.json` | | Output dir | `.gstack/qa-reports/` | `Output to /tmp/qa` | +| Evidence layout | flat (default) | `--evidence-per-finding`, `evidence per finding`, `one folder per bug` | | Scope | Full app (or diff-scoped) | `Focus on the billing page` | | Auth | None | `Sign in to user@example.com`, `Import cookies from cookies.json` | @@ -1051,6 +1052,74 @@ $B snapshot -i -a -o "$REPORT_DIR/screenshots/issue-002.png" **Write each issue to the report immediately** using the template format from `qa/templates/qa-report-template.md`. +#### Evidence layout: flat (default) vs per-finding + +By default, evidence files share one `screenshots/` directory and the report references them by filename (`issue-001-step-1.png`, etc). This stays compact and works well for 1-5 findings. + +When the run is invoked with **`--evidence-per-finding`**, switch to one folder per finding: + +``` +.gstack/qa-reports/ +└── qa-report-{domain}-{YYYY-MM-DD}/ + ├── REPORT.md # top-level report (same content as flat mode) + ├── findings/ + │ ├── 001-critical-checkout-500-on-submit/ + │ │ ├── finding.md # severity + repro + env + expected/actual + │ │ ├── step-1.png # before action + │ │ ├── step-2.png # after action + │ │ ├── result.png # final state + │ │ └── repro.webm # OPTIONAL — present iff `$B record` was active + │ ├── 002-high-search-no-results/ + │ │ └── ... + │ └── 003-low-cosmetic-spacing/ + │ └── ... + └── baseline.json # unchanged +``` + +**finding.md** (per-finding) MUST have this shape: + +```markdown +# {NNN}: {Title} + +**Severity:** critical / high / medium / low / cosmetic +**Category:** functional / visual / ux / accessibility / performance / content / console / links +**Page:** `` +**Detected:** {YYYY-MM-DD HH:mm} ({tier} mode) + +## What's wrong +{one paragraph plain-language description} + +## Repro steps +1. {step} +2. {step} — see `step-1.png` +3. {step} — see `step-2.png` + +## Expected vs actual +- **Expected:** {what should happen} +- **Actual:** {what does happen} + +## Environment +- Browser: Chromium {version} +- Viewport: {WxH} +- Auth: {persona/cookie name or "none"} + +## Evidence +- `step-1.png` — before the action +- `step-2.png` — after the action +- `result.png` — final state +- `repro.webm` — full interactive repro (if recorded) +``` + +**When per-finding is the right call:** +- Run produces ≥5 findings (the flat layout gets noisy past that). +- Any finding is critical or high severity — those tickets travel further, need self-contained evidence. +- An interactive bug needs video evidence — `record start` then `record stop` (Playwright `recordVideo`) writes a `.webm`; move it into the corresponding finding folder during Phase 6. +- Findings will be handed off as Linear/Jira tickets — each folder zips into a self-contained attachment. + +**When per-finding is overkill:** quick smoke runs, 1-2 findings, regression-mode reruns where the baseline is the canonical artifact. Stick with the flat layout. + +Top-level `REPORT.md` content is identical between the two layouts; only the on-disk filing differs. + ### Phase 6: Wrap Up 1. **Compute health score** using the rubric below @@ -1175,6 +1244,8 @@ Write to `~/.gstack/projects/{slug}/{user}-{branch}-test-outcome-{datetime}.md` ### Output Structure +**Default (flat layout):** + ``` .gstack/qa-reports/ ├── qa-report-{domain}-{YYYY-MM-DD}.md # Structured report @@ -1188,6 +1259,8 @@ Write to `~/.gstack/projects/{slug}/{user}-{branch}-test-outcome-{datetime}.md` Report filenames use the domain and date: `qa-report-myapp-com-2026-03-12.md` +**With `--evidence-per-finding`:** one folder per finding under a per-run report dir. Each folder is self-contained — `finding.md` plus step screenshots and (if `$B record` was active) a `repro.webm`. See "Evidence layout: flat vs per-finding" in the Document phase above for the full structure and `finding.md` template. + --- ## Capture Learnings diff --git a/qa/SKILL.md b/qa/SKILL.md index 0b56e53e2..4fed20ec6 100644 --- a/qa/SKILL.md +++ b/qa/SKILL.md @@ -824,6 +824,7 @@ You are a QA engineer AND a bug-fix engineer. Test web applications like a real | Tier | Standard | `--quick`, `--exhaustive` | | Mode | full | `--regression .gstack/qa-reports/baseline.json` | | Output dir | `.gstack/qa-reports/` | `Output to /tmp/qa` | +| Evidence layout | flat (default) | `--evidence-per-finding`, `evidence per finding`, `one folder per bug` | | Scope | Full app (or diff-scoped) | `Focus on the billing page` | | Auth | None | `Sign in to user@example.com`, `Import cookies from cookies.json` | @@ -1285,6 +1286,74 @@ $B snapshot -i -a -o "$REPORT_DIR/screenshots/issue-002.png" **Write each issue to the report immediately** using the template format from `qa/templates/qa-report-template.md`. +#### Evidence layout: flat (default) vs per-finding + +By default, evidence files share one `screenshots/` directory and the report references them by filename (`issue-001-step-1.png`, etc). This stays compact and works well for 1-5 findings. + +When the run is invoked with **`--evidence-per-finding`**, switch to one folder per finding: + +``` +.gstack/qa-reports/ +└── qa-report-{domain}-{YYYY-MM-DD}/ + ├── REPORT.md # top-level report (same content as flat mode) + ├── findings/ + │ ├── 001-critical-checkout-500-on-submit/ + │ │ ├── finding.md # severity + repro + env + expected/actual + │ │ ├── step-1.png # before action + │ │ ├── step-2.png # after action + │ │ ├── result.png # final state + │ │ └── repro.webm # OPTIONAL — present iff `$B record` was active + │ ├── 002-high-search-no-results/ + │ │ └── ... + │ └── 003-low-cosmetic-spacing/ + │ └── ... + └── baseline.json # unchanged +``` + +**finding.md** (per-finding) MUST have this shape: + +```markdown +# {NNN}: {Title} + +**Severity:** critical / high / medium / low / cosmetic +**Category:** functional / visual / ux / accessibility / performance / content / console / links +**Page:** `` +**Detected:** {YYYY-MM-DD HH:mm} ({tier} mode) + +## What's wrong +{one paragraph plain-language description} + +## Repro steps +1. {step} +2. {step} — see `step-1.png` +3. {step} — see `step-2.png` + +## Expected vs actual +- **Expected:** {what should happen} +- **Actual:** {what does happen} + +## Environment +- Browser: Chromium {version} +- Viewport: {WxH} +- Auth: {persona/cookie name or "none"} + +## Evidence +- `step-1.png` — before the action +- `step-2.png` — after the action +- `result.png` — final state +- `repro.webm` — full interactive repro (if recorded) +``` + +**When per-finding is the right call:** +- Run produces ≥5 findings (the flat layout gets noisy past that). +- Any finding is critical or high severity — those tickets travel further, need self-contained evidence. +- An interactive bug needs video evidence — `record start` then `record stop` (Playwright `recordVideo`) writes a `.webm`; move it into the corresponding finding folder during Phase 6. +- Findings will be handed off as Linear/Jira tickets — each folder zips into a self-contained attachment. + +**When per-finding is overkill:** quick smoke runs, 1-2 findings, regression-mode reruns where the baseline is the canonical artifact. Stick with the flat layout. + +Top-level `REPORT.md` content is identical between the two layouts; only the on-disk filing differs. + ### Phase 6: Wrap Up 1. **Compute health score** using the rubric below @@ -1399,6 +1468,8 @@ Record baseline health score at end of Phase 6. ## Output Structure +**Default (flat layout):** + ``` .gstack/qa-reports/ ├── qa-report-{domain}-{YYYY-MM-DD}.md # Structured report @@ -1414,6 +1485,8 @@ Record baseline health score at end of Phase 6. Report filenames use the domain and date: `qa-report-myapp-com-2026-03-12.md` +**With `--evidence-per-finding`:** one folder per finding under a per-run report dir. Each folder is self-contained — `finding.md` plus step screenshots, fix before/after, and (if `$B record` was active) a `repro.webm`. See "Evidence layout: flat vs per-finding" in the Document phase below for the full structure and `finding.md` template. + --- ## Phase 7: Triage From b70483495bcd37b5369afd4d5fb07dafaaecdbaa Mon Sep 17 00:00:00 2001 From: Tim White Date: Mon, 20 Jul 2026 14:33:28 -0700 Subject: [PATCH 3/3] fix(qa,qa-only): define evidence layout edge cases; drop unavailable recording Remove $B record / repro.webm advertising (browse has no recorder yet). Document flat-default vs --evidence-per-finding nested layout, deterministic collision handling (no silent overwrite; sequential NNN and -2/-3 suffixes), and missing/unreadable asset rules (report and continue; never invent files). Add focused contract tests for both /qa and /qa-only. --- qa-only/SKILL.md | 49 ++++++++++++++---- qa-only/SKILL.md.tmpl | 2 +- qa/SKILL.md | 49 ++++++++++++++---- qa/SKILL.md.tmpl | 2 +- scripts/resolvers/utility.ts | 47 +++++++++++++---- test/gen-skill-docs.test.ts | 95 +++++++++++++++++++++++++++++++++++ test/skill-validation.test.ts | 30 +++++++++++ 7 files changed, 243 insertions(+), 31 deletions(-) diff --git a/qa-only/SKILL.md b/qa-only/SKILL.md index 73e80c885..28bf3d405 100644 --- a/qa-only/SKILL.md +++ b/qa-only/SKILL.md @@ -1054,9 +1054,9 @@ $B snapshot -i -a -o "$REPORT_DIR/screenshots/issue-002.png" #### Evidence layout: flat (default) vs per-finding -By default, evidence files share one `screenshots/` directory and the report references them by filename (`issue-001-step-1.png`, etc). This stays compact and works well for 1-5 findings. +**Default is flat.** Evidence files share one `screenshots/` directory and the report references them by filename (`issue-001-step-1.png`, etc). Flat is used unless the user explicitly opts in. This stays compact and works well for 1-5 findings. -When the run is invoked with **`--evidence-per-finding`**, switch to one folder per finding: +**Opt-in nested layout.** When the run is invoked with **`--evidence-per-finding`** (or natural-language variants: `evidence per finding`, `one folder per bug`), switch to one folder per finding: ``` .gstack/qa-reports/ @@ -1065,10 +1065,9 @@ When the run is invoked with **`--evidence-per-finding`**, switch to one folder ├── findings/ │ ├── 001-critical-checkout-500-on-submit/ │ │ ├── finding.md # severity + repro + env + expected/actual - │ │ ├── step-1.png # before action - │ │ ├── step-2.png # after action - │ │ ├── result.png # final state - │ │ └── repro.webm # OPTIONAL — present iff `$B record` was active + │ │ ├── step-1.png # before action (when capture succeeded) + │ │ ├── step-2.png # after action (when capture succeeded) + │ │ └── result.png # final state (when capture succeeded) │ ├── 002-high-search-no-results/ │ │ └── ... │ └── 003-low-cosmetic-spacing/ @@ -1076,6 +1075,36 @@ When the run is invoked with **`--evidence-per-finding`**, switch to one folder └── baseline.json # unchanged ``` +**Directory / file creation and naming:** + +1. **Flat (default):** + - Ensure `.gstack/qa-reports/screenshots/` exists (`mkdir -p`). + - Report file: `qa-report-{domain}-{YYYY-MM-DD}.md` under the output dir. + - Evidence files: `issue-{NNN}-step-{K}.png`, `issue-{NNN}-result.png` (and for /qa fix loop: `issue-{NNN}-before.png` / `issue-{NNN}-after.png`). + - `{NNN}` is a sequential zero-padded index starting at `001`. Assign the next free index by scanning existing `issue-*.png` / report issue IDs and taking max + 1. +2. **Per-finding (opt-in):** + - Create a per-run report directory: `qa-report-{domain}-{YYYY-MM-DD}/` with `findings/` inside (`mkdir -p`). + - Top-level report is `REPORT.md` inside that directory (not a sibling `.md` file). + - Each finding folder: `{NNN}-{severity}-{kebab-slug}/` where `{NNN}` is sequential (`001`, `002`, …), severity is lowercase (`critical`/`high`/`medium`/`low`/`cosmetic`), and slug is a short kebab-case title stem (alphanumeric + hyphens only, max ~40 chars). + - Inside each finding folder write `finding.md` first, then only the screenshot files that capture actually produced (`step-1.png`, `step-2.png`, `result.png`; /qa fix loop may add `before.png` / `after.png`). + - Create a finding folder only when that finding is documented — do not pre-create empty finding dirs. + +**Collisions (must not silently overwrite evidence):** + +- Never overwrite an existing evidence file or finding directory. If a target path already exists, pick a deterministic free name: + - **Sequential IDs first:** prefer the next free `{NNN}` (flat filenames and nested folder prefixes). Scan existing siblings, take max + 1. + - **Same-day report path collision:** if `qa-report-{domain}-{YYYY-MM-DD}.md` (flat) or `qa-report-{domain}-{YYYY-MM-DD}/` (nested) already exists from an earlier run, append `-2`, then `-3`, etc. (`qa-report-{domain}-{YYYY-MM-DD}-2.md` / `…-2/`) until the path is free. Do not clobber the prior run. + - **Nested folder slug collision under the same NNN:** if `findings/{NNN}-{severity}-{slug}/` exists, append `-2`, `-3`, … to the folder name before writing. + - **Flat filename collision:** if `issue-{NNN}-….png` exists for the chosen NNN, bump NNN (auto-increment) rather than overwriting. +- This matches the existing regression-test collision rule: check existing names, take max number + 1 / next free suffix — never silent overwrite. + +**Missing or unreadable evidence assets:** + +- Capture failures are normal (timeout, navigation error, permission). **Do not invent placeholder image files** and do not write empty/dummy PNGs. +- In `finding.md` / the report **Evidence** section, list only files that exist on disk after a successful write. For a failed capture, note it inline, e.g. `_(capture failed: {short reason})_`, and continue the run. +- If a referenced path is unreadable at report time, treat it as missing: omit the file link, note `_(unreadable: {path})_`, continue. Never block the rest of documentation on one bad asset. +- Top-level `REPORT.md` / flat report screenshot counts must reflect files that actually exist. + **finding.md** (per-finding) MUST have this shape: ```markdown @@ -1107,18 +1136,18 @@ When the run is invoked with **`--evidence-per-finding`**, switch to one folder - `step-1.png` — before the action - `step-2.png` — after the action - `result.png` — final state -- `repro.webm` — full interactive repro (if recorded) ``` +(Only list evidence files that exist. If a step capture failed, replace that bullet with `_(capture failed: {reason})_`.) + **When per-finding is the right call:** - Run produces ≥5 findings (the flat layout gets noisy past that). - Any finding is critical or high severity — those tickets travel further, need self-contained evidence. -- An interactive bug needs video evidence — `record start` then `record stop` (Playwright `recordVideo`) writes a `.webm`; move it into the corresponding finding folder during Phase 6. - Findings will be handed off as Linear/Jira tickets — each folder zips into a self-contained attachment. **When per-finding is overkill:** quick smoke runs, 1-2 findings, regression-mode reruns where the baseline is the canonical artifact. Stick with the flat layout. -Top-level `REPORT.md` content is identical between the two layouts; only the on-disk filing differs. +Top-level `REPORT.md` content is identical between the two layouts; only the on-disk filing differs. Screenshots remain the evidence format — video recording is not part of this layout. ### Phase 6: Wrap Up @@ -1259,7 +1288,7 @@ Write to `~/.gstack/projects/{slug}/{user}-{branch}-test-outcome-{datetime}.md` Report filenames use the domain and date: `qa-report-myapp-com-2026-03-12.md` -**With `--evidence-per-finding`:** one folder per finding under a per-run report dir. Each folder is self-contained — `finding.md` plus step screenshots and (if `$B record` was active) a `repro.webm`. See "Evidence layout: flat vs per-finding" in the Document phase above for the full structure and `finding.md` template. +**With `--evidence-per-finding`:** one folder per finding under a per-run report dir. Each folder is self-contained — `finding.md` plus step screenshots when those captures succeeded. Default remains flat (`screenshots/` + single report `.md`). Nested mode must not silently overwrite prior evidence; see "Evidence layout: flat vs per-finding" in the Document phase above for naming, collision, and missing-asset rules. --- diff --git a/qa-only/SKILL.md.tmpl b/qa-only/SKILL.md.tmpl index cec4d9bc3..e23eaec5a 100644 --- a/qa-only/SKILL.md.tmpl +++ b/qa-only/SKILL.md.tmpl @@ -107,7 +107,7 @@ Write to `~/.gstack/projects/{slug}/{user}-{branch}-test-outcome-{datetime}.md` Report filenames use the domain and date: `qa-report-myapp-com-2026-03-12.md` -**With `--evidence-per-finding`:** one folder per finding under a per-run report dir. Each folder is self-contained — `finding.md` plus step screenshots and (if `$B record` was active) a `repro.webm`. See "Evidence layout: flat vs per-finding" in the Document phase above for the full structure and `finding.md` template. +**With `--evidence-per-finding`:** one folder per finding under a per-run report dir. Each folder is self-contained — `finding.md` plus step screenshots when those captures succeeded. Default remains flat (`screenshots/` + single report `.md`). Nested mode must not silently overwrite prior evidence; see "Evidence layout: flat vs per-finding" in the Document phase above for naming, collision, and missing-asset rules. --- diff --git a/qa/SKILL.md b/qa/SKILL.md index 4fed20ec6..d79259fef 100644 --- a/qa/SKILL.md +++ b/qa/SKILL.md @@ -1288,9 +1288,9 @@ $B snapshot -i -a -o "$REPORT_DIR/screenshots/issue-002.png" #### Evidence layout: flat (default) vs per-finding -By default, evidence files share one `screenshots/` directory and the report references them by filename (`issue-001-step-1.png`, etc). This stays compact and works well for 1-5 findings. +**Default is flat.** Evidence files share one `screenshots/` directory and the report references them by filename (`issue-001-step-1.png`, etc). Flat is used unless the user explicitly opts in. This stays compact and works well for 1-5 findings. -When the run is invoked with **`--evidence-per-finding`**, switch to one folder per finding: +**Opt-in nested layout.** When the run is invoked with **`--evidence-per-finding`** (or natural-language variants: `evidence per finding`, `one folder per bug`), switch to one folder per finding: ``` .gstack/qa-reports/ @@ -1299,10 +1299,9 @@ When the run is invoked with **`--evidence-per-finding`**, switch to one folder ├── findings/ │ ├── 001-critical-checkout-500-on-submit/ │ │ ├── finding.md # severity + repro + env + expected/actual - │ │ ├── step-1.png # before action - │ │ ├── step-2.png # after action - │ │ ├── result.png # final state - │ │ └── repro.webm # OPTIONAL — present iff `$B record` was active + │ │ ├── step-1.png # before action (when capture succeeded) + │ │ ├── step-2.png # after action (when capture succeeded) + │ │ └── result.png # final state (when capture succeeded) │ ├── 002-high-search-no-results/ │ │ └── ... │ └── 003-low-cosmetic-spacing/ @@ -1310,6 +1309,36 @@ When the run is invoked with **`--evidence-per-finding`**, switch to one folder └── baseline.json # unchanged ``` +**Directory / file creation and naming:** + +1. **Flat (default):** + - Ensure `.gstack/qa-reports/screenshots/` exists (`mkdir -p`). + - Report file: `qa-report-{domain}-{YYYY-MM-DD}.md` under the output dir. + - Evidence files: `issue-{NNN}-step-{K}.png`, `issue-{NNN}-result.png` (and for /qa fix loop: `issue-{NNN}-before.png` / `issue-{NNN}-after.png`). + - `{NNN}` is a sequential zero-padded index starting at `001`. Assign the next free index by scanning existing `issue-*.png` / report issue IDs and taking max + 1. +2. **Per-finding (opt-in):** + - Create a per-run report directory: `qa-report-{domain}-{YYYY-MM-DD}/` with `findings/` inside (`mkdir -p`). + - Top-level report is `REPORT.md` inside that directory (not a sibling `.md` file). + - Each finding folder: `{NNN}-{severity}-{kebab-slug}/` where `{NNN}` is sequential (`001`, `002`, …), severity is lowercase (`critical`/`high`/`medium`/`low`/`cosmetic`), and slug is a short kebab-case title stem (alphanumeric + hyphens only, max ~40 chars). + - Inside each finding folder write `finding.md` first, then only the screenshot files that capture actually produced (`step-1.png`, `step-2.png`, `result.png`; /qa fix loop may add `before.png` / `after.png`). + - Create a finding folder only when that finding is documented — do not pre-create empty finding dirs. + +**Collisions (must not silently overwrite evidence):** + +- Never overwrite an existing evidence file or finding directory. If a target path already exists, pick a deterministic free name: + - **Sequential IDs first:** prefer the next free `{NNN}` (flat filenames and nested folder prefixes). Scan existing siblings, take max + 1. + - **Same-day report path collision:** if `qa-report-{domain}-{YYYY-MM-DD}.md` (flat) or `qa-report-{domain}-{YYYY-MM-DD}/` (nested) already exists from an earlier run, append `-2`, then `-3`, etc. (`qa-report-{domain}-{YYYY-MM-DD}-2.md` / `…-2/`) until the path is free. Do not clobber the prior run. + - **Nested folder slug collision under the same NNN:** if `findings/{NNN}-{severity}-{slug}/` exists, append `-2`, `-3`, … to the folder name before writing. + - **Flat filename collision:** if `issue-{NNN}-….png` exists for the chosen NNN, bump NNN (auto-increment) rather than overwriting. +- This matches the existing regression-test collision rule: check existing names, take max number + 1 / next free suffix — never silent overwrite. + +**Missing or unreadable evidence assets:** + +- Capture failures are normal (timeout, navigation error, permission). **Do not invent placeholder image files** and do not write empty/dummy PNGs. +- In `finding.md` / the report **Evidence** section, list only files that exist on disk after a successful write. For a failed capture, note it inline, e.g. `_(capture failed: {short reason})_`, and continue the run. +- If a referenced path is unreadable at report time, treat it as missing: omit the file link, note `_(unreadable: {path})_`, continue. Never block the rest of documentation on one bad asset. +- Top-level `REPORT.md` / flat report screenshot counts must reflect files that actually exist. + **finding.md** (per-finding) MUST have this shape: ```markdown @@ -1341,18 +1370,18 @@ When the run is invoked with **`--evidence-per-finding`**, switch to one folder - `step-1.png` — before the action - `step-2.png` — after the action - `result.png` — final state -- `repro.webm` — full interactive repro (if recorded) ``` +(Only list evidence files that exist. If a step capture failed, replace that bullet with `_(capture failed: {reason})_`.) + **When per-finding is the right call:** - Run produces ≥5 findings (the flat layout gets noisy past that). - Any finding is critical or high severity — those tickets travel further, need self-contained evidence. -- An interactive bug needs video evidence — `record start` then `record stop` (Playwright `recordVideo`) writes a `.webm`; move it into the corresponding finding folder during Phase 6. - Findings will be handed off as Linear/Jira tickets — each folder zips into a self-contained attachment. **When per-finding is overkill:** quick smoke runs, 1-2 findings, regression-mode reruns where the baseline is the canonical artifact. Stick with the flat layout. -Top-level `REPORT.md` content is identical between the two layouts; only the on-disk filing differs. +Top-level `REPORT.md` content is identical between the two layouts; only the on-disk filing differs. Screenshots remain the evidence format — video recording is not part of this layout. ### Phase 6: Wrap Up @@ -1485,7 +1514,7 @@ Record baseline health score at end of Phase 6. Report filenames use the domain and date: `qa-report-myapp-com-2026-03-12.md` -**With `--evidence-per-finding`:** one folder per finding under a per-run report dir. Each folder is self-contained — `finding.md` plus step screenshots, fix before/after, and (if `$B record` was active) a `repro.webm`. See "Evidence layout: flat vs per-finding" in the Document phase below for the full structure and `finding.md` template. +**With `--evidence-per-finding`:** one folder per finding under a per-run report dir. Each folder is self-contained — `finding.md` plus step screenshots and fix before/after when those captures succeeded. Default remains flat (`screenshots/` + single report `.md`). Nested mode must not silently overwrite prior evidence; see "Evidence layout: flat vs per-finding" in the Document phase (Phases 1-6) for naming, collision, and missing-asset rules. --- diff --git a/qa/SKILL.md.tmpl b/qa/SKILL.md.tmpl index d076e6230..c7bece91a 100644 --- a/qa/SKILL.md.tmpl +++ b/qa/SKILL.md.tmpl @@ -145,7 +145,7 @@ Record baseline health score at end of Phase 6. Report filenames use the domain and date: `qa-report-myapp-com-2026-03-12.md` -**With `--evidence-per-finding`:** one folder per finding under a per-run report dir. Each folder is self-contained — `finding.md` plus step screenshots, fix before/after, and (if `$B record` was active) a `repro.webm`. See "Evidence layout: flat vs per-finding" in the Document phase below for the full structure and `finding.md` template. +**With `--evidence-per-finding`:** one folder per finding under a per-run report dir. Each folder is self-contained — `finding.md` plus step screenshots and fix before/after when those captures succeeded. Default remains flat (`screenshots/` + single report `.md`). Nested mode must not silently overwrite prior evidence; see "Evidence layout: flat vs per-finding" in the Document phase (Phases 1-6) for naming, collision, and missing-asset rules. --- diff --git a/scripts/resolvers/utility.ts b/scripts/resolvers/utility.ts index 4be68c843..088285548 100644 --- a/scripts/resolvers/utility.ts +++ b/scripts/resolvers/utility.ts @@ -259,9 +259,9 @@ $B snapshot -i -a -o "$REPORT_DIR/screenshots/issue-002.png" #### Evidence layout: flat (default) vs per-finding -By default, evidence files share one \`screenshots/\` directory and the report references them by filename (\`issue-001-step-1.png\`, etc). This stays compact and works well for 1-5 findings. +**Default is flat.** Evidence files share one \`screenshots/\` directory and the report references them by filename (\`issue-001-step-1.png\`, etc). Flat is used unless the user explicitly opts in. This stays compact and works well for 1-5 findings. -When the run is invoked with **\`--evidence-per-finding\`**, switch to one folder per finding: +**Opt-in nested layout.** When the run is invoked with **\`--evidence-per-finding\`** (or natural-language variants: \`evidence per finding\`, \`one folder per bug\`), switch to one folder per finding: \`\`\` .gstack/qa-reports/ @@ -270,10 +270,9 @@ When the run is invoked with **\`--evidence-per-finding\`**, switch to one folde ├── findings/ │ ├── 001-critical-checkout-500-on-submit/ │ │ ├── finding.md # severity + repro + env + expected/actual - │ │ ├── step-1.png # before action - │ │ ├── step-2.png # after action - │ │ ├── result.png # final state - │ │ └── repro.webm # OPTIONAL — present iff \`$B record\` was active + │ │ ├── step-1.png # before action (when capture succeeded) + │ │ ├── step-2.png # after action (when capture succeeded) + │ │ └── result.png # final state (when capture succeeded) │ ├── 002-high-search-no-results/ │ │ └── ... │ └── 003-low-cosmetic-spacing/ @@ -281,6 +280,36 @@ When the run is invoked with **\`--evidence-per-finding\`**, switch to one folde └── baseline.json # unchanged \`\`\` +**Directory / file creation and naming:** + +1. **Flat (default):** + - Ensure \`.gstack/qa-reports/screenshots/\` exists (\`mkdir -p\`). + - Report file: \`qa-report-{domain}-{YYYY-MM-DD}.md\` under the output dir. + - Evidence files: \`issue-{NNN}-step-{K}.png\`, \`issue-{NNN}-result.png\` (and for /qa fix loop: \`issue-{NNN}-before.png\` / \`issue-{NNN}-after.png\`). + - \`{NNN}\` is a sequential zero-padded index starting at \`001\`. Assign the next free index by scanning existing \`issue-*.png\` / report issue IDs and taking max + 1. +2. **Per-finding (opt-in):** + - Create a per-run report directory: \`qa-report-{domain}-{YYYY-MM-DD}/\` with \`findings/\` inside (\`mkdir -p\`). + - Top-level report is \`REPORT.md\` inside that directory (not a sibling \`.md\` file). + - Each finding folder: \`{NNN}-{severity}-{kebab-slug}/\` where \`{NNN}\` is sequential (\`001\`, \`002\`, …), severity is lowercase (\`critical\`/\`high\`/\`medium\`/\`low\`/\`cosmetic\`), and slug is a short kebab-case title stem (alphanumeric + hyphens only, max ~40 chars). + - Inside each finding folder write \`finding.md\` first, then only the screenshot files that capture actually produced (\`step-1.png\`, \`step-2.png\`, \`result.png\`; /qa fix loop may add \`before.png\` / \`after.png\`). + - Create a finding folder only when that finding is documented — do not pre-create empty finding dirs. + +**Collisions (must not silently overwrite evidence):** + +- Never overwrite an existing evidence file or finding directory. If a target path already exists, pick a deterministic free name: + - **Sequential IDs first:** prefer the next free \`{NNN}\` (flat filenames and nested folder prefixes). Scan existing siblings, take max + 1. + - **Same-day report path collision:** if \`qa-report-{domain}-{YYYY-MM-DD}.md\` (flat) or \`qa-report-{domain}-{YYYY-MM-DD}/\` (nested) already exists from an earlier run, append \`-2\`, then \`-3\`, etc. (\`qa-report-{domain}-{YYYY-MM-DD}-2.md\` / \`…-2/\`) until the path is free. Do not clobber the prior run. + - **Nested folder slug collision under the same NNN:** if \`findings/{NNN}-{severity}-{slug}/\` exists, append \`-2\`, \`-3\`, … to the folder name before writing. + - **Flat filename collision:** if \`issue-{NNN}-….png\` exists for the chosen NNN, bump NNN (auto-increment) rather than overwriting. +- This matches the existing regression-test collision rule: check existing names, take max number + 1 / next free suffix — never silent overwrite. + +**Missing or unreadable evidence assets:** + +- Capture failures are normal (timeout, navigation error, permission). **Do not invent placeholder image files** and do not write empty/dummy PNGs. +- In \`finding.md\` / the report **Evidence** section, list only files that exist on disk after a successful write. For a failed capture, note it inline, e.g. \`_(capture failed: {short reason})_\`, and continue the run. +- If a referenced path is unreadable at report time, treat it as missing: omit the file link, note \`_(unreadable: {path})_\`, continue. Never block the rest of documentation on one bad asset. +- Top-level \`REPORT.md\` / flat report screenshot counts must reflect files that actually exist. + **finding.md** (per-finding) MUST have this shape: \`\`\`markdown @@ -312,18 +341,18 @@ When the run is invoked with **\`--evidence-per-finding\`**, switch to one folde - \`step-1.png\` — before the action - \`step-2.png\` — after the action - \`result.png\` — final state -- \`repro.webm\` — full interactive repro (if recorded) \`\`\` +(Only list evidence files that exist. If a step capture failed, replace that bullet with \`_(capture failed: {reason})_\`.) + **When per-finding is the right call:** - Run produces ≥5 findings (the flat layout gets noisy past that). - Any finding is critical or high severity — those tickets travel further, need self-contained evidence. -- An interactive bug needs video evidence — \`record start\` then \`record stop\` (Playwright \`recordVideo\`) writes a \`.webm\`; move it into the corresponding finding folder during Phase 6. - Findings will be handed off as Linear/Jira tickets — each folder zips into a self-contained attachment. **When per-finding is overkill:** quick smoke runs, 1-2 findings, regression-mode reruns where the baseline is the canonical artifact. Stick with the flat layout. -Top-level \`REPORT.md\` content is identical between the two layouts; only the on-disk filing differs. +Top-level \`REPORT.md\` content is identical between the two layouts; only the on-disk filing differs. Screenshots remain the evidence format — video recording is not part of this layout. ### Phase 6: Wrap Up diff --git a/test/gen-skill-docs.test.ts b/test/gen-skill-docs.test.ts index 4bf8abeee..1e523f107 100644 --- a/test/gen-skill-docs.test.ts +++ b/test/gen-skill-docs.test.ts @@ -476,6 +476,101 @@ describe('gen-skill-docs', () => { }); }); +/** + * Evidence layout contract for /qa and /qa-only. + * + * Shared methodology (via {{QA_METHODOLOGY}}) plus each leaf's Setup row and + * Output Structure pointer must stay aligned: flat default, opt-in nested + * layout, no unavailable recording claims, explicit collision and missing-asset + * rules. + */ +describe('QA evidence layout contract (qa + qa-only)', () => { + const qaContent = () => fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md'), 'utf-8'); + const qaOnlyContent = () => fs.readFileSync(path.join(ROOT, 'qa-only', 'SKILL.md'), 'utf-8'); + const both = () => [ + { name: 'qa', content: qaContent() }, + { name: 'qa-only', content: qaOnlyContent() }, + ]; + + test('both modes document flat default and --evidence-per-finding opt-in', () => { + for (const { name, content } of both()) { + expect(content, name).toContain('--evidence-per-finding'); + expect(content, name).toMatch(/Evidence layout.*flat \(default\)/s); + expect(content, name).toContain('Evidence layout: flat (default) vs per-finding'); + expect(content, name).toContain('findings/'); + expect(content, name).toContain('finding.md'); + // Flat remains the default on-disk shape + expect(content, name).toContain('screenshots/'); + expect(content, name).toMatch(/Default \(flat layout\)/); + } + }); + + test('both modes define finding folder naming and nested tree shape', () => { + for (const { name, content } of both()) { + // Canonical nested paths from the shared methodology section + expect(content, name).toContain('001-critical-checkout-500-on-submit/'); + expect(content, name).toContain('step-1.png'); + expect(content, name).toContain('result.png'); + expect(content, name).toContain('REPORT.md'); + // finding.md shape fields + expect(content, name).toContain('**Severity:**'); + expect(content, name).toContain('## Repro steps'); + expect(content, name).toContain('## Evidence'); + } + }); + + test('both modes ban unavailable video recording claims', () => { + for (const { name, content } of both()) { + // $B record / recordVideo / repro.webm are not implemented in browse — + // skills must not advertise them (reviewer feedback on PR #1484). + expect(content, name).not.toMatch(/\$B record\b/); + expect(content, name).not.toMatch(/\brepro\.webm\b/); + expect(content, name).not.toMatch(/\brecordVideo\b/); + expect(content, name).not.toMatch(/`record start`/); + expect(content, name).not.toMatch(/`record stop`/); + } + }); + + test('both modes specify collision handling (no silent overwrite)', () => { + for (const { name, content } of both()) { + expect(content, name).toMatch(/collision|must not silently overwrite|never overwrite/i); + // Deterministic conflict resolution: sequential NNN and/or -2, -3 suffix + expect(content, name).toMatch(/-2|-3|auto-increment|sequential/i); + } + }); + + test('both modes specify missing/unreadable evidence asset behavior', () => { + for (const { name, content } of both()) { + expect(content, name).toMatch(/missing|unreadable|capture failed/i); + // Do not invent placeholder files; report and continue + expect(content, name).toMatch(/do not invent|must not invent|never invent|omit/i); + } + }); + + test('leaf templates carry Setup row and Output Structure pointer without recording', () => { + const qaTmpl = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md.tmpl'), 'utf-8'); + const qaOnlyTmpl = fs.readFileSync(path.join(ROOT, 'qa-only', 'SKILL.md.tmpl'), 'utf-8'); + for (const { name, tmpl } of [ + { name: 'qa', tmpl: qaTmpl }, + { name: 'qa-only', tmpl: qaOnlyTmpl }, + ]) { + expect(tmpl, name).toContain('| Evidence layout | flat (default) |'); + expect(tmpl, name).toContain('`--evidence-per-finding`'); + expect(tmpl, name).toMatch(/With `--evidence-per-finding`:/); + expect(tmpl, name).not.toMatch(/\$B record\b/); + expect(tmpl, name).not.toMatch(/\brepro\.webm\b/); + } + }); + + test('/qa nested pointer keeps fix before/after; /qa-only does not claim fix assets', () => { + const qaTmpl = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md.tmpl'), 'utf-8'); + const qaOnlyTmpl = fs.readFileSync(path.join(ROOT, 'qa-only', 'SKILL.md.tmpl'), 'utf-8'); + expect(qaTmpl).toMatch(/fix before\/after|before\/after/); + // qa-only is report-only — no fix-loop evidence language required + expect(qaOnlyTmpl).toContain('finding.md'); + }); +}); + describe('BASE_BRANCH_DETECT resolver', () => { // Find a generated SKILL.md that uses the placeholder (ship is guaranteed to) const shipContent = fs.readFileSync(path.join(ROOT, 'ship', 'SKILL.md'), 'utf-8'); diff --git a/test/skill-validation.test.ts b/test/skill-validation.test.ts index 53c7c33aa..63a26d447 100644 --- a/test/skill-validation.test.ts +++ b/test/skill-validation.test.ts @@ -421,6 +421,36 @@ describe('QA skill structure validation', () => { expect(qaContent).toContain('screenshots/'); expect(qaContent).toContain('.gstack/qa-reports/'); }); + + test('evidence layout: flat default, opt-in nested, no recording claims', () => { + expect(qaContent).toContain('--evidence-per-finding'); + expect(qaContent).toContain('Evidence layout: flat (default) vs per-finding'); + expect(qaContent).toContain('findings/'); + expect(qaContent).toContain('finding.md'); + // Unavailable browse capability — must not advertise + expect(qaContent).not.toMatch(/\brepro\.webm\b/); + expect(qaContent).not.toMatch(/\$B record\b/); + expect(qaContent).not.toMatch(/\brecordVideo\b/); + // Edge cases required by the layout contract + expect(qaContent).toMatch(/must not silently overwrite|never overwrite/i); + expect(qaContent).toMatch(/missing|unreadable|capture failed/i); + }); +}); + +describe('QA-only evidence layout contract', () => { + const qaOnlyContent = fs.readFileSync(path.join(ROOT, 'qa-only', 'SKILL.md'), 'utf-8'); + + test('mirrors nested evidence contract without recording claims', () => { + expect(qaOnlyContent).toContain('--evidence-per-finding'); + expect(qaOnlyContent).toContain('Evidence layout: flat (default) vs per-finding'); + expect(qaOnlyContent).toContain('findings/'); + expect(qaOnlyContent).toContain('finding.md'); + expect(qaOnlyContent).not.toMatch(/\brepro\.webm\b/); + expect(qaOnlyContent).not.toMatch(/\$B record\b/); + expect(qaOnlyContent).not.toMatch(/\brecordVideo\b/); + expect(qaOnlyContent).toMatch(/must not silently overwrite|never overwrite/i); + expect(qaOnlyContent).toMatch(/missing|unreadable|capture failed/i); + }); }); // --- Part 7: Greptile history format consistency (A3) ---