From 992a8f22b2ea4fcb4c6977846e84f83ebff86420 Mon Sep 17 00:00:00 2001 From: fedster99 Date: Sun, 19 Jul 2026 06:43:27 +0200 Subject: [PATCH] feat(ship): add complete engineering handoffs --- ship/SKILL.md | 69 +++++++++++++- ship/SKILL.md.tmpl | 69 +++++++++++++- test/fixtures/golden/claude-ship-SKILL.md | 69 +++++++++++++- test/fixtures/golden/codex-ship-SKILL.md | 69 +++++++++++++- test/fixtures/golden/factory-ship-SKILL.md | 69 +++++++++++++- test/helpers/touchfiles.ts | 2 + test/ship-completion-report.test.ts | 74 +++++++++++++++ test/skill-coverage-matrix.ts | 7 +- test/skill-llm-eval.test.ts | 100 ++++++++++++++++++++- 9 files changed, 515 insertions(+), 13 deletions(-) create mode 100644 test/ship-completion-report.test.ts diff --git a/ship/SKILL.md b/ship/SKILL.md index eadffaa8f..aaa7396b0 100644 --- a/ship/SKILL.md +++ b/ship/SKILL.md @@ -843,7 +843,7 @@ branch name wherever the instructions say "the base branch" or ``. # Ship: Fully Automated Ship Workflow -You are running the `/ship` workflow. This is a **non-interactive, fully automated** workflow. Do NOT ask for confirmation at any step. The user said `/ship` which means DO IT. Run straight through and output the PR URL at the end. +You are running the `/ship` workflow. This is a **non-interactive, fully automated** workflow. Do NOT ask for confirmation at any step. The user said `/ship` which means DO IT. Run straight through and include the PR URL in the final engineering handoff. **Only stop for:** - On the base branch (abort) @@ -1246,7 +1246,11 @@ Before pushing, re-verify if code changed during Steps 4-6: 2. **Build verification:** If the project has a build step, run it. Paste output. -3. **Rationalization prevention:** +3. **UI evidence gate:** For a user-visible UI diff, produce the matched + Before/After evidence from Step 22 before push. If unavailable, STOP; do not + push or create/update the PR/MR. + +4. **Rationalization prevention:** - "Should work now" → RUN IT. - "I'm confident" → Confidence is not evidence. - "I already tested earlier" → Code changed since then. Test again. @@ -1415,3 +1419,64 @@ through `gstack-version-bump`; never hand-roll the VERSION/package.json write. - **Never push without fresh verification evidence.** If code changed after Step 5 tests, re-run before pushing. - **Step 7 generates coverage tests.** They must pass before committing. Never commit failing tests. - **The goal is: user says `/ship`, next thing they see is the review + PR URL + auto-synced docs.** + +--- + +## Step 22: Full engineering handoff + +End every ship attempt with this report, even when stopped. Use each heading; +write `Not applicable` briefly when a category does not apply. + +```markdown +## Engineering summary + +### Outcome +What shipped (version, branch, commit, PR/MR URL), or what blocked it. + +### Problem and root cause +What was wrong or missing, why, and the supporting evidence. + +### Investigation and decisions +Key findings, alternatives, decisions, and why this approach won. + +### Implementation +Changed behavior, components, data paths, APIs, migrations, infrastructure, and docs. + +### Verification +Fresh test/build/typecheck/lint/behavior/review/CI/deploy evidence; mark pass/fail/skipped. + +### Risks and operational impact +Tradeoffs, compatibility, rollout/migration, observability, rollback. + +### Remaining work +Unfinished work. Write `None` when the completion contract is fully satisfied. + +### Decision required +The user's next decision. Write `None` when no decision is required. +``` + +### UI before/after evidence + +For user-visible UI changes, include matched **Before** and **After** screenshots: + +1. Use a pre-change capture only if it faithfully represents the base; otherwise run the merge-base/base revision in an isolated worktree or use its known deployment. +2. Capture the current branch after implementation and verification. +3. Match route, application state, data, viewport, theme, and zoom. Use safe test data; never expose credentials, customer data, or private information. +4. Show `![Before](path)` and `![After](path)` inline; include both artifact paths. +5. Do not commit screenshot artifacts unless the repository explicitly requires it. + +If a faithful comparison is impossible, use the same report with Outcome blocked, +mark UI evidence incomplete, and STOP before Step 17. Do not push, create/update +the PR/MR, claim UI completion, or offer a waiver. + +### Put simply + +Final response section; nothing follows. Be intuitive, extremely concise, precise: + +```markdown +### Put simply + +- **Why:** The problem this solves. +- **What:** What is now different for the user or system. +- **How:** The essential mechanism, without unnecessary implementation detail. +``` diff --git a/ship/SKILL.md.tmpl b/ship/SKILL.md.tmpl index 068ac4fe5..15460e69a 100644 --- a/ship/SKILL.md.tmpl +++ b/ship/SKILL.md.tmpl @@ -34,7 +34,7 @@ triggers: # Ship: Fully Automated Ship Workflow -You are running the `/ship` workflow. This is a **non-interactive, fully automated** workflow. Do NOT ask for confirmation at any step. The user said `/ship` which means DO IT. Run straight through and output the PR URL at the end. +You are running the `/ship` workflow. This is a **non-interactive, fully automated** workflow. Do NOT ask for confirmation at any step. The user said `/ship` which means DO IT. Run straight through and include the PR URL in the final engineering handoff. **Only stop for:** - On the base branch (abort) @@ -367,7 +367,11 @@ Before pushing, re-verify if code changed during Steps 4-6: 2. **Build verification:** If the project has a build step, run it. Paste output. -3. **Rationalization prevention:** +3. **UI evidence gate:** For a user-visible UI diff, produce the matched + Before/After evidence from Step 22 before push. If unavailable, STOP; do not + push or create/update the PR/MR. + +4. **Rationalization prevention:** - "Should work now" → RUN IT. - "I'm confident" → Confidence is not evidence. - "I already tested earlier" → Code changed since then. Test again. @@ -535,3 +539,64 @@ through `gstack-version-bump`; never hand-roll the VERSION/package.json write. - **Never push without fresh verification evidence.** If code changed after Step 5 tests, re-run before pushing. - **Step 7 generates coverage tests.** They must pass before committing. Never commit failing tests. - **The goal is: user says `/ship`, next thing they see is the review + PR URL + auto-synced docs.** + +--- + +## Step 22: Full engineering handoff + +End every ship attempt with this report, even when stopped. Use each heading; +write `Not applicable` briefly when a category does not apply. + +```markdown +## Engineering summary + +### Outcome +What shipped (version, branch, commit, PR/MR URL), or what blocked it. + +### Problem and root cause +What was wrong or missing, why, and the supporting evidence. + +### Investigation and decisions +Key findings, alternatives, decisions, and why this approach won. + +### Implementation +Changed behavior, components, data paths, APIs, migrations, infrastructure, and docs. + +### Verification +Fresh test/build/typecheck/lint/behavior/review/CI/deploy evidence; mark pass/fail/skipped. + +### Risks and operational impact +Tradeoffs, compatibility, rollout/migration, observability, rollback. + +### Remaining work +Unfinished work. Write `None` when the completion contract is fully satisfied. + +### Decision required +The user's next decision. Write `None` when no decision is required. +``` + +### UI before/after evidence + +For user-visible UI changes, include matched **Before** and **After** screenshots: + +1. Use a pre-change capture only if it faithfully represents the base; otherwise run the merge-base/base revision in an isolated worktree or use its known deployment. +2. Capture the current branch after implementation and verification. +3. Match route, application state, data, viewport, theme, and zoom. Use safe test data; never expose credentials, customer data, or private information. +4. Show `![Before](path)` and `![After](path)` inline; include both artifact paths. +5. Do not commit screenshot artifacts unless the repository explicitly requires it. + +If a faithful comparison is impossible, use the same report with Outcome blocked, +mark UI evidence incomplete, and STOP before Step 17. Do not push, create/update +the PR/MR, claim UI completion, or offer a waiver. + +### Put simply + +Final response section; nothing follows. Be intuitive, extremely concise, precise: + +```markdown +### Put simply + +- **Why:** The problem this solves. +- **What:** What is now different for the user or system. +- **How:** The essential mechanism, without unnecessary implementation detail. +``` diff --git a/test/fixtures/golden/claude-ship-SKILL.md b/test/fixtures/golden/claude-ship-SKILL.md index eadffaa8f..aaa7396b0 100644 --- a/test/fixtures/golden/claude-ship-SKILL.md +++ b/test/fixtures/golden/claude-ship-SKILL.md @@ -843,7 +843,7 @@ branch name wherever the instructions say "the base branch" or ``. # Ship: Fully Automated Ship Workflow -You are running the `/ship` workflow. This is a **non-interactive, fully automated** workflow. Do NOT ask for confirmation at any step. The user said `/ship` which means DO IT. Run straight through and output the PR URL at the end. +You are running the `/ship` workflow. This is a **non-interactive, fully automated** workflow. Do NOT ask for confirmation at any step. The user said `/ship` which means DO IT. Run straight through and include the PR URL in the final engineering handoff. **Only stop for:** - On the base branch (abort) @@ -1246,7 +1246,11 @@ Before pushing, re-verify if code changed during Steps 4-6: 2. **Build verification:** If the project has a build step, run it. Paste output. -3. **Rationalization prevention:** +3. **UI evidence gate:** For a user-visible UI diff, produce the matched + Before/After evidence from Step 22 before push. If unavailable, STOP; do not + push or create/update the PR/MR. + +4. **Rationalization prevention:** - "Should work now" → RUN IT. - "I'm confident" → Confidence is not evidence. - "I already tested earlier" → Code changed since then. Test again. @@ -1415,3 +1419,64 @@ through `gstack-version-bump`; never hand-roll the VERSION/package.json write. - **Never push without fresh verification evidence.** If code changed after Step 5 tests, re-run before pushing. - **Step 7 generates coverage tests.** They must pass before committing. Never commit failing tests. - **The goal is: user says `/ship`, next thing they see is the review + PR URL + auto-synced docs.** + +--- + +## Step 22: Full engineering handoff + +End every ship attempt with this report, even when stopped. Use each heading; +write `Not applicable` briefly when a category does not apply. + +```markdown +## Engineering summary + +### Outcome +What shipped (version, branch, commit, PR/MR URL), or what blocked it. + +### Problem and root cause +What was wrong or missing, why, and the supporting evidence. + +### Investigation and decisions +Key findings, alternatives, decisions, and why this approach won. + +### Implementation +Changed behavior, components, data paths, APIs, migrations, infrastructure, and docs. + +### Verification +Fresh test/build/typecheck/lint/behavior/review/CI/deploy evidence; mark pass/fail/skipped. + +### Risks and operational impact +Tradeoffs, compatibility, rollout/migration, observability, rollback. + +### Remaining work +Unfinished work. Write `None` when the completion contract is fully satisfied. + +### Decision required +The user's next decision. Write `None` when no decision is required. +``` + +### UI before/after evidence + +For user-visible UI changes, include matched **Before** and **After** screenshots: + +1. Use a pre-change capture only if it faithfully represents the base; otherwise run the merge-base/base revision in an isolated worktree or use its known deployment. +2. Capture the current branch after implementation and verification. +3. Match route, application state, data, viewport, theme, and zoom. Use safe test data; never expose credentials, customer data, or private information. +4. Show `![Before](path)` and `![After](path)` inline; include both artifact paths. +5. Do not commit screenshot artifacts unless the repository explicitly requires it. + +If a faithful comparison is impossible, use the same report with Outcome blocked, +mark UI evidence incomplete, and STOP before Step 17. Do not push, create/update +the PR/MR, claim UI completion, or offer a waiver. + +### Put simply + +Final response section; nothing follows. Be intuitive, extremely concise, precise: + +```markdown +### Put simply + +- **Why:** The problem this solves. +- **What:** What is now different for the user or system. +- **How:** The essential mechanism, without unnecessary implementation detail. +``` diff --git a/test/fixtures/golden/codex-ship-SKILL.md b/test/fixtures/golden/codex-ship-SKILL.md index d99630c4b..a9494af1e 100644 --- a/test/fixtures/golden/codex-ship-SKILL.md +++ b/test/fixtures/golden/codex-ship-SKILL.md @@ -829,7 +829,7 @@ branch name wherever the instructions say "the base branch" or ``. # Ship: Fully Automated Ship Workflow -You are running the `/ship` workflow. This is a **non-interactive, fully automated** workflow. Do NOT ask for confirmation at any step. The user said `/ship` which means DO IT. Run straight through and output the PR URL at the end. +You are running the `/ship` workflow. This is a **non-interactive, fully automated** workflow. Do NOT ask for confirmation at any step. The user said `/ship` which means DO IT. Run straight through and include the PR URL in the final engineering handoff. **Only stop for:** - On the base branch (abort) @@ -2429,7 +2429,11 @@ Before pushing, re-verify if code changed during Steps 4-6: 2. **Build verification:** If the project has a build step, run it. Paste output. -3. **Rationalization prevention:** +3. **UI evidence gate:** For a user-visible UI diff, produce the matched + Before/After evidence from Step 22 before push. If unavailable, STOP; do not + push or create/update the PR/MR. + +4. **Rationalization prevention:** - "Should work now" → RUN IT. - "I'm confident" → Confidence is not evidence. - "I already tested earlier" → Code changed since then. Test again. @@ -2801,3 +2805,64 @@ through `gstack-version-bump`; never hand-roll the VERSION/package.json write. - **Never push without fresh verification evidence.** If code changed after Step 5 tests, re-run before pushing. - **Step 7 generates coverage tests.** They must pass before committing. Never commit failing tests. - **The goal is: user says `/ship`, next thing they see is the review + PR URL + auto-synced docs.** + +--- + +## Step 22: Full engineering handoff + +End every ship attempt with this report, even when stopped. Use each heading; +write `Not applicable` briefly when a category does not apply. + +```markdown +## Engineering summary + +### Outcome +What shipped (version, branch, commit, PR/MR URL), or what blocked it. + +### Problem and root cause +What was wrong or missing, why, and the supporting evidence. + +### Investigation and decisions +Key findings, alternatives, decisions, and why this approach won. + +### Implementation +Changed behavior, components, data paths, APIs, migrations, infrastructure, and docs. + +### Verification +Fresh test/build/typecheck/lint/behavior/review/CI/deploy evidence; mark pass/fail/skipped. + +### Risks and operational impact +Tradeoffs, compatibility, rollout/migration, observability, rollback. + +### Remaining work +Unfinished work. Write `None` when the completion contract is fully satisfied. + +### Decision required +The user's next decision. Write `None` when no decision is required. +``` + +### UI before/after evidence + +For user-visible UI changes, include matched **Before** and **After** screenshots: + +1. Use a pre-change capture only if it faithfully represents the base; otherwise run the merge-base/base revision in an isolated worktree or use its known deployment. +2. Capture the current branch after implementation and verification. +3. Match route, application state, data, viewport, theme, and zoom. Use safe test data; never expose credentials, customer data, or private information. +4. Show `![Before](path)` and `![After](path)` inline; include both artifact paths. +5. Do not commit screenshot artifacts unless the repository explicitly requires it. + +If a faithful comparison is impossible, use the same report with Outcome blocked, +mark UI evidence incomplete, and STOP before Step 17. Do not push, create/update +the PR/MR, claim UI completion, or offer a waiver. + +### Put simply + +Final response section; nothing follows. Be intuitive, extremely concise, precise: + +```markdown +### Put simply + +- **Why:** The problem this solves. +- **What:** What is now different for the user or system. +- **How:** The essential mechanism, without unnecessary implementation detail. +``` diff --git a/test/fixtures/golden/factory-ship-SKILL.md b/test/fixtures/golden/factory-ship-SKILL.md index a2acad24f..ce9b755f8 100644 --- a/test/fixtures/golden/factory-ship-SKILL.md +++ b/test/fixtures/golden/factory-ship-SKILL.md @@ -831,7 +831,7 @@ branch name wherever the instructions say "the base branch" or ``. # Ship: Fully Automated Ship Workflow -You are running the `/ship` workflow. This is a **non-interactive, fully automated** workflow. Do NOT ask for confirmation at any step. The user said `/ship` which means DO IT. Run straight through and output the PR URL at the end. +You are running the `/ship` workflow. This is a **non-interactive, fully automated** workflow. Do NOT ask for confirmation at any step. The user said `/ship` which means DO IT. Run straight through and include the PR URL in the final engineering handoff. **Only stop for:** - On the base branch (abort) @@ -2835,7 +2835,11 @@ Before pushing, re-verify if code changed during Steps 4-6: 2. **Build verification:** If the project has a build step, run it. Paste output. -3. **Rationalization prevention:** +3. **UI evidence gate:** For a user-visible UI diff, produce the matched + Before/After evidence from Step 22 before push. If unavailable, STOP; do not + push or create/update the PR/MR. + +4. **Rationalization prevention:** - "Should work now" → RUN IT. - "I'm confident" → Confidence is not evidence. - "I already tested earlier" → Code changed since then. Test again. @@ -3207,3 +3211,64 @@ through `gstack-version-bump`; never hand-roll the VERSION/package.json write. - **Never push without fresh verification evidence.** If code changed after Step 5 tests, re-run before pushing. - **Step 7 generates coverage tests.** They must pass before committing. Never commit failing tests. - **The goal is: user says `/ship`, next thing they see is the review + PR URL + auto-synced docs.** + +--- + +## Step 22: Full engineering handoff + +End every ship attempt with this report, even when stopped. Use each heading; +write `Not applicable` briefly when a category does not apply. + +```markdown +## Engineering summary + +### Outcome +What shipped (version, branch, commit, PR/MR URL), or what blocked it. + +### Problem and root cause +What was wrong or missing, why, and the supporting evidence. + +### Investigation and decisions +Key findings, alternatives, decisions, and why this approach won. + +### Implementation +Changed behavior, components, data paths, APIs, migrations, infrastructure, and docs. + +### Verification +Fresh test/build/typecheck/lint/behavior/review/CI/deploy evidence; mark pass/fail/skipped. + +### Risks and operational impact +Tradeoffs, compatibility, rollout/migration, observability, rollback. + +### Remaining work +Unfinished work. Write `None` when the completion contract is fully satisfied. + +### Decision required +The user's next decision. Write `None` when no decision is required. +``` + +### UI before/after evidence + +For user-visible UI changes, include matched **Before** and **After** screenshots: + +1. Use a pre-change capture only if it faithfully represents the base; otherwise run the merge-base/base revision in an isolated worktree or use its known deployment. +2. Capture the current branch after implementation and verification. +3. Match route, application state, data, viewport, theme, and zoom. Use safe test data; never expose credentials, customer data, or private information. +4. Show `![Before](path)` and `![After](path)` inline; include both artifact paths. +5. Do not commit screenshot artifacts unless the repository explicitly requires it. + +If a faithful comparison is impossible, use the same report with Outcome blocked, +mark UI evidence incomplete, and STOP before Step 17. Do not push, create/update +the PR/MR, claim UI completion, or offer a waiver. + +### Put simply + +Final response section; nothing follows. Be intuitive, extremely concise, precise: + +```markdown +### Put simply + +- **Why:** The problem this solves. +- **What:** What is now different for the user or system. +- **How:** The essential mechanism, without unnecessary implementation detail. +``` diff --git a/test/helpers/touchfiles.ts b/test/helpers/touchfiles.ts index 33ba9ad97..7b61dc797 100644 --- a/test/helpers/touchfiles.ts +++ b/test/helpers/touchfiles.ts @@ -772,6 +772,8 @@ export const LLM_JUDGE_TOUCHFILES: Record = { // Ship & Release 'ship/SKILL.md workflow': ['ship/SKILL.md', 'ship/SKILL.md.tmpl'], + 'ship/SKILL.md completion handoff': ['ship/SKILL.md', 'ship/SKILL.md.tmpl'], + 'ship/SKILL.md completion behavior': ['ship/SKILL.md', 'ship/SKILL.md.tmpl'], 'document-release/SKILL.md workflow': ['document-release/SKILL.md', 'document-release/SKILL.md.tmpl'], // Plan Reviews diff --git a/test/ship-completion-report.test.ts b/test/ship-completion-report.test.ts new file mode 100644 index 000000000..c52291dea --- /dev/null +++ b/test/ship-completion-report.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, test } from 'bun:test'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +const root = join(import.meta.dir, '..'); + +const skillPaths = [ + 'ship/SKILL.md.tmpl', + 'ship/SKILL.md', + 'test/fixtures/golden/claude-ship-SKILL.md', + 'test/fixtures/golden/codex-ship-SKILL.md', + 'test/fixtures/golden/factory-ship-SKILL.md', +]; + +const engineeringHeadings = [ + '### Outcome', + '### Problem and root cause', + '### Investigation and decisions', + '### Implementation', + '### Verification', + '### Risks and operational impact', + '### Remaining work', + '### Decision required', +]; + +describe('ship completion report', () => { + for (const relativePath of skillPaths) { + test(`${relativePath} requires the complete two-layer handoff`, () => { + const skill = readFileSync(join(root, relativePath), 'utf8'); + const handoff = skill.indexOf('## Step 22: Full engineering handoff'); + const engineeringSummary = skill.indexOf('## Engineering summary', handoff); + const uiEvidence = skill.indexOf('### UI before/after evidence', handoff); + const putSimply = skill.lastIndexOf('### Put simply'); + const uiEvidenceGate = skill.indexOf('3. **UI evidence gate:**'); + const pushStep = skill.indexOf('## Step 17: Push'); + + expect(handoff).toBeGreaterThan(-1); + expect(engineeringSummary).toBeGreaterThan(handoff); + expect(skill).toContain('Use each heading'); + + let previousHeading = engineeringSummary; + for (const heading of engineeringHeadings) { + const headingIndex = skill.indexOf(heading, previousHeading); + expect(headingIndex).toBeGreaterThan(previousHeading); + previousHeading = headingIndex; + } + + expect(uiEvidence).toBeGreaterThan(previousHeading); + expect(skill).toContain('Write `None` when the completion contract is fully satisfied.'); + expect(skill).toContain('Write `None` when no decision is required.'); + expect(skill).toContain('matched **Before** and **After** screenshots'); + expect(skill).toContain('merge-base/base revision in an isolated worktree'); + expect(skill).toContain('current branch after implementation and verification'); + expect(skill).toContain('route, application state, data, viewport, theme, and zoom'); + expect(skill).toContain('Use safe test data; never expose credentials, customer data, or private information.'); + expect(skill).toContain('`![Before](path)` and `![After](path)` inline'); + expect(skill).toContain('include both artifact paths'); + expect(skill).toContain('Do not commit screenshot artifacts unless the repository explicitly requires it.'); + expect(skill).toContain('claim UI completion'); + expect(uiEvidenceGate).toBeGreaterThan(-1); + expect(pushStep).toBeGreaterThan(uiEvidenceGate); + expect(skill).toMatch(/do not\s+push or create\/update the PR\/MR/); + expect(skill).toContain('STOP before Step 17'); + expect(skill).toContain('or offer a waiver'); + expect(putSimply).toBeGreaterThan(uiEvidence); + expect(skill.slice(putSimply)).toContain('- **Why:**'); + expect(skill.slice(putSimply)).toContain('- **What:**'); + expect(skill.slice(putSimply)).toContain('- **How:**'); + expect(skill.slice(handoff).match(/^### .+$/gm)?.at(-1)).toBe('### Put simply'); + expect(skill.slice(handoff).match(/^## Step /gm)?.length).toBe(1); + expect(skill).not.toContain('output the PR URL at the end'); + }); + } +}); diff --git a/test/skill-coverage-matrix.ts b/test/skill-coverage-matrix.ts index 7359afbce..0291597b9 100644 --- a/test/skill-coverage-matrix.ts +++ b/test/skill-coverage-matrix.ts @@ -39,7 +39,12 @@ export interface SkillCoverage { export const SKILL_COVERAGE: Record = { // ─── Core loop ────────────────────────────────────────────── ship: { - gate: ['test/skill-e2e-ship-idempotency.test.ts', 'test/skill-coverage-floor.test.ts'], + gate: [ + 'test/skill-e2e-ship-idempotency.test.ts', + 'test/ship-completion-report.test.ts', + 'test/skill-llm-eval.test.ts', + 'test/skill-coverage-floor.test.ts', + ], periodic: ['test/skill-e2e-workflow.test.ts'], }, review: { diff --git a/test/skill-llm-eval.test.ts b/test/skill-llm-eval.test.ts index 17b1adf0d..8914727b2 100644 --- a/test/skill-llm-eval.test.ts +++ b/test/skill-llm-eval.test.ts @@ -537,6 +537,7 @@ async function runWorkflowJudge(opts: { judgeContext: string; judgeGoal: string; thresholds?: { clarity: number; completeness: number; actionability: number }; + includeCarvedSections?: boolean; }) { const t0 = Date.now(); const defaults = { clarity: 4, completeness: 3, actionability: 4 }; @@ -549,7 +550,7 @@ async function runWorkflowJudge(opts: { let content = fs.readFileSync(path.join(ROOT, opts.skillPath), 'utf-8'); const secDir = path.join(ROOT, path.dirname(opts.skillPath), 'sections'); const sectionBodies: string[] = []; - if (fs.existsSync(secDir)) { + if (opts.includeCarvedSections !== false && fs.existsSync(secDir)) { for (const f of fs.readdirSync(secDir).sort()) { if (f.endsWith('.md') && !f.endsWith('.md.tmpl')) { const body = fs.readFileSync(path.join(secDir, f), 'utf-8'); @@ -617,7 +618,12 @@ ${section}`); } // Block 1: Ship & Release skills -describeIfSelected('Ship & Release skill evals', ['ship/SKILL.md workflow', 'document-release/SKILL.md workflow'], () => { +describeIfSelected('Ship & Release skill evals', [ + 'ship/SKILL.md workflow', + 'ship/SKILL.md completion handoff', + 'ship/SKILL.md completion behavior', + 'document-release/SKILL.md workflow', +], () => { testIfSelected('ship/SKILL.md workflow', async () => { await runWorkflowJudge({ testName: 'ship/SKILL.md workflow', @@ -630,6 +636,96 @@ describeIfSelected('Ship & Release skill evals', ['ship/SKILL.md workflow', 'doc }); }, 30_000); + testIfSelected('ship/SKILL.md completion handoff', async () => { + await runWorkflowJudge({ + testName: 'ship/SKILL.md completion handoff', + suite: 'Ship & Release skill evals', + skillPath: 'ship/SKILL.md', + startMarker: '## Step 22: Full engineering handoff', + endMarker: null, + judgeContext: 'the final user-facing handoff contract for a ship workflow', + judgeGoal: + 'how to report the complete engineering outcome and evidence, require matched before/after screenshots only for UI changes, disclose incomplete UI evidence, and finish with an extremely concise Why/What/How explanation', + thresholds: { clarity: 4, completeness: 4, actionability: 4 }, + includeCarvedSections: false, + }); + }, 30_000); + + testIfSelected('ship/SKILL.md completion behavior', async () => { + const t0 = Date.now(); + const skill = fs.readFileSync(path.join(ROOT, 'ship', 'SKILL.md'), 'utf-8'); + const handoffStart = skill.indexOf('## Step 22: Full engineering handoff'); + expect(handoffStart).toBeGreaterThan(-1); + const handoff = skill.slice(handoffStart); + const client = new Anthropic(); + const response = await client.messages.create({ + model: 'claude-sonnet-4-6', + max_tokens: 4096, + messages: [{ + role: 'user', + content: `Follow the ship handoff instructions below and produce three final user-facing responses. +Return ONLY JSON with string fields "non_ui", "ui_complete", and "ui_incomplete". Do not use a JSON code fence. + +SCENARIO non_ui: v1.2.3.4 shipped on feat/report at commit abc123; PR https://example.test/pr/1; root cause was a missing output contract; implementation added it; 505 tests passed; no UI changed; no remaining work or decision. + +SCENARIO ui_complete: v1.2.3.4 shipped on feat/ui at commit def456; PR https://example.test/pr/2; a button alignment bug was fixed; 505 tests passed; matched evidence is Before /tmp/before.png and After /tmp/after.png on the same route, state, data, viewport, theme, and zoom; no remaining work or decision. + +SCENARIO ui_incomplete: a UI changed, but the base revision cannot be rendered, so faithful Before evidence is unavailable; the workflow reached the UI evidence gate before push and must report the blocker without claiming completion or inventing a PR URL. + +INSTRUCTIONS: +${handoff}`, + }], + }); + + const text = response.content[0].type === 'text' ? response.content[0].text : ''; + const jsonMatch = text.match(/\{[\s\S]*\}/); + if (!jsonMatch) throw new Error(`Completion behavior eval returned non-JSON: ${text.slice(0, 300)}`); + const outputs = JSON.parse(jsonMatch[0]) as Record; + const headings = [ + '## Engineering summary', + '### Outcome', + '### Problem and root cause', + '### Investigation and decisions', + '### Implementation', + '### Verification', + '### Risks and operational impact', + '### Remaining work', + '### Decision required', + '### Put simply', + ]; + + for (const key of ['non_ui', 'ui_complete', 'ui_incomplete']) { + const output = outputs[key]; + expect(typeof output).toBe('string'); + for (const heading of headings) expect(output).toContain(heading); + expect(output.match(/^### .+$/gm)?.at(-1)).toBe('### Put simply'); + expect(output.slice(output.lastIndexOf('### Put simply'))).toContain('- **Why:**'); + expect(output.slice(output.lastIndexOf('### Put simply'))).toContain('- **What:**'); + expect(output.slice(output.lastIndexOf('### Put simply'))).toContain('- **How:**'); + } + + expect(outputs.non_ui).not.toMatch(/!\[[^\]]*(Before|After)/i); + expect(outputs.ui_complete).toContain('/tmp/before.png'); + expect(outputs.ui_complete).toContain('/tmp/after.png'); + expect(outputs.ui_complete).toMatch(/Before[\s\S]*After/i); + expect(outputs.ui_complete).toMatch(/!\[Before\]\([^)]*before\.png\)/i); + expect(outputs.ui_complete).toMatch(/!\[After\]\([^)]*after\.png\)/i); + expect(outputs.ui_incomplete).toMatch(/incomplete/i); + expect(outputs.ui_incomplete).toMatch(/stop|block|cannot claim|not complete/i); + expect(outputs.ui_incomplete).not.toContain('https://example.test/pr/'); + expect(outputs.ui_incomplete).not.toMatch(/waiv/i); + + evalCollector?.addTest({ + name: 'ship/SKILL.md completion behavior', + suite: 'Ship & Release skill evals', + tier: 'llm-judge', + passed: true, + duration_ms: Date.now() - t0, + cost_usd: 0.08, + judge_reasoning: 'Generated non-UI, complete-UI, and blocked-UI handoffs satisfied the deterministic output contract.', + }); + }, 60_000); + testIfSelected('document-release/SKILL.md workflow', async () => { await runWorkflowJudge({ testName: 'document-release/SKILL.md workflow',