From 44cd3037f0dd5a1df0cf457d80b586df5a265361 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Tue, 11 Aug 2026 17:43:56 -0700 Subject: [PATCH] feat(pty-runner): scope-gate question/auto-select detectors + observation flags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two render-shape-anchored detectors (whitespace-squished, like the Pattern-4/5 collapsed-form handling): isScopeGateQuestionVisible requires the question text PLUS option-body text (native AskUserQuestion renders numbered options, prose fallback renders lettered — the option body appears in both; narration doesn't), and isScopeGateAutoSelectVisible requires the announcement prefix PLUS the selected-B token. runPlanSkillObservation gains scopeGateQuestionObserved / scopeGateAutoSelectObserved high-water flags (attached at every return path) so paid smokes can assert gate behavior across the whole run instead of the lossy 2KB evidence tail. runPlanSkillFloorCheck no longer counts a scope-gate render toward auqObserved (tail-scoped exclusion) — the floor measures FINDING-driven questions, and the gate could fire inside the 3s pre-target window. Unit fixtures pin clean/native/collapsed positives, narration negatives, and the verbatim template announcement string (template rewording fails here first, before the paid smokes degrade to vacuous asserts). Co-Authored-By: Claude Fable 5 --- test/helpers/claude-pty-runner.ts | 83 ++++++++++++++++++++- test/helpers/claude-pty-runner.unit.test.ts | 82 ++++++++++++++++++++ 2 files changed, 164 insertions(+), 1 deletion(-) diff --git a/test/helpers/claude-pty-runner.ts b/test/helpers/claude-pty-runner.ts index 2f507c66c..49e7fc8cf 100644 --- a/test/helpers/claude-pty-runner.ts +++ b/test/helpers/claude-pty-runner.ts @@ -618,6 +618,43 @@ export function isProseAUQVisible(visible: string): boolean { return false; } +// --------------------------------------------------------------------------- +// Scope-gate render detectors (plan-eng-review / plan-design-review) +// --------------------------------------------------------------------------- +// +// Both anchor on the RENDER SHAPE, not bare keywords, so model narration +// about the gate ("normally I'd ask what should I review…") stays false. +// Matching is whitespace-squished + lowercased because stripAnsi collapses +// TTY cursor-positioning escapes unpredictably (the same failure mode the +// Pattern-4/5 collapsed-form handling above exists for). + +/** + * True when the scope-gate QUESTION is actually rendered: the question text + * plus option A's body text. Option-body anchoring (not `A)`/`B)` markers) + * because native AskUserQuestion renders NUMBERED options in the TTY while + * the --disallowedTools prose fallback renders lettered ones — the option + * body appears in both renders; narration rarely quotes both the question + * and an option body. + */ +export function isScopeGateQuestionVisible(visible: string): boolean { + const squished = visible.replace(/\s+/g, '').toLowerCase(); + return squished.includes('whatshouldireview') && squished.includes('currentbranchdiff'); +} + +/** + * True when the plan-mode auto-select announcement is rendered: + * "Scope gate: plan mode — auto-selected B (reviewing )." + * Requires BOTH the announcement prefix and the selected-B token so + * narration ("in plan mode I'd auto-select B") stays false. + */ +export function isScopeGateAutoSelectVisible(visible: string): boolean { + const squished = visible.replace(/\s+/g, '').toLowerCase(); + return ( + squished.includes('scopegate:planmode') && + (squished.includes('auto-selectedb') || squished.includes('autoselectedb')) + ); +} + /** * Parse a rendered numbered-option list out of the visible TTY text. * @@ -1516,6 +1553,20 @@ export interface PlanSkillObservation { * Haiku judge fallback rather than the regex detector. */ waitingEverObserved?: boolean; + /** + * High-water-mark flag: did the scope-gate QUESTION ("What should I + * review?" plus option-body text) ever render during the run? Same + * lossy-2KB-evidence rationale as proseAUQEverObserved. The plan-mode + * smokes assert this stays false (gate bypassed via auto-select B); the + * no-op regression asserts it fires outside plan mode. + */ + scopeGateQuestionObserved?: boolean; + /** + * High-water-mark flag: did the plan-mode auto-select announcement + * ("Scope gate: plan mode — auto-selected B …") ever render? The + * plan-mode smokes assert true; the no-op regression asserts false. + */ + scopeGateAutoSelectObserved?: boolean; } /** @@ -1619,6 +1670,8 @@ export async function runPlanSkillObservation(opts: { // even if the current state is 'working'. let proseAUQEverObserved = false; let waitingEverObserved = false; + let scopeGateQuestionObserved = false; + let scopeGateAutoSelectObserved = false; const JUDGE_AFTER_MS = 60_000; const JUDGE_INTERVAL_MS = 30_000; while (Date.now() - start < budgetMs) { @@ -1631,6 +1684,8 @@ export async function runPlanSkillObservation(opts: { summary: `claude exited (code=${session.exitCode()}) before reaching a terminal outcome`, evidence: visible.slice(-2000), elapsedMs: Date.now() - startedAt, + scopeGateQuestionObserved, + scopeGateAutoSelectObserved, }; } if (visible.includes('Unknown command:')) { @@ -1639,6 +1694,8 @@ export async function runPlanSkillObservation(opts: { summary: `claude rejected /${opts.skillName} as unknown command (skill not registered in this cwd)`, evidence: visible.slice(-2000), elapsedMs: Date.now() - startedAt, + scopeGateQuestionObserved, + scopeGateAutoSelectObserved, }; } @@ -1652,6 +1709,15 @@ export async function runPlanSkillObservation(opts: { tag: 'prose-auq-surfaced', }); } + // Scope-gate render tracking (same high-water shape). Full-run + // detection matters because the 2KB evidence tail usually scrolls + // past the gate render before the outcome fires. + if (!scopeGateQuestionObserved && isScopeGateQuestionVisible(visible)) { + scopeGateQuestionObserved = true; + } + if (!scopeGateAutoSelectObserved && isScopeGateAutoSelectVisible(visible)) { + scopeGateAutoSelectObserved = true; + } const classified = classifyVisible(visible, { strictPlanWrites: !!opts.initialPlanContent, @@ -1663,6 +1729,8 @@ export async function runPlanSkillObservation(opts: { elapsedMs: Date.now() - startedAt, proseAUQEverObserved, waitingEverObserved, + scopeGateQuestionObserved, + scopeGateAutoSelectObserved, }; // Capture the plan file path on any outcome where one may have been // written. Gating only on 'plan_ready' missed two cases: (1) the @@ -1693,6 +1761,8 @@ export async function runPlanSkillObservation(opts: { summary: `LLM judge: ${lastJudgeVerdict.reasoning} (state=waiting after ${Math.round(elapsed / 1000)}s)`, evidence: visible.slice(-2000), elapsedMs: Date.now() - startedAt, + scopeGateQuestionObserved, + scopeGateAutoSelectObserved, }; } } @@ -1716,6 +1786,8 @@ export async function runPlanSkillObservation(opts: { elapsedMs: Date.now() - startedAt, proseAUQEverObserved, waitingEverObserved, + scopeGateQuestionObserved, + scopeGateAutoSelectObserved, }; } return { @@ -1729,6 +1801,8 @@ export async function runPlanSkillObservation(opts: { elapsedMs: Date.now() - startedAt, proseAUQEverObserved, waitingEverObserved, + scopeGateQuestionObserved, + scopeGateAutoSelectObserved, }; } finally { await session.close(); @@ -2129,10 +2203,17 @@ export async function runPlanSkillFloorCheck(opts: { // OR via prose-rendered options under --disallowedTools when no MCP // variant is callable (isProseAUQVisible). Both surface the question // to the user; the bug we're catching is "fired zero AUQs." + // + // Scope-gate renders do NOT count: the gate's "What should I review?" + // can fire inside the 3s pre-target window and would trivially satisfy + // the floor, but the floor measures FINDING-driven questions. The + // exclusion is TAIL-scoped so an early gate render that has scrolled + // out doesn't suppress a later, real finding AUQ. const tail = visible.slice(-TAIL_SCAN_BYTES); if ( (isNumberedOptionListVisible(visible) || isProseAUQVisible(visible)) && - !isPermissionDialogVisible(tail) + !isPermissionDialogVisible(tail) && + !isScopeGateQuestionVisible(tail) ) { return { auqObserved: true, diff --git a/test/helpers/claude-pty-runner.unit.test.ts b/test/helpers/claude-pty-runner.unit.test.ts index df4e5fad1..37f027938 100644 --- a/test/helpers/claude-pty-runner.unit.test.ts +++ b/test/helpers/claude-pty-runner.unit.test.ts @@ -28,6 +28,8 @@ import { isPermissionDialogVisible, isNumberedOptionListVisible, isProseAUQVisible, + isScopeGateQuestionVisible, + isScopeGateAutoSelectVisible, isPlanReadyVisible, parseNumberedOptions, classifyVisible, @@ -194,6 +196,86 @@ describe('isNumberedOptionListVisible', () => { }); }); +describe('scope-gate render detectors', () => { + // The verbatim announcement string from the plan-eng/plan-design SKILL.md + // templates. If the template rewording drifts, THIS fixture fails first — + // before the paid plan-mode smokes silently degrade to vacuous asserts. + const TEMPLATE_ANNOUNCEMENT = + 'Scope gate: plan mode — auto-selected B (reviewing ).'; + + describe('isScopeGateQuestionVisible', () => { + test('matches the clean prose gate render (question + option bodies)', () => { + const sample = ` +What should I review? +A) The current branch diff — the work in progress on this branch. +B) A plan or design doc I'll paste or point you to. +C) A specific file, directory, or path. +Recommendation: A when a branch diff exists, otherwise B. +`; + expect(isScopeGateQuestionVisible(sample)).toBe(true); + }); + + test('matches the native numbered render (no lettered markers)', () => { + const sample = ` + What should I review? + + ❯ 1. The current branch diff — the work in progress on this branch. + 2. A plan or design doc I'll paste or point you to. + 3. A specific file, directory, or path. +`; + expect(isScopeGateQuestionVisible(sample)).toBe(true); + }); + + test('matches the PTY-collapsed render (stripAnsi squished spaces)', () => { + const sample = 'WhatshouldIreview?A)Thecurrentbranchdiff—theworkinprogress'; + expect(isScopeGateQuestionVisible(sample)).toBe(true); + }); + + test('stays false on narration quoting only the question', () => { + const sample = + "Normally I'd ask 'What should I review?' but plan mode is active, so I'm proceeding."; + expect(isScopeGateQuestionVisible(sample)).toBe(false); + }); + + test('stays false on unrelated review prose', () => { + const sample = 'I will review the current branch diff and report findings.'; + expect(isScopeGateQuestionVisible(sample)).toBe(false); + }); + }); + + describe('isScopeGateAutoSelectVisible', () => { + test('matches the verbatim template announcement', () => { + expect(isScopeGateAutoSelectVisible(TEMPLATE_ANNOUNCEMENT)).toBe(true); + }); + + test('matches a real announcement with a concrete target', () => { + const sample = + 'Scope gate: plan mode — auto-selected B (reviewing ~/.claude/plans/my-feature.md). Running the Design Doc Check next.'; + expect(isScopeGateAutoSelectVisible(sample)).toBe(true); + }); + + test('matches the PTY-collapsed announcement', () => { + const sample = 'Scopegate:planmode—auto-selectedB(reviewingPLAN.md).'; + expect(isScopeGateAutoSelectVisible(sample)).toBe(true); + }); + + test('stays false on narration about the behavior', () => { + const sample = "In plan mode I'd auto-select B and review the active plan."; + expect(isScopeGateAutoSelectVisible(sample)).toBe(false); + }); + + test('stays false on AUTO_DECIDE preamble output', () => { + const sample = 'Auto-decided scope question → B (your preference). Change with /plan-tune.'; + expect(isScopeGateAutoSelectVisible(sample)).toBe(false); + }); + + test('stays false on a bare "selected B" without the announcement prefix', () => { + const sample = 'I selected B as the review target.'; + expect(isScopeGateAutoSelectVisible(sample)).toBe(false); + }); + }); +}); + describe('isProseAUQVisible', () => { test('matches 4 lettered options A) B) C) D) at line starts (plan-eng prose AUQ shape)', () => { const sample = `