mirror of https://github.com/garrytan/gstack.git
test(pty): detect markdown bold-bullet prose AUQs (fixes office-hours smoke)
office-hours auto-mode renders its mode question as `- **Building a startup**`
markdown bullets (office-hours/SKILL.md.tmpl:102) with no letter/number marker.
isProseAUQVisible only matched `A)`-style lettered or `1.`-style numbered
options, so the question went undetected: the model surfaced it at ~2m19s
(well under the 300s budget) but the harness kept scoring the run "working"
off the spinner glyphs and timed out — a false timeout on a question that was
already on screen.
Add Pattern 3: when an interrogative line ('?') is present AND 3+ bold-bullet
markers (`- **`) appear in the 4KB tail, classify as a prose AUQ. Bold is the
discriminator vs incidental prose bullets; the line anchor is dropped (stripAnsi
can collapse option lines) and the existing `❯ 1.` cursor gate still defers to a
live native list. Wires through the existing classifyVisible 'asked' path and the
timeout high-water-mark, so office-hours now classifies 'asked' instead of
'timeout'. Five unit cases: the office-hours render passes; no-'?', <3-bullet,
plain-bullet, and native-cursor cases stay false.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
6216c3e326
commit
f436aab0db
|
|
@ -504,6 +504,9 @@ ${tail}
|
|||
* for plan-eng / plan-design / plan-devex prose AUQ
|
||||
* - 3+ distinct numbered options (1. 2. 3.) at line starts WITHOUT a
|
||||
* `❯<spaces>1.` cursor — typical for autoplan / office-hours prose AUQ
|
||||
* - 3+ markdown bold-bullet options (`- **label**`) following an
|
||||
* interrogative line — office-hours renders its mode question this way
|
||||
* (`> - **Building a startup**`), which has no letter/number marker
|
||||
*
|
||||
* Used by classifyVisible and runPlanSkillFloorCheck to return outcome='asked'
|
||||
* (or auq_observed) instead of letting the harness time out when the model
|
||||
|
|
@ -547,7 +550,24 @@ export function isProseAUQVisible(visible: string): boolean {
|
|||
while ((nm = numberedRe.exec(tail)) !== null) {
|
||||
if (nm[1]) numberedHits.add(nm[1]);
|
||||
}
|
||||
return numberedHits.size >= 2;
|
||||
if (numberedHits.size >= 2) return true;
|
||||
|
||||
// Pattern 3: markdown bold-bullet option list. office-hours renders its
|
||||
// mode question as `> - **Building a startup**` lines under
|
||||
// --disallowedTools — no letter/number marker, so Patterns 1-2 miss it,
|
||||
// and the model keeps a spinner up so the Haiku judge scores it 'working'
|
||||
// and the run times out despite the question being on screen.
|
||||
// Require both: an interrogative line (the question stem ends in '?') AND
|
||||
// 3+ bold-bullet markers. The bold (`- **`) requirement is what separates
|
||||
// an option list from incidental prose bullets; the line anchor is dropped
|
||||
// because stripAnsi can collapse option lines (see Pattern 1 note), so we
|
||||
// count markers anywhere in the tail. The `❯ 1.` cursor gate above already
|
||||
// excludes a live native list.
|
||||
if (/\?/.test(tail)) {
|
||||
const boldBulletHits = (tail.match(/[-*•]\s+\*\*/g) || []).length;
|
||||
if (boldBulletHits >= 3) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -291,6 +291,64 @@ This refers to (see option B) above and also to point A) earlier.
|
|||
expect(isProseAUQVisible('Just some plain text output from the model.')).toBe(false);
|
||||
expect(isProseAUQVisible('')).toBe(false);
|
||||
});
|
||||
|
||||
// Pattern 3: markdown bold-bullet options — office-hours renders its mode
|
||||
// question this way under --disallowedTools, with no letter/number marker.
|
||||
test('matches office-hours markdown bold-bullet mode question (Pattern 3)', () => {
|
||||
const sample = `
|
||||
> Before we dig in — what's your goal with this?
|
||||
>
|
||||
> - **Building a startup** (or thinking about it)
|
||||
> - **Intrapreneurship** — internal project at a company, need to ship fast
|
||||
> - **Hackathon / demo** — time-boxed, need to impress
|
||||
> - **Open source / research** — building for a community
|
||||
> - **Learning** — teaching yourself to code
|
||||
❯
|
||||
`;
|
||||
expect(isProseAUQVisible(sample)).toBe(true);
|
||||
});
|
||||
|
||||
test('bold-bullets require a preceding interrogative — no "?" => false', () => {
|
||||
// 3+ bold bullets but no question stem: this is a feature list, not an AUQ.
|
||||
const sample = `
|
||||
Here is what shipped:
|
||||
- **Faster builds** via caching
|
||||
- **Smaller binaries** through tree-shaking
|
||||
- **Better errors** with source maps
|
||||
`;
|
||||
expect(isProseAUQVisible(sample)).toBe(false);
|
||||
});
|
||||
|
||||
test('a question with fewer than 3 bold bullets stays false (guard)', () => {
|
||||
const sample = `
|
||||
Which approach do you prefer?
|
||||
- **Option one** is simpler
|
||||
- **Option two** is faster
|
||||
`;
|
||||
expect(isProseAUQVisible(sample)).toBe(false);
|
||||
});
|
||||
|
||||
test('plain (non-bold) bullets after a question do not trigger Pattern 3', () => {
|
||||
// Only bold bullets count — plain "- text" prose lists are too common.
|
||||
const sample = `
|
||||
What should we do about this?
|
||||
- run the tests
|
||||
- ship the fix
|
||||
- file a follow-up
|
||||
`;
|
||||
expect(isProseAUQVisible(sample)).toBe(false);
|
||||
});
|
||||
|
||||
test('Pattern 3 still defers to a live native cursor list (❯ 1.)', () => {
|
||||
const sample = `
|
||||
> What's your goal?
|
||||
❯ 1. **Building a startup**
|
||||
2. **Intrapreneurship**
|
||||
3. **Hackathon**
|
||||
`;
|
||||
// The ❯1. cursor gate fires first — native list handling owns this.
|
||||
expect(isProseAUQVisible(sample)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('classifyVisible (runtime path through the runner classifier)', () => {
|
||||
|
|
|
|||
Loading…
Reference in New Issue