diff --git a/scrape/SKILL.md b/scrape/SKILL.md index dc965ec5f..c335da916 100644 --- a/scrape/SKILL.md +++ b/scrape/SKILL.md @@ -859,7 +859,11 @@ zero-arg skills) and run: $B skill run [--arg key=value ...] ``` -Emit the JSON the skill prints to stdout. Stop. +Emit the JSON the skill prints to stdout — but sanity-check it against the +intent first. A matched skill can return stale or empty data; if the result +is empty, malformed, or clearly does not cover what the intent asked for, +the skill is likely out of date, so fall through to the prototype path +instead of emitting it. Otherwise emit it and stop. If matching is ambiguous (two skills could plausibly fit), pick the narrower-tier one (project > global > bundled — `$B skill list` shows the @@ -879,11 +883,40 @@ No match. Drive the page using `$B` primitives: 4. `$B links` — when the intent is to gather URLs. 5. Iterate: try a selector, check the output, refine. -Emit the result as JSON on stdout (one document, not pretty-printed). -Use a stable shape — typically `{ "items": [...], "count": N }` or -similar — so downstream consumers can treat it as data. +Assemble the result as JSON (one document, not pretty-printed). Use a +stable shape — typically `{ "items": [...], "count": N }` or similar — so +downstream consumers can treat it as data. Do not emit it yet — it has to +clear the evidence gate first. -## Step 5 — Skillify nudge +## Step 5 — Verify the prototype result against the intent + +A plausible-looking JSON shape is not proof you got the data right. Before +emitting the prototype result as the answer, check it against the intent: + +1. **Decompose the intent into checkpoints.** Each datum, field, filter, + count, or ranking the intent named is one checkpoint. "Product names and + prices for items under $50, cheapest first" is four checkpoints: names + present, prices present, every price under $50, order ascending by price. +2. **Back each checkpoint with evidence.** Evidence is a populated field in + the result (not null, not an empty string, not a placeholder) or a + capture that shows it on the page — `$B snapshot --text` for a value, + `$B screenshot` for a visible state. A filter or ranking checkpoint is + satisfied when the returned data itself demonstrates it — every row + genuinely under $50, the array genuinely ordered by price — whether the + page applied it or you filtered and sorted the extracted rows yourself. + What fails is asserting an order or filter the data cannot back: a + "cheapest first" you never actually sorted, or a "most reviewed" with no + review-count field to sort on. +3. **Be harsh.** An empty `items: []` passes only if the page genuinely has + nothing matching, and you confirm that from a capture rather than + assuming it. Partial rows (half the records missing a price) or a count + that disagrees with an explicit quantity in the intent fail the gate. + +If every checkpoint is backed, emit the JSON and move to the nudge. If any +checkpoint cannot be backed, treat it as a prototype failure (see "When the +prototype fails") — never emit partial data as the answer. + +## Step 6 — Skillify nudge After a successful prototype, append exactly one line: @@ -915,7 +948,8 @@ after 3-4 selector attempts: ## Output discipline The match path returns whatever JSON the matched skill emits. The -prototype path returns whatever JSON you construct. In both cases: +prototype path returns the JSON you construct, once it clears the Step 5 +evidence gate. In both cases: - One JSON document, on stdout. - Stderr (or chat) is for logs and the skillify nudge. diff --git a/scrape/SKILL.md.tmpl b/scrape/SKILL.md.tmpl index 4cb4f17c0..7d25b11ba 100644 --- a/scrape/SKILL.md.tmpl +++ b/scrape/SKILL.md.tmpl @@ -85,7 +85,11 @@ zero-arg skills) and run: $B skill run [--arg key=value ...] ``` -Emit the JSON the skill prints to stdout. Stop. +Emit the JSON the skill prints to stdout — but sanity-check it against the +intent first. A matched skill can return stale or empty data; if the result +is empty, malformed, or clearly does not cover what the intent asked for, +the skill is likely out of date, so fall through to the prototype path +instead of emitting it. Otherwise emit it and stop. If matching is ambiguous (two skills could plausibly fit), pick the narrower-tier one (project > global > bundled — `$B skill list` shows the @@ -105,11 +109,40 @@ No match. Drive the page using `$B` primitives: 4. `$B links` — when the intent is to gather URLs. 5. Iterate: try a selector, check the output, refine. -Emit the result as JSON on stdout (one document, not pretty-printed). -Use a stable shape — typically `{ "items": [...], "count": N }` or -similar — so downstream consumers can treat it as data. +Assemble the result as JSON (one document, not pretty-printed). Use a +stable shape — typically `{ "items": [...], "count": N }` or similar — so +downstream consumers can treat it as data. Do not emit it yet — it has to +clear the evidence gate first. -## Step 5 — Skillify nudge +## Step 5 — Verify the prototype result against the intent + +A plausible-looking JSON shape is not proof you got the data right. Before +emitting the prototype result as the answer, check it against the intent: + +1. **Decompose the intent into checkpoints.** Each datum, field, filter, + count, or ranking the intent named is one checkpoint. "Product names and + prices for items under $50, cheapest first" is four checkpoints: names + present, prices present, every price under $50, order ascending by price. +2. **Back each checkpoint with evidence.** Evidence is a populated field in + the result (not null, not an empty string, not a placeholder) or a + capture that shows it on the page — `$B snapshot --text` for a value, + `$B screenshot` for a visible state. A filter or ranking checkpoint is + satisfied when the returned data itself demonstrates it — every row + genuinely under $50, the array genuinely ordered by price — whether the + page applied it or you filtered and sorted the extracted rows yourself. + What fails is asserting an order or filter the data cannot back: a + "cheapest first" you never actually sorted, or a "most reviewed" with no + review-count field to sort on. +3. **Be harsh.** An empty `items: []` passes only if the page genuinely has + nothing matching, and you confirm that from a capture rather than + assuming it. Partial rows (half the records missing a price) or a count + that disagrees with an explicit quantity in the intent fail the gate. + +If every checkpoint is backed, emit the JSON and move to the nudge. If any +checkpoint cannot be backed, treat it as a prototype failure (see "When the +prototype fails") — never emit partial data as the answer. + +## Step 6 — Skillify nudge After a successful prototype, append exactly one line: @@ -141,7 +174,8 @@ after 3-4 selector attempts: ## Output discipline The match path returns whatever JSON the matched skill emits. The -prototype path returns whatever JSON you construct. In both cases: +prototype path returns the JSON you construct, once it clears the Step 5 +evidence gate. In both cases: - One JSON document, on stdout. - Stderr (or chat) is for logs and the skillify nudge. diff --git a/test/helpers/touchfiles.ts b/test/helpers/touchfiles.ts index 33ba9ad97..b75682373 100644 --- a/test/helpers/touchfiles.ts +++ b/test/helpers/touchfiles.ts @@ -345,6 +345,9 @@ export const E2E_TOUCHFILES: Record = { 'scrape-prototype-path': [ 'scrape/**', 'browse/src/browser-skills.ts', 'browse/src/browser-skill-commands.ts', ], + 'scrape-evidence-gate': [ + 'scrape/**', 'browse/src/browser-skills.ts', 'browse/src/browser-skill-commands.ts', + ], 'skillify-happy-path': [ 'skillify/**', 'scrape/**', 'browse/src/browser-skill-write.ts', 'browse/src/browser-skills.ts', 'browse/src/browser-skill-commands.ts', @@ -720,6 +723,8 @@ export const E2E_TIERS: Record = { // Browser-skills Phase 2a — gate (D1/D3 contracts must not silently break) 'scrape-match-path': 'gate', 'scrape-prototype-path': 'gate', + // Evidence gate asserts on model judgment (unbackable checkpoint) — periodic. + 'scrape-evidence-gate': 'periodic', 'skillify-happy-path': 'gate', 'skillify-provenance-refusal': 'gate', 'skillify-approval-reject': 'gate', diff --git a/test/skill-e2e-skillify.test.ts b/test/skill-e2e-skillify.test.ts index d5a02bd35..deaf6119a 100644 --- a/test/skill-e2e-skillify.test.ts +++ b/test/skill-e2e-skillify.test.ts @@ -1,7 +1,7 @@ /** * Browser-skills Phase 2a — gate-tier E2E for /scrape and /skillify. * - * Five scenarios cover the productivity loop and the contracts locked + * Six scenarios cover the productivity loop and the contracts locked * during the v1.19.0.0 plan review: * * D1 — /skillify provenance guard (scenario 4) @@ -21,8 +21,12 @@ * /scrape refuses with the D1 message; nothing on disk. * 5. skillify-approval-reject — /scrape then /skillify but reject in * the approval gate; temp dir is removed, nothing at final path. + * 6. scrape-evidence-gate — /scrape against a fixture missing a + * requested field; the Step 5 evidence gate must stop the agent from + * emitting fabricated/partial data as a successful result. * - * All five run gate-tier (~$0.50–$1.50 each, ~$5 total per CI). + * Scenarios 1-5 run gate-tier (~$0.50–$1.50 each, ~$5 total per CI); + * scenario 6 runs periodic-tier (its assertion rests on model judgment). * Set EVALS=1 to enable. Set EVALS_MODEL to override (default sonnet-4-6). */ @@ -177,6 +181,7 @@ const PROTOTYPE_FIXTURE_HTML = ` describeIfSelected('Browser-skills Phase 2a E2E (/scrape + /skillify)', [ 'scrape-match-path', 'scrape-prototype-path', + 'scrape-evidence-gate', 'skillify-happy-path', 'skillify-provenance-refusal', 'skillify-approval-reject', @@ -459,4 +464,54 @@ Use HOME=${workDir}. Do NOT commit the skill.`, expect(stagingLeftovers.length).toBe(0); try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {} }, 420_000); + + // ── 6. /scrape evidence gate: unbackable checkpoint blocks success ──── + testConcurrentIfSelected('scrape-evidence-gate', async () => { + const { workDir, gstackHome } = setupSkillifyWorkdir('evidence', ['scrape']); + + // Fixture has titles + scores but NO price data. The intent asks for + // prices, so the "prices" checkpoint cannot be backed by the page — the + // Step 5 gate must stop the agent from emitting fabricated/null prices. + const fixturePath = path.join(workDir, 'fixture.html'); + fs.writeFileSync(fixturePath, PROTOTYPE_FIXTURE_HTML); + const fileUrl = `file://${fixturePath}`; + + const result = await runSkillTest({ + prompt: `Run /scrape product names and PRICES from ${fileUrl}. +Invoke /scrape via the Skill tool and follow its phases, including the +Step 5 evidence gate (decompose the intent into checkpoints and back each +with evidence from the page). Drive the page with $B, then report your +result. Do NOT use AskUserQuestion.`, + workingDirectory: workDir, + env: { GSTACK_HOME: gstackHome }, + maxTurns: 18, + allowedTools: ['Skill', 'Bash', 'Read'], + timeout: 180_000, + testName: 'scrape-evidence-gate', + runId, + }); + + logCost('scrape-evidence-gate', result); + + const cmds = bashCommands(result); + const wentToFixture = cmds.some(c => c.includes(fileUrl)); + const fetchedHtml = cmds.some(c => /\bgoto\b|\bhtml\b|\btext\b/.test(c)); + const surface = fullSurface(result); + // The gate must hold: the page has no prices, so the agent must surface + // that the price checkpoint is unbackable rather than emit fabricated or + // null prices as a finished result. Assert it acknowledged the gap. + const flaggedMissingPrice = /price/i.test(surface) + && /(no price|not (present|found|available)|isn'?t (present|available)|missing|unavailable|absent|couldn'?t find|could not find|cannot find|n\/a)/i.test(surface); + const exitOk = ['success', 'error_max_turns'].includes(result.exitReason); + + recordE2E(evalCollector, 'scrape evidence gate blocks unbackable checkpoint', 'Phase 2a E2E', result, { + passed: exitOk && wentToFixture && fetchedHtml && flaggedMissingPrice, + }); + + expect(exitOk).toBe(true); + expect(wentToFixture).toBe(true); + expect(fetchedHtml).toBe(true); + expect(flaggedMissingPrice).toBe(true); + try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {} + }, 240_000); });