mirror of https://github.com/garrytan/gstack.git
1290 lines
56 KiB
TypeScript
1290 lines
56 KiB
TypeScript
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
|
import { runSkillTest } from './helpers/session-runner';
|
|
import {
|
|
ROOT, browseBin, runId, evalsEnabled,
|
|
describeIfSelected, testConcurrentIfSelected,
|
|
copyDirSync, setupBrowseShims, logCost, recordE2E,
|
|
createEvalCollector, finalizeEvalCollector,
|
|
setupPlanEngReviewFixture, matchesUnnegated,
|
|
} from './helpers/e2e-helpers';
|
|
import { judgePosture } from './helpers/llm-judge';
|
|
import { spawnSync } from 'child_process';
|
|
import * as fs from 'fs';
|
|
import * as path from 'path';
|
|
import * as os from 'os';
|
|
|
|
const evalCollector = createEvalCollector('e2e-plan');
|
|
|
|
// --- Plan CEO Review E2E ---
|
|
|
|
describeIfSelected('Plan CEO Review E2E', ['plan-ceo-review'], () => {
|
|
let planDir: string;
|
|
|
|
beforeAll(() => {
|
|
planDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-plan-ceo-'));
|
|
const run = (cmd: string, args: string[]) =>
|
|
spawnSync(cmd, args, { cwd: planDir, stdio: 'pipe', timeout: 5000 });
|
|
|
|
// Init git repo (CEO review SKILL.md has a "System Audit" step that runs git)
|
|
run('git', ['init', '-b', 'main']);
|
|
run('git', ['config', 'user.email', 'test@test.com']);
|
|
run('git', ['config', 'user.name', 'Test']);
|
|
|
|
// Create a simple plan document for the agent to review
|
|
fs.writeFileSync(path.join(planDir, 'plan.md'), `# Plan: Add User Dashboard
|
|
|
|
## Context
|
|
We're building a new user dashboard that shows recent activity, notifications, and quick actions.
|
|
|
|
## Changes
|
|
1. New React component \`UserDashboard\` in \`src/components/\`
|
|
2. REST API endpoint \`GET /api/dashboard\` returning user stats
|
|
3. PostgreSQL query for activity aggregation
|
|
4. Redis cache layer for dashboard data (5min TTL)
|
|
|
|
## Architecture
|
|
- Frontend: React + TailwindCSS
|
|
- Backend: Express.js REST API
|
|
- Database: PostgreSQL with existing user/activity tables
|
|
- Cache: Redis for dashboard aggregates
|
|
|
|
## Open questions
|
|
- Should we use WebSocket for real-time updates?
|
|
- How do we handle users with 100k+ activity records?
|
|
`);
|
|
|
|
run('git', ['add', '.']);
|
|
run('git', ['commit', '-m', 'add plan']);
|
|
|
|
// Copy plan-ceo-review skill
|
|
fs.mkdirSync(path.join(planDir, 'plan-ceo-review'), { recursive: true });
|
|
fs.copyFileSync(
|
|
path.join(ROOT, 'plan-ceo-review', 'SKILL.md'),
|
|
path.join(planDir, 'plan-ceo-review', 'SKILL.md'),
|
|
);
|
|
// Carved skills (v2 plan T9): copy sections/ so the review workflow + report template are present.
|
|
{ const _sec = path.join(ROOT, 'plan-ceo-review', 'sections'); if (fs.existsSync(_sec)) fs.cpSync(_sec, path.join(planDir, 'plan-ceo-review', 'sections'), { recursive: true }); }
|
|
});
|
|
|
|
afterAll(() => {
|
|
try { fs.rmSync(planDir, { recursive: true, force: true }); } catch {}
|
|
});
|
|
|
|
testConcurrentIfSelected('plan-ceo-review', async () => {
|
|
const result = await runSkillTest({
|
|
prompt: `Read plan-ceo-review/SKILL.md for the review workflow.
|
|
|
|
Read plan.md — that's the plan to review. This is a standalone plan document, not a codebase — skip any codebase exploration or system audit steps.
|
|
|
|
Choose HOLD SCOPE mode. Skip any AskUserQuestion calls — this is non-interactive.
|
|
Write your complete review directly to ${planDir}/review-output.md
|
|
|
|
Focus on reviewing the plan content: architecture, error handling, security, and performance.`,
|
|
workingDirectory: planDir,
|
|
maxTurns: 15,
|
|
timeout: 360_000,
|
|
testName: 'plan-ceo-review',
|
|
runId,
|
|
model: 'claude-opus-4-7',
|
|
});
|
|
|
|
logCost('/plan-ceo-review', result);
|
|
recordE2E(evalCollector, '/plan-ceo-review', 'Plan CEO Review E2E', result, {
|
|
passed: ['success', 'error_max_turns'].includes(result.exitReason),
|
|
});
|
|
// Accept error_max_turns — the CEO review is very thorough and may exceed turns
|
|
expect(['success', 'error_max_turns']).toContain(result.exitReason);
|
|
|
|
// Verify the review was written
|
|
const reviewPath = path.join(planDir, 'review-output.md');
|
|
if (fs.existsSync(reviewPath)) {
|
|
const review = fs.readFileSync(reviewPath, 'utf-8');
|
|
expect(review.length).toBeGreaterThan(200);
|
|
}
|
|
}, 420_000);
|
|
});
|
|
|
|
// --- Plan CEO Review (SELECTIVE EXPANSION) E2E ---
|
|
|
|
describeIfSelected('Plan CEO Review SELECTIVE EXPANSION E2E', ['plan-ceo-review-selective'], () => {
|
|
let planDir: string;
|
|
|
|
beforeAll(() => {
|
|
planDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-plan-ceo-sel-'));
|
|
const run = (cmd: string, args: string[]) =>
|
|
spawnSync(cmd, args, { cwd: planDir, stdio: 'pipe', timeout: 5000 });
|
|
|
|
run('git', ['init', '-b', 'main']);
|
|
run('git', ['config', 'user.email', 'test@test.com']);
|
|
run('git', ['config', 'user.name', 'Test']);
|
|
|
|
fs.writeFileSync(path.join(planDir, 'plan.md'), `# Plan: Add User Dashboard
|
|
|
|
## Context
|
|
We're building a new user dashboard that shows recent activity, notifications, and quick actions.
|
|
|
|
## Changes
|
|
1. New React component \`UserDashboard\` in \`src/components/\`
|
|
2. REST API endpoint \`GET /api/dashboard\` returning user stats
|
|
3. PostgreSQL query for activity aggregation
|
|
4. Redis cache layer for dashboard data (5min TTL)
|
|
|
|
## Architecture
|
|
- Frontend: React + TailwindCSS
|
|
- Backend: Express.js REST API
|
|
- Database: PostgreSQL with existing user/activity tables
|
|
- Cache: Redis for dashboard aggregates
|
|
|
|
## Open questions
|
|
- Should we use WebSocket for real-time updates?
|
|
- How do we handle users with 100k+ activity records?
|
|
`);
|
|
|
|
run('git', ['add', '.']);
|
|
run('git', ['commit', '-m', 'add plan']);
|
|
|
|
fs.mkdirSync(path.join(planDir, 'plan-ceo-review'), { recursive: true });
|
|
fs.copyFileSync(
|
|
path.join(ROOT, 'plan-ceo-review', 'SKILL.md'),
|
|
path.join(planDir, 'plan-ceo-review', 'SKILL.md'),
|
|
);
|
|
// Carved skills (v2 plan T9): copy sections/ so the review workflow + report template are present.
|
|
{ const _sec = path.join(ROOT, 'plan-ceo-review', 'sections'); if (fs.existsSync(_sec)) fs.cpSync(_sec, path.join(planDir, 'plan-ceo-review', 'sections'), { recursive: true }); }
|
|
});
|
|
|
|
afterAll(() => {
|
|
try { fs.rmSync(planDir, { recursive: true, force: true }); } catch {}
|
|
});
|
|
|
|
testConcurrentIfSelected('plan-ceo-review-selective', async () => {
|
|
const result = await runSkillTest({
|
|
prompt: `Read plan-ceo-review/SKILL.md for the review workflow.
|
|
|
|
Read plan.md — that's the plan to review. This is a standalone plan document, not a codebase — skip any codebase exploration or system audit steps.
|
|
|
|
Choose SELECTIVE EXPANSION mode. Skip any AskUserQuestion calls — this is non-interactive.
|
|
For the cherry-pick ceremony, accept all expansion proposals automatically.
|
|
Write your complete review directly to ${planDir}/review-output-selective.md
|
|
|
|
Focus on reviewing the plan content: architecture, error handling, security, and performance.`,
|
|
workingDirectory: planDir,
|
|
maxTurns: 15,
|
|
timeout: 360_000,
|
|
testName: 'plan-ceo-review-selective',
|
|
runId,
|
|
model: 'claude-opus-4-7',
|
|
});
|
|
|
|
logCost('/plan-ceo-review (SELECTIVE)', result);
|
|
recordE2E(evalCollector, '/plan-ceo-review-selective', 'Plan CEO Review SELECTIVE EXPANSION E2E', result, {
|
|
passed: ['success', 'error_max_turns'].includes(result.exitReason),
|
|
});
|
|
expect(['success', 'error_max_turns']).toContain(result.exitReason);
|
|
|
|
const reviewPath = path.join(planDir, 'review-output-selective.md');
|
|
if (fs.existsSync(reviewPath)) {
|
|
const review = fs.readFileSync(reviewPath, 'utf-8');
|
|
expect(review.length).toBeGreaterThan(200);
|
|
}
|
|
}, 420_000);
|
|
});
|
|
|
|
// --- Plan CEO Review SCOPE EXPANSION energy (V1.1 mode-posture regression gate) ---
|
|
|
|
describeIfSelected('Plan CEO Review Expansion Energy E2E', ['plan-ceo-review-expansion-energy'], () => {
|
|
let planDir: string;
|
|
|
|
beforeAll(() => {
|
|
planDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-plan-ceo-exp-'));
|
|
const run = (cmd: string, args: string[]) =>
|
|
spawnSync(cmd, args, { cwd: planDir, stdio: 'pipe', timeout: 5000 });
|
|
|
|
run('git', ['init', '-b', 'main']);
|
|
run('git', ['config', 'user.email', 'test@test.com']);
|
|
run('git', ['config', 'user.name', 'Test']);
|
|
|
|
// Use the shared fixture so expansion-energy regressions are reproducible.
|
|
const fixture = fs.readFileSync(
|
|
path.join(ROOT, 'test', 'fixtures', 'mode-posture', 'expansion-plan.md'),
|
|
'utf-8',
|
|
);
|
|
fs.writeFileSync(path.join(planDir, 'plan.md'), fixture);
|
|
|
|
run('git', ['add', '.']);
|
|
run('git', ['commit', '-m', 'add plan']);
|
|
|
|
fs.mkdirSync(path.join(planDir, 'plan-ceo-review'), { recursive: true });
|
|
fs.copyFileSync(
|
|
path.join(ROOT, 'plan-ceo-review', 'SKILL.md'),
|
|
path.join(planDir, 'plan-ceo-review', 'SKILL.md'),
|
|
);
|
|
// Carved skills (v2 plan T9): copy sections/ so the review workflow + report template are present.
|
|
{ const _sec = path.join(ROOT, 'plan-ceo-review', 'sections'); if (fs.existsSync(_sec)) fs.cpSync(_sec, path.join(planDir, 'plan-ceo-review', 'sections'), { recursive: true }); }
|
|
});
|
|
|
|
afterAll(() => {
|
|
try { fs.rmSync(planDir, { recursive: true, force: true }); } catch {}
|
|
});
|
|
|
|
testConcurrentIfSelected('plan-ceo-review-expansion-energy', async () => {
|
|
const result = await runSkillTest({
|
|
prompt: `Read plan-ceo-review/SKILL.md for the review workflow.
|
|
|
|
Read plan.md — that's the plan to review. This is a standalone plan document, not a codebase — skip any codebase exploration or system audit steps.
|
|
|
|
Choose SCOPE EXPANSION mode. Skip any AskUserQuestion calls — this is non-interactive. Auto-approve the ideal-architecture approach in 0C-bis. For 0D, run all three analyses (10x check, platonic ideal, delight opportunities), then emit exactly 2 concrete expansion proposals in the opt-in ceremony.
|
|
|
|
Write your expansion proposals to ${planDir}/proposals.md with ONLY the proposal text — no conversational wrapper, no review summary, no mode analysis. Each proposal separated by "---".`,
|
|
workingDirectory: planDir,
|
|
maxTurns: 15,
|
|
timeout: 360_000,
|
|
testName: 'plan-ceo-review-expansion-energy',
|
|
runId,
|
|
model: 'claude-opus-4-7',
|
|
});
|
|
|
|
logCost('/plan-ceo-review (EXPANSION ENERGY)', result);
|
|
recordE2E(evalCollector, '/plan-ceo-review-expansion-energy', 'Plan CEO Review Expansion Energy E2E', result, {
|
|
passed: ['success', 'error_max_turns'].includes(result.exitReason),
|
|
});
|
|
// Transient API failure escape hatch — see /plan-review-report for the
|
|
// full rationale. Same shape: error_api with 0 turns means the API call
|
|
// never reached the model, so nothing the test verifies could have run.
|
|
if (result.exitReason === 'error_api' && result.costEstimate?.turnsUsed === 0) {
|
|
console.warn('[transient] /plan-ceo-review-expansion-energy: error_api with 0 turns — treating as inconclusive');
|
|
return;
|
|
}
|
|
expect(['success', 'error_max_turns']).toContain(result.exitReason);
|
|
|
|
const proposalsPath = path.join(planDir, 'proposals.md');
|
|
if (!fs.existsSync(proposalsPath)) {
|
|
throw new Error('Agent did not emit proposals.md — expansion energy eval requires proposal output');
|
|
}
|
|
const proposalText = fs.readFileSync(proposalsPath, 'utf-8');
|
|
expect(proposalText.length).toBeGreaterThan(200);
|
|
|
|
const scores = await judgePosture('expansion', proposalText);
|
|
console.log('Expansion energy scores:', JSON.stringify(scores, null, 2));
|
|
// Pass threshold: 4/5 on both axes (good — matches posture with minor weakness).
|
|
expect(scores.axis_a).toBeGreaterThanOrEqual(4); // surface_framing
|
|
expect(scores.axis_b).toBeGreaterThanOrEqual(4); // decision_preservation
|
|
}, 600_000);
|
|
});
|
|
|
|
// --- Plan Eng Review E2E ---
|
|
|
|
describeIfSelected('Plan Eng Review E2E', ['plan-eng-review'], () => {
|
|
let planDir: string;
|
|
|
|
beforeAll(() => {
|
|
planDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-plan-eng-'));
|
|
const run = (cmd: string, args: string[]) =>
|
|
spawnSync(cmd, args, { cwd: planDir, stdio: 'pipe', timeout: 5000 });
|
|
|
|
run('git', ['init', '-b', 'main']);
|
|
run('git', ['config', 'user.email', 'test@test.com']);
|
|
run('git', ['config', 'user.name', 'Test']);
|
|
|
|
// Create a plan with more engineering detail
|
|
fs.writeFileSync(path.join(planDir, 'plan.md'), `# Plan: Migrate Auth to JWT
|
|
|
|
## Context
|
|
Replace session-cookie auth with JWT tokens. Currently using express-session + Redis store.
|
|
|
|
## Changes
|
|
1. Add \`jsonwebtoken\` package
|
|
2. New middleware \`auth/jwt-verify.ts\` replacing \`auth/session-check.ts\`
|
|
3. Login endpoint returns { accessToken, refreshToken }
|
|
4. Refresh endpoint rotates tokens
|
|
5. Migration script to invalidate existing sessions
|
|
|
|
## Files Modified
|
|
| File | Change |
|
|
|------|--------|
|
|
| auth/jwt-verify.ts | NEW: JWT verification middleware |
|
|
| auth/session-check.ts | DELETED |
|
|
| routes/login.ts | Return JWT instead of setting cookie |
|
|
| routes/refresh.ts | NEW: Token refresh endpoint |
|
|
| middleware/index.ts | Swap session-check for jwt-verify |
|
|
|
|
## Error handling
|
|
- Expired token: 401 with \`token_expired\` code
|
|
- Invalid token: 401 with \`invalid_token\` code
|
|
- Refresh with revoked token: 403
|
|
|
|
## Not in scope
|
|
- OAuth/OIDC integration
|
|
- Rate limiting on refresh endpoint
|
|
`);
|
|
|
|
run('git', ['add', '.']);
|
|
run('git', ['commit', '-m', 'add plan']);
|
|
|
|
// Copy plan-eng-review skill
|
|
fs.mkdirSync(path.join(planDir, 'plan-eng-review'), { recursive: true });
|
|
fs.copyFileSync(
|
|
path.join(ROOT, 'plan-eng-review', 'SKILL.md'),
|
|
path.join(planDir, 'plan-eng-review', 'SKILL.md'),
|
|
);
|
|
// Carved skills (v2 plan T9): copy sections/ so the review workflow + report template are present.
|
|
{ const _sec = path.join(ROOT, 'plan-eng-review', 'sections'); if (fs.existsSync(_sec)) fs.cpSync(_sec, path.join(planDir, 'plan-eng-review', 'sections'), { recursive: true }); }
|
|
});
|
|
|
|
afterAll(() => {
|
|
try { fs.rmSync(planDir, { recursive: true, force: true }); } catch {}
|
|
});
|
|
|
|
testConcurrentIfSelected('plan-eng-review', async () => {
|
|
const result = await runSkillTest({
|
|
prompt: `Read plan-eng-review/SKILL.md for the review workflow.
|
|
|
|
Read plan.md — that's the plan to review. This is a standalone plan document, not a codebase — skip any codebase exploration steps.
|
|
|
|
Proceed directly to the full review. Skip any AskUserQuestion calls — this is non-interactive.
|
|
Write your complete review directly to ${planDir}/review-output.md
|
|
|
|
Focus on architecture, code quality, tests, and performance sections.`,
|
|
workingDirectory: planDir,
|
|
maxTurns: 15,
|
|
timeout: 360_000,
|
|
testName: 'plan-eng-review',
|
|
runId,
|
|
model: 'claude-opus-4-7',
|
|
});
|
|
|
|
logCost('/plan-eng-review', result);
|
|
recordE2E(evalCollector, '/plan-eng-review', 'Plan Eng Review E2E', result, {
|
|
passed: ['success', 'error_max_turns'].includes(result.exitReason),
|
|
});
|
|
expect(['success', 'error_max_turns']).toContain(result.exitReason);
|
|
|
|
// Verify the review was written
|
|
const reviewPath = path.join(planDir, 'review-output.md');
|
|
if (fs.existsSync(reviewPath)) {
|
|
const review = fs.readFileSync(reviewPath, 'utf-8');
|
|
expect(review.length).toBeGreaterThan(200);
|
|
}
|
|
}, 420_000);
|
|
});
|
|
|
|
// --- Plan Eng Review Data-Model Bias Regression E2E ---
|
|
//
|
|
// Regression test for the data-model bias fix: verifies that /plan-eng-review
|
|
// recommends a separate model instead of inlining columns + JSONField when
|
|
// a plan tries to merge polymorphic variant data into an existing model "to
|
|
// keep the diff right-sized." Before the fix, the skill's "right-sized diff"
|
|
// preference pushed the AI toward accepting the inline approach; after the
|
|
// fix, the data-model exception bullet + Data model review checklist should
|
|
// make the AI push back and recommend normalization citing SRP/3NF.
|
|
|
|
describeIfSelected('Plan Eng Review Data-Model Bias E2E', ['plan-eng-review-data-model-bias'], () => {
|
|
let planDir: string;
|
|
|
|
beforeAll(() => {
|
|
// Synthetic plan designed to trip the old bias: four clearly-polymorphic
|
|
// tier variants + feature-flag bag that the user proposes to inline onto
|
|
// User rather than create a new model. After the fix, the skill should
|
|
// recommend a separate SubscriptionTier model and push back on the
|
|
// JSONField for tier_features.
|
|
planDir = setupPlanEngReviewFixture('skill-e2e-plan-eng-dmb-', `# Plan: Add Subscription Tiers to User Model
|
|
|
|
## Context
|
|
I want to add a 'subscription tier' feature to my Django app. Each user can
|
|
be on a free, basic, premium, or enterprise tier. Each tier has:
|
|
- a monthly price
|
|
- a feature set (list of feature flags)
|
|
- a max-users limit
|
|
- an SLA tier (none / standard / premium)
|
|
|
|
## Current state
|
|
The User model has ~15 fields (email, name, created_at, etc).
|
|
|
|
## Proposed changes
|
|
I'm thinking of just adding these columns directly to the User model so I
|
|
don't have to deal with another table:
|
|
|
|
- tier_name: CharField with choices=['free', 'basic', 'premium', 'enterprise']
|
|
- tier_price: DecimalField
|
|
- tier_features: JSONField (list of feature flag strings, varies per tier)
|
|
- tier_max_users: IntegerField
|
|
- tier_sla: CharField with choices=['none', 'standard', 'premium']
|
|
|
|
This keeps the diff minimal — no new model, no new migration beyond the one
|
|
column-add, no new FK navigation in the codebase.
|
|
|
|
## Open questions
|
|
- Should feature flags be a separate table or stay as JSONField?
|
|
- Any concerns with this approach?
|
|
`);
|
|
});
|
|
|
|
afterAll(() => {
|
|
try { fs.rmSync(planDir, { recursive: true, force: true }); } catch {}
|
|
});
|
|
|
|
testConcurrentIfSelected('plan-eng-review-data-model-bias', async () => {
|
|
const result = await runSkillTest({
|
|
prompt: `Read plan-eng-review/SKILL.md for the review workflow.
|
|
|
|
Read plan.md — that's the plan to review. This is a standalone plan document, not a codebase — skip any codebase exploration steps.
|
|
|
|
Proceed directly to the architecture review section. Skip any AskUserQuestion calls — this is non-interactive. Write your complete architecture review directly to ${planDir}/review-output.md
|
|
|
|
Focus specifically on the data model design in the plan. Apply the data model review checklist from the skill.`,
|
|
workingDirectory: planDir,
|
|
maxTurns: 15,
|
|
timeout: 360_000,
|
|
testName: 'plan-eng-review-data-model-bias',
|
|
runId,
|
|
model: 'claude-opus-4-6',
|
|
});
|
|
|
|
logCost('/plan-eng-review data-model-bias', result);
|
|
|
|
// Verify the review was written
|
|
const reviewPath = path.join(planDir, 'review-output.md');
|
|
const review = fs.existsSync(reviewPath)
|
|
? fs.readFileSync(reviewPath, 'utf-8')
|
|
: '';
|
|
|
|
// Behavioral assertions: the review should recommend a separate tier model
|
|
// and push back on JSONField for the feature set. Matching is deliberately
|
|
// loose (case-insensitive substrings) to tolerate wording variance while
|
|
// still catching the bias regression. Strip markdown formatting chars
|
|
// (backticks, asterisks) first — LLMs commonly wrap identifiers like
|
|
// `SubscriptionTier` in backticks, which would otherwise break the \s*
|
|
// assumptions below.
|
|
const lower = review.toLowerCase();
|
|
const unformatted = review.replace(/[`*_]/g, '');
|
|
// matchesUnnegated so a (would-be-regression) "I would NOT recommend a
|
|
// separate tier model here" answer doesn't get counted as a pass just
|
|
// for containing the words — consistent with the sibling counterexample
|
|
// tests below.
|
|
const recommendsSeparateModel =
|
|
matchesUnnegated(unformatted, /separate\s+(subscription\s*)?tier\s*model/i) ||
|
|
matchesUnnegated(unformatted, /new\s+(subscription\s*)?tier\s*model/i) ||
|
|
matchesUnnegated(unformatted, /subscriptiontier\s*model/i) ||
|
|
matchesUnnegated(unformatted, /extract[\s\S]*tier[\s\S]*model/i) ||
|
|
matchesUnnegated(unformatted, /tier\s*(?:table|model)\s*(?:with|per)/i);
|
|
const pushesBackOnJsonField =
|
|
lower.includes('jsonfield') &&
|
|
(lower.includes('feature') || lower.includes('polymorph') || lower.includes('explicit'));
|
|
const citesNormalization =
|
|
/normali[sz]/i.test(review) ||
|
|
/\bsrp\b/i.test(review) ||
|
|
/single\s+responsibility/i.test(review) ||
|
|
/\b3nf\b/i.test(review) ||
|
|
/normal\s+form/i.test(review);
|
|
|
|
recordE2E(evalCollector, '/plan-eng-review-data-model-bias', 'Plan Eng Review Data-Model Bias E2E', result, {
|
|
passed:
|
|
['success', 'error_max_turns'].includes(result.exitReason) &&
|
|
review.length > 200 &&
|
|
recommendsSeparateModel &&
|
|
pushesBackOnJsonField &&
|
|
citesNormalization,
|
|
});
|
|
|
|
expect(['success', 'error_max_turns']).toContain(result.exitReason);
|
|
expect(review.length).toBeGreaterThan(200);
|
|
expect(recommendsSeparateModel).toBe(true);
|
|
expect(pushesBackOnJsonField).toBe(true);
|
|
expect(citesNormalization).toBe(true);
|
|
}, 420_000);
|
|
});
|
|
|
|
// --- Plan Eng Review Data-Model Legitimate-JSON Counterexample E2E ---
|
|
//
|
|
// Positive-case regression: the data-model bias fix narrowed "JSONField is not
|
|
// an escape hatch for polymorphism" to explicitly exempt genuinely legitimate
|
|
// uses (third-party payload caches, opaque preference bags, schemas still
|
|
// being discovered). This verifies the narrowing actually works — that
|
|
// /plan-eng-review does NOT push back on a JSONField that squarely matches
|
|
// one of its own listed legitimate cases (caching a third-party webhook
|
|
// payload verbatim, schema controlled entirely by the external producer).
|
|
|
|
describeIfSelected('Plan Eng Review Data-Model Legitimate JSON E2E', ['plan-eng-review-data-model-legitimate-json'], () => {
|
|
let planDir: string;
|
|
|
|
beforeAll(() => {
|
|
// Synthetic plan designed to NOT trip the JSONField guidance: the payload
|
|
// is a verbatim third-party webhook body, schema owned by Stripe (not us),
|
|
// never queried into by key — the textbook legitimate JSONField case.
|
|
planDir = setupPlanEngReviewFixture('skill-e2e-plan-eng-json-', `# Plan: Add Stripe Webhook Event Log
|
|
|
|
## Context
|
|
We need to store incoming Stripe webhook events for audit and replay. Stripe
|
|
sends many event types (payment_intent.succeeded, charge.refunded,
|
|
customer.subscription.updated, etc.), each with a different JSON body shape
|
|
that Stripe controls and can change without notice.
|
|
|
|
## Proposed changes
|
|
Add a WebhookEvent model:
|
|
- id: primary key
|
|
- event_type: CharField (e.g. "payment_intent.succeeded")
|
|
- stripe_event_id: CharField, unique
|
|
- received_at: DateTimeField
|
|
- payload: JSONField — the verbatim JSON body Stripe sent, unmodified
|
|
|
|
We will never query into specific payload keys from application code; the
|
|
only consumer is a manual replay tool that re-POSTs the raw payload to our
|
|
webhook handler for reprocessing. The field exists purely to cache Stripe's
|
|
response verbatim for audit/replay, not to model our own domain state.
|
|
|
|
## Open questions
|
|
- Is storing the whole event as JSONField the right call here, or should we
|
|
break out the commonly-used fields into columns?
|
|
`);
|
|
});
|
|
|
|
afterAll(() => {
|
|
try { fs.rmSync(planDir, { recursive: true, force: true }); } catch {}
|
|
});
|
|
|
|
testConcurrentIfSelected('plan-eng-review-data-model-legitimate-json', async () => {
|
|
const result = await runSkillTest({
|
|
prompt: `Read plan-eng-review/SKILL.md for the review workflow.
|
|
|
|
Read plan.md — that's the plan to review. This is a standalone plan document, not a codebase — skip any codebase exploration steps.
|
|
|
|
Proceed directly to the architecture review section. Skip any AskUserQuestion calls — this is non-interactive. Write your complete architecture review directly to ${planDir}/review-output.md
|
|
|
|
Focus specifically on the data model design in the plan. Apply the data model review checklist from the skill.`,
|
|
workingDirectory: planDir,
|
|
maxTurns: 15,
|
|
timeout: 360_000,
|
|
testName: 'plan-eng-review-data-model-legitimate-json',
|
|
runId,
|
|
model: 'claude-opus-4-6',
|
|
});
|
|
|
|
logCost('/plan-eng-review data-model-legitimate-json', result);
|
|
|
|
const reviewPath = path.join(planDir, 'review-output.md');
|
|
const review = fs.existsSync(reviewPath)
|
|
? fs.readFileSync(reviewPath, 'utf-8')
|
|
: '';
|
|
const lower = review.toLowerCase();
|
|
|
|
// Deliberately loose, case-insensitive matching (same tolerance as the
|
|
// bias-regression test above) to survive wording variance while still
|
|
// catching an over-eager JSONField warning. matchesUnnegated() ignores
|
|
// matches preceded by a negation word — otherwise a CORRECT "I would NOT
|
|
// extract the payload into columns" answer would trip this check, since
|
|
// it contains "extract...payload...column" just like an incorrect one.
|
|
// Includes normali[sz]e alongside promote/split/convert/extract/move since
|
|
// that's the exact vocabulary the companion template edits use.
|
|
const discussesThePayloadField = lower.includes('payload') || lower.includes('jsonfield');
|
|
const pushesToSplitTheField =
|
|
matchesUnnegated(review, /(promote|split|convert|extract|move|normali[sz]e)[^.]{0,80}payload[^.]{0,80}(column|field)/i) ||
|
|
matchesUnnegated(review, /payload[^.]{0,80}(promote|split|convert to (explicit )?column|explicit column|normali[sz]e)/i);
|
|
const acknowledgesLegitimateUse =
|
|
/legitimate|appropriate|reasonable|correct (choice|call|approach)|fine as[- ](is|it)|acceptable|(third[- ]party|external).{0,40}(payload|blob|response)|verbatim|schemaless/i.test(review);
|
|
|
|
recordE2E(evalCollector, '/plan-eng-review-data-model-legitimate-json', 'Plan Eng Review Data-Model Legitimate JSON E2E', result, {
|
|
passed:
|
|
['success', 'error_max_turns'].includes(result.exitReason) &&
|
|
review.length > 200 &&
|
|
discussesThePayloadField &&
|
|
!pushesToSplitTheField &&
|
|
acknowledgesLegitimateUse,
|
|
});
|
|
|
|
expect(['success', 'error_max_turns']).toContain(result.exitReason);
|
|
expect(review.length).toBeGreaterThan(200);
|
|
expect(discussesThePayloadField).toBe(true);
|
|
expect(pushesToSplitTheField).toBe(false);
|
|
expect(acknowledgesLegitimateUse).toBe(true);
|
|
}, 420_000);
|
|
});
|
|
|
|
// --- Plan Eng Review Data-Model Measured-Denormalization Counterexample E2E ---
|
|
//
|
|
// Positive-case regression: the data-model bias fix narrowed "normalize
|
|
// first" to explicitly exempt denormalization backed by a stated
|
|
// measurement (profiled hot path, load test, documented query cost). This
|
|
// verifies /plan-eng-review accepts a denormalized snapshot field when the
|
|
// plan cites a concrete measurement, instead of reflexively recommending a
|
|
// return to a fully-normalized live-join design.
|
|
|
|
describeIfSelected('Plan Eng Review Data-Model Measured Denormalization E2E', ['plan-eng-review-data-model-measured-denorm'], () => {
|
|
let planDir: string;
|
|
|
|
beforeAll(() => {
|
|
// Synthetic plan designed to NOT trip the normalize-first guidance: the
|
|
// denormalization is scoped to one read path and backed by a stated
|
|
// measurement (APM p95 + load test), matching the fix's own counterexample.
|
|
planDir = setupPlanEngReviewFixture('skill-e2e-plan-eng-denorm-', `# Plan: Add OrderSummary Read Snapshot for Checkout Confirmation
|
|
|
|
## Context
|
|
The checkout confirmation page currently joins Order -> Customer ->
|
|
ShippingAddress (3 tables) to render the confirmation. APM (New Relic) shows
|
|
this read path at p95 180ms under current traffic; a load test at 2x
|
|
projected peak traffic showed p95 climbing to 340ms, driven almost entirely
|
|
by this join according to the query plan.
|
|
|
|
## Proposed changes
|
|
Add an OrderSummary model that snapshots customer_name and
|
|
shipping_address_line onto the Order row at order-creation time, used ONLY
|
|
by the checkout confirmation read path. Order, Customer, and
|
|
ShippingAddress remain fully normalized everywhere else in the codebase —
|
|
this is a single, measured, scoped denormalization for one hot read path,
|
|
not a general schema change.
|
|
|
|
## Open questions
|
|
- Is this the right call, or should we keep reading live via the FK chain
|
|
and instead add caching/indexing to fix the latency?
|
|
`);
|
|
});
|
|
|
|
afterAll(() => {
|
|
try { fs.rmSync(planDir, { recursive: true, force: true }); } catch {}
|
|
});
|
|
|
|
testConcurrentIfSelected('plan-eng-review-data-model-measured-denorm', async () => {
|
|
const result = await runSkillTest({
|
|
prompt: `Read plan-eng-review/SKILL.md for the review workflow.
|
|
|
|
Read plan.md — that's the plan to review. This is a standalone plan document, not a codebase — skip any codebase exploration steps.
|
|
|
|
Proceed directly to the architecture review section. Skip any AskUserQuestion calls — this is non-interactive. Write your complete architecture review directly to ${planDir}/review-output.md
|
|
|
|
Focus specifically on the data model design in the plan. Apply the data model review checklist from the skill.`,
|
|
workingDirectory: planDir,
|
|
maxTurns: 15,
|
|
timeout: 360_000,
|
|
testName: 'plan-eng-review-data-model-measured-denorm',
|
|
runId,
|
|
model: 'claude-opus-4-6',
|
|
});
|
|
|
|
logCost('/plan-eng-review data-model-measured-denorm', result);
|
|
|
|
const reviewPath = path.join(planDir, 'review-output.md');
|
|
const review = fs.existsSync(reviewPath)
|
|
? fs.readFileSync(reviewPath, 'utf-8')
|
|
: '';
|
|
const lower = review.toLowerCase();
|
|
|
|
const discussesTheDenormalization = lower.includes('denormal') || lower.includes('ordersummary');
|
|
// Broadened per review feedback: the original pattern missed common
|
|
// rejection phrasings ("instead of denormalizing", "cache/index instead")
|
|
// and its "normali[sz]e...instead|back|it" clause was loose enough to
|
|
// false-positive on approving language ("stays normalized elsewhere;
|
|
// keep it that way") — tightened to require an explicit join/read/query
|
|
// + instead framing.
|
|
const rejectsAsUnjustified =
|
|
/(premature|not[- ]yet justified|lacks?[- ]a measurement|no measurement|remove[^.]{0,40}(denormali[sz]ation|snapshot)|normali[sz]e[^.]{0,20}(this|the)?[^.]{0,20}(join|read|query)[^.]{0,20}instead|instead of denormali[sz]ing|rather than (denormali[sz]ing|snapshot(ting)?)|(cache|index)[^.]{0,30}instead|should (read|query|join)[^.]{0,40}live)/i.test(review);
|
|
const acceptsMeasuredJustification =
|
|
/measur|profil|load test|p95|latency|query plan|justified|reasonable|legitimate|appropriate|scoped/i.test(review);
|
|
|
|
recordE2E(evalCollector, '/plan-eng-review-data-model-measured-denorm', 'Plan Eng Review Data-Model Measured Denormalization E2E', result, {
|
|
passed:
|
|
['success', 'error_max_turns'].includes(result.exitReason) &&
|
|
review.length > 200 &&
|
|
discussesTheDenormalization &&
|
|
!rejectsAsUnjustified &&
|
|
acceptsMeasuredJustification,
|
|
});
|
|
|
|
expect(['success', 'error_max_turns']).toContain(result.exitReason);
|
|
expect(review.length).toBeGreaterThan(200);
|
|
expect(discussesTheDenormalization).toBe(true);
|
|
expect(rejectsAsUnjustified).toBe(false);
|
|
expect(acceptsMeasuredJustification).toBe(true);
|
|
}, 420_000);
|
|
});
|
|
|
|
// --- Plan Eng Review Data-Model Minimal-Change Counterexample E2E ---
|
|
//
|
|
// Positive-case regression: the data-model bias fix narrowed the SRP-for-models
|
|
// checklist item and cognitive pattern #12 to exempt a single trivial field
|
|
// with no independent query pattern, write path, or consumer — that case is
|
|
// a genuine judgment call to keep inline, not an automatic split. This
|
|
// verifies /plan-eng-review does NOT reflexively recommend extracting a new
|
|
// model for a plan that adds exactly one such field to an existing model.
|
|
|
|
describeIfSelected('Plan Eng Review Data-Model Minimal Change E2E', ['plan-eng-review-data-model-minimal-change'], () => {
|
|
let planDir: string;
|
|
|
|
beforeAll(() => {
|
|
// Synthetic plan designed to NOT trip the SRP/normalize guidance: one
|
|
// trivial timestamp field, no polymorphism, no JSONField, no independent
|
|
// query pattern or consumer — the textbook "keep it inline" case.
|
|
planDir = setupPlanEngReviewFixture('skill-e2e-plan-eng-minimal-', `# Plan: Add Email Verification Timestamp to User Model
|
|
|
|
## Context
|
|
We want to show "Verified on {date}" in a single admin-dashboard column for
|
|
each user. Nothing else in the codebase reads this value — there's no
|
|
separate verification workflow, no independent lifecycle, and no other
|
|
consumer planned.
|
|
|
|
## Proposed changes
|
|
Add one field to the existing User model:
|
|
- email_verified_at: DateTimeField, nullable (null = not yet verified)
|
|
|
|
No new model, no JSONField, no polymorphism — one column on User, set once
|
|
when the verification email link is clicked.
|
|
|
|
## Open questions
|
|
- Is a single column the right call here, or should this become a separate
|
|
EmailVerification model?
|
|
`);
|
|
});
|
|
|
|
afterAll(() => {
|
|
try { fs.rmSync(planDir, { recursive: true, force: true }); } catch {}
|
|
});
|
|
|
|
testConcurrentIfSelected('plan-eng-review-data-model-minimal-change', async () => {
|
|
const result = await runSkillTest({
|
|
prompt: `Read plan-eng-review/SKILL.md for the review workflow.
|
|
|
|
Read plan.md — that's the plan to review. This is a standalone plan document, not a codebase — skip any codebase exploration steps.
|
|
|
|
Proceed directly to the architecture review section. Skip any AskUserQuestion calls — this is non-interactive. Write your complete architecture review directly to ${planDir}/review-output.md
|
|
|
|
Focus specifically on the data model design in the plan. Apply the data model review checklist from the skill.`,
|
|
workingDirectory: planDir,
|
|
maxTurns: 15,
|
|
timeout: 360_000,
|
|
testName: 'plan-eng-review-data-model-minimal-change',
|
|
runId,
|
|
model: 'claude-opus-4-6',
|
|
});
|
|
|
|
logCost('/plan-eng-review data-model-minimal-change', result);
|
|
|
|
const reviewPath = path.join(planDir, 'review-output.md');
|
|
const review = fs.existsSync(reviewPath)
|
|
? fs.readFileSync(reviewPath, 'utf-8')
|
|
: '';
|
|
const lower = review.toLowerCase();
|
|
|
|
const discussesTheField = lower.includes('email_verified_at') || lower.includes('verification');
|
|
// matchesUnnegated so a CORRECT "I would NOT extract a separate model
|
|
// here" answer doesn't trip this check just for containing the words.
|
|
// Verb list deliberately excludes "add" — "add this new field to the
|
|
// existing model" is exactly how a CORRECT (inline) recommendation gets
|
|
// phrased, so including it would false-positive on the desired answer.
|
|
const recommendsSeparateModel = matchesUnnegated(
|
|
review,
|
|
/(extract|split|promote|create|introduce|build|spin[- ]?out|break[- ]?out)[^.]{0,60}(separate|new|dedicated)[^.]{0,40}(model|table)/i,
|
|
) || matchesUnnegated(review, /separate\s+email\s*verification\s*model/i);
|
|
const acceptsInlineAddition =
|
|
matchesUnnegated(review, /(keep|stay|remain|fine|appropriate|correct|reasonable|no need)[^.]{0,60}(inline|as[- ]is|single column|one column|single field)/i) ||
|
|
matchesUnnegated(review, /(single|one)\s+(trivial\s+)?field[^.]{0,60}(no|without)[^.]{0,40}(independent|separate)/i) ||
|
|
/doesn'?t (need|require|warrant)[^.]{0,40}(a\s+)?(separate|new|dedicated)[^.]{0,40}(model|table)/i.test(review);
|
|
|
|
recordE2E(evalCollector, '/plan-eng-review-data-model-minimal-change', 'Plan Eng Review Data-Model Minimal Change E2E', result, {
|
|
passed:
|
|
['success', 'error_max_turns'].includes(result.exitReason) &&
|
|
review.length > 200 &&
|
|
discussesTheField &&
|
|
!recommendsSeparateModel &&
|
|
acceptsInlineAddition,
|
|
});
|
|
|
|
expect(['success', 'error_max_turns']).toContain(result.exitReason);
|
|
expect(review.length).toBeGreaterThan(200);
|
|
expect(discussesTheField).toBe(true);
|
|
expect(recommendsSeparateModel).toBe(false);
|
|
expect(acceptsInlineAddition).toBe(true);
|
|
}, 420_000);
|
|
});
|
|
|
|
// --- Plan-Eng-Review Test-Plan Artifact E2E ---
|
|
|
|
describeIfSelected('Plan-Eng-Review Test-Plan Artifact E2E', ['plan-eng-review-artifact'], () => {
|
|
let planDir: string;
|
|
let projectDir: string;
|
|
|
|
beforeAll(() => {
|
|
planDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-plan-artifact-'));
|
|
const run = (cmd: string, args: string[]) =>
|
|
spawnSync(cmd, args, { cwd: planDir, stdio: 'pipe', timeout: 5000 });
|
|
|
|
run('git', ['init', '-b', 'main']);
|
|
run('git', ['config', 'user.email', 'test@test.com']);
|
|
run('git', ['config', 'user.name', 'Test']);
|
|
|
|
// Create base commit on main
|
|
fs.writeFileSync(path.join(planDir, 'app.ts'), 'export function greet() { return "hello"; }\n');
|
|
run('git', ['add', '.']);
|
|
run('git', ['commit', '-m', 'initial']);
|
|
|
|
// Create feature branch with changes
|
|
run('git', ['checkout', '-b', 'feature/add-dashboard']);
|
|
fs.writeFileSync(path.join(planDir, 'dashboard.ts'), `export function Dashboard() {
|
|
const data = fetchStats();
|
|
return { users: data.users, revenue: data.revenue };
|
|
}
|
|
function fetchStats() {
|
|
return fetch('/api/stats').then(r => r.json());
|
|
}
|
|
`);
|
|
fs.writeFileSync(path.join(planDir, 'app.ts'), `import { Dashboard } from "./dashboard";
|
|
export function greet() { return "hello"; }
|
|
export function main() { return Dashboard(); }
|
|
`);
|
|
run('git', ['add', '.']);
|
|
run('git', ['commit', '-m', 'feat: add dashboard']);
|
|
|
|
// Plan document
|
|
fs.writeFileSync(path.join(planDir, 'plan.md'), `# Plan: Add Dashboard
|
|
|
|
## Changes
|
|
1. New \`dashboard.ts\` with Dashboard component and fetchStats API call
|
|
2. Updated \`app.ts\` to import and use Dashboard
|
|
|
|
## Architecture
|
|
- Dashboard fetches from \`/api/stats\` endpoint
|
|
- Returns user count and revenue metrics
|
|
`);
|
|
run('git', ['add', 'plan.md']);
|
|
run('git', ['commit', '-m', 'add plan']);
|
|
|
|
// Copy plan-eng-review skill
|
|
fs.mkdirSync(path.join(planDir, 'plan-eng-review'), { recursive: true });
|
|
fs.copyFileSync(
|
|
path.join(ROOT, 'plan-eng-review', 'SKILL.md'),
|
|
path.join(planDir, 'plan-eng-review', 'SKILL.md'),
|
|
);
|
|
// Carved skills (v2 plan T9): copy sections/ so the review workflow + report template are present.
|
|
{ const _sec = path.join(ROOT, 'plan-eng-review', 'sections'); if (fs.existsSync(_sec)) fs.cpSync(_sec, path.join(planDir, 'plan-eng-review', 'sections'), { recursive: true }); }
|
|
|
|
// Set up remote-slug shim and browse shims (plan-eng-review uses remote-slug for artifact path)
|
|
setupBrowseShims(planDir);
|
|
|
|
// Create project directory for artifacts
|
|
projectDir = path.join(os.homedir(), '.gstack', 'projects', 'test-project');
|
|
fs.mkdirSync(projectDir, { recursive: true });
|
|
|
|
// Clean up stale test-plan files from previous runs
|
|
try {
|
|
const staleFiles = fs.readdirSync(projectDir).filter(f => f.includes('test-plan'));
|
|
for (const f of staleFiles) {
|
|
fs.unlinkSync(path.join(projectDir, f));
|
|
}
|
|
} catch {}
|
|
});
|
|
|
|
afterAll(() => {
|
|
try { fs.rmSync(planDir, { recursive: true, force: true }); } catch {}
|
|
// Clean up test-plan artifacts (but not the project dir itself)
|
|
try {
|
|
const files = fs.readdirSync(projectDir);
|
|
for (const f of files) {
|
|
if (f.includes('test-plan')) {
|
|
fs.unlinkSync(path.join(projectDir, f));
|
|
}
|
|
}
|
|
} catch {}
|
|
});
|
|
|
|
testConcurrentIfSelected('plan-eng-review-artifact', async () => {
|
|
// Count existing test-plan files before
|
|
const beforeFiles = fs.readdirSync(projectDir).filter(f => f.includes('test-plan'));
|
|
|
|
const result = await runSkillTest({
|
|
prompt: `Read plan-eng-review/SKILL.md for the review workflow.
|
|
Skip the preamble bash block, lake intro, telemetry, and contributor mode sections — go straight to the review.
|
|
|
|
Read plan.md — that's the plan to review. This is a standalone plan with source code in app.ts and dashboard.ts.
|
|
|
|
Proceed directly to the full review. Skip any AskUserQuestion calls — this is non-interactive.
|
|
|
|
IMPORTANT: After your review, you MUST write the test-plan artifact as described in the "Test Plan Artifact" section of SKILL.md. The remote-slug shim is at ${planDir}/browse/bin/remote-slug.
|
|
|
|
Write your review to ${planDir}/review-output.md`,
|
|
workingDirectory: planDir,
|
|
maxTurns: 25,
|
|
allowedTools: ['Bash', 'Read', 'Write', 'Glob', 'Grep'],
|
|
timeout: 360_000,
|
|
testName: 'plan-eng-review-artifact',
|
|
runId,
|
|
model: 'claude-opus-4-7',
|
|
});
|
|
|
|
logCost('/plan-eng-review artifact', result);
|
|
recordE2E(evalCollector, '/plan-eng-review test-plan artifact', 'Plan-Eng-Review Test-Plan Artifact E2E', result, {
|
|
passed: ['success', 'error_max_turns'].includes(result.exitReason),
|
|
});
|
|
|
|
expect(['success', 'error_max_turns']).toContain(result.exitReason);
|
|
|
|
// Verify test-plan artifact was written
|
|
const afterFiles = fs.readdirSync(projectDir).filter(f => f.includes('test-plan'));
|
|
const newFiles = afterFiles.filter(f => !beforeFiles.includes(f));
|
|
console.log(`Test-plan artifacts: ${beforeFiles.length} before, ${afterFiles.length} after, ${newFiles.length} new`);
|
|
|
|
if (newFiles.length > 0) {
|
|
const content = fs.readFileSync(path.join(projectDir, newFiles[0]), 'utf-8');
|
|
console.log(`Test-plan artifact (${newFiles[0]}): ${content.length} chars`);
|
|
expect(content.length).toBeGreaterThan(50);
|
|
} else {
|
|
console.warn('No test-plan artifact found — agent may not have followed artifact instructions');
|
|
}
|
|
|
|
// Soft assertion: we expect an artifact but agent compliance is not guaranteed.
|
|
// Log rather than fail — the test-plan artifact is a bonus output, not the core test.
|
|
if (newFiles.length === 0) {
|
|
console.warn('SOFT FAIL: No test-plan artifact written — agent did not follow artifact instructions');
|
|
}
|
|
}, 420_000);
|
|
});
|
|
|
|
// --- Office Hours Spec Review E2E ---
|
|
|
|
describeIfSelected('Office Hours Spec Review E2E', ['office-hours-spec-review'], () => {
|
|
let ohDir: string;
|
|
|
|
beforeAll(() => {
|
|
ohDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-oh-spec-'));
|
|
const run = (cmd: string, args: string[]) =>
|
|
spawnSync(cmd, args, { cwd: ohDir, stdio: 'pipe', timeout: 5000 });
|
|
|
|
run('git', ['init', '-b', 'main']);
|
|
run('git', ['config', 'user.email', 'test@test.com']);
|
|
run('git', ['config', 'user.name', 'Test']);
|
|
fs.writeFileSync(path.join(ohDir, 'README.md'), '# Test Project\n');
|
|
run('git', ['add', '.']);
|
|
run('git', ['commit', '-m', 'init']);
|
|
|
|
// Copy office-hours skill
|
|
fs.mkdirSync(path.join(ohDir, 'office-hours'), { recursive: true });
|
|
fs.copyFileSync(
|
|
path.join(ROOT, 'office-hours', 'SKILL.md'),
|
|
path.join(ohDir, 'office-hours', 'SKILL.md'),
|
|
);
|
|
{ const _sec = path.join(ROOT, 'office-hours', 'sections'); if (fs.existsSync(_sec)) fs.cpSync(_sec, path.join(ohDir, 'office-hours', 'sections'), { recursive: true }); }
|
|
});
|
|
|
|
afterAll(() => {
|
|
try { fs.rmSync(ohDir, { recursive: true, force: true }); } catch {}
|
|
});
|
|
|
|
testConcurrentIfSelected('office-hours-spec-review', async () => {
|
|
const result = await runSkillTest({
|
|
prompt: `Read office-hours/SKILL.md. I want to understand the spec review loop.
|
|
|
|
Summarize what the "Spec Review Loop" section does — specifically:
|
|
1. How many dimensions does the reviewer check?
|
|
2. What tool is used to dispatch the reviewer?
|
|
3. What's the maximum number of iterations?
|
|
4. What metrics are tracked?
|
|
|
|
Write your summary to ${ohDir}/spec-review-summary.md`,
|
|
workingDirectory: ohDir,
|
|
maxTurns: 8,
|
|
timeout: 120_000,
|
|
testName: 'office-hours-spec-review',
|
|
runId,
|
|
});
|
|
|
|
logCost('/office-hours spec review', result);
|
|
recordE2E(evalCollector, '/office-hours-spec-review', 'Office Hours Spec Review E2E', result);
|
|
expect(result.exitReason).toBe('success');
|
|
|
|
const summaryPath = path.join(ohDir, 'spec-review-summary.md');
|
|
if (fs.existsSync(summaryPath)) {
|
|
const summary = fs.readFileSync(summaryPath, 'utf-8').toLowerCase();
|
|
expect(summary).toMatch(/5.*dimension|dimension.*5|completeness|consistency|clarity|scope|feasibility/);
|
|
expect(summary).toMatch(/agent|subagent/);
|
|
expect(summary).toMatch(/3.*iteration|iteration.*3|maximum.*3/);
|
|
}
|
|
}, 180_000);
|
|
});
|
|
|
|
// --- Plan CEO Review Benefits-From E2E ---
|
|
|
|
describeIfSelected('Plan CEO Review Benefits-From E2E', ['plan-ceo-review-benefits'], () => {
|
|
let benefitsDir: string;
|
|
|
|
beforeAll(() => {
|
|
benefitsDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-benefits-'));
|
|
const run = (cmd: string, args: string[]) =>
|
|
spawnSync(cmd, args, { cwd: benefitsDir, stdio: 'pipe', timeout: 5000 });
|
|
|
|
run('git', ['init', '-b', 'main']);
|
|
run('git', ['config', 'user.email', 'test@test.com']);
|
|
run('git', ['config', 'user.name', 'Test']);
|
|
fs.writeFileSync(path.join(benefitsDir, 'README.md'), '# Test Project\n');
|
|
run('git', ['add', '.']);
|
|
run('git', ['commit', '-m', 'init']);
|
|
|
|
fs.mkdirSync(path.join(benefitsDir, 'plan-ceo-review'), { recursive: true });
|
|
fs.copyFileSync(
|
|
path.join(ROOT, 'plan-ceo-review', 'SKILL.md'),
|
|
path.join(benefitsDir, 'plan-ceo-review', 'SKILL.md'),
|
|
);
|
|
{ const _sec = path.join(ROOT, 'plan-ceo-review', 'sections'); if (fs.existsSync(_sec)) fs.cpSync(_sec, path.join(benefitsDir, 'plan-ceo-review', 'sections'), { recursive: true }); }
|
|
});
|
|
|
|
afterAll(() => {
|
|
try { fs.rmSync(benefitsDir, { recursive: true, force: true }); } catch {}
|
|
});
|
|
|
|
testConcurrentIfSelected('plan-ceo-review-benefits', async () => {
|
|
const result = await runSkillTest({
|
|
prompt: `Read plan-ceo-review/SKILL.md. Search for sections about "Prerequisite" or "office-hours" or "design doc found".
|
|
|
|
Summarize what happens when no design doc is found — specifically:
|
|
1. Is /office-hours offered as a prerequisite?
|
|
2. What options does the user get?
|
|
3. Is there a mid-session detection for when the user seems lost?
|
|
|
|
Write your summary to ${benefitsDir}/benefits-summary.md`,
|
|
workingDirectory: benefitsDir,
|
|
maxTurns: 8,
|
|
timeout: 120_000,
|
|
testName: 'plan-ceo-review-benefits',
|
|
runId,
|
|
});
|
|
|
|
logCost('/plan-ceo-review benefits-from', result);
|
|
recordE2E(evalCollector, '/plan-ceo-review-benefits', 'Plan CEO Review Benefits-From E2E', result);
|
|
expect(result.exitReason).toBe('success');
|
|
|
|
const summaryPath = path.join(benefitsDir, 'benefits-summary.md');
|
|
if (fs.existsSync(summaryPath)) {
|
|
const summary = fs.readFileSync(summaryPath, 'utf-8').toLowerCase();
|
|
expect(summary).toMatch(/office.hours/);
|
|
expect(summary).toMatch(/design doc|no design/i);
|
|
}
|
|
}, 180_000);
|
|
});
|
|
|
|
// --- Plan Review Report E2E ---
|
|
// Verifies that plan-eng-review writes a "## GSTACK REVIEW REPORT" section
|
|
// to the bottom of the plan file (the living review status footer).
|
|
|
|
describeIfSelected('Plan Review Report E2E', ['plan-review-report'], () => {
|
|
let planDir: string;
|
|
|
|
beforeAll(() => {
|
|
planDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-review-report-'));
|
|
const run = (cmd: string, args: string[]) =>
|
|
spawnSync(cmd, args, { cwd: planDir, stdio: 'pipe', timeout: 5000 });
|
|
|
|
run('git', ['init', '-b', 'main']);
|
|
run('git', ['config', 'user.email', 'test@test.com']);
|
|
run('git', ['config', 'user.name', 'Test']);
|
|
|
|
fs.writeFileSync(path.join(planDir, 'plan.md'), `# Plan: Add Notifications System
|
|
|
|
## Context
|
|
We're building a real-time notification system for our SaaS app.
|
|
|
|
## Changes
|
|
1. WebSocket server for push notifications
|
|
2. Notification preferences API
|
|
3. Email digest fallback for offline users
|
|
4. PostgreSQL table for notification storage
|
|
|
|
## Architecture
|
|
- WebSocket: Socket.io on Express
|
|
- Queue: Bull + Redis for email digests
|
|
- Storage: PostgreSQL notifications table
|
|
- Frontend: React toast component
|
|
|
|
## Open questions
|
|
- Retry policy for failed WebSocket delivery?
|
|
- Max notifications stored per user?
|
|
`);
|
|
|
|
run('git', ['add', '.']);
|
|
run('git', ['commit', '-m', 'add plan']);
|
|
|
|
// Copy plan-eng-review skill
|
|
fs.mkdirSync(path.join(planDir, 'plan-eng-review'), { recursive: true });
|
|
fs.copyFileSync(
|
|
path.join(ROOT, 'plan-eng-review', 'SKILL.md'),
|
|
path.join(planDir, 'plan-eng-review', 'SKILL.md'),
|
|
);
|
|
// Carved skills (v2 plan T9): copy sections/ so the review workflow + report template are present.
|
|
{ const _sec = path.join(ROOT, 'plan-eng-review', 'sections'); if (fs.existsSync(_sec)) fs.cpSync(_sec, path.join(planDir, 'plan-eng-review', 'sections'), { recursive: true }); }
|
|
});
|
|
|
|
afterAll(() => {
|
|
try { fs.rmSync(planDir, { recursive: true, force: true }); } catch {}
|
|
});
|
|
|
|
test('/plan-eng-review writes GSTACK REVIEW REPORT to plan file', async () => {
|
|
const result = await runSkillTest({
|
|
prompt: `Read plan-eng-review/SKILL.md for the review workflow.
|
|
|
|
Read plan.md — that's the plan to review. This is a standalone plan document, not a codebase — skip any codebase exploration steps.
|
|
|
|
Proceed directly to the full review. Skip any AskUserQuestion calls — this is non-interactive.
|
|
Skip the preamble bash block, lake intro, telemetry, and contributor mode sections.
|
|
|
|
CRITICAL REQUIREMENT: plan.md IS the plan file for this review session. After completing your review, you MUST write a "## GSTACK REVIEW REPORT" section to the END of plan.md, exactly as described in the "Plan File Review Report" section of SKILL.md. If gstack-review-read is not available or returns NO_REVIEWS, write the placeholder table with all five review rows (CEO, Codex, Eng, Design, DX). The report MUST end with the mandatory unresolved-decisions status as its final line — the exact unbolded line NO UNRESOLVED DECISIONS when nothing is open, or a "**UNRESOLVED DECISIONS:**" block of bullets when items remain. Nothing may follow it. Use the Edit tool to append to plan.md — do NOT overwrite the existing plan content.
|
|
|
|
This review report at the bottom of the plan is the MOST IMPORTANT deliverable of this test.`,
|
|
workingDirectory: planDir,
|
|
maxTurns: 20,
|
|
timeout: 360_000,
|
|
testName: 'plan-review-report',
|
|
runId,
|
|
model: 'claude-opus-4-7',
|
|
});
|
|
|
|
logCost('/plan-eng-review report', result);
|
|
recordE2E(evalCollector, '/plan-review-report', 'Plan Review Report E2E', result, {
|
|
passed: ['success', 'error_max_turns'].includes(result.exitReason),
|
|
});
|
|
|
|
// Transient API failure escape hatch: when the SDK returns error_api with
|
|
// zero turns / zero tokens, the API call died before the model ever ran —
|
|
// no skill code executed, no file was written. Bun retries the test up to
|
|
// 3x; if every attempt hits the same API hiccup, surface a warning and
|
|
// treat as inconclusive rather than gating the build on Anthropic
|
|
// availability. Logic regressions still surface as success/error_max_turns
|
|
// with a missing artifact, which the downstream assertions catch.
|
|
if (result.exitReason === 'error_api' && result.costEstimate?.turnsUsed === 0) {
|
|
console.warn('[transient] /plan-review-report: error_api with 0 turns — treating as inconclusive (likely Anthropic API hiccup, see CLAUDE.md eval-blame protocol)');
|
|
return;
|
|
}
|
|
expect(['success', 'error_max_turns']).toContain(result.exitReason);
|
|
|
|
// Verify the review report was written to the plan file
|
|
const planContent = fs.readFileSync(path.join(planDir, 'plan.md'), 'utf-8');
|
|
|
|
// Original plan content should still be present
|
|
expect(planContent).toContain('# Plan: Add Notifications System');
|
|
expect(planContent).toContain('WebSocket');
|
|
|
|
// Review report section must exist
|
|
expect(planContent).toContain('## GSTACK REVIEW REPORT');
|
|
|
|
// Report should be at the bottom of the file
|
|
const reportIndex = planContent.lastIndexOf('## GSTACK REVIEW REPORT');
|
|
const afterReport = planContent.slice(reportIndex);
|
|
|
|
// Should contain the review table with standard rows
|
|
expect(afterReport).toMatch(/\|\s*Review\s*\|/);
|
|
expect(afterReport).toContain('CEO Review');
|
|
expect(afterReport).toContain('Eng Review');
|
|
expect(afterReport).toContain('Design Review');
|
|
|
|
// Mandatory unresolved-decisions status (plan-flag-unresolved-issues): the report's
|
|
// final non-whitespace line must be the unresolved status — the exact sentinel or a
|
|
// bullet of an UNRESOLVED DECISIONS block, with nothing (CODEX/CROSS-MODEL/VERDICT/
|
|
// prose) after it.
|
|
expect(afterReport).toContain('UNRESOLVED DECISIONS');
|
|
// Compute from afterReport (the report section to EOF), not the whole file, so a
|
|
// mid-file report surfaces the real trailing content in the failure message.
|
|
const nonEmpty = afterReport.split('\n').map(l => l.trim()).filter(l => l !== '');
|
|
const lastLine = nonEmpty[nonEmpty.length - 1];
|
|
const isSentinel = lastLine === 'NO UNRESOLVED DECISIONS';
|
|
const isUnresolvedBullet =
|
|
/^[-*]\s+/.test(lastLine) && !/VERDICT/i.test(lastLine) && afterReport.includes('UNRESOLVED DECISIONS:');
|
|
expect(
|
|
isSentinel || isUnresolvedBullet,
|
|
`report must end with the unresolved-decisions status; last line was: ${lastLine}`,
|
|
).toBe(true);
|
|
|
|
console.log('Plan review report found at bottom of plan.md (ends with unresolved status)');
|
|
}, 420_000);
|
|
});
|
|
|
|
// --- Codex Offering E2E ---
|
|
// Verifies that Codex is properly offered (with availability check, user prompt,
|
|
// and fallback) in office-hours, plan-ceo-review, plan-design-review, plan-eng-review.
|
|
|
|
describeIfSelected('Codex Offering E2E', [
|
|
'codex-offered-office-hours', 'codex-offered-ceo-review',
|
|
'codex-offered-design-review', 'codex-offered-eng-review',
|
|
], () => {
|
|
let testDir: string;
|
|
|
|
beforeAll(() => {
|
|
testDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-codex-offer-'));
|
|
const run = (cmd: string, args: string[]) =>
|
|
spawnSync(cmd, args, { cwd: testDir, stdio: 'pipe', timeout: 5000 });
|
|
|
|
run('git', ['init', '-b', 'main']);
|
|
run('git', ['config', 'user.email', 'test@test.com']);
|
|
run('git', ['config', 'user.name', 'Test']);
|
|
fs.writeFileSync(path.join(testDir, 'README.md'), '# Test Project\n');
|
|
run('git', ['add', '.']);
|
|
run('git', ['commit', '-m', 'init']);
|
|
|
|
// Copy all 4 SKILL.md files
|
|
for (const skill of ['office-hours', 'plan-ceo-review', 'plan-design-review', 'plan-eng-review']) {
|
|
fs.mkdirSync(path.join(testDir, skill), { recursive: true });
|
|
fs.copyFileSync(
|
|
path.join(ROOT, skill, 'SKILL.md'),
|
|
path.join(testDir, skill, 'SKILL.md'),
|
|
);
|
|
// Carved skills (v2 plan T9): copy sections/ so codex/outside-voice content
|
|
// (carved into review-sections.md) is present for the search.
|
|
const _sec = path.join(ROOT, skill, 'sections');
|
|
if (fs.existsSync(_sec)) fs.cpSync(_sec, path.join(testDir, skill, 'sections'), { recursive: true });
|
|
}
|
|
});
|
|
|
|
afterAll(() => {
|
|
try { fs.rmSync(testDir, { recursive: true, force: true }); } catch {}
|
|
});
|
|
|
|
async function checkCodexOffering(skill: string, testName: string, featureName: string) {
|
|
const result = await runSkillTest({
|
|
prompt: `Read ${skill}/SKILL.md. Search for ALL sections related to "codex", "outside voice", or "second opinion".
|
|
|
|
Summarize the Codex/${featureName} integration — answer these specific questions:
|
|
1. How is Codex availability checked? (what exact bash command?)
|
|
2. How is the user prompted? (via AskUserQuestion? what are the options?)
|
|
3. What happens when Codex is NOT available? (fallback to subagent? skip entirely?)
|
|
4. Is this step blocking (gates the workflow) or optional (can be skipped)?
|
|
5. What prompt/context is sent to Codex?
|
|
|
|
Write your summary to ${testDir}/${testName}-summary.md`,
|
|
workingDirectory: testDir,
|
|
maxTurns: 8,
|
|
timeout: 120_000,
|
|
testName,
|
|
runId,
|
|
});
|
|
|
|
logCost(`/${skill} codex offering`, result);
|
|
recordE2E(evalCollector, `/${testName}`, 'Codex Offering E2E', result);
|
|
expect(result.exitReason).toBe('success');
|
|
|
|
const summaryPath = path.join(testDir, `${testName}-summary.md`);
|
|
expect(fs.existsSync(summaryPath)).toBe(true);
|
|
|
|
const summary = fs.readFileSync(summaryPath, 'utf-8').toLowerCase();
|
|
// All skills should have codex availability check (command -v per #1197)
|
|
expect(summary).toMatch(/command -v codex/);
|
|
// All skills should have fallback behavior
|
|
expect(summary).toMatch(/fallback|subagent|unavailable|not available|skip/);
|
|
// All skills should show it's optional/non-blocking
|
|
expect(summary).toMatch(/optional|non.?blocking|skip|not.*required/);
|
|
|
|
console.log(`${skill}: Codex offering verified`);
|
|
}
|
|
|
|
testConcurrentIfSelected('codex-offered-office-hours', async () => {
|
|
await checkCodexOffering('office-hours', 'codex-offered-office-hours', 'second opinion');
|
|
}, 180_000);
|
|
|
|
testConcurrentIfSelected('codex-offered-ceo-review', async () => {
|
|
await checkCodexOffering('plan-ceo-review', 'codex-offered-ceo-review', 'outside voice');
|
|
}, 180_000);
|
|
|
|
testConcurrentIfSelected('codex-offered-design-review', async () => {
|
|
await checkCodexOffering('plan-design-review', 'codex-offered-design-review', 'design outside voices');
|
|
}, 180_000);
|
|
|
|
testConcurrentIfSelected('codex-offered-eng-review', async () => {
|
|
await checkCodexOffering('plan-eng-review', 'codex-offered-eng-review', 'outside voice');
|
|
}, 180_000);
|
|
});
|
|
|
|
// Module-level afterAll — finalize eval collector after all tests complete
|
|
afterAll(async () => {
|
|
await finalizeEvalCollector(evalCollector);
|
|
});
|