RomanNum3ral
/
gstack
mirror of https://github.com/garrytan/gstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

test(evals): retro E2E passes require the report on disk 

Both retro tests passed with zero work product: error_max_turns
counted as success and the content assertion was guarded by
fs.existsSync — a run that burned 30 turns and wrote nothing recorded
green (red team). The report is now load-bearing for pass/fail.
This commit is contained in:
Garry Tan 2026-08-15 17:02:04 -07:00
parent 6ef93fd69f
commit 3dc2cde346
No known key found for this signature in database
GPG Key ID: C1F69E85C74EFE1D
1 changed files with 20 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
test/skill-e2e-retro.test.ts
View File

@ -80,17 +80,18 @@ Write your retrospective to ${dir}/retro-output.md`,
});
logCost('/retro base-branch', result);
// The report is the work product: a run that exits max-turns without
// writing it is a FAIL, not a pass — otherwise this test cannot detect
// the most basic regression (the skill stops producing its report).
const retroPath = path.join(dir, 'retro-output.md');
const wroteReport = fs.existsSync(retroPath);
recordE2E(evalCollector, '/retro default branch detection', 'Base branch detection', result, {
passed: ['success', 'error_max_turns'].includes(result.exitReason),
passed: ['success', 'error_max_turns'].includes(result.exitReason) && wroteReport,
});
expect(['success', 'error_max_turns']).toContain(result.exitReason);
// Verify retro output was produced
const retroPath = path.join(dir, 'retro-output.md');
if (fs.existsSync(retroPath)) {
const content = fs.readFileSync(retroPath, 'utf-8');
expect(content.length).toBeGreaterThan(100);
}
expect(wroteReport).toBe(true);
const content = fs.readFileSync(retroPath, 'utf-8');
expect(content.length).toBeGreaterThan(100);
}, 480_000);
});
@ -165,18 +166,18 @@ Analyze the git history and produce the narrative report as described in the SKI
});
logCost('/retro', result);
recordE2E(evalCollector, '/retro', 'Retro E2E', result, {
passed: ['success', 'error_max_turns'].includes(result.exitReason),
});
// Accept error_max_turns — retro does many git commands to analyze history
expect(['success', 'error_max_turns']).toContain(result.exitReason);
// Verify the retro was written
// Accept error_max_turns (retro does many git commands to analyze
// history) — but only WITH the report on disk. The report is the work
// product; max-turns with nothing written is a fail.
const retroPath = path.join(retroDir, 'retro-output.md');
if (fs.existsSync(retroPath)) {
const retro = fs.readFileSync(retroPath, 'utf-8');
expect(retro.length).toBeGreaterThan(100);
}
const wroteReport = fs.existsSync(retroPath);
recordE2E(evalCollector, '/retro', 'Retro E2E', result, {
passed: ['success', 'error_max_turns'].includes(result.exitReason) && wroteReport,
});
expect(['success', 'error_max_turns']).toContain(result.exitReason);
expect(wroteReport).toBe(true);
const retro = fs.readFileSync(retroPath, 'utf-8');
expect(retro.length).toBeGreaterThan(100);
}, 420_000);
});