RomanNum3ral
/
gstack
mirror of https://github.com/garrytan/gstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

test(e2e): setup-gbrain remote — hermetic env via opts, evidence on failure, output-scoped classifier 

Three separate defects stacked on this one test: the ambient
GBRAIN_MCP_TOKEN/GSTACK_HOME/PATH mutations never reached the child
(hermetic-env scrubs them by allowlist — broken since hermetic env
landed; the child correctly stopped at Step 4c with NEEDS_CONTEXT),
failures discarded the in-memory transcript so every triage started
blind, and the wrote-findings-before-asking classifier scanned the full
event stream where the child's own Read of the skill file always
contains the review-report phrase. Env now goes via opts.env, failures
dump bash commands + final text, and the classifier scans assistant
output only. Green in 67s with all seven asserts.
This commit is contained in:
Garry Tan 2026-08-15 02:43:39 -07:00
parent 4cbb9e1306
commit dd29d80be0
No known key found for this signature in database
GPG Key ID: C1F69E85C74EFE1D
1 changed files with 29 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
test/skill-e2e-setup-gbrain-remote.test.ts
View File

@ -130,15 +130,16 @@ describeE2E('/setup-gbrain Path 4 (Remote MCP) — happy path', () => {
const askUserQuestions: Array<{ input: Record<string, unknown> }> = [];
const binary = resolveClaudeBinary();
// Ambient env mutations. Restored in finally.
const orig = {
gstackHome: process.env.GSTACK_HOME,
pathEnv: process.env.PATH,
mcpToken: process.env.GBRAIN_MCP_TOKEN,
// Per-test child env, passed via opts.env (merges last over the complete
// hermetic env). Ambient process.env mutations DO NOT reach children:
// hermetic-env scrubs GBRAIN_*/GSTACK_* by allowlist — this test's token
// silently never arrived from the day hermetic env landed, and the child
// correctly stopped at Step 4c with NEEDS_CONTEXT.
const childEnv = {
GSTACK_HOME: gstackHome,
GBRAIN_MCP_TOKEN: SECRET_TOKEN,
PATH: `${fakeBinDir}:${path.join(path.resolve(import.meta.dir, '..'), 'bin')}:${process.env.PATH ?? '/usr/bin:/bin:/opt/homebrew/bin'}`,
};
process.env.GSTACK_HOME = gstackHome;
process.env.PATH = `${fakeBinDir}:${path.join(path.resolve(import.meta.dir, '..'), 'bin')}:${process.env.PATH ?? '/usr/bin:/bin:/opt/homebrew/bin'}`;
process.env.GBRAIN_MCP_TOKEN = SECRET_TOKEN;
let modelTextOutput = '';
@ -146,6 +147,7 @@ describeE2E('/setup-gbrain Path 4 (Remote MCP) — happy path', () => {
const skillPath = path.resolve(import.meta.dir, '..', 'setup-gbrain', 'SKILL.md');
const result = await runAgentSdkTest({
systemPrompt: { type: 'preset', preset: 'claude_code' },
env: childEnv,
userPrompt:
`Read the skill file at ${skillPath} and follow Path 4 (Remote MCP) only. ` +
`Use this MCP URL: ${stubServer.url}. ` +
@ -195,6 +197,19 @@ describeE2E('/setup-gbrain Path 4 (Remote MCP) — happy path', () => {
// Assertion 2: claude mcp add was called with --transport http.
const calls = fs.existsSync(callLog) ? fs.readFileSync(callLog, 'utf-8') : '';
if (!/mcp add.*--transport http/.test(calls)) {
// Failure evidence: without this, the transcript dies with the test
// and every triage pass starts blind (three did).
const bashCmds = result.toolCalls
.filter((t) => t.tool === 'Bash')
.map((t) => String((t.input as { command?: string })?.command ?? '').slice(0, 200));
console.error(
`[setup-gbrain-remote] mcp-add never hit the fake shim.\n` +
`exitReason=${result.exitReason} turns=${result.turnsUsed}\n` +
`--- bash commands (${bashCmds.length}) ---\n${bashCmds.join('\n')}\n` +
`--- final text (last 1500 chars) ---\n${result.output.slice(-1500)}`,
);
}
expect(calls).toMatch(/mcp add.*--transport http/);
// Assertion 3: the secret token NEVER appears in the final CLAUDE.md.
@ -207,14 +222,16 @@ describeE2E('/setup-gbrain Path 4 (Remote MCP) — happy path', () => {
// Assertion 5: classifier — the model didn't write findings before
// asking. The Path 4 prose has 5 STOP gates; if any of them got
// skipped, that's the wrote_findings_before_asking pattern.
const wroteBefore = /## GSTACK REVIEW REPORT|critical_gaps/i.test(modelTextOutput);
// Scan the ASSISTANT's text only: modelTextOutput serializes every
// event including the child's Read of the skill file, whose generated
// footer contains the literal "GSTACK REVIEW REPORT" — a guaranteed
// false positive on both trees.
const wroteBefore = /## GSTACK REVIEW REPORT|critical_gaps/i.test(result.output);
// Setup-gbrain doesn't have a review report contract, so this is
// a structural shape check, not a hard failure mode.
expect(wroteBefore).toBe(false);
} finally {
if (orig.gstackHome === undefined) delete process.env.GSTACK_HOME; else process.env.GSTACK_HOME = orig.gstackHome;
if (orig.pathEnv === undefined) delete process.env.PATH; else process.env.PATH = orig.pathEnv;
if (orig.mcpToken === undefined) delete process.env.GBRAIN_MCP_TOKEN; else process.env.GBRAIN_MCP_TOKEN = orig.mcpToken;
// (no ambient process.env mutations to restore — env goes via opts.env)
await stubServer.close();
fs.rmSync(gstackHome, { recursive: true, force: true });
fs.rmSync(fakeBinDir, { recursive: true, force: true });