mirror of https://github.com/garrytan/gstack.git
test(e2e): periodic-tier repairs from the failure triage
Each fix traces to a receipt: brain-privacy-gate staged config never reached the hermetic child (ambient GSTACK_HOME is scrubbed) and the operator's remote-mode gbrain suppressed the gate — both now injected per-test; ship-idempotency threw away its evidence on the timeout path and ran a 600s budget its own subject can exceed (now 900s, evidence captured); auto-decide-preserved gets the same headroom its sibling plan-ceo tests got; context-skills' hides-checks scanned bash output where an ls legitimately names old checkpoints (final-text scope now); design names the missing section instead of a bare count and learns the easing/duration/micro-interaction synonyms; qa-workflow's collector afterAll gets an explicit 60s hook timeout.
This commit is contained in:
parent
11b3e517d7
commit
fa783470a6
|
|
@ -111,7 +111,7 @@ describeE2E('AUTO_DECIDE opt-in preserved under Conductor flags (periodic)', ()
|
||||||
skillName: 'plan-ceo-review',
|
skillName: 'plan-ceo-review',
|
||||||
inPlanMode: true,
|
inPlanMode: true,
|
||||||
extraArgs: ['--disallowedTools', 'AskUserQuestion'],
|
extraArgs: ['--disallowedTools', 'AskUserQuestion'],
|
||||||
timeoutMs: 300_000,
|
timeoutMs: 540_000,
|
||||||
env: { GSTACK_HOME: tmpHome, CONDUCTOR_WORKSPACE_PATH: tmpHome },
|
env: { GSTACK_HOME: tmpHome, CONDUCTOR_WORKSPACE_PATH: tmpHome },
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -135,5 +135,5 @@ describeE2E('AUTO_DECIDE opt-in preserved under Conductor flags (periodic)', ()
|
||||||
} finally {
|
} finally {
|
||||||
try { fs.rmSync(tmpHome, { recursive: true, force: true }); } catch { /* best-effort */ }
|
try { fs.rmSync(tmpHome, { recursive: true, force: true }); } catch { /* best-effort */ }
|
||||||
}
|
}
|
||||||
}, 360_000);
|
}, 660_000);
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -34,6 +34,12 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
|
||||||
// Stage a fresh GSTACK_HOME with artifacts_sync_mode_prompted=false.
|
// Stage a fresh GSTACK_HOME with artifacts_sync_mode_prompted=false.
|
||||||
const gstackHome = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-gstack-'));
|
const gstackHome = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-gstack-'));
|
||||||
const fakeBinDir = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-bin-'));
|
const fakeBinDir = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-bin-'));
|
||||||
|
// Fresh HOME with NO ~/.claude.json: on a machine where gbrain is
|
||||||
|
// registered type=http, the preamble's remote-mode detection reads the
|
||||||
|
// operator's ~/.claude.json and echoes "ARTIFACTS_SYNC: remote-mode" —
|
||||||
|
// and the local privacy gate legitimately never fires. An empty HOME
|
||||||
|
// makes the detection find nothing.
|
||||||
|
const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-home-'));
|
||||||
|
|
||||||
// Seed the config so the gate's condition passes.
|
// Seed the config so the gate's condition passes.
|
||||||
fs.writeFileSync(
|
fs.writeFileSync(
|
||||||
|
|
@ -59,12 +65,16 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
|
||||||
const askUserQuestions: Array<{ input: Record<string, unknown> }> = [];
|
const askUserQuestions: Array<{ input: Record<string, unknown> }> = [];
|
||||||
const binary = resolveClaudeBinary();
|
const binary = resolveClaudeBinary();
|
||||||
|
|
||||||
// Ambient env mutations — restored in finally so other tests in the file
|
// Per-test env, merged LAST by the hermetic env builder (safe post-v1.39:
|
||||||
// don't inherit them.
|
// the runner always passes a COMPLETE hermetic env, so overrides can't
|
||||||
const origGstackHome = process.env.GSTACK_HOME;
|
// break auth). Ambient process.env.GSTACK_HOME mutation does NOT work
|
||||||
const origPath = process.env.PATH;
|
// here — hermetic-env scrubs GSTACK_* and repoints GSTACK_HOME at its
|
||||||
process.env.GSTACK_HOME = gstackHome;
|
// own singleton dir, so the staged config would never reach the child.
|
||||||
process.env.PATH = `${fakeBinDir}:${process.env.PATH ?? '/usr/bin:/bin:/opt/homebrew/bin'}`;
|
const childEnv = {
|
||||||
|
GSTACK_HOME: gstackHome,
|
||||||
|
HOME: tempHome,
|
||||||
|
PATH: `${fakeBinDir}:${process.env.PATH ?? '/usr/bin:/bin:/opt/homebrew/bin'}`,
|
||||||
|
};
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Pick a small skill with the preamble and load it via Read to force
|
// Pick a small skill with the preamble and load it via Read to force
|
||||||
|
|
@ -85,12 +95,7 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
|
||||||
workingDirectory: gstackHome,
|
workingDirectory: gstackHome,
|
||||||
maxTurns: 10,
|
maxTurns: 10,
|
||||||
allowedTools: ['Read', 'Grep', 'Glob', 'Bash'],
|
allowedTools: ['Read', 'Grep', 'Glob', 'Bash'],
|
||||||
// NOTE: do NOT pass `env:` here. When the Agent SDK gets an explicit
|
env: childEnv,
|
||||||
// env object, its auth pipeline doesn't pick up ANTHROPIC_API_KEY the
|
|
||||||
// same way as when env is undefined (SDK-internal detail, verified
|
|
||||||
// against the plan-mode-no-op test which passes no env and auths
|
|
||||||
// cleanly). Instead, mutate process.env before the call so the SDK
|
|
||||||
// inherits our overrides ambiently.
|
|
||||||
...(binary ? { pathToClaudeCodeExecutable: binary } : {}),
|
...(binary ? { pathToClaudeCodeExecutable: binary } : {}),
|
||||||
canUseTool: async (toolName, input) => {
|
canUseTool: async (toolName, input) => {
|
||||||
if (toolName === 'AskUserQuestion') {
|
if (toolName === 'AskUserQuestion') {
|
||||||
|
|
@ -141,13 +146,9 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
|
||||||
// (The preamble is supposed to be idempotent within a session.)
|
// (The preamble is supposed to be idempotent within a session.)
|
||||||
expect(privacyQuestions.length).toBe(1);
|
expect(privacyQuestions.length).toBe(1);
|
||||||
} finally {
|
} finally {
|
||||||
// Restore ambient env before other tests.
|
|
||||||
if (origGstackHome === undefined) delete process.env.GSTACK_HOME;
|
|
||||||
else process.env.GSTACK_HOME = origGstackHome;
|
|
||||||
if (origPath === undefined) delete process.env.PATH;
|
|
||||||
else process.env.PATH = origPath;
|
|
||||||
fs.rmSync(gstackHome, { recursive: true, force: true });
|
fs.rmSync(gstackHome, { recursive: true, force: true });
|
||||||
fs.rmSync(fakeBinDir, { recursive: true, force: true });
|
fs.rmSync(fakeBinDir, { recursive: true, force: true });
|
||||||
|
fs.rmSync(tempHome, { recursive: true, force: true });
|
||||||
}
|
}
|
||||||
}, 180_000);
|
}, 180_000);
|
||||||
|
|
||||||
|
|
@ -155,6 +156,10 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
|
||||||
// Same staging, but prompted=true this time. Gate should be silent.
|
// Same staging, but prompted=true this time. Gate should be silent.
|
||||||
const gstackHome = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-off-'));
|
const gstackHome = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-off-'));
|
||||||
const fakeBinDir = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-off-bin-'));
|
const fakeBinDir = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-off-bin-'));
|
||||||
|
// Fresh HOME without a .claude.json — same rationale as the first test:
|
||||||
|
// without it the operator's ~/.claude.json flips the preamble into
|
||||||
|
// remote-mode and this negative test passes vacuously.
|
||||||
|
const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-off-home-'));
|
||||||
|
|
||||||
fs.writeFileSync(
|
fs.writeFileSync(
|
||||||
path.join(gstackHome, 'config.yaml'),
|
path.join(gstackHome, 'config.yaml'),
|
||||||
|
|
@ -171,11 +176,13 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
|
||||||
const askUserQuestions: Array<{ input: Record<string, unknown> }> = [];
|
const askUserQuestions: Array<{ input: Record<string, unknown> }> = [];
|
||||||
const binary = resolveClaudeBinary();
|
const binary = resolveClaudeBinary();
|
||||||
|
|
||||||
// Ambient env mutations (see note on the first test).
|
// Per-test env, merged LAST by the hermetic env builder (see note on the
|
||||||
const origGstackHome = process.env.GSTACK_HOME;
|
// first test — ambient GSTACK_HOME mutation is scrubbed by hermetic-env).
|
||||||
const origPath = process.env.PATH;
|
const childEnv = {
|
||||||
process.env.GSTACK_HOME = gstackHome;
|
GSTACK_HOME: gstackHome,
|
||||||
process.env.PATH = `${fakeBinDir}:${process.env.PATH ?? '/usr/bin:/bin:/opt/homebrew/bin'}`;
|
HOME: tempHome,
|
||||||
|
PATH: `${fakeBinDir}:${process.env.PATH ?? '/usr/bin:/bin:/opt/homebrew/bin'}`,
|
||||||
|
};
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await runAgentSdkTest({
|
await runAgentSdkTest({
|
||||||
|
|
@ -185,6 +192,7 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
|
||||||
workingDirectory: gstackHome,
|
workingDirectory: gstackHome,
|
||||||
maxTurns: 4,
|
maxTurns: 4,
|
||||||
allowedTools: ['Read', 'Grep', 'Glob', 'Bash'],
|
allowedTools: ['Read', 'Grep', 'Glob', 'Bash'],
|
||||||
|
env: childEnv,
|
||||||
...(binary ? { pathToClaudeCodeExecutable: binary } : {}),
|
...(binary ? { pathToClaudeCodeExecutable: binary } : {}),
|
||||||
canUseTool: async (toolName, input) => {
|
canUseTool: async (toolName, input) => {
|
||||||
if (toolName === 'AskUserQuestion') {
|
if (toolName === 'AskUserQuestion') {
|
||||||
|
|
@ -216,12 +224,9 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
|
||||||
});
|
});
|
||||||
expect(privacyQuestions.length).toBe(0);
|
expect(privacyQuestions.length).toBe(0);
|
||||||
} finally {
|
} finally {
|
||||||
if (origGstackHome === undefined) delete process.env.GSTACK_HOME;
|
|
||||||
else process.env.GSTACK_HOME = origGstackHome;
|
|
||||||
if (origPath === undefined) delete process.env.PATH;
|
|
||||||
else process.env.PATH = origPath;
|
|
||||||
fs.rmSync(gstackHome, { recursive: true, force: true });
|
fs.rmSync(gstackHome, { recursive: true, force: true });
|
||||||
fs.rmSync(fakeBinDir, { recursive: true, force: true });
|
fs.rmSync(fakeBinDir, { recursive: true, force: true });
|
||||||
|
fs.rmSync(tempHome, { recursive: true, force: true });
|
||||||
}
|
}
|
||||||
}, 180_000);
|
}, 180_000);
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -447,8 +447,13 @@ Do NOT use AskUserQuestion.`,
|
||||||
// Match by filename timestamp (stable, unambiguous) plus a looser
|
// Match by filename timestamp (stable, unambiguous) plus a looser
|
||||||
// prose check.
|
// prose check.
|
||||||
const showsMain = /20260101-120000|main-work/.test(out);
|
const showsMain = /20260101-120000|main-work/.test(out);
|
||||||
const hidesAlpha = !/20260202-120000/.test(out);
|
// Hide checks scope to the FINAL text output only: fullOutputSurface
|
||||||
const hidesBeta = !/20260303-120000/.test(out);
|
// includes bash tool_results, and a legitimate `ls` of the checkpoint
|
||||||
|
// dir lists every branch's filename. The filtering under test happens
|
||||||
|
// in the user-facing list, not in the agent's intermediate reads.
|
||||||
|
const finalText = result.output ?? '';
|
||||||
|
const hidesAlpha = !/20260202-120000|LISTCURR_ALPHA_TOKEN/.test(finalText);
|
||||||
|
const hidesBeta = !/20260303-120000|LISTCURR_BETA_TOKEN/.test(finalText);
|
||||||
const routed = skillCalls(result).includes('context-save');
|
const routed = skillCalls(result).includes('context-save');
|
||||||
const exitOk = ['success', 'error_max_turns'].includes(result.exitReason);
|
const exitOk = ['success', 'error_max_turns'].includes(result.exitReason);
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -126,7 +126,7 @@ Write DESIGN.md and CLAUDE.md (or update it) in the working directory.`,
|
||||||
'Color': ['color', 'colour', 'palette', 'colors'],
|
'Color': ['color', 'colour', 'palette', 'colors'],
|
||||||
'Spacing': ['spacing', 'space', 'whitespace', 'gap'],
|
'Spacing': ['spacing', 'space', 'whitespace', 'gap'],
|
||||||
'Layout': ['layout', 'grid', 'structure', 'composition'],
|
'Layout': ['layout', 'grid', 'structure', 'composition'],
|
||||||
'Motion': ['motion', 'animation', 'transition', 'movement'],
|
'Motion': ['motion', 'animation', 'transition', 'movement', 'easing', 'duration', 'micro-interaction'],
|
||||||
};
|
};
|
||||||
const missingSections = Object.entries(sectionSynonyms).filter(
|
const missingSections = Object.entries(sectionSynonyms).filter(
|
||||||
([_, synonyms]) => !synonyms.some(s => designContent.toLowerCase().includes(s))
|
([_, synonyms]) => !synonyms.some(s => designContent.toLowerCase().includes(s))
|
||||||
|
|
@ -152,7 +152,9 @@ Write DESIGN.md and CLAUDE.md (or update it) in the working directory.`,
|
||||||
expect(['success', 'error_max_turns']).toContain(result.exitReason);
|
expect(['success', 'error_max_turns']).toContain(result.exitReason);
|
||||||
expect(designExists).toBe(true);
|
expect(designExists).toBe(true);
|
||||||
if (designExists) {
|
if (designExists) {
|
||||||
expect(missingSections).toHaveLength(0);
|
// join() so a failure names the offending section(s) — a bare
|
||||||
|
// toHaveLength(0) failure never prints WHICH synonym set missed.
|
||||||
|
expect(missingSections.join(', ')).toBe('');
|
||||||
}
|
}
|
||||||
if (claudeExists) {
|
if (claudeExists) {
|
||||||
const claude = fs.readFileSync(claudePath, 'utf-8');
|
const claude = fs.readFileSync(claudePath, 'utf-8');
|
||||||
|
|
|
||||||
|
|
@ -406,7 +406,9 @@ Do NOT fix any bugs. Do NOT use AskUserQuestion — just pick vitest.`,
|
||||||
}, 120_000);
|
}, 120_000);
|
||||||
});
|
});
|
||||||
|
|
||||||
// Module-level afterAll — finalize eval collector after all tests complete
|
// Module-level afterAll — finalize eval collector after all tests complete.
|
||||||
|
// Explicit 60s timeout: finalize does a JSON save + cross-run comparison and
|
||||||
|
// has been observed at 6.26s, past bun's 5s default hook timeout.
|
||||||
afterAll(async () => {
|
afterAll(async () => {
|
||||||
await finalizeEvalCollector(evalCollector);
|
await finalizeEvalCollector(evalCollector);
|
||||||
});
|
}, 60_000);
|
||||||
|
|
|
||||||
|
|
@ -158,7 +158,7 @@ describeE2E('/ship idempotency E2E (periodic, real-PTY)', () => {
|
||||||
const session = await launchClaudePty({
|
const session = await launchClaudePty({
|
||||||
permissionMode: 'plan',
|
permissionMode: 'plan',
|
||||||
cwd: fixture.workTree,
|
cwd: fixture.workTree,
|
||||||
timeoutMs: 720_000,
|
timeoutMs: 1_080_000,
|
||||||
// Disable network-y pieces so the agent can't reach actual github.
|
// Disable network-y pieces so the agent can't reach actual github.
|
||||||
env: { GH_TOKEN: 'mock-not-real', NO_COLOR: '1' },
|
env: { GH_TOKEN: 'mock-not-real', NO_COLOR: '1' },
|
||||||
seedSkills: true,
|
seedSkills: true,
|
||||||
|
|
@ -172,7 +172,7 @@ describeE2E('/ship idempotency E2E (periodic, real-PTY)', () => {
|
||||||
const since = session.mark();
|
const since = session.mark();
|
||||||
session.send('/ship\r');
|
session.send('/ship\r');
|
||||||
|
|
||||||
const budgetMs = 600_000;
|
const budgetMs = 900_000;
|
||||||
const start = Date.now();
|
const start = Date.now();
|
||||||
let lastPermSig = '';
|
let lastPermSig = '';
|
||||||
while (Date.now() - start < budgetMs) {
|
while (Date.now() - start < budgetMs) {
|
||||||
|
|
@ -234,6 +234,12 @@ describeE2E('/ship idempotency E2E (periodic, real-PTY)', () => {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Budget exhausted without a terminal signal: capture the tail NOW,
|
||||||
|
// while the session is still alive. Only the break paths above set
|
||||||
|
// evidence — without this, the timeout throw ships evidence: "".
|
||||||
|
if (outcome === 'timeout') {
|
||||||
|
evidence = session.visibleSince(since).slice(-3000);
|
||||||
|
}
|
||||||
} finally {
|
} finally {
|
||||||
await session.close();
|
await session.close();
|
||||||
}
|
}
|
||||||
|
|
@ -273,6 +279,6 @@ describeE2E('/ship idempotency E2E (periodic, real-PTY)', () => {
|
||||||
try { fs.rmSync(path.dirname(fixture.workTree), { recursive: true, force: true }); } catch { /* ignore */ }
|
try { fs.rmSync(path.dirname(fixture.workTree), { recursive: true, force: true }); } catch { /* ignore */ }
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
900_000, // 15 min wall clock
|
1_200_000, // 20 min wall clock
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue