mirror of https://github.com/garrytan/gstack.git
test(e2e): periodic-tier repairs from the failure triage
Each fix traces to a receipt: brain-privacy-gate staged config never reached the hermetic child (ambient GSTACK_HOME is scrubbed) and the operator's remote-mode gbrain suppressed the gate — both now injected per-test; ship-idempotency threw away its evidence on the timeout path and ran a 600s budget its own subject can exceed (now 900s, evidence captured); auto-decide-preserved gets the same headroom its sibling plan-ceo tests got; context-skills' hides-checks scanned bash output where an ls legitimately names old checkpoints (final-text scope now); design names the missing section instead of a bare count and learns the easing/duration/micro-interaction synonyms; qa-workflow's collector afterAll gets an explicit 60s hook timeout.
This commit is contained in:
parent
11b3e517d7
commit
fa783470a6
|
|
@ -111,7 +111,7 @@ describeE2E('AUTO_DECIDE opt-in preserved under Conductor flags (periodic)', ()
|
|||
skillName: 'plan-ceo-review',
|
||||
inPlanMode: true,
|
||||
extraArgs: ['--disallowedTools', 'AskUserQuestion'],
|
||||
timeoutMs: 300_000,
|
||||
timeoutMs: 540_000,
|
||||
env: { GSTACK_HOME: tmpHome, CONDUCTOR_WORKSPACE_PATH: tmpHome },
|
||||
});
|
||||
|
||||
|
|
@ -135,5 +135,5 @@ describeE2E('AUTO_DECIDE opt-in preserved under Conductor flags (periodic)', ()
|
|||
} finally {
|
||||
try { fs.rmSync(tmpHome, { recursive: true, force: true }); } catch { /* best-effort */ }
|
||||
}
|
||||
}, 360_000);
|
||||
}, 660_000);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -34,6 +34,12 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
|
|||
// Stage a fresh GSTACK_HOME with artifacts_sync_mode_prompted=false.
|
||||
const gstackHome = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-gstack-'));
|
||||
const fakeBinDir = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-bin-'));
|
||||
// Fresh HOME with NO ~/.claude.json: on a machine where gbrain is
|
||||
// registered type=http, the preamble's remote-mode detection reads the
|
||||
// operator's ~/.claude.json and echoes "ARTIFACTS_SYNC: remote-mode" —
|
||||
// and the local privacy gate legitimately never fires. An empty HOME
|
||||
// makes the detection find nothing.
|
||||
const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-home-'));
|
||||
|
||||
// Seed the config so the gate's condition passes.
|
||||
fs.writeFileSync(
|
||||
|
|
@ -59,12 +65,16 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
|
|||
const askUserQuestions: Array<{ input: Record<string, unknown> }> = [];
|
||||
const binary = resolveClaudeBinary();
|
||||
|
||||
// Ambient env mutations — restored in finally so other tests in the file
|
||||
// don't inherit them.
|
||||
const origGstackHome = process.env.GSTACK_HOME;
|
||||
const origPath = process.env.PATH;
|
||||
process.env.GSTACK_HOME = gstackHome;
|
||||
process.env.PATH = `${fakeBinDir}:${process.env.PATH ?? '/usr/bin:/bin:/opt/homebrew/bin'}`;
|
||||
// Per-test env, merged LAST by the hermetic env builder (safe post-v1.39:
|
||||
// the runner always passes a COMPLETE hermetic env, so overrides can't
|
||||
// break auth). Ambient process.env.GSTACK_HOME mutation does NOT work
|
||||
// here — hermetic-env scrubs GSTACK_* and repoints GSTACK_HOME at its
|
||||
// own singleton dir, so the staged config would never reach the child.
|
||||
const childEnv = {
|
||||
GSTACK_HOME: gstackHome,
|
||||
HOME: tempHome,
|
||||
PATH: `${fakeBinDir}:${process.env.PATH ?? '/usr/bin:/bin:/opt/homebrew/bin'}`,
|
||||
};
|
||||
|
||||
try {
|
||||
// Pick a small skill with the preamble and load it via Read to force
|
||||
|
|
@ -85,12 +95,7 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
|
|||
workingDirectory: gstackHome,
|
||||
maxTurns: 10,
|
||||
allowedTools: ['Read', 'Grep', 'Glob', 'Bash'],
|
||||
// NOTE: do NOT pass `env:` here. When the Agent SDK gets an explicit
|
||||
// env object, its auth pipeline doesn't pick up ANTHROPIC_API_KEY the
|
||||
// same way as when env is undefined (SDK-internal detail, verified
|
||||
// against the plan-mode-no-op test which passes no env and auths
|
||||
// cleanly). Instead, mutate process.env before the call so the SDK
|
||||
// inherits our overrides ambiently.
|
||||
env: childEnv,
|
||||
...(binary ? { pathToClaudeCodeExecutable: binary } : {}),
|
||||
canUseTool: async (toolName, input) => {
|
||||
if (toolName === 'AskUserQuestion') {
|
||||
|
|
@ -141,13 +146,9 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
|
|||
// (The preamble is supposed to be idempotent within a session.)
|
||||
expect(privacyQuestions.length).toBe(1);
|
||||
} finally {
|
||||
// Restore ambient env before other tests.
|
||||
if (origGstackHome === undefined) delete process.env.GSTACK_HOME;
|
||||
else process.env.GSTACK_HOME = origGstackHome;
|
||||
if (origPath === undefined) delete process.env.PATH;
|
||||
else process.env.PATH = origPath;
|
||||
fs.rmSync(gstackHome, { recursive: true, force: true });
|
||||
fs.rmSync(fakeBinDir, { recursive: true, force: true });
|
||||
fs.rmSync(tempHome, { recursive: true, force: true });
|
||||
}
|
||||
}, 180_000);
|
||||
|
||||
|
|
@ -155,6 +156,10 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
|
|||
// Same staging, but prompted=true this time. Gate should be silent.
|
||||
const gstackHome = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-off-'));
|
||||
const fakeBinDir = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-off-bin-'));
|
||||
// Fresh HOME without a .claude.json — same rationale as the first test:
|
||||
// without it the operator's ~/.claude.json flips the preamble into
|
||||
// remote-mode and this negative test passes vacuously.
|
||||
const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-off-home-'));
|
||||
|
||||
fs.writeFileSync(
|
||||
path.join(gstackHome, 'config.yaml'),
|
||||
|
|
@ -171,11 +176,13 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
|
|||
const askUserQuestions: Array<{ input: Record<string, unknown> }> = [];
|
||||
const binary = resolveClaudeBinary();
|
||||
|
||||
// Ambient env mutations (see note on the first test).
|
||||
const origGstackHome = process.env.GSTACK_HOME;
|
||||
const origPath = process.env.PATH;
|
||||
process.env.GSTACK_HOME = gstackHome;
|
||||
process.env.PATH = `${fakeBinDir}:${process.env.PATH ?? '/usr/bin:/bin:/opt/homebrew/bin'}`;
|
||||
// Per-test env, merged LAST by the hermetic env builder (see note on the
|
||||
// first test — ambient GSTACK_HOME mutation is scrubbed by hermetic-env).
|
||||
const childEnv = {
|
||||
GSTACK_HOME: gstackHome,
|
||||
HOME: tempHome,
|
||||
PATH: `${fakeBinDir}:${process.env.PATH ?? '/usr/bin:/bin:/opt/homebrew/bin'}`,
|
||||
};
|
||||
|
||||
try {
|
||||
await runAgentSdkTest({
|
||||
|
|
@ -185,6 +192,7 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
|
|||
workingDirectory: gstackHome,
|
||||
maxTurns: 4,
|
||||
allowedTools: ['Read', 'Grep', 'Glob', 'Bash'],
|
||||
env: childEnv,
|
||||
...(binary ? { pathToClaudeCodeExecutable: binary } : {}),
|
||||
canUseTool: async (toolName, input) => {
|
||||
if (toolName === 'AskUserQuestion') {
|
||||
|
|
@ -216,12 +224,9 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
|
|||
});
|
||||
expect(privacyQuestions.length).toBe(0);
|
||||
} finally {
|
||||
if (origGstackHome === undefined) delete process.env.GSTACK_HOME;
|
||||
else process.env.GSTACK_HOME = origGstackHome;
|
||||
if (origPath === undefined) delete process.env.PATH;
|
||||
else process.env.PATH = origPath;
|
||||
fs.rmSync(gstackHome, { recursive: true, force: true });
|
||||
fs.rmSync(fakeBinDir, { recursive: true, force: true });
|
||||
fs.rmSync(tempHome, { recursive: true, force: true });
|
||||
}
|
||||
}, 180_000);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -447,8 +447,13 @@ Do NOT use AskUserQuestion.`,
|
|||
// Match by filename timestamp (stable, unambiguous) plus a looser
|
||||
// prose check.
|
||||
const showsMain = /20260101-120000|main-work/.test(out);
|
||||
const hidesAlpha = !/20260202-120000/.test(out);
|
||||
const hidesBeta = !/20260303-120000/.test(out);
|
||||
// Hide checks scope to the FINAL text output only: fullOutputSurface
|
||||
// includes bash tool_results, and a legitimate `ls` of the checkpoint
|
||||
// dir lists every branch's filename. The filtering under test happens
|
||||
// in the user-facing list, not in the agent's intermediate reads.
|
||||
const finalText = result.output ?? '';
|
||||
const hidesAlpha = !/20260202-120000|LISTCURR_ALPHA_TOKEN/.test(finalText);
|
||||
const hidesBeta = !/20260303-120000|LISTCURR_BETA_TOKEN/.test(finalText);
|
||||
const routed = skillCalls(result).includes('context-save');
|
||||
const exitOk = ['success', 'error_max_turns'].includes(result.exitReason);
|
||||
|
||||
|
|
|
|||
|
|
@ -126,7 +126,7 @@ Write DESIGN.md and CLAUDE.md (or update it) in the working directory.`,
|
|||
'Color': ['color', 'colour', 'palette', 'colors'],
|
||||
'Spacing': ['spacing', 'space', 'whitespace', 'gap'],
|
||||
'Layout': ['layout', 'grid', 'structure', 'composition'],
|
||||
'Motion': ['motion', 'animation', 'transition', 'movement'],
|
||||
'Motion': ['motion', 'animation', 'transition', 'movement', 'easing', 'duration', 'micro-interaction'],
|
||||
};
|
||||
const missingSections = Object.entries(sectionSynonyms).filter(
|
||||
([_, synonyms]) => !synonyms.some(s => designContent.toLowerCase().includes(s))
|
||||
|
|
@ -152,7 +152,9 @@ Write DESIGN.md and CLAUDE.md (or update it) in the working directory.`,
|
|||
expect(['success', 'error_max_turns']).toContain(result.exitReason);
|
||||
expect(designExists).toBe(true);
|
||||
if (designExists) {
|
||||
expect(missingSections).toHaveLength(0);
|
||||
// join() so a failure names the offending section(s) — a bare
|
||||
// toHaveLength(0) failure never prints WHICH synonym set missed.
|
||||
expect(missingSections.join(', ')).toBe('');
|
||||
}
|
||||
if (claudeExists) {
|
||||
const claude = fs.readFileSync(claudePath, 'utf-8');
|
||||
|
|
|
|||
|
|
@ -406,7 +406,9 @@ Do NOT fix any bugs. Do NOT use AskUserQuestion — just pick vitest.`,
|
|||
}, 120_000);
|
||||
});
|
||||
|
||||
// Module-level afterAll — finalize eval collector after all tests complete
|
||||
// Module-level afterAll — finalize eval collector after all tests complete.
|
||||
// Explicit 60s timeout: finalize does a JSON save + cross-run comparison and
|
||||
// has been observed at 6.26s, past bun's 5s default hook timeout.
|
||||
afterAll(async () => {
|
||||
await finalizeEvalCollector(evalCollector);
|
||||
});
|
||||
}, 60_000);
|
||||
|
|
|
|||
|
|
@ -158,7 +158,7 @@ describeE2E('/ship idempotency E2E (periodic, real-PTY)', () => {
|
|||
const session = await launchClaudePty({
|
||||
permissionMode: 'plan',
|
||||
cwd: fixture.workTree,
|
||||
timeoutMs: 720_000,
|
||||
timeoutMs: 1_080_000,
|
||||
// Disable network-y pieces so the agent can't reach actual github.
|
||||
env: { GH_TOKEN: 'mock-not-real', NO_COLOR: '1' },
|
||||
seedSkills: true,
|
||||
|
|
@ -172,7 +172,7 @@ describeE2E('/ship idempotency E2E (periodic, real-PTY)', () => {
|
|||
const since = session.mark();
|
||||
session.send('/ship\r');
|
||||
|
||||
const budgetMs = 600_000;
|
||||
const budgetMs = 900_000;
|
||||
const start = Date.now();
|
||||
let lastPermSig = '';
|
||||
while (Date.now() - start < budgetMs) {
|
||||
|
|
@ -234,6 +234,12 @@ describeE2E('/ship idempotency E2E (periodic, real-PTY)', () => {
|
|||
break;
|
||||
}
|
||||
}
|
||||
// Budget exhausted without a terminal signal: capture the tail NOW,
|
||||
// while the session is still alive. Only the break paths above set
|
||||
// evidence — without this, the timeout throw ships evidence: "".
|
||||
if (outcome === 'timeout') {
|
||||
evidence = session.visibleSince(since).slice(-3000);
|
||||
}
|
||||
} finally {
|
||||
await session.close();
|
||||
}
|
||||
|
|
@ -273,6 +279,6 @@ describeE2E('/ship idempotency E2E (periodic, real-PTY)', () => {
|
|||
try { fs.rmSync(path.dirname(fixture.workTree), { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
}
|
||||
},
|
||||
900_000, // 15 min wall clock
|
||||
1_200_000, // 20 min wall clock
|
||||
);
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in New Issue