test(e2e): periodic-tier repairs from the failure triage

Each fix traces to a receipt: brain-privacy-gate staged config never
reached the hermetic child (ambient GSTACK_HOME is scrubbed) and the
operator's remote-mode gbrain suppressed the gate — both now injected
per-test; ship-idempotency threw away its evidence on the timeout path
and ran a 600s budget its own subject can exceed (now 900s, evidence
captured); auto-decide-preserved gets the same headroom its sibling
plan-ceo tests got; context-skills' hides-checks scanned bash output
where an ls legitimately names old checkpoints (final-text scope now);
design names the missing section instead of a bare count and learns the
easing/duration/micro-interaction synonyms; qa-workflow's collector
afterAll gets an explicit 60s hook timeout.
This commit is contained in:
Garry Tan 2026-08-14 21:22:37 -07:00
parent 11b3e517d7
commit fa783470a6
No known key found for this signature in database
GPG Key ID: C1F69E85C74EFE1D
6 changed files with 57 additions and 37 deletions

View File

@ -111,7 +111,7 @@ describeE2E('AUTO_DECIDE opt-in preserved under Conductor flags (periodic)', ()
skillName: 'plan-ceo-review',
inPlanMode: true,
extraArgs: ['--disallowedTools', 'AskUserQuestion'],
timeoutMs: 300_000,
timeoutMs: 540_000,
env: { GSTACK_HOME: tmpHome, CONDUCTOR_WORKSPACE_PATH: tmpHome },
});
@ -135,5 +135,5 @@ describeE2E('AUTO_DECIDE opt-in preserved under Conductor flags (periodic)', ()
} finally {
try { fs.rmSync(tmpHome, { recursive: true, force: true }); } catch { /* best-effort */ }
}
}, 360_000);
}, 660_000);
});

View File

@ -34,6 +34,12 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
// Stage a fresh GSTACK_HOME with artifacts_sync_mode_prompted=false.
const gstackHome = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-gstack-'));
const fakeBinDir = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-bin-'));
// Fresh HOME with NO ~/.claude.json: on a machine where gbrain is
// registered type=http, the preamble's remote-mode detection reads the
// operator's ~/.claude.json and echoes "ARTIFACTS_SYNC: remote-mode" —
// and the local privacy gate legitimately never fires. An empty HOME
// makes the detection find nothing.
const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-home-'));
// Seed the config so the gate's condition passes.
fs.writeFileSync(
@ -59,12 +65,16 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
const askUserQuestions: Array<{ input: Record<string, unknown> }> = [];
const binary = resolveClaudeBinary();
// Ambient env mutations — restored in finally so other tests in the file
// don't inherit them.
const origGstackHome = process.env.GSTACK_HOME;
const origPath = process.env.PATH;
process.env.GSTACK_HOME = gstackHome;
process.env.PATH = `${fakeBinDir}:${process.env.PATH ?? '/usr/bin:/bin:/opt/homebrew/bin'}`;
// Per-test env, merged LAST by the hermetic env builder (safe post-v1.39:
// the runner always passes a COMPLETE hermetic env, so overrides can't
// break auth). Ambient process.env.GSTACK_HOME mutation does NOT work
// here — hermetic-env scrubs GSTACK_* and repoints GSTACK_HOME at its
// own singleton dir, so the staged config would never reach the child.
const childEnv = {
GSTACK_HOME: gstackHome,
HOME: tempHome,
PATH: `${fakeBinDir}:${process.env.PATH ?? '/usr/bin:/bin:/opt/homebrew/bin'}`,
};
try {
// Pick a small skill with the preamble and load it via Read to force
@ -85,12 +95,7 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
workingDirectory: gstackHome,
maxTurns: 10,
allowedTools: ['Read', 'Grep', 'Glob', 'Bash'],
// NOTE: do NOT pass `env:` here. When the Agent SDK gets an explicit
// env object, its auth pipeline doesn't pick up ANTHROPIC_API_KEY the
// same way as when env is undefined (SDK-internal detail, verified
// against the plan-mode-no-op test which passes no env and auths
// cleanly). Instead, mutate process.env before the call so the SDK
// inherits our overrides ambiently.
env: childEnv,
...(binary ? { pathToClaudeCodeExecutable: binary } : {}),
canUseTool: async (toolName, input) => {
if (toolName === 'AskUserQuestion') {
@ -141,13 +146,9 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
// (The preamble is supposed to be idempotent within a session.)
expect(privacyQuestions.length).toBe(1);
} finally {
// Restore ambient env before other tests.
if (origGstackHome === undefined) delete process.env.GSTACK_HOME;
else process.env.GSTACK_HOME = origGstackHome;
if (origPath === undefined) delete process.env.PATH;
else process.env.PATH = origPath;
fs.rmSync(gstackHome, { recursive: true, force: true });
fs.rmSync(fakeBinDir, { recursive: true, force: true });
fs.rmSync(tempHome, { recursive: true, force: true });
}
}, 180_000);
@ -155,6 +156,10 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
// Same staging, but prompted=true this time. Gate should be silent.
const gstackHome = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-off-'));
const fakeBinDir = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-off-bin-'));
// Fresh HOME without a .claude.json — same rationale as the first test:
// without it the operator's ~/.claude.json flips the preamble into
// remote-mode and this negative test passes vacuously.
const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-off-home-'));
fs.writeFileSync(
path.join(gstackHome, 'config.yaml'),
@ -171,11 +176,13 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
const askUserQuestions: Array<{ input: Record<string, unknown> }> = [];
const binary = resolveClaudeBinary();
// Ambient env mutations (see note on the first test).
const origGstackHome = process.env.GSTACK_HOME;
const origPath = process.env.PATH;
process.env.GSTACK_HOME = gstackHome;
process.env.PATH = `${fakeBinDir}:${process.env.PATH ?? '/usr/bin:/bin:/opt/homebrew/bin'}`;
// Per-test env, merged LAST by the hermetic env builder (see note on the
// first test — ambient GSTACK_HOME mutation is scrubbed by hermetic-env).
const childEnv = {
GSTACK_HOME: gstackHome,
HOME: tempHome,
PATH: `${fakeBinDir}:${process.env.PATH ?? '/usr/bin:/bin:/opt/homebrew/bin'}`,
};
try {
await runAgentSdkTest({
@ -185,6 +192,7 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
workingDirectory: gstackHome,
maxTurns: 4,
allowedTools: ['Read', 'Grep', 'Glob', 'Bash'],
env: childEnv,
...(binary ? { pathToClaudeCodeExecutable: binary } : {}),
canUseTool: async (toolName, input) => {
if (toolName === 'AskUserQuestion') {
@ -216,12 +224,9 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
});
expect(privacyQuestions.length).toBe(0);
} finally {
if (origGstackHome === undefined) delete process.env.GSTACK_HOME;
else process.env.GSTACK_HOME = origGstackHome;
if (origPath === undefined) delete process.env.PATH;
else process.env.PATH = origPath;
fs.rmSync(gstackHome, { recursive: true, force: true });
fs.rmSync(fakeBinDir, { recursive: true, force: true });
fs.rmSync(tempHome, { recursive: true, force: true });
}
}, 180_000);
});

View File

@ -447,8 +447,13 @@ Do NOT use AskUserQuestion.`,
// Match by filename timestamp (stable, unambiguous) plus a looser
// prose check.
const showsMain = /20260101-120000|main-work/.test(out);
const hidesAlpha = !/20260202-120000/.test(out);
const hidesBeta = !/20260303-120000/.test(out);
// Hide checks scope to the FINAL text output only: fullOutputSurface
// includes bash tool_results, and a legitimate `ls` of the checkpoint
// dir lists every branch's filename. The filtering under test happens
// in the user-facing list, not in the agent's intermediate reads.
const finalText = result.output ?? '';
const hidesAlpha = !/20260202-120000|LISTCURR_ALPHA_TOKEN/.test(finalText);
const hidesBeta = !/20260303-120000|LISTCURR_BETA_TOKEN/.test(finalText);
const routed = skillCalls(result).includes('context-save');
const exitOk = ['success', 'error_max_turns'].includes(result.exitReason);

View File

@ -126,7 +126,7 @@ Write DESIGN.md and CLAUDE.md (or update it) in the working directory.`,
'Color': ['color', 'colour', 'palette', 'colors'],
'Spacing': ['spacing', 'space', 'whitespace', 'gap'],
'Layout': ['layout', 'grid', 'structure', 'composition'],
'Motion': ['motion', 'animation', 'transition', 'movement'],
'Motion': ['motion', 'animation', 'transition', 'movement', 'easing', 'duration', 'micro-interaction'],
};
const missingSections = Object.entries(sectionSynonyms).filter(
([_, synonyms]) => !synonyms.some(s => designContent.toLowerCase().includes(s))
@ -152,7 +152,9 @@ Write DESIGN.md and CLAUDE.md (or update it) in the working directory.`,
expect(['success', 'error_max_turns']).toContain(result.exitReason);
expect(designExists).toBe(true);
if (designExists) {
expect(missingSections).toHaveLength(0);
// join() so a failure names the offending section(s) — a bare
// toHaveLength(0) failure never prints WHICH synonym set missed.
expect(missingSections.join(', ')).toBe('');
}
if (claudeExists) {
const claude = fs.readFileSync(claudePath, 'utf-8');

View File

@ -406,7 +406,9 @@ Do NOT fix any bugs. Do NOT use AskUserQuestion — just pick vitest.`,
}, 120_000);
});
// Module-level afterAll — finalize eval collector after all tests complete
// Module-level afterAll — finalize eval collector after all tests complete.
// Explicit 60s timeout: finalize does a JSON save + cross-run comparison and
// has been observed at 6.26s, past bun's 5s default hook timeout.
afterAll(async () => {
await finalizeEvalCollector(evalCollector);
});
}, 60_000);

View File

@ -158,7 +158,7 @@ describeE2E('/ship idempotency E2E (periodic, real-PTY)', () => {
const session = await launchClaudePty({
permissionMode: 'plan',
cwd: fixture.workTree,
timeoutMs: 720_000,
timeoutMs: 1_080_000,
// Disable network-y pieces so the agent can't reach actual github.
env: { GH_TOKEN: 'mock-not-real', NO_COLOR: '1' },
seedSkills: true,
@ -172,7 +172,7 @@ describeE2E('/ship idempotency E2E (periodic, real-PTY)', () => {
const since = session.mark();
session.send('/ship\r');
const budgetMs = 600_000;
const budgetMs = 900_000;
const start = Date.now();
let lastPermSig = '';
while (Date.now() - start < budgetMs) {
@ -234,6 +234,12 @@ describeE2E('/ship idempotency E2E (periodic, real-PTY)', () => {
break;
}
}
// Budget exhausted without a terminal signal: capture the tail NOW,
// while the session is still alive. Only the break paths above set
// evidence — without this, the timeout throw ships evidence: "".
if (outcome === 'timeout') {
evidence = session.visibleSince(since).slice(-3000);
}
} finally {
await session.close();
}
@ -273,6 +279,6 @@ describeE2E('/ship idempotency E2E (periodic, real-PTY)', () => {
try { fs.rmSync(path.dirname(fixture.workTree), { recursive: true, force: true }); } catch { /* ignore */ }
}
},
900_000, // 15 min wall clock
1_200_000, // 20 min wall clock
);
});