test(e2e): periodic-tier repairs from the failure triage

Each fix traces to a receipt: brain-privacy-gate staged config never
reached the hermetic child (ambient GSTACK_HOME is scrubbed) and the
operator's remote-mode gbrain suppressed the gate — both now injected
per-test; ship-idempotency threw away its evidence on the timeout path
and ran a 600s budget its own subject can exceed (now 900s, evidence
captured); auto-decide-preserved gets the same headroom its sibling
plan-ceo tests got; context-skills' hides-checks scanned bash output
where an ls legitimately names old checkpoints (final-text scope now);
design names the missing section instead of a bare count and learns the
easing/duration/micro-interaction synonyms; qa-workflow's collector
afterAll gets an explicit 60s hook timeout.
This commit is contained in:
Garry Tan 2026-08-14 21:22:37 -07:00
parent 11b3e517d7
commit fa783470a6
No known key found for this signature in database
GPG Key ID: C1F69E85C74EFE1D
6 changed files with 57 additions and 37 deletions

View File

@ -111,7 +111,7 @@ describeE2E('AUTO_DECIDE opt-in preserved under Conductor flags (periodic)', ()
skillName: 'plan-ceo-review', skillName: 'plan-ceo-review',
inPlanMode: true, inPlanMode: true,
extraArgs: ['--disallowedTools', 'AskUserQuestion'], extraArgs: ['--disallowedTools', 'AskUserQuestion'],
timeoutMs: 300_000, timeoutMs: 540_000,
env: { GSTACK_HOME: tmpHome, CONDUCTOR_WORKSPACE_PATH: tmpHome }, env: { GSTACK_HOME: tmpHome, CONDUCTOR_WORKSPACE_PATH: tmpHome },
}); });
@ -135,5 +135,5 @@ describeE2E('AUTO_DECIDE opt-in preserved under Conductor flags (periodic)', ()
} finally { } finally {
try { fs.rmSync(tmpHome, { recursive: true, force: true }); } catch { /* best-effort */ } try { fs.rmSync(tmpHome, { recursive: true, force: true }); } catch { /* best-effort */ }
} }
}, 360_000); }, 660_000);
}); });

View File

@ -34,6 +34,12 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
// Stage a fresh GSTACK_HOME with artifacts_sync_mode_prompted=false. // Stage a fresh GSTACK_HOME with artifacts_sync_mode_prompted=false.
const gstackHome = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-gstack-')); const gstackHome = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-gstack-'));
const fakeBinDir = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-bin-')); const fakeBinDir = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-bin-'));
// Fresh HOME with NO ~/.claude.json: on a machine where gbrain is
// registered type=http, the preamble's remote-mode detection reads the
// operator's ~/.claude.json and echoes "ARTIFACTS_SYNC: remote-mode" —
// and the local privacy gate legitimately never fires. An empty HOME
// makes the detection find nothing.
const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-home-'));
// Seed the config so the gate's condition passes. // Seed the config so the gate's condition passes.
fs.writeFileSync( fs.writeFileSync(
@ -59,12 +65,16 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
const askUserQuestions: Array<{ input: Record<string, unknown> }> = []; const askUserQuestions: Array<{ input: Record<string, unknown> }> = [];
const binary = resolveClaudeBinary(); const binary = resolveClaudeBinary();
// Ambient env mutations — restored in finally so other tests in the file // Per-test env, merged LAST by the hermetic env builder (safe post-v1.39:
// don't inherit them. // the runner always passes a COMPLETE hermetic env, so overrides can't
const origGstackHome = process.env.GSTACK_HOME; // break auth). Ambient process.env.GSTACK_HOME mutation does NOT work
const origPath = process.env.PATH; // here — hermetic-env scrubs GSTACK_* and repoints GSTACK_HOME at its
process.env.GSTACK_HOME = gstackHome; // own singleton dir, so the staged config would never reach the child.
process.env.PATH = `${fakeBinDir}:${process.env.PATH ?? '/usr/bin:/bin:/opt/homebrew/bin'}`; const childEnv = {
GSTACK_HOME: gstackHome,
HOME: tempHome,
PATH: `${fakeBinDir}:${process.env.PATH ?? '/usr/bin:/bin:/opt/homebrew/bin'}`,
};
try { try {
// Pick a small skill with the preamble and load it via Read to force // Pick a small skill with the preamble and load it via Read to force
@ -85,12 +95,7 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
workingDirectory: gstackHome, workingDirectory: gstackHome,
maxTurns: 10, maxTurns: 10,
allowedTools: ['Read', 'Grep', 'Glob', 'Bash'], allowedTools: ['Read', 'Grep', 'Glob', 'Bash'],
// NOTE: do NOT pass `env:` here. When the Agent SDK gets an explicit env: childEnv,
// env object, its auth pipeline doesn't pick up ANTHROPIC_API_KEY the
// same way as when env is undefined (SDK-internal detail, verified
// against the plan-mode-no-op test which passes no env and auths
// cleanly). Instead, mutate process.env before the call so the SDK
// inherits our overrides ambiently.
...(binary ? { pathToClaudeCodeExecutable: binary } : {}), ...(binary ? { pathToClaudeCodeExecutable: binary } : {}),
canUseTool: async (toolName, input) => { canUseTool: async (toolName, input) => {
if (toolName === 'AskUserQuestion') { if (toolName === 'AskUserQuestion') {
@ -141,13 +146,9 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
// (The preamble is supposed to be idempotent within a session.) // (The preamble is supposed to be idempotent within a session.)
expect(privacyQuestions.length).toBe(1); expect(privacyQuestions.length).toBe(1);
} finally { } finally {
// Restore ambient env before other tests.
if (origGstackHome === undefined) delete process.env.GSTACK_HOME;
else process.env.GSTACK_HOME = origGstackHome;
if (origPath === undefined) delete process.env.PATH;
else process.env.PATH = origPath;
fs.rmSync(gstackHome, { recursive: true, force: true }); fs.rmSync(gstackHome, { recursive: true, force: true });
fs.rmSync(fakeBinDir, { recursive: true, force: true }); fs.rmSync(fakeBinDir, { recursive: true, force: true });
fs.rmSync(tempHome, { recursive: true, force: true });
} }
}, 180_000); }, 180_000);
@ -155,6 +156,10 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
// Same staging, but prompted=true this time. Gate should be silent. // Same staging, but prompted=true this time. Gate should be silent.
const gstackHome = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-off-')); const gstackHome = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-off-'));
const fakeBinDir = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-off-bin-')); const fakeBinDir = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-off-bin-'));
// Fresh HOME without a .claude.json — same rationale as the first test:
// without it the operator's ~/.claude.json flips the preamble into
// remote-mode and this negative test passes vacuously.
const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-off-home-'));
fs.writeFileSync( fs.writeFileSync(
path.join(gstackHome, 'config.yaml'), path.join(gstackHome, 'config.yaml'),
@ -171,11 +176,13 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
const askUserQuestions: Array<{ input: Record<string, unknown> }> = []; const askUserQuestions: Array<{ input: Record<string, unknown> }> = [];
const binary = resolveClaudeBinary(); const binary = resolveClaudeBinary();
// Ambient env mutations (see note on the first test). // Per-test env, merged LAST by the hermetic env builder (see note on the
const origGstackHome = process.env.GSTACK_HOME; // first test — ambient GSTACK_HOME mutation is scrubbed by hermetic-env).
const origPath = process.env.PATH; const childEnv = {
process.env.GSTACK_HOME = gstackHome; GSTACK_HOME: gstackHome,
process.env.PATH = `${fakeBinDir}:${process.env.PATH ?? '/usr/bin:/bin:/opt/homebrew/bin'}`; HOME: tempHome,
PATH: `${fakeBinDir}:${process.env.PATH ?? '/usr/bin:/bin:/opt/homebrew/bin'}`,
};
try { try {
await runAgentSdkTest({ await runAgentSdkTest({
@ -185,6 +192,7 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
workingDirectory: gstackHome, workingDirectory: gstackHome,
maxTurns: 4, maxTurns: 4,
allowedTools: ['Read', 'Grep', 'Glob', 'Bash'], allowedTools: ['Read', 'Grep', 'Glob', 'Bash'],
env: childEnv,
...(binary ? { pathToClaudeCodeExecutable: binary } : {}), ...(binary ? { pathToClaudeCodeExecutable: binary } : {}),
canUseTool: async (toolName, input) => { canUseTool: async (toolName, input) => {
if (toolName === 'AskUserQuestion') { if (toolName === 'AskUserQuestion') {
@ -216,12 +224,9 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
}); });
expect(privacyQuestions.length).toBe(0); expect(privacyQuestions.length).toBe(0);
} finally { } finally {
if (origGstackHome === undefined) delete process.env.GSTACK_HOME;
else process.env.GSTACK_HOME = origGstackHome;
if (origPath === undefined) delete process.env.PATH;
else process.env.PATH = origPath;
fs.rmSync(gstackHome, { recursive: true, force: true }); fs.rmSync(gstackHome, { recursive: true, force: true });
fs.rmSync(fakeBinDir, { recursive: true, force: true }); fs.rmSync(fakeBinDir, { recursive: true, force: true });
fs.rmSync(tempHome, { recursive: true, force: true });
} }
}, 180_000); }, 180_000);
}); });

View File

@ -447,8 +447,13 @@ Do NOT use AskUserQuestion.`,
// Match by filename timestamp (stable, unambiguous) plus a looser // Match by filename timestamp (stable, unambiguous) plus a looser
// prose check. // prose check.
const showsMain = /20260101-120000|main-work/.test(out); const showsMain = /20260101-120000|main-work/.test(out);
const hidesAlpha = !/20260202-120000/.test(out); // Hide checks scope to the FINAL text output only: fullOutputSurface
const hidesBeta = !/20260303-120000/.test(out); // includes bash tool_results, and a legitimate `ls` of the checkpoint
// dir lists every branch's filename. The filtering under test happens
// in the user-facing list, not in the agent's intermediate reads.
const finalText = result.output ?? '';
const hidesAlpha = !/20260202-120000|LISTCURR_ALPHA_TOKEN/.test(finalText);
const hidesBeta = !/20260303-120000|LISTCURR_BETA_TOKEN/.test(finalText);
const routed = skillCalls(result).includes('context-save'); const routed = skillCalls(result).includes('context-save');
const exitOk = ['success', 'error_max_turns'].includes(result.exitReason); const exitOk = ['success', 'error_max_turns'].includes(result.exitReason);

View File

@ -126,7 +126,7 @@ Write DESIGN.md and CLAUDE.md (or update it) in the working directory.`,
'Color': ['color', 'colour', 'palette', 'colors'], 'Color': ['color', 'colour', 'palette', 'colors'],
'Spacing': ['spacing', 'space', 'whitespace', 'gap'], 'Spacing': ['spacing', 'space', 'whitespace', 'gap'],
'Layout': ['layout', 'grid', 'structure', 'composition'], 'Layout': ['layout', 'grid', 'structure', 'composition'],
'Motion': ['motion', 'animation', 'transition', 'movement'], 'Motion': ['motion', 'animation', 'transition', 'movement', 'easing', 'duration', 'micro-interaction'],
}; };
const missingSections = Object.entries(sectionSynonyms).filter( const missingSections = Object.entries(sectionSynonyms).filter(
([_, synonyms]) => !synonyms.some(s => designContent.toLowerCase().includes(s)) ([_, synonyms]) => !synonyms.some(s => designContent.toLowerCase().includes(s))
@ -152,7 +152,9 @@ Write DESIGN.md and CLAUDE.md (or update it) in the working directory.`,
expect(['success', 'error_max_turns']).toContain(result.exitReason); expect(['success', 'error_max_turns']).toContain(result.exitReason);
expect(designExists).toBe(true); expect(designExists).toBe(true);
if (designExists) { if (designExists) {
expect(missingSections).toHaveLength(0); // join() so a failure names the offending section(s) — a bare
// toHaveLength(0) failure never prints WHICH synonym set missed.
expect(missingSections.join(', ')).toBe('');
} }
if (claudeExists) { if (claudeExists) {
const claude = fs.readFileSync(claudePath, 'utf-8'); const claude = fs.readFileSync(claudePath, 'utf-8');

View File

@ -406,7 +406,9 @@ Do NOT fix any bugs. Do NOT use AskUserQuestion — just pick vitest.`,
}, 120_000); }, 120_000);
}); });
// Module-level afterAll — finalize eval collector after all tests complete // Module-level afterAll — finalize eval collector after all tests complete.
// Explicit 60s timeout: finalize does a JSON save + cross-run comparison and
// has been observed at 6.26s, past bun's 5s default hook timeout.
afterAll(async () => { afterAll(async () => {
await finalizeEvalCollector(evalCollector); await finalizeEvalCollector(evalCollector);
}); }, 60_000);

View File

@ -158,7 +158,7 @@ describeE2E('/ship idempotency E2E (periodic, real-PTY)', () => {
const session = await launchClaudePty({ const session = await launchClaudePty({
permissionMode: 'plan', permissionMode: 'plan',
cwd: fixture.workTree, cwd: fixture.workTree,
timeoutMs: 720_000, timeoutMs: 1_080_000,
// Disable network-y pieces so the agent can't reach actual github. // Disable network-y pieces so the agent can't reach actual github.
env: { GH_TOKEN: 'mock-not-real', NO_COLOR: '1' }, env: { GH_TOKEN: 'mock-not-real', NO_COLOR: '1' },
seedSkills: true, seedSkills: true,
@ -172,7 +172,7 @@ describeE2E('/ship idempotency E2E (periodic, real-PTY)', () => {
const since = session.mark(); const since = session.mark();
session.send('/ship\r'); session.send('/ship\r');
const budgetMs = 600_000; const budgetMs = 900_000;
const start = Date.now(); const start = Date.now();
let lastPermSig = ''; let lastPermSig = '';
while (Date.now() - start < budgetMs) { while (Date.now() - start < budgetMs) {
@ -234,6 +234,12 @@ describeE2E('/ship idempotency E2E (periodic, real-PTY)', () => {
break; break;
} }
} }
// Budget exhausted without a terminal signal: capture the tail NOW,
// while the session is still alive. Only the break paths above set
// evidence — without this, the timeout throw ships evidence: "".
if (outcome === 'timeout') {
evidence = session.visibleSince(since).slice(-3000);
}
} finally { } finally {
await session.close(); await session.close();
} }
@ -273,6 +279,6 @@ describeE2E('/ship idempotency E2E (periodic, real-PTY)', () => {
try { fs.rmSync(path.dirname(fixture.workTree), { recursive: true, force: true }); } catch { /* ignore */ } try { fs.rmSync(path.dirname(fixture.workTree), { recursive: true, force: true }); } catch { /* ignore */ }
} }
}, },
900_000, // 15 min wall clock 1_200_000, // 20 min wall clock
); );
}); });