gstack/test/skill-e2e-brain-privacy-gate.test.ts

/**
 * Privacy-gate E2E (periodic tier, paid).
 *
 * The gbrain-sync preamble block instructs the model to fire a one-time
 * AskUserQuestion when:
 *   - `BRAIN_SYNC: off` in the preamble echo (sync mode not on)
 *   - config `artifacts_sync_mode_prompted` is "false"
 *   - gbrain is detected on the host (binary on PATH or `gbrain doctor`
 *     --fast --json succeeds)
 *
 * This test stages all three conditions (via env + a fake `gbrain` binary
 * on PATH), runs a cheap gstack skill through the Agent SDK, intercepts
 * every tool use via canUseTool, and asserts: one of the AskUserQuestions
 * fired by the preamble is the privacy gate with its distinctive prose
 * and three options (full / artifacts-only / decline).
 *
 * Cost: ~$0.30-$0.50 per run. Periodic tier (EVALS=1 EVALS_TIER=periodic).
 *
 * See scripts/resolvers/preamble/generate-brain-sync-block.ts for the
 * prose contract this test locks in.
 */

import { test, expect } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { runAgentSdkTest, passThroughNonAskUserQuestion, resolveClaudeBinary } from './helpers/agent-sdk-runner';

const describeE2E = describeE2ETier('periodic');

describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
  test('gstack skill preamble fires the 3-option AskUserQuestion when gbrain is detected', async () => {
    // Stage a fresh GSTACK_HOME with artifacts_sync_mode_prompted=false.
    const gstackHome = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-gstack-'));
    const fakeBinDir = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-bin-'));
    // Fresh HOME with NO ~/.claude.json: on a machine where gbrain is
    // registered type=http, the preamble's remote-mode detection reads the
    // operator's ~/.claude.json and echoes "ARTIFACTS_SYNC: remote-mode" —
    // and the local privacy gate legitimately never fires. An empty HOME
    // makes the detection find nothing.
    const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-home-'));

    // Seed the config so the gate's condition passes.
    fs.writeFileSync(
      path.join(gstackHome, 'config.yaml'),
      'artifacts_sync_mode: off\nartifacts_sync_mode_prompted: false\n',
      { mode: 0o600 }
    );

    // Fake `gbrain` binary that makes the host-detection probe succeed.
    // The preamble checks `gbrain doctor --fast --json` OR `which gbrain`.
    // Either branch counts as "gbrain detected."
    fs.writeFileSync(
      path.join(fakeBinDir, 'gbrain'),
      '#!/bin/bash\n' +
        'case "$1" in\n' +
        '  doctor) echo \'{"status":"ok","schema_version":2}\' ; exit 0 ;;\n' +
        '  --version) echo "0.18.2" ; exit 0 ;;\n' +
        '  *) exit 0 ;;\n' +
        'esac\n',
      { mode: 0o755 }
    );

    const askUserQuestions: Array<{ input: Record<string, unknown> }> = [];
    const binary = resolveClaudeBinary();

    // Per-test env, merged LAST by the hermetic env builder (safe post-v1.39:
    // the runner always passes a COMPLETE hermetic env, so overrides can't
    // break auth). Ambient process.env.GSTACK_HOME mutation does NOT work
    // here — hermetic-env scrubs GSTACK_* and repoints GSTACK_HOME at its
    // own singleton dir, so the staged config would never reach the child.
    const childEnv = {
      GSTACK_HOME: gstackHome,
      HOME: tempHome,
      PATH: `${fakeBinDir}:${process.env.PATH ?? '/usr/bin:/bin:/opt/homebrew/bin'}`,
    };

    try {
      // Pick a small skill with the preamble and load it via Read to force
      // the model to execute every preamble directive. A narrow "run /learn"
      // prompt often gets reduced to a direct action, skipping the preamble
      // gates. Mirror the plan-mode-no-op test pattern: ask the model to
      // follow the skill's instructions in full.
      const learnSkill = path.resolve(
        import.meta.dir,
        '..',
        'learn',
        'SKILL.md'
      );
      await runAgentSdkTest({
        systemPrompt: { type: 'preset', preset: 'claude_code' },
        userPrompt:
          `Read the skill file at ${learnSkill} and follow its instructions from the top, including every preamble directive. Execute every bash block. If any AskUserQuestion fires, present it.`,
        workingDirectory: gstackHome,
        maxTurns: 10,
        allowedTools: ['Read', 'Grep', 'Glob', 'Bash'],
        env: childEnv,
        ...(binary ? { pathToClaudeCodeExecutable: binary } : {}),
        canUseTool: async (toolName, input) => {
          if (toolName === 'AskUserQuestion') {
            askUserQuestions.push({ input });
            // Auto-answer "Decline — keep everything local" (option C)
            // so the skill can continue without actually turning on sync.
            const q = (input.questions as Array<{
              question: string;
              options: Array<{ label: string }>;
            }>)[0];
            const decline =
              q.options.find((o) => /decline|keep everything local|no thanks/i.test(o.label)) ??
              q.options[q.options.length - 1]!;
            return {
              behavior: 'allow',
              updatedInput: {
                questions: input.questions,
                answers: { [q.question]: decline.label },
              },
            };
          }
          return passThroughNonAskUserQuestion(toolName, input);
        },
      });

      // Assertion 1: the privacy gate fired.
      const privacyQuestions = askUserQuestions.filter((aq) => {
        const qs = aq.input.questions as Array<{ question: string }>;
        return qs.some(
          (q) =>
            /publish.*session memory|private github repo|gbrain indexes/i.test(q.question)
        );
      });
      expect(privacyQuestions.length).toBeGreaterThanOrEqual(1);

      // Assertion 2: the question has the three expected options.
      const gate = privacyQuestions[0]!.input.questions as Array<{
        question: string;
        options: Array<{ label: string }>;
      }>;
      const labels = gate[0]!.options.map((o) => o.label.toLowerCase()).join(' | ');
      // Full / artifacts-only / decline are the three canonical options.
      expect(labels).toMatch(/everything|allowlisted|full/);
      expect(labels).toMatch(/artifact/);
      expect(labels).toMatch(/decline|local|no thanks/);

      // Assertion 3: the gate should NOT fire twice in one run.
      // (The preamble is supposed to be idempotent within a session.)
      expect(privacyQuestions.length).toBe(1);
    } finally {
      fs.rmSync(gstackHome, { recursive: true, force: true });
      fs.rmSync(fakeBinDir, { recursive: true, force: true });
      fs.rmSync(tempHome, { recursive: true, force: true });
    }
  }, 180_000);

  test('privacy gate does NOT fire when artifacts_sync_mode_prompted is already true', async () => {
    // Same staging, but prompted=true this time. Gate should be silent.
    const gstackHome = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-off-'));
    const fakeBinDir = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-off-bin-'));
    // Fresh HOME without a .claude.json — same rationale as the first test:
    // without it the operator's ~/.claude.json flips the preamble into
    // remote-mode and this negative test passes vacuously.
    const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), 'privacy-gate-off-home-'));

    fs.writeFileSync(
      path.join(gstackHome, 'config.yaml'),
      'artifacts_sync_mode: off\nartifacts_sync_mode_prompted: true\n',
      { mode: 0o600 }
    );

    fs.writeFileSync(
      path.join(fakeBinDir, 'gbrain'),
      '#!/bin/bash\necho \'{"status":"ok"}\'\nexit 0\n',
      { mode: 0o755 }
    );

    const askUserQuestions: Array<{ input: Record<string, unknown> }> = [];
    const binary = resolveClaudeBinary();

    // Per-test env, merged LAST by the hermetic env builder (see note on the
    // first test — ambient GSTACK_HOME mutation is scrubbed by hermetic-env).
    const childEnv = {
      GSTACK_HOME: gstackHome,
      HOME: tempHome,
      PATH: `${fakeBinDir}:${process.env.PATH ?? '/usr/bin:/bin:/opt/homebrew/bin'}`,
    };

    try {
      await runAgentSdkTest({
        systemPrompt: { type: 'preset', preset: 'claude_code' },
        userPrompt:
          'Run /learn with no arguments. Just report the learnings count.',
        workingDirectory: gstackHome,
        maxTurns: 4,
        allowedTools: ['Read', 'Grep', 'Glob', 'Bash'],
        env: childEnv,
        ...(binary ? { pathToClaudeCodeExecutable: binary } : {}),
        canUseTool: async (toolName, input) => {
          if (toolName === 'AskUserQuestion') {
            askUserQuestions.push({ input });
            // Pass through whatever the model asks; don't prefer anything.
            const q = (input.questions as Array<{
              question: string;
              options: Array<{ label: string }>;
            }>)[0];
            return {
              behavior: 'allow',
              updatedInput: {
                questions: input.questions,
                answers: { [q.question]: q.options[0]!.label },
              },
            };
          }
          return passThroughNonAskUserQuestion(toolName, input);
        },
      });

      // No AskUserQuestion should have matched the privacy gate's prose.
      const privacyQuestions = askUserQuestions.filter((aq) => {
        const qs = aq.input.questions as Array<{ question: string }>;
        return qs.some(
          (q) =>
            /publish.*session memory|private github repo|gbrain indexes/i.test(q.question)
        );
      });
      expect(privacyQuestions.length).toBe(0);
    } finally {
      fs.rmSync(gstackHome, { recursive: true, force: true });
      fs.rmSync(fakeBinDir, { recursive: true, force: true });
      fs.rmSync(tempHome, { recursive: true, force: true });
    }
  }, 180_000);
});