gstack/test/skill-e2e-triage.test.ts

/**
 * /ship test-failure ownership triage E2E.
 *
 * Rehomed VERBATIM from the pre-split monolith (test/skill-e2e.test.ts,
 * deleted on this branch): the monolith's filename never matched the paid
 * glob (`test/skill-e2e-*.test.ts` — note the hyphen), so this GATE-tier
 * test (`ship-triage` in E2E_TIERS) silently never executed after the
 * v1.56 split.
 *
 * DRIFT WARNING (attribution for the first paid run after rehoming): the
 * prompt references "Test Failure Ownership Triage (Steps T1-T4)" — no
 * such section exists in the current generated ship/SKILL.md (the skill
 * drifted while this test was a zombie). The body is copied faithfully
 * (no behavioral edits), so a failure here indicts the drift, not the
 * move. The only change vs the monolith body: the staged ship/SKILL.md is
 * extracted via test/helpers/skill-fixture.ts (extractSkillBody — full
 * skill-specific body, shared preamble dropped) per CLAUDE.md
 * "E2E test fixtures: extract, don't copy".
 */

import { test, expect, beforeAll, afterAll } from 'bun:test';
import { runSkillTest } from './helpers/session-runner';
import {
  ROOT, runId,
  describeIfSelected,
  copyDirSync, logCost, recordE2E,
  createEvalCollector, finalizeEvalCollector,
} from './helpers/e2e-helpers';
import { extractSkillBody } from './helpers/skill-fixture';
import { spawnSync } from 'child_process';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';

const evalCollector = createEvalCollector('e2e-triage');

// --- Triage E2E ---

describeIfSelected('Test Failure Triage E2E', ['ship-triage'], () => {
  let triageDir: string;

  beforeAll(() => {
    triageDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-triage-'));

    // Copy ship skill files, then replace the SKILL.md with the extracted
    // skill body (extract, don't copy).
    copyDirSync(path.join(ROOT, 'ship'), path.join(triageDir, 'ship'));
    fs.writeFileSync(
      path.join(triageDir, 'ship', 'SKILL.md'),
      extractSkillBody(path.join(ROOT, 'ship')),
    );

    const run = (cmd: string, args: string[]) =>
      spawnSync(cmd, args, { cwd: triageDir, stdio: 'pipe', timeout: 5000 });

    // Init git repo
    run('git', ['init', '-b', 'main']);
    run('git', ['config', 'user.email', 'test@test.com']);
    run('git', ['config', 'user.name', 'Test']);

    // Create a project with a pre-existing test failure on main
    fs.writeFileSync(path.join(triageDir, 'package.json'), JSON.stringify({
      name: 'triage-test-app',
      version: '1.0.0',
      scripts: { test: 'node test/run.js' },
    }, null, 2));

    fs.mkdirSync(path.join(triageDir, 'src'), { recursive: true });
    fs.mkdirSync(path.join(triageDir, 'test'), { recursive: true });

    // Source with a bug that exists on main (pre-existing)
    fs.writeFileSync(path.join(triageDir, 'src', 'math.js'), `
module.exports = {
  add: (a, b) => a + b,
  divide: (a, b) => a / b,  // BUG: no zero-division check (pre-existing)
};
`);

    // Test file that catches the pre-existing bug
    fs.writeFileSync(path.join(triageDir, 'test', 'math.test.js'), `
const { add, divide } = require('../src/math');

// This test passes
if (add(2, 3) !== 5) { console.error('FAIL: add(2,3) should be 5'); process.exit(1); }
console.log('PASS: add');

// This test FAILS — pre-existing bug (divide by zero returns Infinity, not an error)
try {
  const result = divide(10, 0);
  if (result === Infinity) { console.error('FAIL: divide(10,0) should throw, got Infinity'); process.exit(1); }
} catch(e) {
  console.log('PASS: divide zero check');
}
`);

    // Test runner — each test in a subprocess so one failure doesn't kill the other
    fs.writeFileSync(path.join(triageDir, 'test', 'run.js'), `
const { execSync } = require('child_process');
const path = require('path');
let failures = 0;
for (const f of ['math.test.js', 'string.test.js']) {
  try {
    execSync('node ' + path.join(__dirname, f), { stdio: 'inherit' });
  } catch (e) {
    failures++;
  }
}
if (failures > 0) process.exit(1);
`);

    // Commit on main with the pre-existing bug
    run('git', ['add', '.']);
    run('git', ['commit', '-m', 'initial: math utils with tests']);

    // Create feature branch
    run('git', ['checkout', '-b', 'feature/string-utils']);

    // Add new code with a new bug (in-branch)
    fs.writeFileSync(path.join(triageDir, 'src', 'string.js'), `
module.exports = {
  capitalize: (s) => s.charAt(0).toUpperCase() + s.slice(1),
  reverse: (s) => s.split('').reverse().join(''),
  truncate: (s, len) => s.substring(0, len),  // BUG: no null check (in-branch)
};
`);

    // Add test that catches the in-branch bug
    fs.writeFileSync(path.join(triageDir, 'test', 'string.test.js'), `
const { capitalize, reverse, truncate } = require('../src/string');

if (capitalize('hello') !== 'Hello') { console.error('FAIL: capitalize'); process.exit(1); }
console.log('PASS: capitalize');

if (reverse('abc') !== 'cba') { console.error('FAIL: reverse'); process.exit(1); }
console.log('PASS: reverse');

// This test FAILS — in-branch bug (null input causes TypeError)
try {
  truncate(null, 5);
  console.log('PASS: truncate null');
} catch(e) {
  console.error('FAIL: truncate(null, 5) threw: ' + e.message);
  process.exit(1);
}
`);

    run('git', ['add', '.']);
    run('git', ['commit', '-m', 'feat: add string utilities']);
  });

  afterAll(() => {
    try { fs.rmSync(triageDir, { recursive: true, force: true }); } catch {}
  });

  test('/ship triage correctly classifies in-branch vs pre-existing failures', async () => {
    const result = await runSkillTest({
      prompt: `Read the file ship/SKILL.md for the ship workflow instructions.

You are on the feature/string-utils branch. The base branch is main.
This is a test project — there is no remote, no PR to create.

Run the tests first:
\`\`\`bash
cd ${triageDir} && node test/run.js
\`\`\`

The tests will fail. Now run ONLY the Test Failure Ownership Triage (Steps T1-T4) from the ship workflow.

For each failing test, classify it as:
- **In-branch**: caused by changes on this branch (feature/string-utils)
- **Pre-existing**: existed before this branch (present on main)

Use git diff origin/main...HEAD (or git diff main...HEAD since there's no remote) to determine which files changed on this branch.

Output your classification for each failure clearly, labeling each as "IN-BRANCH" or "PRE-EXISTING" with your reasoning.

This is a solo repo (REPO_MODE=solo). For pre-existing failures, recommend fixing now.`,
      workingDirectory: triageDir,
      maxTurns: 20,
      allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
      timeout: 180_000,
      testName: 'ship-triage',
      runId,
    });

    logCost('/ship triage', result);

    const output = result.output || '';
    const outputLower = output.toLowerCase();

    // The triage should identify the string/truncate failure as in-branch
    const hasInBranch = outputLower.includes('in-branch') || outputLower.includes('in branch') || outputLower.includes('introduced');
    // The triage should identify the math/divide failure as pre-existing
    const hasPreExisting = outputLower.includes('pre-existing') || outputLower.includes('pre existing') || outputLower.includes('existed before');

    console.log(`Output identifies IN-BRANCH failures: ${hasInBranch}`);
    console.log(`Output identifies PRE-EXISTING failures: ${hasPreExisting}`);

    // Check that the string/truncate bug is classified as in-branch
    const mentionsTruncate = outputLower.includes('truncate') || outputLower.includes('string');
    const mentionsDivide = outputLower.includes('divide') || outputLower.includes('math');

    console.log(`Mentions truncate/string (in-branch bug): ${mentionsTruncate}`);
    console.log(`Mentions divide/math (pre-existing bug): ${mentionsDivide}`);

    // Verify BOTH failure classes are exercised (not just detected):
    // The test runner must have actually run both test files
    const ranMathTest = output.includes('math.test') || output.includes('FAIL: divide');
    const ranStringTest = output.includes('string.test') || output.includes('FAIL: truncate');
    console.log(`Ran math test file (pre-existing failure): ${ranMathTest}`);
    console.log(`Ran string test file (in-branch failure): ${ranStringTest}`);

    recordE2E(evalCollector, '/ship triage', 'Test Failure Triage E2E', result, {
      passed: result.exitReason === 'success' && hasInBranch && hasPreExisting,
      has_in_branch_classification: hasInBranch,
      has_pre_existing_classification: hasPreExisting,
      mentions_truncate: mentionsTruncate,
      mentions_divide: mentionsDivide,
      ran_both_test_files: ranMathTest && ranStringTest,
    });

    expect(result.exitReason).toBe('success');
    // Must classify at least one failure as in-branch AND one as pre-existing
    expect(hasInBranch).toBe(true);
    expect(hasPreExisting).toBe(true);
    // Must mention the specific bugs
    expect(mentionsTruncate).toBe(true);
    expect(mentionsDivide).toBe(true);
    // Must have actually run both test files (exercises both failure classes)
    expect(ranMathTest).toBe(true);
    expect(ranStringTest).toBe(true);
  }, 240_000);
});

// Module-level afterAll — finalize eval collector after all tests complete
afterAll(async () => {
  await finalizeEvalCollector(evalCollector);
});