mirror of https://github.com/garrytan/gstack.git
evals: kill the 3,372-line zombie monolith; revive 4 never-run tests
test/skill-e2e.test.ts survived the v1.56 split as a zombie: the paid glob needs the skill-e2e-* hyphen, so with EVALS=1 NOTHING has executed it for ~8 releases — and it held the ONLY implementations of four map-registered tests: review-coverage-audit (gate), plan-eng-coverage- audit (gate), ship-triage (gate), ship-idempotency (periodic). Three gate tests silently never ran — the exact 0%-execution class this branch exists to kill. Rehomed into test/skill-e2e-coverage-audit.test.ts, -triage.test.ts, and -ship-idempotency-sdk.test.ts with bodies byte-identical modulo collector wiring and fixture extraction (drift observed in the skills since v1.56 is DOCUMENTED in each header, not fixed — their first paid run in 8 releases must attribute failures to drift, not to this move). All 24 other monolith names were true duplicates of the split files — dropped with the monolith. Matrix rows added to both eval workflows; the paid glob's zombie-exclusion is now a commented regression pin. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
parent
9b8f8d7767
commit
1e0ff1e1e2
|
|
@ -86,11 +86,16 @@ jobs:
|
|||
file: test/skill-e2e-review.test.ts
|
||||
- name: e2e-retro
|
||||
file: test/skill-e2e-retro.test.ts
|
||||
# e2e-review-attribution is gate-only (all three of its tests are
|
||||
# gate-tier) — deliberately absent here; an all-skip shard would just
|
||||
# burn a container boot weekly.
|
||||
# e2e-review-attribution, e2e-coverage-audit, and e2e-triage are
|
||||
# gate-only (every test they hold is gate-tier) — deliberately absent
|
||||
# here; an all-skip shard would just burn a container boot weekly.
|
||||
- name: e2e-workflow
|
||||
file: test/skill-e2e-workflow.test.ts
|
||||
# Rehomed from the deleted pre-split monolith: the SDK-harness
|
||||
# ship-idempotency check (periodic; the real-PTY variant runs via
|
||||
# the periodic sharded runner).
|
||||
- name: e2e-ship-idempotency-sdk
|
||||
file: test/skill-e2e-ship-idempotency-sdk.test.ts
|
||||
- name: e2e-routing
|
||||
file: test/skill-routing-e2e.test.ts
|
||||
- name: e2e-codex
|
||||
|
|
|
|||
|
|
@ -112,6 +112,14 @@ jobs:
|
|||
file: test/skill-e2e-review-attribution.test.ts
|
||||
- name: e2e-workflow
|
||||
file: test/skill-e2e-workflow.test.ts
|
||||
# Rehomed from the deleted pre-split monolith (its filename never
|
||||
# matched the skill-e2e-* glob, so these gate tests silently never
|
||||
# ran). Both files hold gate-tier tests: review/plan-eng coverage
|
||||
# audits and the /ship failure-ownership triage.
|
||||
- name: e2e-coverage-audit
|
||||
file: test/skill-e2e-coverage-audit.test.ts
|
||||
- name: e2e-triage
|
||||
file: test/skill-e2e-triage.test.ts
|
||||
- name: e2e-routing
|
||||
file: test/skill-routing-e2e.test.ts
|
||||
- name: e2e-codex
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
*
|
||||
* Spawns `codex exec` with skills installed in a temp HOME, parses JSONL
|
||||
* output, and validates structured results. Follows the same pattern as
|
||||
* skill-e2e.test.ts but adapted for Codex CLI.
|
||||
* the skill-e2e-*.test.ts suites but adapted for Codex CLI.
|
||||
*
|
||||
* Prerequisites:
|
||||
* - `codex` binary installed (npm install -g @openai/codex)
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
/**
|
||||
* Shared helpers for E2E test files.
|
||||
*
|
||||
* Extracted from the monolithic skill-e2e.test.ts to support splitting
|
||||
* tests across multiple files by category.
|
||||
* Extracted from the (since-deleted) pre-split monolith to support
|
||||
* splitting tests across multiple skill-e2e-*.test.ts files by category.
|
||||
*/
|
||||
|
||||
import '../../lib/conductor-env-shim';
|
||||
|
|
|
|||
|
|
@ -34,6 +34,9 @@ describe('paid test enumeration', () => {
|
|||
expect(isPaidTestFile('test/codex-e2e.test.ts')).toBe(true);
|
||||
expect(isPaidTestFile('test/skill-e2e-triage-audit.test.ts')).toBe(true);
|
||||
// Outside the globs: no dash, extra suffix, or a free test.
|
||||
// 'test/skill-e2e.test.ts' is the DELETED pre-split monolith's name,
|
||||
// kept here as a regression pin: its glob-invisibility is exactly how
|
||||
// two gate tests went unexecuted for ~8 releases before the rehoming.
|
||||
expect(isPaidTestFile('test/skill-e2e.test.ts')).toBe(false);
|
||||
expect(isPaidTestFile('test/codex-e2e-recommendation-substance.test.ts')).toBe(false);
|
||||
expect(isPaidTestFile('test/paid-shards.test.ts')).toBe(false);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,191 @@
|
|||
/**
|
||||
* Coverage-audit E2E — /review and /plan-eng-review coverage-diagram flows.
|
||||
*
|
||||
* Rehomed VERBATIM from the pre-split monolith (test/skill-e2e.test.ts,
|
||||
* deleted on this branch): the monolith's filename never matched the paid
|
||||
* glob (`test/skill-e2e-*.test.ts` — note the hyphen), so these two GATE-tier
|
||||
* tests (`review-coverage-audit`, `plan-eng-coverage-audit` in E2E_TIERS)
|
||||
* silently never executed after the v1.56 split.
|
||||
*
|
||||
* DRIFT WARNING (attribution for the first paid run after rehoming): the
|
||||
* prompts reference "Step 4.75 (Test Coverage Diagram)" in review/SKILL.md
|
||||
* and a "Test Coverage Audit" section in plan-eng-review/SKILL.md. NEITHER
|
||||
* section exists in the current generated skills — the skills drifted while
|
||||
* these tests were zombies. Test bodies are copied faithfully (no behavioral
|
||||
* edits), so a failure here indicts the ~8 releases of drift, not the move.
|
||||
* The only change vs the monolith bodies: the staged SKILL.md fixtures are
|
||||
* extracted via test/helpers/skill-fixture.ts (extractSkillBody — full
|
||||
* skill-specific body, shared preamble dropped) per CLAUDE.md
|
||||
* "E2E test fixtures: extract, don't copy".
|
||||
*/
|
||||
|
||||
import { test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId,
|
||||
describeIfSelected,
|
||||
copyDirSync, logCost, recordE2E,
|
||||
createEvalCollector, finalizeEvalCollector,
|
||||
} from './helpers/e2e-helpers';
|
||||
import { extractSkillBody } from './helpers/skill-fixture';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
|
||||
const evalCollector = createEvalCollector('e2e-coverage-audit');
|
||||
|
||||
// --- Review Coverage Audit E2E ---
|
||||
|
||||
describeIfSelected('Review Coverage Audit E2E', ['review-coverage-audit'], () => {
|
||||
let reviewCoverageDir: string;
|
||||
|
||||
beforeAll(() => {
|
||||
reviewCoverageDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-review-coverage-'));
|
||||
|
||||
// Copy review skill files, then replace the SKILL.md with the extracted
|
||||
// skill body (extract, don't copy — the checklists/specialists in the
|
||||
// dir are small hand-written files and stay whole).
|
||||
copyDirSync(path.join(ROOT, 'review'), path.join(reviewCoverageDir, 'review'));
|
||||
fs.writeFileSync(
|
||||
path.join(reviewCoverageDir, 'review', 'SKILL.md'),
|
||||
extractSkillBody(path.join(ROOT, 'review')),
|
||||
);
|
||||
|
||||
// Use shared fixture for billing project with coverage gaps
|
||||
const { createCoverageAuditFixture } = require('./fixtures/coverage-audit-fixture');
|
||||
createCoverageAuditFixture(reviewCoverageDir);
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
try { fs.rmSync(reviewCoverageDir, { recursive: true, force: true }); } catch {}
|
||||
});
|
||||
|
||||
test('/review Step 4.75 produces coverage diagram', async () => {
|
||||
const result = await runSkillTest({
|
||||
prompt: `Read the file review/SKILL.md for the review workflow instructions.
|
||||
|
||||
You are on the feature/billing branch. The base branch is main.
|
||||
This is a test project — there is no remote, no PR to create.
|
||||
|
||||
ONLY run Step 4.75 (Test Coverage Diagram) from the review workflow.
|
||||
Skip all other steps (scope drift, checklist, design review, fix-first, etc.).
|
||||
|
||||
The source code is in ${reviewCoverageDir}/src/billing.ts.
|
||||
Existing tests are in ${reviewCoverageDir}/test/billing.test.ts.
|
||||
|
||||
Produce the ASCII coverage diagram showing which code paths are tested and which have gaps.
|
||||
Output the diagram directly.`,
|
||||
workingDirectory: reviewCoverageDir,
|
||||
maxTurns: 15,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
|
||||
timeout: 120_000,
|
||||
testName: 'review-coverage-audit',
|
||||
runId,
|
||||
});
|
||||
|
||||
logCost('/review coverage audit', result);
|
||||
recordE2E(evalCollector, '/review Step 4.75 coverage audit', 'Review Coverage Audit E2E', result, {
|
||||
passed: result.exitReason === 'success',
|
||||
});
|
||||
|
||||
expect(result.exitReason).toBe('success');
|
||||
|
||||
// Check output contains coverage diagram elements
|
||||
const output = result.output || '';
|
||||
const outputLower = output.toLowerCase();
|
||||
const hasGap = outputLower.includes('gap') || outputLower.includes('no test');
|
||||
const hasTested = outputLower.includes('tested') || output.includes('✓') || output.includes('★');
|
||||
const hasCoverage = outputLower.includes('coverage') || outputLower.includes('paths tested');
|
||||
|
||||
console.log(`Output has GAP markers: ${hasGap}`);
|
||||
console.log(`Output has TESTED markers: ${hasTested}`);
|
||||
console.log(`Output has coverage summary: ${hasCoverage}`);
|
||||
|
||||
// The agent MUST produce a coverage diagram with gap and tested markers
|
||||
expect(hasGap || hasTested).toBe(true);
|
||||
|
||||
// At minimum, the agent should have read the source and test files
|
||||
const readCalls = result.toolCalls.filter(tc => tc.tool === 'Read');
|
||||
expect(readCalls.length).toBeGreaterThan(0);
|
||||
}, 180_000);
|
||||
});
|
||||
|
||||
// --- Plan Eng Review Coverage Audit E2E ---
|
||||
|
||||
describeIfSelected('Plan Eng Review Coverage Audit E2E', ['plan-eng-coverage-audit'], () => {
|
||||
let planCoverageDir: string;
|
||||
|
||||
beforeAll(() => {
|
||||
planCoverageDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-plan-coverage-'));
|
||||
|
||||
// Copy plan-eng-review skill files, then replace the SKILL.md with the
|
||||
// extracted skill body (extract, don't copy).
|
||||
copyDirSync(path.join(ROOT, 'plan-eng-review'), path.join(planCoverageDir, 'plan-eng-review'));
|
||||
fs.writeFileSync(
|
||||
path.join(planCoverageDir, 'plan-eng-review', 'SKILL.md'),
|
||||
extractSkillBody(path.join(ROOT, 'plan-eng-review')),
|
||||
);
|
||||
|
||||
// Use shared fixture for billing project with coverage gaps
|
||||
const { createCoverageAuditFixture } = require('./fixtures/coverage-audit-fixture');
|
||||
createCoverageAuditFixture(planCoverageDir);
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
try { fs.rmSync(planCoverageDir, { recursive: true, force: true }); } catch {}
|
||||
});
|
||||
|
||||
test('/plan-eng-review coverage audit traces plan codepaths', async () => {
|
||||
const result = await runSkillTest({
|
||||
prompt: `Read the file plan-eng-review/SKILL.md for the plan review workflow instructions.
|
||||
|
||||
You are on the feature/billing branch. The base branch is main.
|
||||
This is a test project — there is no remote, no PR to create.
|
||||
|
||||
ONLY run the Test Coverage Audit section from the plan review workflow.
|
||||
Skip all other steps (architecture, code quality, performance, etc.).
|
||||
|
||||
The source code is in ${planCoverageDir}/src/billing.ts.
|
||||
Existing tests are in ${planCoverageDir}/test/billing.test.ts.
|
||||
|
||||
Produce the ASCII coverage diagram showing which code paths are tested and which have gaps.
|
||||
Output the diagram directly.`,
|
||||
workingDirectory: planCoverageDir,
|
||||
maxTurns: 15,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
|
||||
timeout: 120_000,
|
||||
testName: 'plan-eng-coverage-audit',
|
||||
runId,
|
||||
});
|
||||
|
||||
logCost('/plan-eng-review coverage audit', result);
|
||||
recordE2E(evalCollector, '/plan-eng-review coverage audit', 'Plan Eng Review Coverage Audit E2E', result, {
|
||||
passed: result.exitReason === 'success',
|
||||
});
|
||||
|
||||
expect(result.exitReason).toBe('success');
|
||||
|
||||
// Check output contains coverage diagram elements
|
||||
const output = result.output || '';
|
||||
const outputLower = output.toLowerCase();
|
||||
const hasGap = outputLower.includes('gap') || outputLower.includes('no test');
|
||||
const hasTested = outputLower.includes('tested') || output.includes('✓') || output.includes('★');
|
||||
const hasCoverage = outputLower.includes('coverage') || outputLower.includes('paths tested');
|
||||
|
||||
console.log(`Output has GAP markers: ${hasGap}`);
|
||||
console.log(`Output has TESTED markers: ${hasTested}`);
|
||||
console.log(`Output has coverage summary: ${hasCoverage}`);
|
||||
|
||||
// The agent MUST produce a coverage diagram with gap and tested markers
|
||||
expect(hasGap || hasTested).toBe(true);
|
||||
|
||||
// At minimum, the agent should have read the source and test files
|
||||
const readCalls = result.toolCalls.filter(tc => tc.tool === 'Read');
|
||||
expect(readCalls.length).toBeGreaterThan(0);
|
||||
}, 180_000);
|
||||
});
|
||||
|
||||
// Module-level afterAll — finalize eval collector after all tests complete
|
||||
afterAll(async () => {
|
||||
await finalizeEvalCollector(evalCollector);
|
||||
});
|
||||
|
|
@ -0,0 +1,139 @@
|
|||
/**
|
||||
* /ship idempotency guard E2E — SDK-harness variant (#649).
|
||||
*
|
||||
* Rehomed VERBATIM from the pre-split monolith (test/skill-e2e.test.ts,
|
||||
* deleted on this branch): the monolith's filename never matched the paid
|
||||
* glob (`test/skill-e2e-*.test.ts` — note the hyphen), so this periodic
|
||||
* test (`ship-idempotency` in E2E_TIERS) silently never executed after
|
||||
* the v1.56 split. The real-PTY variant lives in
|
||||
* test/skill-e2e-ship-idempotency.test.ts (`ship-idempotency-pty`) and
|
||||
* exercises the actual /ship skill end-to-end; this one is the synthetic
|
||||
* SDK-harness check the PTY variant's header contrasts itself against.
|
||||
*
|
||||
* DRIFT WARNING (attribution for the first paid run after rehoming): the
|
||||
* fixture slices ship/SKILL.md on the markers '## Step 4: Version bump',
|
||||
* '## Step 7: Push', and '## Step 8.5'. The current generated skill numbers
|
||||
* these Step 12 (Version bump) and Step 17 (Push) — every indexOf returns
|
||||
* -1 and ship-steps.md ends up essentially empty. The body is copied
|
||||
* faithfully (no behavioral edits, per the rehoming integrity rule), so a
|
||||
* failure here indicts the ~8 releases of drift, not the move. The fix
|
||||
* (repoint the markers or use test/helpers/skill-fixture.ts with the
|
||||
* current section names) is a deliberate follow-up, not smuggled into the
|
||||
* rehoming commit.
|
||||
*/
|
||||
|
||||
import { expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId,
|
||||
describeIfSelected, testConcurrentIfSelected,
|
||||
logCost, recordE2E,
|
||||
createEvalCollector, finalizeEvalCollector,
|
||||
} from './helpers/e2e-helpers';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
|
||||
const evalCollector = createEvalCollector('e2e-ship-idempotency-sdk');
|
||||
|
||||
// --- Ship idempotency (#649) ---
|
||||
describeIfSelected('Ship idempotency', ['ship-idempotency'], () => {
|
||||
let idempDir: string;
|
||||
const gitRun = (args: string[], cwd: string) =>
|
||||
spawnSync('git', args, { cwd, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
beforeAll(() => {
|
||||
idempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-ship-idemp-'));
|
||||
|
||||
// Create git repo with initial commit on main
|
||||
gitRun(['init', '-b', 'main'], idempDir);
|
||||
gitRun(['config', 'user.email', 'test@test.com'], idempDir);
|
||||
gitRun(['config', 'user.name', 'Test'], idempDir);
|
||||
|
||||
fs.writeFileSync(path.join(idempDir, 'app.ts'), 'console.log("v1");\n');
|
||||
fs.writeFileSync(path.join(idempDir, 'VERSION'), '0.1.0.0\n');
|
||||
fs.writeFileSync(path.join(idempDir, 'CHANGELOG.md'), '# Changelog\n');
|
||||
gitRun(['add', '.'], idempDir);
|
||||
gitRun(['commit', '-m', 'initial'], idempDir);
|
||||
|
||||
// Create feature branch with changes
|
||||
gitRun(['checkout', '-b', 'feat/my-feature'], idempDir);
|
||||
fs.writeFileSync(path.join(idempDir, 'app.ts'), 'console.log("v2");\n');
|
||||
gitRun(['add', 'app.ts'], idempDir);
|
||||
gitRun(['commit', '-m', 'feat: update to v2'], idempDir);
|
||||
|
||||
// Simulate prior /ship run: bump VERSION and write CHANGELOG entry
|
||||
fs.writeFileSync(path.join(idempDir, 'VERSION'), '0.2.0.0\n');
|
||||
fs.writeFileSync(path.join(idempDir, 'CHANGELOG.md'),
|
||||
'# Changelog\n\n## [0.2.0.0] — 2026-03-30\n\n- Updated app to v2\n');
|
||||
gitRun(['add', 'VERSION', 'CHANGELOG.md'], idempDir);
|
||||
gitRun(['commit', '-m', 'chore: bump version to 0.2.0.0'], idempDir);
|
||||
|
||||
// Extract just the idempotency-relevant sections from ship/SKILL.md
|
||||
const full = fs.readFileSync(path.join(ROOT, 'ship', 'SKILL.md'), 'utf-8');
|
||||
const step4Start = full.indexOf('## Step 4: Version bump');
|
||||
const step4End = full.indexOf('\n---\n', step4Start);
|
||||
const step7Start = full.indexOf('## Step 7: Push');
|
||||
const step8End = full.indexOf('## Step 8.5');
|
||||
const extracted = [
|
||||
full.slice(step4Start, step4End > step4Start ? step4End : step4Start + 500),
|
||||
full.slice(step7Start, step8End > step7Start ? step8End : step7Start + 500),
|
||||
].join('\n\n---\n\n');
|
||||
fs.writeFileSync(path.join(idempDir, 'ship-steps.md'), extracted);
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
try { fs.rmSync(idempDir, { recursive: true, force: true }); } catch {}
|
||||
});
|
||||
|
||||
testConcurrentIfSelected('ship-idempotency', async () => {
|
||||
const result = await runSkillTest({
|
||||
prompt: `You are in a git repo on branch feat/my-feature. A prior /ship run already:
|
||||
- Bumped VERSION from 0.1.0.0 to 0.2.0.0
|
||||
- Wrote a CHANGELOG entry for 0.2.0.0
|
||||
- But the push/PR step failed
|
||||
|
||||
Read ship-steps.md for the idempotency check instructions from the ship workflow.
|
||||
|
||||
Run ONLY the idempotency checks described in Steps 4 and 7. Do NOT actually push or create PRs (there is no remote).
|
||||
|
||||
After running the checks, write a report to ${idempDir}/idemp-result.md containing:
|
||||
- Whether VERSION was detected as ALREADY_BUMPED or not
|
||||
- Whether the push was detected as ALREADY_PUSHED or PUSH_NEEDED
|
||||
- The current VERSION value (should still be 0.2.0.0)
|
||||
|
||||
Do NOT modify VERSION or CHANGELOG. Only run the detection checks and report.`,
|
||||
workingDirectory: idempDir,
|
||||
maxTurns: 10,
|
||||
timeout: 60_000,
|
||||
testName: 'ship-idempotency',
|
||||
runId,
|
||||
});
|
||||
|
||||
logCost('/ship idempotency', result);
|
||||
recordE2E(evalCollector, '/ship idempotency guard', 'Ship idempotency', result);
|
||||
expect(result.exitReason).toBe('success');
|
||||
|
||||
// Verify VERSION was NOT modified
|
||||
const version = fs.readFileSync(path.join(idempDir, 'VERSION'), 'utf-8').trim();
|
||||
expect(version).toBe('0.2.0.0');
|
||||
|
||||
// Verify CHANGELOG was NOT duplicated
|
||||
const changelog = fs.readFileSync(path.join(idempDir, 'CHANGELOG.md'), 'utf-8');
|
||||
const versionEntries = (changelog.match(/## \[0\.2\.0\.0\]/g) || []).length;
|
||||
expect(versionEntries).toBe(1);
|
||||
|
||||
// Check the result report if it was written
|
||||
const reportPath = path.join(idempDir, 'idemp-result.md');
|
||||
if (fs.existsSync(reportPath)) {
|
||||
const report = fs.readFileSync(reportPath, 'utf-8');
|
||||
expect(report.toLowerCase()).toContain('already_bumped');
|
||||
}
|
||||
}, 120_000);
|
||||
});
|
||||
|
||||
// Module-level afterAll — finalize eval collector after all tests complete
|
||||
afterAll(async () => {
|
||||
await finalizeEvalCollector(evalCollector);
|
||||
});
|
||||
|
|
@ -11,8 +11,9 @@
|
|||
* 4. Does NOT append a duplicate CHANGELOG [0.0.2] entry
|
||||
* 5. Does NOT create a new "chore: bump version" commit
|
||||
*
|
||||
* Why real-PTY: the existing ship-idempotency test in skill-e2e.test.ts
|
||||
* uses the SDK harness with a synthetic prompt asking the agent to "run
|
||||
* Why real-PTY: the ship-idempotency test in
|
||||
* test/skill-e2e-ship-idempotency-sdk.test.ts uses the SDK harness with a
|
||||
* synthetic prompt asking the agent to "run
|
||||
* ONLY the idempotency checks." This test exercises the actual /ship
|
||||
* skill end-to-end against a real git fixture so a regression that
|
||||
* silently re-bumps despite the check passing would be caught.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,238 @@
|
|||
/**
|
||||
* /ship test-failure ownership triage E2E.
|
||||
*
|
||||
* Rehomed VERBATIM from the pre-split monolith (test/skill-e2e.test.ts,
|
||||
* deleted on this branch): the monolith's filename never matched the paid
|
||||
* glob (`test/skill-e2e-*.test.ts` — note the hyphen), so this GATE-tier
|
||||
* test (`ship-triage` in E2E_TIERS) silently never executed after the
|
||||
* v1.56 split.
|
||||
*
|
||||
* DRIFT WARNING (attribution for the first paid run after rehoming): the
|
||||
* prompt references "Test Failure Ownership Triage (Steps T1-T4)" — no
|
||||
* such section exists in the current generated ship/SKILL.md (the skill
|
||||
* drifted while this test was a zombie). The body is copied faithfully
|
||||
* (no behavioral edits), so a failure here indicts the drift, not the
|
||||
* move. The only change vs the monolith body: the staged ship/SKILL.md is
|
||||
* extracted via test/helpers/skill-fixture.ts (extractSkillBody — full
|
||||
* skill-specific body, shared preamble dropped) per CLAUDE.md
|
||||
* "E2E test fixtures: extract, don't copy".
|
||||
*/
|
||||
|
||||
import { test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId,
|
||||
describeIfSelected,
|
||||
copyDirSync, logCost, recordE2E,
|
||||
createEvalCollector, finalizeEvalCollector,
|
||||
} from './helpers/e2e-helpers';
|
||||
import { extractSkillBody } from './helpers/skill-fixture';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
|
||||
const evalCollector = createEvalCollector('e2e-triage');
|
||||
|
||||
// --- Triage E2E ---
|
||||
|
||||
describeIfSelected('Test Failure Triage E2E', ['ship-triage'], () => {
|
||||
let triageDir: string;
|
||||
|
||||
beforeAll(() => {
|
||||
triageDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-triage-'));
|
||||
|
||||
// Copy ship skill files, then replace the SKILL.md with the extracted
|
||||
// skill body (extract, don't copy).
|
||||
copyDirSync(path.join(ROOT, 'ship'), path.join(triageDir, 'ship'));
|
||||
fs.writeFileSync(
|
||||
path.join(triageDir, 'ship', 'SKILL.md'),
|
||||
extractSkillBody(path.join(ROOT, 'ship')),
|
||||
);
|
||||
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: triageDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
// Init git repo
|
||||
run('git', ['init', '-b', 'main']);
|
||||
run('git', ['config', 'user.email', 'test@test.com']);
|
||||
run('git', ['config', 'user.name', 'Test']);
|
||||
|
||||
// Create a project with a pre-existing test failure on main
|
||||
fs.writeFileSync(path.join(triageDir, 'package.json'), JSON.stringify({
|
||||
name: 'triage-test-app',
|
||||
version: '1.0.0',
|
||||
scripts: { test: 'node test/run.js' },
|
||||
}, null, 2));
|
||||
|
||||
fs.mkdirSync(path.join(triageDir, 'src'), { recursive: true });
|
||||
fs.mkdirSync(path.join(triageDir, 'test'), { recursive: true });
|
||||
|
||||
// Source with a bug that exists on main (pre-existing)
|
||||
fs.writeFileSync(path.join(triageDir, 'src', 'math.js'), `
|
||||
module.exports = {
|
||||
add: (a, b) => a + b,
|
||||
divide: (a, b) => a / b, // BUG: no zero-division check (pre-existing)
|
||||
};
|
||||
`);
|
||||
|
||||
// Test file that catches the pre-existing bug
|
||||
fs.writeFileSync(path.join(triageDir, 'test', 'math.test.js'), `
|
||||
const { add, divide } = require('../src/math');
|
||||
|
||||
// This test passes
|
||||
if (add(2, 3) !== 5) { console.error('FAIL: add(2,3) should be 5'); process.exit(1); }
|
||||
console.log('PASS: add');
|
||||
|
||||
// This test FAILS — pre-existing bug (divide by zero returns Infinity, not an error)
|
||||
try {
|
||||
const result = divide(10, 0);
|
||||
if (result === Infinity) { console.error('FAIL: divide(10,0) should throw, got Infinity'); process.exit(1); }
|
||||
} catch(e) {
|
||||
console.log('PASS: divide zero check');
|
||||
}
|
||||
`);
|
||||
|
||||
// Test runner — each test in a subprocess so one failure doesn't kill the other
|
||||
fs.writeFileSync(path.join(triageDir, 'test', 'run.js'), `
|
||||
const { execSync } = require('child_process');
|
||||
const path = require('path');
|
||||
let failures = 0;
|
||||
for (const f of ['math.test.js', 'string.test.js']) {
|
||||
try {
|
||||
execSync('node ' + path.join(__dirname, f), { stdio: 'inherit' });
|
||||
} catch (e) {
|
||||
failures++;
|
||||
}
|
||||
}
|
||||
if (failures > 0) process.exit(1);
|
||||
`);
|
||||
|
||||
// Commit on main with the pre-existing bug
|
||||
run('git', ['add', '.']);
|
||||
run('git', ['commit', '-m', 'initial: math utils with tests']);
|
||||
|
||||
// Create feature branch
|
||||
run('git', ['checkout', '-b', 'feature/string-utils']);
|
||||
|
||||
// Add new code with a new bug (in-branch)
|
||||
fs.writeFileSync(path.join(triageDir, 'src', 'string.js'), `
|
||||
module.exports = {
|
||||
capitalize: (s) => s.charAt(0).toUpperCase() + s.slice(1),
|
||||
reverse: (s) => s.split('').reverse().join(''),
|
||||
truncate: (s, len) => s.substring(0, len), // BUG: no null check (in-branch)
|
||||
};
|
||||
`);
|
||||
|
||||
// Add test that catches the in-branch bug
|
||||
fs.writeFileSync(path.join(triageDir, 'test', 'string.test.js'), `
|
||||
const { capitalize, reverse, truncate } = require('../src/string');
|
||||
|
||||
if (capitalize('hello') !== 'Hello') { console.error('FAIL: capitalize'); process.exit(1); }
|
||||
console.log('PASS: capitalize');
|
||||
|
||||
if (reverse('abc') !== 'cba') { console.error('FAIL: reverse'); process.exit(1); }
|
||||
console.log('PASS: reverse');
|
||||
|
||||
// This test FAILS — in-branch bug (null input causes TypeError)
|
||||
try {
|
||||
truncate(null, 5);
|
||||
console.log('PASS: truncate null');
|
||||
} catch(e) {
|
||||
console.error('FAIL: truncate(null, 5) threw: ' + e.message);
|
||||
process.exit(1);
|
||||
}
|
||||
`);
|
||||
|
||||
run('git', ['add', '.']);
|
||||
run('git', ['commit', '-m', 'feat: add string utilities']);
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
try { fs.rmSync(triageDir, { recursive: true, force: true }); } catch {}
|
||||
});
|
||||
|
||||
test('/ship triage correctly classifies in-branch vs pre-existing failures', async () => {
|
||||
const result = await runSkillTest({
|
||||
prompt: `Read the file ship/SKILL.md for the ship workflow instructions.
|
||||
|
||||
You are on the feature/string-utils branch. The base branch is main.
|
||||
This is a test project — there is no remote, no PR to create.
|
||||
|
||||
Run the tests first:
|
||||
\`\`\`bash
|
||||
cd ${triageDir} && node test/run.js
|
||||
\`\`\`
|
||||
|
||||
The tests will fail. Now run ONLY the Test Failure Ownership Triage (Steps T1-T4) from the ship workflow.
|
||||
|
||||
For each failing test, classify it as:
|
||||
- **In-branch**: caused by changes on this branch (feature/string-utils)
|
||||
- **Pre-existing**: existed before this branch (present on main)
|
||||
|
||||
Use git diff origin/main...HEAD (or git diff main...HEAD since there's no remote) to determine which files changed on this branch.
|
||||
|
||||
Output your classification for each failure clearly, labeling each as "IN-BRANCH" or "PRE-EXISTING" with your reasoning.
|
||||
|
||||
This is a solo repo (REPO_MODE=solo). For pre-existing failures, recommend fixing now.`,
|
||||
workingDirectory: triageDir,
|
||||
maxTurns: 20,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
|
||||
timeout: 180_000,
|
||||
testName: 'ship-triage',
|
||||
runId,
|
||||
});
|
||||
|
||||
logCost('/ship triage', result);
|
||||
|
||||
const output = result.output || '';
|
||||
const outputLower = output.toLowerCase();
|
||||
|
||||
// The triage should identify the string/truncate failure as in-branch
|
||||
const hasInBranch = outputLower.includes('in-branch') || outputLower.includes('in branch') || outputLower.includes('introduced');
|
||||
// The triage should identify the math/divide failure as pre-existing
|
||||
const hasPreExisting = outputLower.includes('pre-existing') || outputLower.includes('pre existing') || outputLower.includes('existed before');
|
||||
|
||||
console.log(`Output identifies IN-BRANCH failures: ${hasInBranch}`);
|
||||
console.log(`Output identifies PRE-EXISTING failures: ${hasPreExisting}`);
|
||||
|
||||
// Check that the string/truncate bug is classified as in-branch
|
||||
const mentionsTruncate = outputLower.includes('truncate') || outputLower.includes('string');
|
||||
const mentionsDivide = outputLower.includes('divide') || outputLower.includes('math');
|
||||
|
||||
console.log(`Mentions truncate/string (in-branch bug): ${mentionsTruncate}`);
|
||||
console.log(`Mentions divide/math (pre-existing bug): ${mentionsDivide}`);
|
||||
|
||||
// Verify BOTH failure classes are exercised (not just detected):
|
||||
// The test runner must have actually run both test files
|
||||
const ranMathTest = output.includes('math.test') || output.includes('FAIL: divide');
|
||||
const ranStringTest = output.includes('string.test') || output.includes('FAIL: truncate');
|
||||
console.log(`Ran math test file (pre-existing failure): ${ranMathTest}`);
|
||||
console.log(`Ran string test file (in-branch failure): ${ranStringTest}`);
|
||||
|
||||
recordE2E(evalCollector, '/ship triage', 'Test Failure Triage E2E', result, {
|
||||
passed: result.exitReason === 'success' && hasInBranch && hasPreExisting,
|
||||
has_in_branch_classification: hasInBranch,
|
||||
has_pre_existing_classification: hasPreExisting,
|
||||
mentions_truncate: mentionsTruncate,
|
||||
mentions_divide: mentionsDivide,
|
||||
ran_both_test_files: ranMathTest && ranStringTest,
|
||||
});
|
||||
|
||||
expect(result.exitReason).toBe('success');
|
||||
// Must classify at least one failure as in-branch AND one as pre-existing
|
||||
expect(hasInBranch).toBe(true);
|
||||
expect(hasPreExisting).toBe(true);
|
||||
// Must mention the specific bugs
|
||||
expect(mentionsTruncate).toBe(true);
|
||||
expect(mentionsDivide).toBe(true);
|
||||
// Must have actually run both test files (exercises both failure classes)
|
||||
expect(ranMathTest).toBe(true);
|
||||
expect(ranStringTest).toBe(true);
|
||||
}, 240_000);
|
||||
});
|
||||
|
||||
// Module-level afterAll — finalize eval collector after all tests complete
|
||||
afterAll(async () => {
|
||||
await finalizeEvalCollector(evalCollector);
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -200,7 +200,12 @@ describe('real-skill pins: section lists used by E2E fixtures', () => {
|
|||
});
|
||||
|
||||
describe('real-skill pins: body/head extraction used by E2E fixtures', () => {
|
||||
const BODY_EXTRACTED_SKILLS = ['scrape', 'skillify', 'context-save', 'context-restore'];
|
||||
// scrape/skillify/context-*: skill-e2e-skillify + skill-e2e-context-skills.
|
||||
// review/plan-eng-review/ship: skill-e2e-coverage-audit + skill-e2e-triage.
|
||||
const BODY_EXTRACTED_SKILLS = [
|
||||
'scrape', 'skillify', 'context-save', 'context-restore',
|
||||
'review', 'plan-eng-review', 'ship',
|
||||
];
|
||||
|
||||
for (const skill of BODY_EXTRACTED_SKILLS) {
|
||||
test(`extractSkillBody(${skill}) drops the shared preamble, keeps the flow`, () => {
|
||||
|
|
|
|||
Loading…
Reference in New Issue