From 3dc2cde3467ef64190ca6fba1beb7e2abaaf857b Mon Sep 17 00:00:00 2001
From: Garry Tan <garrytan@gmail.com>
Date: Sat, 15 Aug 2026 17:02:04 -0700
Subject: [PATCH] test(evals): retro E2E passes require the report on disk
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Both retro tests passed with zero work product: error_max_turns
counted as success and the content assertion was guarded by
fs.existsSync — a run that burned 30 turns and wrote nothing recorded
green (red team). The report is now load-bearing for pass/fail.
---
 test/skill-e2e-retro.test.ts | 39 ++++++++++++++++++------------------
 1 file changed, 20 insertions(+), 19 deletions(-)

diff --git a/test/skill-e2e-retro.test.ts b/test/skill-e2e-retro.test.ts
index effc0a524..d8ff31b07 100644
--- a/test/skill-e2e-retro.test.ts
+++ b/test/skill-e2e-retro.test.ts
@@ -80,17 +80,18 @@ Write your retrospective to ${dir}/retro-output.md`,
     });
 
     logCost('/retro base-branch', result);
+    // The report is the work product: a run that exits max-turns without
+    // writing it is a FAIL, not a pass — otherwise this test cannot detect
+    // the most basic regression (the skill stops producing its report).
+    const retroPath = path.join(dir, 'retro-output.md');
+    const wroteReport = fs.existsSync(retroPath);
     recordE2E(evalCollector, '/retro default branch detection', 'Base branch detection', result, {
-      passed: ['success', 'error_max_turns'].includes(result.exitReason),
+      passed: ['success', 'error_max_turns'].includes(result.exitReason) && wroteReport,
     });
     expect(['success', 'error_max_turns']).toContain(result.exitReason);
-
-    // Verify retro output was produced
-    const retroPath = path.join(dir, 'retro-output.md');
-    if (fs.existsSync(retroPath)) {
-      const content = fs.readFileSync(retroPath, 'utf-8');
-      expect(content.length).toBeGreaterThan(100);
-    }
+    expect(wroteReport).toBe(true);
+    const content = fs.readFileSync(retroPath, 'utf-8');
+    expect(content.length).toBeGreaterThan(100);
   }, 480_000);
 });
 
@@ -165,18 +166,18 @@ Analyze the git history and produce the narrative report as described in the SKI
     });
 
     logCost('/retro', result);
-    recordE2E(evalCollector, '/retro', 'Retro E2E', result, {
-      passed: ['success', 'error_max_turns'].includes(result.exitReason),
-    });
-    // Accept error_max_turns — retro does many git commands to analyze history
-    expect(['success', 'error_max_turns']).toContain(result.exitReason);
-
-    // Verify the retro was written
+    // Accept error_max_turns (retro does many git commands to analyze
+    // history) — but only WITH the report on disk. The report is the work
+    // product; max-turns with nothing written is a fail.
     const retroPath = path.join(retroDir, 'retro-output.md');
-    if (fs.existsSync(retroPath)) {
-      const retro = fs.readFileSync(retroPath, 'utf-8');
-      expect(retro.length).toBeGreaterThan(100);
-    }
+    const wroteReport = fs.existsSync(retroPath);
+    recordE2E(evalCollector, '/retro', 'Retro E2E', result, {
+      passed: ['success', 'error_max_turns'].includes(result.exitReason) && wroteReport,
+    });
+    expect(['success', 'error_max_turns']).toContain(result.exitReason);
+    expect(wroteReport).toBe(true);
+    const retro = fs.readFileSync(retroPath, 'utf-8');
+    expect(retro.length).toBeGreaterThan(100);
   }, 420_000);
 });