From 78545d314adcb21132fec71bee8bfe8e000c459b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vi=CC=81ctor=20Falco=CC=81n?= <victoor89@gmail.com>
Date: Sat, 18 Jul 2026 19:25:19 +0200
Subject: [PATCH] fix(stats): withhold the Welch verdict below a small-sample
 floor

The CM/user p-value uses a normal approximation to Student's t, which is
anti-conservative at small n (thin tails can read as significant when a real
t-test would not). Below 30 matured users per arm the report now shows the effect
size and t but withholds the significance verdict, only declaring significance
once both arms clear the floor.
---
 ...SendPriceExperimentFunnelReportCommand.php | 35 ++++++++++++++-----
 ...PriceExperimentFunnelReportCommandTest.php | 32 +++++++++++++----
 2 files changed, 53 insertions(+), 14 deletions(-)

diff --git a/app/Console/Commands/SendPriceExperimentFunnelReportCommand.php b/app/Console/Commands/SendPriceExperimentFunnelReportCommand.php
index 49475275..638cdd68 100644
--- a/app/Console/Commands/SendPriceExperimentFunnelReportCommand.php
+++ b/app/Console/Commands/SendPriceExperimentFunnelReportCommand.php
@@ -26,6 +26,12 @@ class SendPriceExperimentFunnelReportCommand extends Command
         PriceExperiment::HIGH => 'high',
     ];
 
+    /**
+     * Per-arm matured floor below which the Welch p-value (normal approximation to
+     * Student's t) is too anti-conservative to trust, so the verdict is withheld.
+     */
+    private const MIN_WELCH_N = 30;
+
     public function __construct(
         private PriceExperimentFunnelCollector $collector,
         private ProportionSignificance $significance,
@@ -193,14 +199,27 @@ class SendPriceExperimentFunnelReportCommand extends Command
                 $this->cmMean($high), $this->cmVariance($high), $high['assignedMature'],
                 $this->cmMean($control), $this->cmVariance($control), $control['assignedMature'],
             );
-            $significant = $welch['p'] < 0.05;
-            $lines[] = sprintf(
-                '  Δ high−control: %s/user  t=%.2f  p=%.3f %s α=0.05 -> %s.%s',
-                Money::format((int) round($welch['diff']), $currency),
-                $welch['t'], $welch['p'], $significant ? '<' : '≥',
-                $significant ? 'significant' : 'not significant',
-                $significant ? '' : ' Keep running.',
-            );
+            $delta = Money::format((int) round($welch['diff']), $currency);
+            $minN = min((int) $control['assignedMature'], (int) $high['assignedMature']);
+
+            if ($minN < self::MIN_WELCH_N) {
+                // The p-value uses a normal approximation to Student's t; below a
+                // per-arm floor its tails are too thin (anti-conservative), so we
+                // show the effect size but withhold the verdict rather than risk a
+                // small-sample false positive.
+                $lines[] = sprintf(
+                    '  Δ high−control: %s/user  t=%.2f  (small sample: min n=%d < %d → p unreliable, verdict withheld)',
+                    $delta, $welch['t'], $minN, self::MIN_WELCH_N,
+                );
+            } else {
+                $significant = $welch['p'] < 0.05;
+                $lines[] = sprintf(
+                    '  Δ high−control: %s/user  t=%.2f  p=%.3f %s α=0.05 -> %s.%s',
+                    $delta, $welch['t'], $welch['p'], $significant ? '<' : '≥',
+                    $significant ? 'significant' : 'not significant',
+                    $significant ? '' : ' Keep running.',
+                );
+            }
         }
 
         $lines[] = '';
diff --git a/tests/Feature/SendPriceExperimentFunnelReportCommandTest.php b/tests/Feature/SendPriceExperimentFunnelReportCommandTest.php
index 940176f6..5be02025 100644
--- a/tests/Feature/SendPriceExperimentFunnelReportCommandTest.php
+++ b/tests/Feature/SendPriceExperimentFunnelReportCommandTest.php
@@ -159,7 +159,7 @@ it('keeps the split even when a winner is forced, instead of collapsing onto one
         ->and($variants[PriceExperiment::HIGH]['assigned'])->toBe(1);
 });
 
-it('prints the primary CM/user (Welch) and conversion guardrail blocks', function () {
+it('prints the primary CM/user (Welch) and conversion guardrail blocks but withholds the verdict at small n', function () {
     $signup = CarbonImmutable::parse('2026-06-05');
 
     for ($i = 0; $i < 3; $i++) {
@@ -169,11 +169,31 @@ it('prints the primary CM/user (Welch) and conversion guardrail blocks', functio
         priceUser(PriceExperiment::HIGH, $signup);
     }
 
-    artisan('stats:price-experiment-funnel', ['--no-discord' => true])
-        ->expectsOutputToContain('PRIMARY')
-        ->expectsOutputToContain('GUARDRAIL')
-        ->expectsOutputToContain('Fisher exact')
-        ->assertSuccessful();
+    Artisan::call('stats:price-experiment-funnel', ['--no-discord' => true]);
+    $output = Artisan::output();
+
+    expect($output)->toContain('PRIMARY')
+        ->and($output)->toContain('GUARDRAIL')
+        ->and($output)->toContain('Fisher exact')
+        ->and($output)->toContain('verdict withheld') // 6 per arm < MIN_WELCH_N
+        ->and($output)->not->toContain('α=0.05 ->'); // no significance verdict at small n
+});
+
+it('emits a Welch significance verdict once both arms clear the small-sample floor', function () {
+    $signup = CarbonImmutable::parse('2026-06-05');
+
+    // 30 per arm (== MIN_WELCH_N), half net-active payers, so there is real variance.
+    for ($i = 0; $i < 30; $i++) {
+        $sub = $i % 2 === 0 ? ['status' => 'active', 'at' => $signup] : null;
+        priceUser(PriceExperiment::CONTROL, $signup, $sub === null ? null : [...$sub, 'price' => 'price_control']);
+        priceUser(PriceExperiment::HIGH, $signup, $sub === null ? null : [...$sub, 'price' => 'price_high']);
+    }
+
+    Artisan::call('stats:price-experiment-funnel', ['--no-discord' => true]);
+    $output = Artisan::output();
+
+    expect($output)->toContain('α=0.05 ->')
+        ->and($output)->not->toContain('verdict withheld');
 });
 
 it('surfaces an unmapped-price warning in the report, not just the logs', function () {