diff --git a/app/Console/Commands/SendPriceExperimentFunnelReportCommand.php b/app/Console/Commands/SendPriceExperimentFunnelReportCommand.php index e5ac0554..f6a99c90 100644 --- a/app/Console/Commands/SendPriceExperimentFunnelReportCommand.php +++ b/app/Console/Commands/SendPriceExperimentFunnelReportCommand.php @@ -7,6 +7,7 @@ use App\Services\Discord\DiscordWebhook; use App\Services\Stats\BinomialProportion; use App\Services\Stats\PriceExperimentFunnelCollector; use App\Services\Stats\ProportionSignificance; +use App\Services\Stats\SampleRatioMismatch; use App\Services\Stats\WelchTTest; use App\Support\Money; use Carbon\CarbonImmutable; @@ -29,6 +30,7 @@ class SendPriceExperimentFunnelReportCommand extends Command private PriceExperimentFunnelCollector $collector, private ProportionSignificance $significance, private WelchTTest $welch, + private SampleRatioMismatch $srm, ) { parent::__construct(); } @@ -44,7 +46,7 @@ class SendPriceExperimentFunnelReportCommand extends Command return self::SUCCESS; } - foreach ([...$this->tableLines($report), ...$this->significanceLines($report)] as $line) { + foreach ([...$this->tableLines($report), ...$this->srmLines($report), ...$this->significanceLines($report)] as $line) { $this->line($line); } @@ -101,6 +103,39 @@ class SendPriceExperimentFunnelReportCommand extends Command return $lines; } + /** + * Sample-ratio-mismatch check: the split must be ~50/50. A low p means the + * assignment (or the pipeline that filters it) is broken, which invalidates + * every comparison below — so this is checked first, on the raw assigned counts + * and on the matured subset (to catch differential attrition). + * + * @param array{variants: array>} $report + * @return list + */ + private function srmLines(array $report): array + { + $control = $report['variants'][PriceExperiment::CONTROL]; + $high = $report['variants'][PriceExperiment::HIGH]; + $lines = ['', 'SRM (assignment balance, expect 50/50 — a low p means the split is broken):']; + + foreach (['assigned' => 'assigned', 'assignedMature' => 'matured'] as $field => $label) { + $c = (int) $control[$field]; + $h = (int) $high[$field]; + + if ($c + $h === 0) { + $lines[] = sprintf(' %-8s control %d / high %d (n=0)', $label, $c, $h); + + continue; + } + + $srm = $this->srm->evenSplit($c, $h); + $flag = $srm['p'] < 0.01 ? ' ⚠ IMBALANCE — investigate before trusting results' : ''; + $lines[] = sprintf(' %-8s control %d / high %d χ²=%.2f p=%.3f%s', $label, $c, $h, $srm['chiSq'], $srm['p'], $flag); + } + + return $lines; + } + /** * Primary decision test — Welch on contribution-margin-per-user (control vs * high), the metric we maximise — plus the conversion guardrail (Wilson CIs + @@ -210,7 +245,7 @@ class SendPriceExperimentFunnelReportCommand extends Command { return [ 'title' => '💶 Price Experiment — Funnel (control vs high)', - 'description' => "```\n".implode("\n", [...$this->tableLines($report), ...$this->significanceLines($report)])."\n```", + 'description' => "```\n".implode("\n", [...$this->tableLines($report), ...$this->srmLines($report), ...$this->significanceLines($report)])."\n```", 'color' => 0x57F287, 'fields' => [ [ diff --git a/app/Services/Stats/Normal.php b/app/Services/Stats/Normal.php new file mode 100644 index 00000000..72f7bd22 --- /dev/null +++ b/app/Services/Stats/Normal.php @@ -0,0 +1,28 @@ + 0.0, 'p' => 1.0]; + } + + $expected = $total / 2.0; + $chiSq = (($countA - $expected) ** 2 + ($countB - $expected) ** 2) / $expected; + + // χ² with 1 df is Z²; its upper tail is P(|Z| > √χ²). + $p = 2.0 * (1.0 - Normal::cdf(sqrt($chiSq))); + + return ['chiSq' => $chiSq, 'p' => $p]; + } +} diff --git a/app/Services/Stats/WelchTTest.php b/app/Services/Stats/WelchTTest.php index 5d398a3f..d5eec454 100644 --- a/app/Services/Stats/WelchTTest.php +++ b/app/Services/Stats/WelchTTest.php @@ -13,8 +13,8 @@ namespace App\Services\Stats; * returned for context. * * ponytail: normal-approx p-value, exact only for large n. If a future experiment - * needs it at small n, swap normalCdf() for a Student-t CDF (regularised incomplete - * beta) — the statistic and df here are already exact. + * needs it at small n, swap Normal::cdf() for a Student-t CDF (regularised + * incomplete beta) — the statistic and df here are already exact. */ final class WelchTTest { @@ -38,24 +38,8 @@ final class WelchTTest (($varA / $nA) ** 2) / ($nA - 1) + (($varB / $nB) ** 2) / ($nB - 1) ); - $p = 2.0 * (1.0 - $this->normalCdf(abs($t))); + $p = 2.0 * (1.0 - Normal::cdf(abs($t))); return ['diff' => $diff, 'se' => $se, 't' => $t, 'df' => $df, 'p' => $p]; } - - private function normalCdf(float $x): float - { - return 0.5 * (1.0 + $this->erf($x / sqrt(2.0))); - } - - /** Abramowitz & Stegun 7.1.26 — |error| < 1.5e-7. */ - private function erf(float $x): float - { - $sign = $x < 0 ? -1.0 : 1.0; - $x = abs($x); - $t = 1.0 / (1.0 + 0.3275911 * $x); - $y = 1.0 - ((((1.061405429 * $t - 1.453152027) * $t + 1.421413741) * $t - 0.284496736) * $t + 0.254829592) * $t * exp(-$x * $x); - - return $sign * $y; - } } diff --git a/tests/Feature/SendPriceExperimentFunnelReportCommandTest.php b/tests/Feature/SendPriceExperimentFunnelReportCommandTest.php index f050c0a3..86a75b1f 100644 --- a/tests/Feature/SendPriceExperimentFunnelReportCommandTest.php +++ b/tests/Feature/SendPriceExperimentFunnelReportCommandTest.php @@ -175,6 +175,34 @@ it('prints the primary CM/user (Welch) and conversion guardrail blocks', functio ->assertSuccessful(); }); +it('flags a sample-ratio mismatch when the assignment split is lopsided', function () { + $signup = CarbonImmutable::parse('2026-06-05'); + + for ($i = 0; $i < 10; $i++) { + priceUser(PriceExperiment::CONTROL, $signup); + } + priceUser(PriceExperiment::HIGH, $signup); + + artisan('stats:price-experiment-funnel', ['--no-discord' => true]) + ->expectsOutputToContain('SRM') + ->expectsOutputToContain('IMBALANCE') + ->assertSuccessful(); +}); + +it('does not flag SRM when the split is balanced', function () { + $signup = CarbonImmutable::parse('2026-06-05'); + + for ($i = 0; $i < 4; $i++) { + priceUser(PriceExperiment::CONTROL, $signup); + priceUser(PriceExperiment::HIGH, $signup); + } + + artisan('stats:price-experiment-funnel', ['--no-discord' => true]) + ->expectsOutputToContain('SRM') + ->doesntExpectOutputToContain('IMBALANCE') + ->assertSuccessful(); +}); + it('posts the price experiment embed to discord with the monitoring-only warning', function () { config(['services.discord.ai_cohort_webhook_url' => 'https://discord.test/hook']); Http::fake(['discord.test/*' => Http::response('', 204)]); diff --git a/tests/Unit/SampleRatioMismatchTest.php b/tests/Unit/SampleRatioMismatchTest.php new file mode 100644 index 00000000..0d0746e4 --- /dev/null +++ b/tests/Unit/SampleRatioMismatchTest.php @@ -0,0 +1,32 @@ +evenSplit(50, 50); + + expect($result['chiSq'])->toBe(0.0) + ->and($result['p'])->toEqualWithDelta(1.0, 1e-6); +}); + +it('returns a neutral result for empty arms', function () { + $result = (new SampleRatioMismatch)->evenSplit(0, 0); + + expect($result['chiSq'])->toBe(0.0) + ->and($result['p'])->toBe(1.0); +}); + +it('matches the known chi-square for a 60/40 split of 100', function () { + // χ² = ((60−50)² + (40−50)²)/50 = 4 → p = 2·(1 − Φ(2)) = 0.0455. + $result = (new SampleRatioMismatch)->evenSplit(60, 40); + + expect($result['chiSq'])->toEqualWithDelta(4.0, 1e-9) + ->and($result['p'])->toEqualWithDelta(0.0455, 1e-3); +}); + +it('flags a badly lopsided split with a tiny p', function () { + $result = (new SampleRatioMismatch)->evenSplit(90, 10); + + expect($result['chiSq'])->toEqualWithDelta(64.0, 1e-9) + ->and($result['p'])->toBeLessThan(0.001); +});