fix(stats): correct the trial/pricing experiment funnel report (#679)
## Summary Audited `stats:experiment-funnel` end-to-end (data acquisition, queries, per-row counting, financial math, and statistics) and fixed the issues that made it misleading for the win/no-win decision. The report used to rank variants by an **absolute** contribution-margin total that mechanically favoured whichever variant matured fastest, and it declared statistical significance with a normal approximation that is invalid at the small conversion counts this experiment has. ## What changed **Presentation & comparability** - Replace `A2P%` (numerator not a subset of the denominator → could exceed 100%) with `Conv%` = conversions ÷ matured-assigned — always ≤100% and comparable across variants. - Print `MatU` (matured cohort size) so the rate denominators are visible and the table reconciles; revive `ARPU`. - Reframe the guidance: compare on per-user `Conv%`/`ARPU`, not the absolute `MRR`/`Cost`/`Burn`/`CM` totals (which scale with `MatU`). **Data correctness** - Count soft-deleted users (`withTrashed`) — they were assigned a variant and their connections incurred real cost. - Attribute by the deterministic `SubscriptionExperiment::bucket()` instead of the resolved Pennant flag, so setting `force_variant` (the winner rollout switch) no longer collapses the whole report onto one variant. Also stops writing Pennant rows as a side effect. - Resolve `MRR` for subscriptions on rotated/archived Stripe price ids (fetch prices by product), warn on any net-active sub whose price is unmapped, round yearly ÷ 12, and skip foreign-currency prices. - `Burn` counts only users who never earned net revenue (no subscription, or paid-then-refunded) — a paid-then-churned user is no longer booked as connect-and-leave leak. **Statistics** - Measure conversion as "ever charged, net of refund" (time-invariant) instead of a live active-now snapshot that biases older cohorts (which have had longer to churn). - Decide significance with **Fisher's exact test** (exact at any sample size), Bonferroni-corrected over the three arms, instead of the normal-approx z that overstates evidence when expected cell counts fall below 5. Add a Newcombe difference-of-proportions CI and a small-sample caveat. - Extract the inference into `App\Services\Stats\ProportionSignificance` (+ a `BinomialProportion` value object), with unit tests that pin the exact interval and p-value numbers. ## Validation Reconstructed every column in raw SQL against a production dump — all reconcile **to the cent / to the row**. Independent recomputation of the statistics (Wilson, Fisher exact, Newcombe, z) matches the command's output. ## Testing 26 tests (20 feature + 6 unit, 94 assertions). `pint` and `phpstan`/larastan green. ## Note This branch also carries two small pre-existing commits unrelated to the funnel (`fix(categories): fall back to gray…`, `chore(schedule): stop scheduling the stuck cohort report`). Happy to split them into their own PR if preferred.
This commit is contained in:
Víctor Falcón
GitHub
parent
d7963736d1
commit
3db03de86d
|
|
@ -4,7 +4,9 @@ namespace App\Console\Commands;
|
|||
|
||||
use App\Features\SubscriptionExperiment;
|
||||
use App\Services\Discord\DiscordWebhook;
|
||||
use App\Services\Stats\BinomialProportion;
|
||||
use App\Services\Stats\ExperimentFunnelCollector;
|
||||
use App\Services\Stats\ProportionSignificance;
|
||||
use Carbon\CarbonImmutable;
|
||||
use Illuminate\Console\Command;
|
||||
|
||||
|
|
@ -22,8 +24,10 @@ class SendExperimentFunnelReportCommand extends Command
|
|||
SubscriptionExperiment::PAY_NOW => 'pay_now',
|
||||
];
|
||||
|
||||
public function __construct(private ExperimentFunnelCollector $collector)
|
||||
{
|
||||
public function __construct(
|
||||
private ExperimentFunnelCollector $collector,
|
||||
private ProportionSignificance $significance,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
|
|
@ -42,6 +46,10 @@ class SendExperimentFunnelReportCommand extends Command
|
|||
$this->line($line);
|
||||
}
|
||||
|
||||
foreach ($this->significanceLines($report) as $line) {
|
||||
$this->line($line);
|
||||
}
|
||||
|
||||
if ($this->option('no-discord')) {
|
||||
$this->info('Skipped Discord (--no-discord).');
|
||||
|
||||
|
|
@ -66,30 +74,103 @@ class SendExperimentFunnelReportCommand extends Command
|
|||
{
|
||||
$revenue = $report['revenueAvailable'];
|
||||
$currency = $report['currency'];
|
||||
$lines = [sprintf('%-8s %5s %5s %5s %5s %6s %8s %8s %8s %8s', 'Variant', 'Assg', 'Actd', 'Card', 'Paid', 'A2P%', 'MRR', 'Cost', 'Burn', 'CM')];
|
||||
$lines = [sprintf(
|
||||
'%-8s %5s %5s %5s %5s %5s %6s %7s %7s %7s %7s %7s',
|
||||
'Variant', 'Assg', 'Actd', 'Card', 'MatU', 'Conv', 'Conv%', 'ARPU', 'MRR', 'Cost', 'Burn', 'CM',
|
||||
)];
|
||||
|
||||
foreach (self::LABELS as $key => $label) {
|
||||
$row = $report['variants'][$key];
|
||||
$mature = $row['activatedMature'] > 0;
|
||||
$mature = $row['assignedMature'] > 0;
|
||||
$showMoney = $revenue && $mature;
|
||||
|
||||
$lines[] = sprintf(
|
||||
'%-8s %5d %5d %5d %5d %6s %8s %8s %8s %8s',
|
||||
'%-8s %5d %5d %5d %5d %5d %6s %7s %7s %7s %7s %7s',
|
||||
$label,
|
||||
$row['assigned'],
|
||||
$row['activated'],
|
||||
$row['subscribed'],
|
||||
$row['activeMature'],
|
||||
$mature ? ((int) round($row['activationToPaidRate'] * 100)).'%' : 'pend',
|
||||
$revenue ? $this->money($row['mrrCents'], $currency) : '—',
|
||||
$row['assignedMature'],
|
||||
$row['convertedMature'],
|
||||
$mature ? ((int) round($row['conversionRate'] * 100)).'%' : 'pend',
|
||||
$showMoney && $row['arpuCents'] !== null ? $this->money($row['arpuCents'], $currency) : '—',
|
||||
$showMoney ? $this->money($row['mrrCents'], $currency) : '—',
|
||||
$mature ? $this->money($row['costCents'], $currency) : '—',
|
||||
$mature ? $this->money($row['wastedCostCents'], $currency) : '—',
|
||||
$revenue && $mature ? $this->money($row['contributionMarginCents'], $currency) : '—',
|
||||
$showMoney ? $this->money($row['contributionMarginCents'], $currency) : '—',
|
||||
);
|
||||
}
|
||||
|
||||
return $lines;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-variant conversion-rate uncertainty (95% Wilson interval) plus the
|
||||
* leader-vs-runner-up verdict from {@see ProportionSignificance} — a Fisher
|
||||
* exact test and a Newcombe difference interval, Bonferroni-corrected — so
|
||||
* "check significance before calling a winner" has the numbers behind it.
|
||||
*
|
||||
* @param array{startedAt: ?CarbonImmutable, currency: string, revenueAvailable: bool, costPerConnectionCents: int, variants: array<string, array<string, mixed>>} $report
|
||||
* @return list<string>
|
||||
*/
|
||||
private function significanceLines(array $report): array
|
||||
{
|
||||
$lines = ['', 'Significance (95% Wilson CI on Conv%, n = MatU):'];
|
||||
$arms = [];
|
||||
|
||||
foreach (self::LABELS as $key => $label) {
|
||||
$row = $report['variants'][$key];
|
||||
$n = (int) $row['assignedMature'];
|
||||
$k = (int) $row['convertedMature'];
|
||||
|
||||
if ($n <= 0) {
|
||||
$lines[] = sprintf(' %-8s pend (n=0)', $label);
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
[$low, $high] = $this->significance->wilsonInterval($k, $n);
|
||||
$lines[] = sprintf(' %-8s %6s [%6s – %6s] (n=%d)', $label, $this->percent($k / $n), $this->percent($low), $this->percent($high), $n);
|
||||
$arms[] = new BinomialProportion($label, $k, $n);
|
||||
}
|
||||
|
||||
if (count($arms) < 2) {
|
||||
$lines[] = 'Not enough matured variants to compare yet.';
|
||||
|
||||
return $lines;
|
||||
}
|
||||
|
||||
usort($arms, fn (BinomialProportion $a, BinomialProportion $b): int => $b->rate() <=> $a->rate());
|
||||
[$leader, $runnerUp] = [$arms[0], $arms[1]];
|
||||
$result = $this->significance->compare($leader, $runnerUp);
|
||||
|
||||
$lines[] = sprintf(
|
||||
'Leader %s vs %s: Δ %+.1f pts (95%% CI %+.1f … %+.1f pts, Newcombe).',
|
||||
$leader->label, $runnerUp->label,
|
||||
($leader->rate() - $runnerUp->rate()) * 100, $result['diffLow'] * 100, $result['diffHigh'] * 100,
|
||||
);
|
||||
$lines[] = sprintf(
|
||||
'Fisher exact p=%.3f %s α=%.3f (Bonferroni×3) -> %s.%s',
|
||||
$result['fisherP'], $result['significant'] ? '<' : '≥', $result['alpha'],
|
||||
$result['significant'] ? 'significant' : 'not significant',
|
||||
$result['significant'] ? '' : ' Keep running.',
|
||||
);
|
||||
|
||||
if ($result['minExpectedCount'] < 5.0) {
|
||||
$lines[] = sprintf(
|
||||
'(Small sample: min expected conversions %.1f < 5, so the normal-approx z=%.2f overstates — exact test used.)',
|
||||
$result['minExpectedCount'], $result['z'],
|
||||
);
|
||||
}
|
||||
|
||||
return $lines;
|
||||
}
|
||||
|
||||
private function percent(float $rate): string
|
||||
{
|
||||
return number_format($rate * 100, 1).'%';
|
||||
}
|
||||
|
||||
private function money(int $cents, string $currency): string
|
||||
{
|
||||
$symbol = match (strtolower($currency)) {
|
||||
|
|
@ -118,17 +199,22 @@ class SendExperimentFunnelReportCommand extends Command
|
|||
'value' => $report['startedAt']->format('D, d M Y').' · new signups split evenly into the three variants.',
|
||||
'inline' => false,
|
||||
],
|
||||
[
|
||||
'name' => '📊 Significance',
|
||||
'value' => "```\n".implode("\n", $this->significanceLines($report))."\n```",
|
||||
'inline' => false,
|
||||
],
|
||||
[
|
||||
'name' => 'Legend',
|
||||
'value' => sprintf(
|
||||
'Assg = signups · Actd = activated (connected a bank or enabled AI = cost triggered) · Card = completed checkout (card on file) · Paid = live non-refunded subs (mature) · A2P%% = Paid ÷ activated (mature) · MRR = monthly run-rate of paid subs (yearly ÷ 12) · Cost = est. connection cost of the mature cohort (%s/connection) · Burn = connection cost of mature users who never converted (money lost) · CM = MRR − Cost · `pend`/`—` = no mature data yet.',
|
||||
'Assg = signups · Actd = activated (connected a bank or enabled AI = cost triggered) · Card = completed checkout (card on file) · MatU = matured assigned (cohort old enough to score for this variant) · Conv = matured users who ever converted (were charged, net of refund) — time-invariant, so it does not shrink as an older cohort has longer to churn · Conv%% = Conv ÷ MatU (always ≤100%%, comparable across variants) · ARPU = MRR ÷ MatU (revenue per matured user) · MRR = monthly run-rate of *currently* paying subs (yearly ÷ 12); Conv above MRR is churn · Cost = est. connection cost of MatU (%s/connection) · Burn = connection cost of matured users who never earned net revenue (connected a bank but never paid, or paid then refunded) · CM = MRR − Cost · `pend`/`—` = no matured data yet.',
|
||||
$this->money($report['costPerConnectionCents'], $report['currency']),
|
||||
),
|
||||
'inline' => false,
|
||||
],
|
||||
[
|
||||
'name' => '⚠️ How to read it',
|
||||
'value' => 'Each variant is gated by its own decision window (control 15d, reduced 7d, pay_now 3d), so pay_now matures first — compare only once all three have mature volume, and check significance before calling a winner. **CM (contribution margin) is the decision metric.** Assg/Actd/Card are lifetime counts; A2P%/Cost/Burn/CM cover only the mature cohort, so the raw Actd→Card→Paid funnel mixes cohorts (immature carded users can\'t be Paid yet) — read it for volume, compare variants on A2P%/CM. Burn is what the connect-and-leave leak costs. Cost is a flat per-connection estimate across all providers, not per-provider billing.',
|
||||
'value' => 'Each variant matures on its own decision window (control 15d, reduced 7d, pay_now 3d, +3d settle), so at any moment MatU differs a lot between variants (pay_now matures first). **Compare variants on Conv% and ARPU — normalized per matured user — not on the absolute MRR/Cost/Burn/CM totals, which scale with MatU and so mechanically favour whichever variant has matured more.** Assg/Actd/Card are lifetime counts; everything from MatU rightward covers the matured cohort only, so the raw Actd→Card→Conv funnel mixes cohorts (immature carded users can\'t have matured yet) — read it for volume. Conv counts anyone ever charged (net of refund), so it is not depressed for older cohorts the way a live-active snapshot would be. Per-user CM is sub-cent at current volume, so treat CM as directional context, not the decision. Check significance (sample size = MatU) before calling a winner. Cost is a flat per-connection estimate across all providers, not per-provider billing.',
|
||||
'inline' => false,
|
||||
],
|
||||
],
|
||||
|
|
|
|||
|
|
@ -0,0 +1,22 @@
|
|||
<?php
|
||||
|
||||
namespace App\Services\Stats;
|
||||
|
||||
/**
|
||||
* One arm of a binomial experiment: a count of successes out of trials, with a
|
||||
* label. Bundles the (successes, trials) pair that the significance methods
|
||||
* would otherwise pass around as loose integers.
|
||||
*/
|
||||
final readonly class BinomialProportion
|
||||
{
|
||||
public function __construct(
|
||||
public string $label,
|
||||
public int $successes,
|
||||
public int $trials,
|
||||
) {}
|
||||
|
||||
public function rate(): float
|
||||
{
|
||||
return $this->trials > 0 ? (float) $this->successes / $this->trials : 0.0;
|
||||
}
|
||||
}
|
||||
|
|
@ -6,9 +6,9 @@ use App\Features\SubscriptionExperiment;
|
|||
use App\Models\User;
|
||||
use Carbon\CarbonImmutable;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Laravel\Cashier\Cashier;
|
||||
use Laravel\Cashier\Subscription;
|
||||
use Laravel\Pennant\Feature;
|
||||
|
||||
class ExperimentFunnelCollector
|
||||
{
|
||||
|
|
@ -20,12 +20,13 @@ class ExperimentFunnelCollector
|
|||
|
||||
/**
|
||||
* Per-variant funnel for the trial/pricing experiment. Users are attributed
|
||||
* by the variant Pennant resolved for them — the same value the runtime
|
||||
* served at checkout/paywall — so the report can't drift from what users
|
||||
* actually experienced (including any QA override or a legacy bucket that
|
||||
* predates the experiment). "Net active" is a live, non-refunded
|
||||
* subscription — an exact, heuristic-free metric that is comparable across
|
||||
* variants once each cohort clears its own decision window.
|
||||
* by SubscriptionExperiment::bucket() — the deterministic crc32 split that is
|
||||
* the single source of truth for assignment — over the in-window signups the
|
||||
* query selects, so it matches the variant each user was served without being
|
||||
* perturbed by the force_variant rollout hook (which pins every user to the
|
||||
* winner once decided) or by Pennant store drift. "Net active" is a live,
|
||||
* non-refunded subscription — an exact, heuristic-free metric that is
|
||||
* comparable across variants once each cohort clears its own decision window.
|
||||
*
|
||||
* The funnel is assigned → activated → carded (subscribed) → net-paying:
|
||||
* "activated" = the user connected a bank or enabled AI, i.e. triggered the
|
||||
|
|
@ -61,7 +62,9 @@ class ExperimentFunnelCollector
|
|||
* refunded: int,
|
||||
* assignedMature: int,
|
||||
* activatedMature: int,
|
||||
* convertedMature: int,
|
||||
* activeMature: int,
|
||||
* conversionRate: ?float,
|
||||
* netActiveRate: ?float,
|
||||
* activationToPaidRate: ?float,
|
||||
* mrrCents: int,
|
||||
|
|
@ -92,8 +95,13 @@ class ExperimentFunnelCollector
|
|||
$excluded = (array) config('ai_suggestions.report.excluded_emails', []);
|
||||
$windows = $this->decisionWindows();
|
||||
$monthlyEquiv = $this->monthlyEquivByPriceId();
|
||||
$missingPrices = [];
|
||||
|
||||
User::query()
|
||||
// Soft-deleted accounts still count: they were assigned a variant and
|
||||
// their bank connections incurred real cost, and deleting the account
|
||||
// is itself an experiment outcome (the strongest "connect and leave").
|
||||
->withTrashed()
|
||||
->where('users.created_at', '>=', $startedAt)
|
||||
->when($excluded !== [], fn ($query) => $query->whereNotIn('email', $excluded))
|
||||
->with(['subscriptions' => fn ($query) => $query->where('type', 'default')])
|
||||
|
|
@ -104,11 +112,16 @@ class ExperimentFunnelCollector
|
|||
'bankingConnections as connection_count' => fn ($query) => $query->withTrashed(),
|
||||
'aiConsents as ai_consent_count',
|
||||
])
|
||||
->chunkById(500, function ($users) use (&$variants, $windows, $now, $monthlyEquiv, $costPerConnectionCents): void {
|
||||
Feature::for($users)->load([SubscriptionExperiment::class]);
|
||||
|
||||
->chunkById(500, function ($users) use (&$variants, &$missingPrices, $windows, $now, $monthlyEquiv, $costPerConnectionCents): void {
|
||||
foreach ($users as $user) {
|
||||
$variant = Feature::for($user)->value(SubscriptionExperiment::class);
|
||||
// Attribute by the deterministic bucket (the single source of
|
||||
// truth in SubscriptionExperiment), not the resolved Pennant
|
||||
// value: the latter is short-circuited by the force_variant
|
||||
// rollout hook, which would collapse every user onto one
|
||||
// variant once a winner is pinned. Every queried user is
|
||||
// in-window, so bucket() equals the variant they were served,
|
||||
// and reading it avoids writing Pennant rows as a side effect.
|
||||
$variant = SubscriptionExperiment::bucket((string) $user->id);
|
||||
|
||||
if (! isset($variants[$variant])) {
|
||||
continue;
|
||||
|
|
@ -130,6 +143,21 @@ class ExperimentFunnelCollector
|
|||
$status = $subscription?->stripe_status;
|
||||
$netActive = $status === 'active' && $subscription->refunded_at === null;
|
||||
|
||||
// "Converted" is time-invariant: the user was ever charged and
|
||||
// not refunded — currently active, or churned after the trial.
|
||||
// Unlike $netActive (a live snapshot), it does not shrink as an
|
||||
// older cohort has more time to cancel, so it is comparable
|
||||
// across variants that matured at different times. Excludes
|
||||
// trial-only cancels (ended on/before the trial → never charged).
|
||||
$converted = $subscription !== null
|
||||
&& $subscription->refunded_at === null
|
||||
&& $status !== 'trialing'
|
||||
&& (
|
||||
$subscription->trial_ends_at === null
|
||||
|| $subscription->ends_at === null
|
||||
|| $subscription->ends_at->greaterThan($subscription->trial_ends_at)
|
||||
);
|
||||
|
||||
if ($subscription !== null) {
|
||||
$row['subscribed']++;
|
||||
$row['trialing'] += $status === 'trialing' ? 1 : 0;
|
||||
|
|
@ -154,10 +182,25 @@ class ExperimentFunnelCollector
|
|||
$row['activatedMature']++;
|
||||
}
|
||||
|
||||
if ($converted) {
|
||||
$row['convertedMature']++;
|
||||
}
|
||||
|
||||
if ($netActive) {
|
||||
$row['activeMature']++;
|
||||
$row['mrrCents'] += (int) ($monthlyEquiv[$subscription->stripe_price] ?? 0);
|
||||
} else {
|
||||
$priceId = (string) $subscription->stripe_price;
|
||||
|
||||
if ($monthlyEquiv !== [] && ! isset($monthlyEquiv[$priceId])) {
|
||||
$missingPrices[$priceId] = true;
|
||||
}
|
||||
|
||||
$row['mrrCents'] += (int) ($monthlyEquiv[$priceId] ?? 0);
|
||||
} elseif ($subscription === null || $subscription->refunded_at !== null) {
|
||||
// Burn = connections of matured users who never earned
|
||||
// net revenue: connected a bank but never carded, or
|
||||
// paid and got refunded. A user who paid and later
|
||||
// churned (canceled, not refunded) did convert, so
|
||||
// their connection cost is not burn.
|
||||
$row['wastedCostCents'] += $connectionCostCents;
|
||||
}
|
||||
}
|
||||
|
|
@ -166,12 +209,21 @@ class ExperimentFunnelCollector
|
|||
}
|
||||
});
|
||||
|
||||
if ($missingPrices !== []) {
|
||||
Log::warning('Experiment funnel: net-active subscriptions on prices absent from the monthly-equivalent map — their MRR is undercounted as 0.', [
|
||||
'price_ids' => array_keys($missingPrices),
|
||||
]);
|
||||
}
|
||||
|
||||
foreach ($variants as $key => $row) {
|
||||
$variants[$key]['conversionRate'] = $row['assignedMature'] > 0
|
||||
? (float) $row['convertedMature'] / $row['assignedMature']
|
||||
: null;
|
||||
$variants[$key]['netActiveRate'] = $row['assignedMature'] > 0
|
||||
? $row['activeMature'] / $row['assignedMature']
|
||||
? (float) $row['activeMature'] / $row['assignedMature']
|
||||
: null;
|
||||
$variants[$key]['activationToPaidRate'] = $row['activatedMature'] > 0
|
||||
? $row['activeMature'] / $row['activatedMature']
|
||||
? (float) $row['activeMature'] / $row['activatedMature']
|
||||
: null;
|
||||
$variants[$key]['arpuCents'] = $row['assignedMature'] > 0
|
||||
? (int) round($row['mrrCents'] / $row['assignedMature'])
|
||||
|
|
@ -189,7 +241,7 @@ class ExperimentFunnelCollector
|
|||
}
|
||||
|
||||
/**
|
||||
* @return array{assigned: int, activated: int, subscribed: int, trialing: int, trialingCanceling: int, active: int, canceled: int, pastDue: int, refunded: int, assignedMature: int, activatedMature: int, activeMature: int, netActiveRate: ?float, activationToPaidRate: ?float, mrrCents: int, arpuCents: ?int, costCents: int, wastedCostCents: int, contributionMarginCents: int}
|
||||
* @return array{assigned: int, activated: int, subscribed: int, trialing: int, trialingCanceling: int, active: int, canceled: int, pastDue: int, refunded: int, assignedMature: int, activatedMature: int, convertedMature: int, activeMature: int, conversionRate: ?float, netActiveRate: ?float, activationToPaidRate: ?float, mrrCents: int, arpuCents: ?int, costCents: int, wastedCostCents: int, contributionMarginCents: int}
|
||||
*/
|
||||
private function emptyRow(): array
|
||||
{
|
||||
|
|
@ -205,7 +257,9 @@ class ExperimentFunnelCollector
|
|||
'refunded' => 0,
|
||||
'assignedMature' => 0,
|
||||
'activatedMature' => 0,
|
||||
'convertedMature' => 0,
|
||||
'activeMature' => 0,
|
||||
'conversionRate' => null,
|
||||
'netActiveRate' => null,
|
||||
'activationToPaidRate' => null,
|
||||
'mrrCents' => 0,
|
||||
|
|
@ -218,8 +272,13 @@ class ExperimentFunnelCollector
|
|||
|
||||
/**
|
||||
* Monthly-equivalent amount (in cents) for each plan price id, from Stripe.
|
||||
* Yearly prices are divided by 12. Cached for an hour; returns [] (revenue
|
||||
* unavailable) if Stripe can't be reached, without caching the failure.
|
||||
* Yearly prices are divided by 12. Fetched by product so that archived,
|
||||
* rotated price ids (Stripe mints a new id and transfers the lookup key on
|
||||
* any amount change) still resolve — otherwise subscriptions on an old id
|
||||
* would silently contribute 0 to MRR. Falls back to the current lookup keys
|
||||
* when no product is configured. Foreign-currency and one-off prices are
|
||||
* skipped. Cached for an hour; returns [] (revenue unavailable) if Stripe
|
||||
* can't be reached, without caching the failure.
|
||||
*
|
||||
* @return array<string, int>
|
||||
*/
|
||||
|
|
@ -231,26 +290,40 @@ class ExperimentFunnelCollector
|
|||
return Cache::get($key);
|
||||
}
|
||||
|
||||
$productId = config('subscriptions.products.pro');
|
||||
$lookups = array_values(array_filter([
|
||||
config('subscriptions.plans.monthly.stripe_lookup_key'),
|
||||
config('subscriptions.plans.yearly.stripe_lookup_key'),
|
||||
]));
|
||||
|
||||
if ($lookups === []) {
|
||||
if ($productId === null && $lookups === []) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$params = $productId !== null
|
||||
? ['product' => $productId, 'limit' => 100]
|
||||
: ['lookup_keys' => $lookups, 'limit' => 10];
|
||||
|
||||
try {
|
||||
$prices = Cashier::stripe()->prices->all(['lookup_keys' => $lookups, 'limit' => 10]);
|
||||
$prices = Cashier::stripe()->prices->all($params);
|
||||
} catch (\Throwable) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$currency = strtolower((string) config('cashier.currency', 'eur'));
|
||||
$map = [];
|
||||
foreach ($prices->data as $price) {
|
||||
if ($price->recurring === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (strtolower((string) ($price->currency ?? $currency)) !== $currency) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$amount = (int) ($price->unit_amount ?? 0);
|
||||
$map[$price->id] = ($price->recurring->interval ?? 'month') === 'year'
|
||||
? intdiv($amount, 12)
|
||||
? (int) round($amount / 12)
|
||||
: $amount;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,149 @@
|
|||
<?php
|
||||
|
||||
namespace App\Services\Stats;
|
||||
|
||||
/**
|
||||
* Frequentist inference on binomial proportions for the experiment funnel:
|
||||
* per-arm Wilson intervals, a Newcombe interval for the difference of two
|
||||
* proportions, and a Fisher exact test for the leader-vs-runner-up comparison.
|
||||
*
|
||||
* Fisher (exact at any sample size) is the decision test because the report's
|
||||
* conversion counts are too small for the two-proportion z normal approximation
|
||||
* to be valid — its expected cell counts fall well below the np>=5 rule.
|
||||
*/
|
||||
final class ProportionSignificance
|
||||
{
|
||||
/** Two-sided 95% standard-normal quantile. */
|
||||
private const Z_95 = 1.96;
|
||||
|
||||
/** @var list<float> memoised log-factorials, index i = log(i!) */
|
||||
private array $logFactorials = [0.0, 0.0];
|
||||
|
||||
/**
|
||||
* Compare the two leading arms: Newcombe difference interval, Fisher exact
|
||||
* p-value, and a family-wise (Bonferroni) corrected significance flag. `z`
|
||||
* and `minExpectedCount` are returned for the small-sample caveat only.
|
||||
*
|
||||
* @return array{alpha: float, diffLow: float, diffHigh: float, fisherP: float, significant: bool, minExpectedCount: float, z: float}
|
||||
*/
|
||||
public function compare(BinomialProportion $leader, BinomialProportion $runnerUp, float $familyAlpha = 0.05, int $comparisons = 3): array
|
||||
{
|
||||
$alpha = $familyAlpha / $comparisons;
|
||||
[$diffLow, $diffHigh] = $this->newcombeDiffInterval($leader, $runnerUp);
|
||||
$fisherP = $this->fisherExactTwoSided(
|
||||
$leader->successes, $leader->trials - $leader->successes,
|
||||
$runnerUp->successes, $runnerUp->trials - $runnerUp->successes,
|
||||
);
|
||||
|
||||
$pooled = ($leader->successes + $runnerUp->successes) / ($leader->trials + $runnerUp->trials);
|
||||
$minExpectedCount = min($leader->trials, $runnerUp->trials) * min($pooled, 1 - $pooled);
|
||||
|
||||
return [
|
||||
'alpha' => $alpha,
|
||||
'diffLow' => $diffLow,
|
||||
'diffHigh' => $diffHigh,
|
||||
'fisherP' => $fisherP,
|
||||
'significant' => $fisherP < $alpha,
|
||||
'minExpectedCount' => $minExpectedCount,
|
||||
'z' => $this->twoProportionZ($leader, $runnerUp),
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Wilson score interval for a binomial proportion — accurate for small n
|
||||
* and near 0/1, where the normal approximation misbehaves.
|
||||
*
|
||||
* @return array{0: float, 1: float} lower and upper bound, clamped to [0, 1]
|
||||
*/
|
||||
public function wilsonInterval(int $successes, int $trials, float $z = self::Z_95): array
|
||||
{
|
||||
$p = $successes / $trials;
|
||||
$z2 = $z * $z;
|
||||
$denom = 1 + $z2 / $trials;
|
||||
$center = ($p + $z2 / (2 * $trials)) / $denom;
|
||||
$margin = ($z / $denom) * sqrt($p * (1 - $p) / $trials + $z2 / (4 * $trials * $trials));
|
||||
|
||||
return [max(0.0, $center - $margin), min(1.0, $center + $margin)];
|
||||
}
|
||||
|
||||
/**
|
||||
* Newcombe (Wilson-based) 95% interval for the difference pA − pB. The
|
||||
* correct object for "is A better than B": overlapping marginal intervals do
|
||||
* NOT imply the difference includes 0.
|
||||
*
|
||||
* @return array{0: float, 1: float}
|
||||
*/
|
||||
public function newcombeDiffInterval(BinomialProportion $a, BinomialProportion $b, float $z = self::Z_95): array
|
||||
{
|
||||
$pA = $a->rate();
|
||||
$pB = $b->rate();
|
||||
[$lA, $uA] = $this->wilsonInterval($a->successes, $a->trials, $z);
|
||||
[$lB, $uB] = $this->wilsonInterval($b->successes, $b->trials, $z);
|
||||
|
||||
$lower = ($pA - $pB) - sqrt(($pA - $lA) ** 2 + ($uB - $pB) ** 2);
|
||||
$upper = ($pA - $pB) + sqrt(($uA - $pA) ** 2 + ($pB - $lB) ** 2);
|
||||
|
||||
return [$lower, $upper];
|
||||
}
|
||||
|
||||
/** Pooled two-proportion z statistic — descriptive only, not the decision test. */
|
||||
public function twoProportionZ(BinomialProportion $a, BinomialProportion $b): float
|
||||
{
|
||||
$pooled = ($a->successes + $b->successes) / ($a->trials + $b->trials);
|
||||
$se = sqrt($pooled * (1 - $pooled) * (1 / $a->trials + 1 / $b->trials));
|
||||
|
||||
return $se > 0.0 ? ($a->rate() - $b->rate()) / $se : 0.0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Two-sided Fisher exact p-value for the 2x2 table [[a, b], [c, d]]
|
||||
* (a/c = successes, b/d = failures). Sums the hypergeometric probabilities
|
||||
* of every same-margin table no more likely than the observed one. Exact at
|
||||
* any sample size — no normal approximation.
|
||||
*/
|
||||
public function fisherExactTwoSided(int $a, int $b, int $c, int $d): float
|
||||
{
|
||||
$rowA = $a + $b;
|
||||
$rowB = $c + $d;
|
||||
$col = $a + $c;
|
||||
$total = $rowA + $rowB;
|
||||
|
||||
if ($rowA === 0 || $rowB === 0 || $col === 0 || $col === $total) {
|
||||
return 1.0;
|
||||
}
|
||||
|
||||
$logProbObserved = $this->hypergeometricLogProb($a, $rowA, $rowB, $col);
|
||||
$p = 0.0;
|
||||
for ($x = max(0, $col - $rowB); $x <= min($col, $rowA); $x++) {
|
||||
$logProb = $this->hypergeometricLogProb($x, $rowA, $rowB, $col);
|
||||
if ($logProb <= $logProbObserved + 1e-7) {
|
||||
$p += exp($logProb);
|
||||
}
|
||||
}
|
||||
|
||||
return min(1.0, $p);
|
||||
}
|
||||
|
||||
private function hypergeometricLogProb(int $x, int $rowA, int $rowB, int $col): float
|
||||
{
|
||||
return $this->logChoose($rowA, $x) + $this->logChoose($rowB, $col - $x) - $this->logChoose($rowA + $rowB, $col);
|
||||
}
|
||||
|
||||
private function logChoose(int $n, int $k): float
|
||||
{
|
||||
if ($k < 0 || $k > $n) {
|
||||
return -INF;
|
||||
}
|
||||
|
||||
return $this->logFactorial($n) - $this->logFactorial($k) - $this->logFactorial($n - $k);
|
||||
}
|
||||
|
||||
private function logFactorial(int $n): float
|
||||
{
|
||||
for ($i = count($this->logFactorials); $i <= $n; $i++) {
|
||||
$this->logFactorials[$i] = $this->logFactorials[$i - 1] + log($i);
|
||||
}
|
||||
|
||||
return $this->logFactorials[$n];
|
||||
}
|
||||
}
|
||||
|
|
@ -7,8 +7,10 @@ use App\Models\User;
|
|||
use App\Services\Stats\ExperimentFunnelCollector;
|
||||
use Carbon\CarbonImmutable;
|
||||
use Illuminate\Support\Carbon;
|
||||
use Illuminate\Support\Facades\Artisan;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Illuminate\Support\Facades\Http;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Support\Str;
|
||||
|
||||
use function Pest\Laravel\artisan;
|
||||
|
|
@ -34,7 +36,7 @@ beforeEach(function () {
|
|||
* with an optional default subscription and any bank connections / AI consent
|
||||
* that mark the user as "activated" and drive the connection-cost columns.
|
||||
*
|
||||
* @param array{status: string, at: CarbonImmutable, endsAt?: CarbonImmutable, refundedAt?: CarbonImmutable}|null $subscription
|
||||
* @param array{status: string, at: CarbonImmutable, endsAt?: CarbonImmutable, trialEndsAt?: CarbonImmutable, refundedAt?: CarbonImmutable}|null $subscription
|
||||
*/
|
||||
function experimentUser(string $variant, CarbonImmutable $signup, ?array $subscription = null, int $connections = 0, bool $aiConsent = false): User
|
||||
{
|
||||
|
|
@ -52,6 +54,7 @@ function experimentUser(string $variant, CarbonImmutable $signup, ?array $subscr
|
|||
'stripe_price' => 'price_test',
|
||||
'created_at' => $subscription['at'],
|
||||
'ends_at' => $subscription['endsAt'] ?? null,
|
||||
'trial_ends_at' => $subscription['trialEndsAt'] ?? null,
|
||||
'refunded_at' => $subscription['refundedAt'] ?? null,
|
||||
]);
|
||||
}
|
||||
|
|
@ -173,6 +176,162 @@ it('computes connection cost, wasted burn and contribution margin', function ()
|
|||
->and($control['activationToPaidRate'])->toBe(0.5); // 1 paid ÷ 2 activated
|
||||
});
|
||||
|
||||
it('keeps the A/B/C split even when a winner is forced, instead of collapsing onto one variant', function () {
|
||||
// force_variant pins the runtime to one variant; the report must still show
|
||||
// the real historical split, not attribute everyone to the forced winner.
|
||||
config(['subscriptions.experiment.force_variant' => SubscriptionExperiment::PAY_NOW]);
|
||||
$signup = CarbonImmutable::parse('2026-06-05');
|
||||
|
||||
experimentUser(SubscriptionExperiment::CONTROL, $signup);
|
||||
experimentUser(SubscriptionExperiment::REDUCED_TRIAL, $signup);
|
||||
experimentUser(SubscriptionExperiment::PAY_NOW, $signup);
|
||||
|
||||
$variants = app(ExperimentFunnelCollector::class)->collect()['variants'];
|
||||
|
||||
expect($variants[SubscriptionExperiment::CONTROL]['assigned'])->toBe(1)
|
||||
->and($variants[SubscriptionExperiment::REDUCED_TRIAL]['assigned'])->toBe(1)
|
||||
->and($variants[SubscriptionExperiment::PAY_NOW]['assigned'])->toBe(1);
|
||||
});
|
||||
|
||||
it('warns instead of silently zeroing MRR when a paid sub is on an unmapped (rotated) price', function () {
|
||||
Log::spy();
|
||||
$signup = CarbonImmutable::parse('2026-06-05');
|
||||
|
||||
// The seeded map only knows 'price_test'; move this paid sub onto a rotated id.
|
||||
$user = experimentUser(SubscriptionExperiment::REDUCED_TRIAL, $signup, ['status' => 'active', 'at' => $signup]);
|
||||
$user->subscriptions()->first()->update(['stripe_price' => 'price_rotated_old']);
|
||||
|
||||
$reduced = app(ExperimentFunnelCollector::class)->collect()['variants'][SubscriptionExperiment::REDUCED_TRIAL];
|
||||
|
||||
expect($reduced['activeMature'])->toBe(1)
|
||||
->and($reduced['mrrCents'])->toBe(0); // unmapped price → 0, but now loudly
|
||||
|
||||
Log::shouldHaveReceived('warning')
|
||||
->withArgs(fn (string $message, array $context = []) => str_contains($message, 'absent from the monthly-equivalent map')
|
||||
&& in_array('price_rotated_old', $context['price_ids'] ?? [], true))
|
||||
->once();
|
||||
});
|
||||
|
||||
it('still counts soft-deleted users so their assignment and connection cost survive', function () {
|
||||
$signup = CarbonImmutable::parse('2026-06-05');
|
||||
|
||||
$user = experimentUser(SubscriptionExperiment::CONTROL, $signup, null, connections: 2);
|
||||
$user->delete(); // soft delete the account
|
||||
|
||||
$control = app(ExperimentFunnelCollector::class)->collect()['variants'][SubscriptionExperiment::CONTROL];
|
||||
|
||||
expect($control['assigned'])->toBe(1)
|
||||
->and($control['assignedMature'])->toBe(1)
|
||||
->and($control['activated'])->toBe(1) // 2 connections
|
||||
->and($control['costCents'])->toBe(80) // 2 × 40, still charged
|
||||
->and($control['wastedCostCents'])->toBe(80); // never paid → pure burn
|
||||
});
|
||||
|
||||
it('reports a matured-cohort conversion capped at 100% and prints the matured denominator', function () {
|
||||
$signup = CarbonImmutable::parse('2026-06-05'); // control matures by the test clock
|
||||
|
||||
// Two paid, matured control users; only one is "activated" (connected a bank).
|
||||
// The old A2P% (Paid ÷ activated = 2 ÷ 1) would print a nonsensical 200%.
|
||||
experimentUser(SubscriptionExperiment::CONTROL, $signup, ['status' => 'active', 'at' => $signup->addDay()], connections: 1);
|
||||
experimentUser(SubscriptionExperiment::CONTROL, $signup, ['status' => 'active', 'at' => $signup->addDay()]);
|
||||
|
||||
$control = app(ExperimentFunnelCollector::class)->collect()['variants'][SubscriptionExperiment::CONTROL];
|
||||
|
||||
expect($control['assignedMature'])->toBe(2)
|
||||
->and($control['activatedMature'])->toBe(1)
|
||||
->and($control['activeMature'])->toBe(2)
|
||||
->and($control['convertedMature'])->toBe(2)
|
||||
->and($control['conversionRate'])->toBe(1.0); // Conv% = Conv ÷ MatU, always ≤ 100%
|
||||
|
||||
Artisan::call('stats:experiment-funnel', ['--no-discord' => true]);
|
||||
$output = Artisan::output();
|
||||
|
||||
expect($output)->toContain('MatU') // matured denominator column is printed
|
||||
->toContain('Conv%')
|
||||
->toContain('100%') // capped, not the old 200% A2P%
|
||||
->not->toContain('200%');
|
||||
});
|
||||
|
||||
it('reports a Wilson confidence interval and defers the verdict while samples are small', function () {
|
||||
$signup = CarbonImmutable::parse('2026-06-05');
|
||||
|
||||
experimentUser(SubscriptionExperiment::CONTROL, $signup, ['status' => 'active', 'at' => $signup->addDay()]);
|
||||
experimentUser(SubscriptionExperiment::CONTROL, $signup);
|
||||
experimentUser(SubscriptionExperiment::REDUCED_TRIAL, $signup, ['status' => 'active', 'at' => $signup->addDay()]);
|
||||
experimentUser(SubscriptionExperiment::REDUCED_TRIAL, $signup);
|
||||
|
||||
Artisan::call('stats:experiment-funnel', ['--no-discord' => true]);
|
||||
$output = Artisan::output();
|
||||
|
||||
expect($output)->toContain('Significance')
|
||||
->toContain('Wilson')
|
||||
->toContain('Fisher exact') // verdict uses the exact test, not the z-approx
|
||||
->toContain('not significant') // equal 50/50 rates, n=2 per arm → nowhere near
|
||||
->toContain('Small sample'); // min expected conversions < 5
|
||||
});
|
||||
|
||||
it('declares significance via the exact test when the separation is real', function () {
|
||||
$signup = CarbonImmutable::parse('2026-06-05');
|
||||
|
||||
// reduced: 5 matured converters; pay_now: 5 matured non-converters (no sub).
|
||||
// Fisher exact on [[5,0],[0,5]] gives p≈0.008 < 0.0167 (Bonferroni) → significant.
|
||||
for ($i = 0; $i < 5; $i++) {
|
||||
experimentUser(SubscriptionExperiment::REDUCED_TRIAL, $signup, ['status' => 'active', 'at' => $signup->addDay()]);
|
||||
experimentUser(SubscriptionExperiment::PAY_NOW, $signup);
|
||||
}
|
||||
|
||||
Artisan::call('stats:experiment-funnel', ['--no-discord' => true]);
|
||||
$output = Artisan::output();
|
||||
|
||||
expect($output)->toContain('Fisher exact')
|
||||
->not->toContain('not significant'); // the exact test clears the corrected bar
|
||||
});
|
||||
|
||||
it('measures conversion as ever-charged, not active-now, so churn does not bias it', function () {
|
||||
$signup = CarbonImmutable::parse('2026-06-05');
|
||||
|
||||
// 1) Still active → converted.
|
||||
experimentUser(SubscriptionExperiment::CONTROL, $signup, ['status' => 'active', 'at' => $signup->addDay()]);
|
||||
// 2) Paid then churned after the trial (charged, not refunded) → converted.
|
||||
experimentUser(SubscriptionExperiment::CONTROL, $signup, [
|
||||
'status' => 'canceled', 'at' => $signup, 'trialEndsAt' => $signup->addDays(15), 'endsAt' => $signup->addDays(40),
|
||||
]);
|
||||
// 3) Canceled on/before the trial end (never charged) → NOT converted.
|
||||
experimentUser(SubscriptionExperiment::CONTROL, $signup, [
|
||||
'status' => 'canceled', 'at' => $signup, 'trialEndsAt' => $signup->addDays(15), 'endsAt' => $signup->addDays(10),
|
||||
]);
|
||||
// 4) Refunded → NOT converted.
|
||||
experimentUser(SubscriptionExperiment::CONTROL, $signup, [
|
||||
'status' => 'canceled', 'at' => $signup, 'endsAt' => $signup->addDay(), 'refundedAt' => $signup->addDay(),
|
||||
]);
|
||||
|
||||
$control = app(ExperimentFunnelCollector::class)->collect()['variants'][SubscriptionExperiment::CONTROL];
|
||||
|
||||
expect($control['assignedMature'])->toBe(4)
|
||||
->and($control['convertedMature'])->toBe(2) // #1 active + #2 churned-after-paying
|
||||
->and($control['activeMature'])->toBe(1) // only #1 is active now
|
||||
->and($control['conversionRate'])->toBe(0.5); // 2 ÷ 4, time-invariant
|
||||
});
|
||||
|
||||
it('excludes churned payers from burn but keeps refunds as burn', function () {
|
||||
$signup = CarbonImmutable::parse('2026-06-05');
|
||||
|
||||
// Paid then canceled (not refunded): converted, so its cost is NOT burn.
|
||||
experimentUser(SubscriptionExperiment::CONTROL, $signup, [
|
||||
'status' => 'canceled', 'at' => $signup, 'endsAt' => $signup->addDays(20),
|
||||
], connections: 2);
|
||||
// Paid then refunded: zero net revenue, so its cost IS burn.
|
||||
experimentUser(SubscriptionExperiment::CONTROL, $signup, [
|
||||
'status' => 'canceled', 'at' => $signup, 'endsAt' => $signup->addDay(), 'refundedAt' => $signup->addDay(),
|
||||
], connections: 3);
|
||||
|
||||
$control = app(ExperimentFunnelCollector::class)->collect()['variants'][SubscriptionExperiment::CONTROL];
|
||||
|
||||
expect($control['costCents'])->toBe(200) // (2 + 3) × 40, all mature connections
|
||||
->and($control['wastedCostCents'])->toBe(120) // only the refunded user's 3 × 40
|
||||
->and($control['refunded'])->toBe(1);
|
||||
});
|
||||
|
||||
it('scales connection cost by the cost-per-connection argument', function () {
|
||||
$signup = CarbonImmutable::parse('2026-06-05');
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,66 @@
|
|||
<?php
|
||||
|
||||
use App\Services\Stats\BinomialProportion;
|
||||
use App\Services\Stats\ProportionSignificance;
|
||||
|
||||
beforeEach(function () {
|
||||
$this->stats = new ProportionSignificance;
|
||||
});
|
||||
|
||||
it('computes the Wilson score interval', function () {
|
||||
[$low, $high] = $this->stats->wilsonInterval(6, 52);
|
||||
expect($low)->toEqualWithDelta(0.0540, 0.0005)
|
||||
->and($high)->toEqualWithDelta(0.2297, 0.0005);
|
||||
|
||||
[$low2, $high2] = $this->stats->wilsonInterval(5, 179);
|
||||
expect($low2)->toEqualWithDelta(0.0120, 0.0005)
|
||||
->and($high2)->toEqualWithDelta(0.0640, 0.0005);
|
||||
});
|
||||
|
||||
it('keeps the Wilson interval inside [0, 1] at the k=0 and k=n boundaries', function () {
|
||||
expect($this->stats->wilsonInterval(0, 30)[0])->toBe(0.0)
|
||||
->and($this->stats->wilsonInterval(30, 30)[1])->toBe(1.0);
|
||||
});
|
||||
|
||||
it('computes a two-sided Fisher exact p-value', function () {
|
||||
// Real experiment table: reduced 6/52 vs pay_now 5/179.
|
||||
expect($this->stats->fisherExactTwoSided(6, 46, 5, 174))->toEqualWithDelta(0.0182, 0.0005);
|
||||
// Clean separation 5/5 vs 0/5 → 2 * C(5,5)/C(10,5) = 2/252.
|
||||
expect($this->stats->fisherExactTwoSided(5, 0, 0, 5))->toEqualWithDelta(0.007936, 0.00001);
|
||||
// Degenerate margin (no successes anywhere) → p = 1.
|
||||
expect($this->stats->fisherExactTwoSided(0, 10, 0, 10))->toBe(1.0);
|
||||
});
|
||||
|
||||
it('computes the Newcombe difference interval and the descriptive z', function () {
|
||||
$reduced = new BinomialProportion('reduced', 6, 52);
|
||||
$payNow = new BinomialProportion('pay_now', 5, 179);
|
||||
|
||||
[$low, $high] = $this->stats->newcombeDiffInterval($reduced, $payNow);
|
||||
expect($low)->toEqualWithDelta(0.0164, 0.0005)
|
||||
->and($high)->toEqualWithDelta(0.2029, 0.0005)
|
||||
->and($this->stats->twoProportionZ($reduced, $payNow))->toEqualWithDelta(2.607, 0.01);
|
||||
});
|
||||
|
||||
it('calls the borderline real comparison NOT significant under Bonferroni', function () {
|
||||
$result = $this->stats->compare(
|
||||
new BinomialProportion('reduced', 6, 52),
|
||||
new BinomialProportion('pay_now', 5, 179),
|
||||
);
|
||||
|
||||
// Fisher p ≈ 0.018 exceeds the corrected bar 0.05/3 ≈ 0.0167 → not a winner yet.
|
||||
expect($result['significant'])->toBeFalse()
|
||||
->and($result['fisherP'])->toEqualWithDelta(0.0182, 0.0005)
|
||||
->and($result['alpha'])->toEqualWithDelta(0.0167, 0.0005)
|
||||
->and($result['minExpectedCount'])->toEqualWithDelta(2.48, 0.05);
|
||||
});
|
||||
|
||||
it('calls a clean separation significant', function () {
|
||||
$result = $this->stats->compare(
|
||||
new BinomialProportion('a', 5, 5),
|
||||
new BinomialProportion('b', 0, 5),
|
||||
);
|
||||
|
||||
// Fisher p ≈ 0.008 clears the corrected bar.
|
||||
expect($result['significant'])->toBeTrue()
|
||||
->and($result['fisherP'])->toEqualWithDelta(0.007936, 0.00001);
|
||||
});
|
||||
Loading…
Reference in New Issue