diff --git a/app/Console/Commands/SendExperimentFunnelReportCommand.php b/app/Console/Commands/SendExperimentFunnelReportCommand.php index f8bc6116..8e0da468 100644 --- a/app/Console/Commands/SendExperimentFunnelReportCommand.php +++ b/app/Console/Commands/SendExperimentFunnelReportCommand.php @@ -68,7 +68,7 @@ class SendExperimentFunnelReportCommand extends Command $currency = $report['currency']; $lines = [sprintf( '%-8s %5s %5s %5s %5s %5s %6s %7s %7s %7s %7s %7s', - 'Variant', 'Assg', 'Actd', 'Card', 'MatU', 'Paid', 'Conv%', 'ARPU', 'MRR', 'Cost', 'Burn', 'CM', + 'Variant', 'Assg', 'Actd', 'Card', 'MatU', 'Conv', 'Conv%', 'ARPU', 'MRR', 'Cost', 'Burn', 'CM', )]; foreach (self::LABELS as $key => $label) { @@ -83,8 +83,8 @@ class SendExperimentFunnelReportCommand extends Command $row['activated'], $row['subscribed'], $row['assignedMature'], - $row['activeMature'], - $mature ? ((int) round($row['netActiveRate'] * 100)).'%' : 'pend', + $row['convertedMature'], + $mature ? ((int) round($row['conversionRate'] * 100)).'%' : 'pend', $money && $row['arpuCents'] !== null ? $this->money($row['arpuCents'], $currency) : '—', $money ? $this->money($row['mrrCents'], $currency) : '—', $mature ? $this->money($row['costCents'], $currency) : '—', @@ -127,14 +127,14 @@ class SendExperimentFunnelReportCommand extends Command [ 'name' => 'Legend', 'value' => sprintf( - 'Assg = signups · Actd = activated (connected a bank or enabled AI = cost triggered) · Card = completed checkout (card on file) · MatU = matured assigned (cohort old enough to score for this variant) · Paid = live non-refunded subs among MatU · Conv%% = Paid ÷ MatU (net conversion, always ≤100%%, comparable across variants) · ARPU = MRR ÷ MatU (revenue per matured user) · MRR = monthly run-rate of paid subs (yearly ÷ 12) · Cost = est. connection cost of MatU (%s/connection) · Burn = connection cost of matured users who never earned net revenue (connected a bank but never paid, or paid then refunded) · CM = MRR − Cost · `pend`/`—` = no matured data yet.', + 'Assg = signups · Actd = activated (connected a bank or enabled AI = cost triggered) · Card = completed checkout (card on file) · MatU = matured assigned (cohort old enough to score for this variant) · Conv = matured users who ever converted (were charged, net of refund) — time-invariant, so it does not shrink as an older cohort has longer to churn · Conv%% = Conv ÷ MatU (always ≤100%%, comparable across variants) · ARPU = MRR ÷ MatU (revenue per matured user) · MRR = monthly run-rate of *currently* paying subs (yearly ÷ 12); Conv above MRR is churn · Cost = est. connection cost of MatU (%s/connection) · Burn = connection cost of matured users who never earned net revenue (connected a bank but never paid, or paid then refunded) · CM = MRR − Cost · `pend`/`—` = no matured data yet.', $this->money($report['costPerConnectionCents'], $report['currency']), ), 'inline' => false, ], [ 'name' => '⚠️ How to read it', - 'value' => 'Each variant matures on its own decision window (control 15d, reduced 7d, pay_now 3d, +3d settle), so at any moment MatU differs a lot between variants (pay_now matures first). **Compare variants on Conv% and ARPU — normalized per matured user — not on the absolute MRR/Cost/Burn/CM totals, which scale with MatU and so mechanically favour whichever variant has matured more.** Assg/Actd/Card are lifetime counts; everything from MatU rightward covers the matured cohort only, so the raw Actd→Card→Paid funnel mixes cohorts (immature carded users can\'t be Paid yet) — read it for volume. Per-user CM is sub-cent at current volume, so treat CM as directional context, not the decision. Check significance (sample size = MatU) before calling a winner. Cost is a flat per-connection estimate across all providers, not per-provider billing.', + 'value' => 'Each variant matures on its own decision window (control 15d, reduced 7d, pay_now 3d, +3d settle), so at any moment MatU differs a lot between variants (pay_now matures first). **Compare variants on Conv% and ARPU — normalized per matured user — not on the absolute MRR/Cost/Burn/CM totals, which scale with MatU and so mechanically favour whichever variant has matured more.** Assg/Actd/Card are lifetime counts; everything from MatU rightward covers the matured cohort only, so the raw Actd→Card→Conv funnel mixes cohorts (immature carded users can\'t have matured yet) — read it for volume. Conv counts anyone ever charged (net of refund), so it is not depressed for older cohorts the way a live-active snapshot would be. Per-user CM is sub-cent at current volume, so treat CM as directional context, not the decision. Check significance (sample size = MatU) before calling a winner. Cost is a flat per-connection estimate across all providers, not per-provider billing.', 'inline' => false, ], ], diff --git a/app/Services/Stats/ExperimentFunnelCollector.php b/app/Services/Stats/ExperimentFunnelCollector.php index 7508c7de..bd4cc2a9 100644 --- a/app/Services/Stats/ExperimentFunnelCollector.php +++ b/app/Services/Stats/ExperimentFunnelCollector.php @@ -62,7 +62,9 @@ class ExperimentFunnelCollector * refunded: int, * assignedMature: int, * activatedMature: int, + * convertedMature: int, * activeMature: int, + * conversionRate: ?float, * netActiveRate: ?float, * activationToPaidRate: ?float, * mrrCents: int, @@ -141,6 +143,21 @@ class ExperimentFunnelCollector $status = $subscription?->stripe_status; $netActive = $status === 'active' && $subscription->refunded_at === null; + // "Converted" is time-invariant: the user was ever charged and + // not refunded — currently active, or churned after the trial. + // Unlike $netActive (a live snapshot), it does not shrink as an + // older cohort has more time to cancel, so it is comparable + // across variants that matured at different times. Excludes + // trial-only cancels (ended on/before the trial → never charged). + $converted = $subscription !== null + && $subscription->refunded_at === null + && $status !== 'trialing' + && ( + $subscription->trial_ends_at === null + || $subscription->ends_at === null + || $subscription->ends_at->greaterThan($subscription->trial_ends_at) + ); + if ($subscription !== null) { $row['subscribed']++; $row['trialing'] += $status === 'trialing' ? 1 : 0; @@ -165,6 +182,10 @@ class ExperimentFunnelCollector $row['activatedMature']++; } + if ($converted) { + $row['convertedMature']++; + } + if ($netActive) { $row['activeMature']++; $priceId = (string) $subscription->stripe_price; @@ -195,6 +216,9 @@ class ExperimentFunnelCollector } foreach ($variants as $key => $row) { + $variants[$key]['conversionRate'] = $row['assignedMature'] > 0 + ? $row['convertedMature'] / $row['assignedMature'] + : null; $variants[$key]['netActiveRate'] = $row['assignedMature'] > 0 ? $row['activeMature'] / $row['assignedMature'] : null; @@ -217,7 +241,7 @@ class ExperimentFunnelCollector } /** - * @return array{assigned: int, activated: int, subscribed: int, trialing: int, trialingCanceling: int, active: int, canceled: int, pastDue: int, refunded: int, assignedMature: int, activatedMature: int, activeMature: int, netActiveRate: ?float, activationToPaidRate: ?float, mrrCents: int, arpuCents: ?int, costCents: int, wastedCostCents: int, contributionMarginCents: int} + * @return array{assigned: int, activated: int, subscribed: int, trialing: int, trialingCanceling: int, active: int, canceled: int, pastDue: int, refunded: int, assignedMature: int, activatedMature: int, convertedMature: int, activeMature: int, conversionRate: ?float, netActiveRate: ?float, activationToPaidRate: ?float, mrrCents: int, arpuCents: ?int, costCents: int, wastedCostCents: int, contributionMarginCents: int} */ private function emptyRow(): array { @@ -233,7 +257,9 @@ class ExperimentFunnelCollector 'refunded' => 0, 'assignedMature' => 0, 'activatedMature' => 0, + 'convertedMature' => 0, 'activeMature' => 0, + 'conversionRate' => null, 'netActiveRate' => null, 'activationToPaidRate' => null, 'mrrCents' => 0, diff --git a/tests/Feature/SendExperimentFunnelReportCommandTest.php b/tests/Feature/SendExperimentFunnelReportCommandTest.php index a007d732..9540ed46 100644 --- a/tests/Feature/SendExperimentFunnelReportCommandTest.php +++ b/tests/Feature/SendExperimentFunnelReportCommandTest.php @@ -35,7 +35,7 @@ beforeEach(function () { * with an optional default subscription and any bank connections / AI consent * that mark the user as "activated" and drive the connection-cost columns. * - * @param array{status: string, at: CarbonImmutable, endsAt?: CarbonImmutable, refundedAt?: CarbonImmutable}|null $subscription + * @param array{status: string, at: CarbonImmutable, endsAt?: CarbonImmutable, trialEndsAt?: CarbonImmutable, refundedAt?: CarbonImmutable}|null $subscription */ function experimentUser(string $variant, CarbonImmutable $signup, ?array $subscription = null, int $connections = 0, bool $aiConsent = false): User { @@ -53,6 +53,7 @@ function experimentUser(string $variant, CarbonImmutable $signup, ?array $subscr 'stripe_price' => 'price_test', 'created_at' => $subscription['at'], 'ends_at' => $subscription['endsAt'] ?? null, + 'trial_ends_at' => $subscription['trialEndsAt'] ?? null, 'refunded_at' => $subscription['refundedAt'] ?? null, ]); } @@ -248,6 +249,32 @@ it('reports a matured-cohort conversion capped at 100% and prints the matured de ->assertSuccessful(); }); +it('measures conversion as ever-charged, not active-now, so churn does not bias it', function () { + $signup = CarbonImmutable::parse('2026-06-05'); + + // 1) Still active → converted. + experimentUser(SubscriptionExperiment::CONTROL, $signup, ['status' => 'active', 'at' => $signup->addDay()]); + // 2) Paid then churned after the trial (charged, not refunded) → converted. + experimentUser(SubscriptionExperiment::CONTROL, $signup, [ + 'status' => 'canceled', 'at' => $signup, 'trialEndsAt' => $signup->addDays(15), 'endsAt' => $signup->addDays(40), + ]); + // 3) Canceled on/before the trial end (never charged) → NOT converted. + experimentUser(SubscriptionExperiment::CONTROL, $signup, [ + 'status' => 'canceled', 'at' => $signup, 'trialEndsAt' => $signup->addDays(15), 'endsAt' => $signup->addDays(10), + ]); + // 4) Refunded → NOT converted. + experimentUser(SubscriptionExperiment::CONTROL, $signup, [ + 'status' => 'canceled', 'at' => $signup, 'endsAt' => $signup->addDay(), 'refundedAt' => $signup->addDay(), + ]); + + $control = app(ExperimentFunnelCollector::class)->collect()['variants'][SubscriptionExperiment::CONTROL]; + + expect($control['assignedMature'])->toBe(4) + ->and($control['convertedMature'])->toBe(2) // #1 active + #2 churned-after-paying + ->and($control['activeMature'])->toBe(1) // only #1 is active now + ->and($control['conversionRate'])->toBe(0.5); // 2 ÷ 4, time-invariant +}); + it('excludes churned payers from burn but keeps refunds as burn', function () { $signup = CarbonImmutable::parse('2026-06-05');