diff --git a/app/Console/Commands/SendExperimentFunnelReportCommand.php b/app/Console/Commands/SendExperimentFunnelReportCommand.php index a065b8c5..e313e67c 100644 --- a/app/Console/Commands/SendExperimentFunnelReportCommand.php +++ b/app/Console/Commands/SendExperimentFunnelReportCommand.php @@ -10,7 +10,9 @@ use Illuminate\Console\Command; class SendExperimentFunnelReportCommand extends Command { - protected $signature = 'stats:experiment-funnel {--no-discord : Print the report to the console only, without posting to Discord}'; + protected $signature = 'stats:experiment-funnel + {--no-discord : Print the report to the console only, without posting to Discord} + {--cost-per-connection=0.4 : Estimated cost (in the Cashier currency) per bank connection, used for the Cost/Burn/CM columns}'; protected $description = 'Post the trial/pricing experiment funnel (per variant) to Discord'; @@ -27,7 +29,8 @@ class SendExperimentFunnelReportCommand extends Command public function handle(): int { - $report = $this->collector->collect(); + $costPerConnectionCents = (int) round(((float) $this->option('cost-per-connection')) * 100); + $report = $this->collector->collect($costPerConnectionCents); if ($report['startedAt'] === null) { $this->warn('Experiment not started — set SUBSCRIPTION_EXPERIMENT_STARTED_AT to begin.'); @@ -56,31 +59,31 @@ class SendExperimentFunnelReportCommand extends Command } /** - * @param array{startedAt: ?CarbonImmutable, currency: string, revenueAvailable: bool, variants: array>} $report + * @param array{startedAt: ?CarbonImmutable, currency: string, revenueAvailable: bool, costPerConnectionCents: int, variants: array>} $report * @return list */ private function tableLines(array $report): array { $revenue = $report['revenueAvailable']; - $lines = [sprintf('%-8s %5s %4s %5s %5s %5s %5s %5s %5s %8s %8s', 'Variant', 'Assg', 'Sub', 'Trl', 'TrlX', 'Actv', 'Cncl', 'Rfnd', 'Net%', 'MRR', 'ARPU')]; + $currency = $report['currency']; + $lines = [sprintf('%-8s %5s %5s %5s %5s %6s %8s %8s %8s %8s', 'Variant', 'Assg', 'Actd', 'Card', 'Paid', 'A2P%', 'MRR', 'Cost', 'Burn', 'CM')]; foreach (self::LABELS as $key => $label) { $row = $report['variants'][$key]; - $mature = $row['assignedMature'] > 0; + $mature = $row['activatedMature'] > 0; $lines[] = sprintf( - '%-8s %5d %4d %5d %5d %5d %5d %5d %5s %8s %8s', + '%-8s %5d %5d %5d %5d %6s %8s %8s %8s %8s', $label, $row['assigned'], + $row['activated'], $row['subscribed'], - $row['trialing'], - $row['trialingCanceling'], - $row['active'], - $row['canceled'], - $row['refunded'], - $mature ? ((int) round($row['netActiveRate'] * 100)).'%' : 'pend', - $revenue ? $this->money($row['mrrCents'], $report['currency']) : '—', - $revenue && $mature ? $this->money((int) $row['arpuCents'], $report['currency']) : '—', + $row['activeMature'], + $mature ? ((int) round($row['activationToPaidRate'] * 100)).'%' : 'pend', + $revenue ? $this->money($row['mrrCents'], $currency) : '—', + $mature ? $this->money($row['costCents'], $currency) : '—', + $mature ? $this->money($row['wastedCostCents'], $currency) : '—', + $revenue && $mature ? $this->money($row['contributionMarginCents'], $currency) : '—', ); } @@ -100,7 +103,7 @@ class SendExperimentFunnelReportCommand extends Command } /** - * @param array{startedAt: ?CarbonImmutable, currency: string, revenueAvailable: bool, variants: array>} $report + * @param array{startedAt: ?CarbonImmutable, currency: string, revenueAvailable: bool, costPerConnectionCents: int, variants: array>} $report * @return array */ private function buildEmbed(array $report): array @@ -117,12 +120,15 @@ class SendExperimentFunnelReportCommand extends Command ], [ 'name' => 'Legend', - 'value' => 'Assg = assigned · Sub = started a plan · Trl = currently in trial · TrlX = of those, already scheduled to cancel at trial end (won\'t convert) · Actv/Cncl = current status · Rfnd = self-service refunds (pay_now) · Net% = live, non-refunded subs ÷ assigned (mature users only) · MRR = monthly run-rate of those subs (yearly ÷ 12) · ARPU = MRR ÷ assigned · `pend`/`—` = no mature data yet.', + 'value' => sprintf( + 'Assg = signups · Actd = activated (connected a bank or enabled AI = cost triggered) · Card = completed checkout (card on file) · Paid = live non-refunded subs (mature) · A2P%% = Paid ÷ activated (mature) · MRR = monthly run-rate of paid subs (yearly ÷ 12) · Cost = est. connection cost of the mature cohort (%s/connection) · Burn = connection cost of mature users who never converted (money lost) · CM = MRR − Cost · `pend`/`—` = no mature data yet.', + $this->money($report['costPerConnectionCents'], $report['currency']), + ), 'inline' => false, ], [ 'name' => '⚠️ How to read it', - 'value' => 'Each variant is gated by its own decision window (control 15d, reduced 7d, pay_now 3d), so pay_now matures first — compare only once all three have mature volume, and check significance before calling a winner. **ARPU is the revenue metric to compare.** MRR is run-rate, so it does not credit pay_now\'s yearly upfront cash; true LTV also needs a churn rate the experiment is too young to have.', + 'value' => 'Each variant is gated by its own decision window (control 15d, reduced 7d, pay_now 3d), so pay_now matures first — compare only once all three have mature volume, and check significance before calling a winner. **CM (contribution margin) is the decision metric.** Assg/Actd/Card are lifetime counts; A2P%/Cost/Burn/CM cover only the mature cohort, so the raw Actd→Card→Paid funnel mixes cohorts (immature carded users can\'t be Paid yet) — read it for volume, compare variants on A2P%/CM. Burn is what the connect-and-leave leak costs. Cost is a flat per-connection estimate across all providers, not per-provider billing.', 'inline' => false, ], ], diff --git a/app/Services/Stats/ExperimentFunnelCollector.php b/app/Services/Stats/ExperimentFunnelCollector.php index 17d1794e..6dec2525 100644 --- a/app/Services/Stats/ExperimentFunnelCollector.php +++ b/app/Services/Stats/ExperimentFunnelCollector.php @@ -27,19 +27,31 @@ class ExperimentFunnelCollector * subscription — an exact, heuristic-free metric that is comparable across * variants once each cohort clears its own decision window. * + * The funnel is assigned → activated → carded (subscribed) → net-paying: + * "activated" = the user connected a bank or enabled AI, i.e. triggered the + * paid infrastructure that costs us money, whether or not they ever paid. The + * gap activated → carded (completed Checkout with a card) is where a user + * connects a bank and walks away without paying — the exact leak the flat + * per-connection cost estimate quantifies. + * * Revenue is the monthly-recurring run-rate (MRR) of the mature, net-active * subscriptions, with yearly plans normalised to a monthly equivalent, plus - * ARPU = MRR ÷ assigned (mature) — the per-signup revenue each variant earns. - * This is run-rate, not realised cash, so it does not credit pay_now's yearly - * upfront payment any differently from a monthly one. Full LTV needs a churn - * rate the experiment is too young to have; ARPU is the proxy until then. + * ARPU = MRR ÷ assigned (mature). Cost is a flat estimate: connections of the + * mature cohort × `$costPerConnectionCents`; "wasted" cost is the same for + * mature users who did not convert (money burned). Contribution margin = + * MRR − cost, the decision metric once every variant has mature volume. This + * is run-rate, not realised cash, so it does not credit pay_now's yearly + * upfront payment any differently from a monthly one. * + * @param int $costPerConnectionCents flat estimated cost per bank connection * @return array{ * startedAt: ?CarbonImmutable, * currency: string, * revenueAvailable: bool, + * costPerConnectionCents: int, * variants: array * } */ - public function collect(): array + public function collect(int $costPerConnectionCents = 40): array { $startedValue = config('subscriptions.experiment.started_at'); $startedAt = $startedValue !== null ? CarbonImmutable::parse($startedValue) : null; @@ -68,7 +85,7 @@ class ExperimentFunnelCollector ]; if ($startedAt === null) { - return ['startedAt' => null, 'currency' => $currency, 'revenueAvailable' => false, 'variants' => $variants]; + return ['startedAt' => null, 'currency' => $currency, 'revenueAvailable' => false, 'costPerConnectionCents' => $costPerConnectionCents, 'variants' => $variants]; } $now = CarbonImmutable::now('UTC'); @@ -81,7 +98,13 @@ class ExperimentFunnelCollector ->when($excluded !== [], fn ($query) => $query->whereNotIn('email', $excluded)) ->with(['subscriptions' => fn ($query) => $query->where('type', 'default')]) ->select(['id', 'created_at']) - ->chunkById(500, function ($users) use (&$variants, $windows, $now, $monthlyEquiv): void { + ->withCount([ + // Cost is incurred by every connection ever opened, so count + // soft-deleted (revoked) ones too — they still cost us money. + 'bankingConnections as connection_count' => fn ($query) => $query->withTrashed(), + 'aiConsents as ai_consent_count', + ]) + ->chunkById(500, function ($users) use (&$variants, $windows, $now, $monthlyEquiv, $costPerConnectionCents): void { Feature::for($users)->load([SubscriptionExperiment::class]); foreach ($users as $user) { @@ -95,6 +118,13 @@ class ExperimentFunnelCollector $row['assigned']++; + $connections = (int) ($user->connection_count ?? 0); + $activated = $connections > 0 || (int) ($user->ai_consent_count ?? 0) > 0; + + if ($activated) { + $row['activated']++; + } + /** @var Subscription|null $subscription */ $subscription = $user->subscriptions->sortByDesc('created_at')->first(); $status = $subscription?->stripe_status; @@ -117,9 +147,18 @@ class ExperimentFunnelCollector if ($mature) { $row['assignedMature']++; + $connectionCostCents = $connections * $costPerConnectionCents; + $row['costCents'] += $connectionCostCents; + + if ($activated) { + $row['activatedMature']++; + } + if ($netActive) { $row['activeMature']++; $row['mrrCents'] += (int) ($monthlyEquiv[$subscription->stripe_price] ?? 0); + } else { + $row['wastedCostCents'] += $connectionCostCents; } } @@ -131,26 +170,32 @@ class ExperimentFunnelCollector $variants[$key]['netActiveRate'] = $row['assignedMature'] > 0 ? $row['activeMature'] / $row['assignedMature'] : null; + $variants[$key]['activationToPaidRate'] = $row['activatedMature'] > 0 + ? $row['activeMature'] / $row['activatedMature'] + : null; $variants[$key]['arpuCents'] = $row['assignedMature'] > 0 ? (int) round($row['mrrCents'] / $row['assignedMature']) : null; + $variants[$key]['contributionMarginCents'] = $row['mrrCents'] - $row['costCents']; } return [ 'startedAt' => $startedAt, 'currency' => $currency, 'revenueAvailable' => $monthlyEquiv !== [], + 'costPerConnectionCents' => $costPerConnectionCents, 'variants' => $variants, ]; } /** - * @return array{assigned: int, subscribed: int, trialing: int, trialingCanceling: int, active: int, canceled: int, pastDue: int, refunded: int, assignedMature: int, activeMature: int, netActiveRate: ?float, mrrCents: int, arpuCents: ?int} + * @return array{assigned: int, activated: int, subscribed: int, trialing: int, trialingCanceling: int, active: int, canceled: int, pastDue: int, refunded: int, assignedMature: int, activatedMature: int, activeMature: int, netActiveRate: ?float, activationToPaidRate: ?float, mrrCents: int, arpuCents: ?int, costCents: int, wastedCostCents: int, contributionMarginCents: int} */ private function emptyRow(): array { return [ 'assigned' => 0, + 'activated' => 0, 'subscribed' => 0, 'trialing' => 0, 'trialingCanceling' => 0, @@ -159,10 +204,15 @@ class ExperimentFunnelCollector 'pastDue' => 0, 'refunded' => 0, 'assignedMature' => 0, + 'activatedMature' => 0, 'activeMature' => 0, 'netActiveRate' => null, + 'activationToPaidRate' => null, 'mrrCents' => 0, 'arpuCents' => null, + 'costCents' => 0, + 'wastedCostCents' => 0, + 'contributionMarginCents' => 0, ]; } diff --git a/tests/Feature/SendExperimentFunnelReportCommandTest.php b/tests/Feature/SendExperimentFunnelReportCommandTest.php index e84c5c22..741dc7bc 100644 --- a/tests/Feature/SendExperimentFunnelReportCommandTest.php +++ b/tests/Feature/SendExperimentFunnelReportCommandTest.php @@ -1,6 +1,8 @@ 0) { + BankingConnection::factory()->count($connections)->for($user)->create(); + } + + if ($aiConsent) { + AiConsent::factory()->for($user)->create(); + } + return $user; } @@ -131,6 +142,48 @@ it('computes MRR and ARPU from the net-active subscriptions', function () { ->and($payNow['arpuCents'])->toBe(0); }); +it('marks a user activated when they connect a bank or enable AI', function () { + $signup = CarbonImmutable::parse('2026-06-05'); + + experimentUser(SubscriptionExperiment::CONTROL, $signup); // neither → not activated + experimentUser(SubscriptionExperiment::CONTROL, $signup, connections: 1); // bank → activated + experimentUser(SubscriptionExperiment::CONTROL, $signup, aiConsent: true); // AI → activated + + $control = app(ExperimentFunnelCollector::class)->collect()['variants'][SubscriptionExperiment::CONTROL]; + + expect($control['assigned'])->toBe(3) + ->and($control['activated'])->toBe(2) + ->and($control['activatedMature'])->toBe(2); +}); + +it('computes connection cost, wasted burn and contribution margin', function () { + $signup = CarbonImmutable::parse('2026-06-05'); + + // Converted control user with 2 connections: cost 2×€0.40, no burn, MRR €3.99. + experimentUser(SubscriptionExperiment::CONTROL, $signup, ['status' => 'active', 'at' => $signup->addDay()], connections: 2); + // Activated non-payer with 3 connections: pure burn, no revenue. + experimentUser(SubscriptionExperiment::CONTROL, $signup, null, connections: 3); + + $control = app(ExperimentFunnelCollector::class)->collect()['variants'][SubscriptionExperiment::CONTROL]; + + expect($control['costCents'])->toBe(200) // (2+3) × 40 + ->and($control['wastedCostCents'])->toBe(120) // 3 × 40, the non-payer + ->and($control['mrrCents'])->toBe(399) + ->and($control['contributionMarginCents'])->toBe(199) // 399 − 200 + ->and($control['activationToPaidRate'])->toBe(0.5); // 1 paid ÷ 2 activated +}); + +it('scales connection cost by the cost-per-connection argument', function () { + $signup = CarbonImmutable::parse('2026-06-05'); + + experimentUser(SubscriptionExperiment::CONTROL, $signup, null, connections: 2); + + $control = app(ExperimentFunnelCollector::class)->collect(100)['variants'][SubscriptionExperiment::CONTROL]; + + expect($control['costCents'])->toBe(200) // 2 × 100 + ->and($control['wastedCostCents'])->toBe(200); +}); + it('marks revenue unavailable when Stripe prices cannot be loaded', function () { Cache::forget('experiment_funnel_monthly_equiv'); Cache::put('experiment_funnel_monthly_equiv', [], now()->addHour()); // empty = unavailable