feat(stats): add unit-economics funnel + connection cost to experiment report (#666)
## What Reworks `stats:experiment-funnel` from a pure conversion funnel into a **contribution-margin** one, so the 3-way trial/pricing experiment can be judged on margin — not just conversion. Signups aren't free: each bank connection costs money per month, so trials that connect banks and never pay are burned cash, which the old report was blind to. ## Funnel Per variant: **assigned → activated → carded → net-paying** - **activated** = connected ≥1 bank connection **OR** gave AI consent (triggered paid infrastructure), whether or not they paid. - **carded** = completed Stripe Checkout (card on file). - **net-paying** = live, non-refunded, mature subscription. The **activated → carded** gap is where a user connects a bank and walks away without paying — the exact leak the report now puts a number on. ## Metrics | Column | Meaning | |---|---| | `A2P%` | net-paying ÷ activated (mature) | | `Cost` | mature-cohort connections × cost/connection | | `Burn` | connection cost of mature non-payers (money lost) | | `CM` | MRR − Cost (the decision metric) | - New `--cost-per-connection` option, default **0.40** per connection. - Connection count includes soft-deleted/revoked connections (they still cost money). - All money/rate columns are gated on each variant's maturity window; `Cost`/`Burn`/`CM` print `—` until a variant has mature volume. Legend notes that raw `Assg`/`Actd`/`Card` are lifetime counts while rates/money are mature-cohort only. - Existing MRR/ARPU/Net% fields stay on the collector; dropped from the printed table. ## Tests Added coverage for activation (bank OR AI), cost/burn/contribution-margin math, and the cost-per-connection argument. Full file green (12 passed).
This commit is contained in:
Víctor Falcón
GitHub
parent
815ca6244c
commit
dada23cd84
|
|
@ -10,7 +10,9 @@ use Illuminate\Console\Command;
|
|||
|
||||
class SendExperimentFunnelReportCommand extends Command
|
||||
{
|
||||
protected $signature = 'stats:experiment-funnel {--no-discord : Print the report to the console only, without posting to Discord}';
|
||||
protected $signature = 'stats:experiment-funnel
|
||||
{--no-discord : Print the report to the console only, without posting to Discord}
|
||||
{--cost-per-connection=0.4 : Estimated cost (in the Cashier currency) per bank connection, used for the Cost/Burn/CM columns}';
|
||||
|
||||
protected $description = 'Post the trial/pricing experiment funnel (per variant) to Discord';
|
||||
|
||||
|
|
@ -27,7 +29,8 @@ class SendExperimentFunnelReportCommand extends Command
|
|||
|
||||
public function handle(): int
|
||||
{
|
||||
$report = $this->collector->collect();
|
||||
$costPerConnectionCents = (int) round(((float) $this->option('cost-per-connection')) * 100);
|
||||
$report = $this->collector->collect($costPerConnectionCents);
|
||||
|
||||
if ($report['startedAt'] === null) {
|
||||
$this->warn('Experiment not started — set SUBSCRIPTION_EXPERIMENT_STARTED_AT to begin.');
|
||||
|
|
@ -56,31 +59,31 @@ class SendExperimentFunnelReportCommand extends Command
|
|||
}
|
||||
|
||||
/**
|
||||
* @param array{startedAt: ?CarbonImmutable, currency: string, revenueAvailable: bool, variants: array<string, array<string, mixed>>} $report
|
||||
* @param array{startedAt: ?CarbonImmutable, currency: string, revenueAvailable: bool, costPerConnectionCents: int, variants: array<string, array<string, mixed>>} $report
|
||||
* @return list<string>
|
||||
*/
|
||||
private function tableLines(array $report): array
|
||||
{
|
||||
$revenue = $report['revenueAvailable'];
|
||||
$lines = [sprintf('%-8s %5s %4s %5s %5s %5s %5s %5s %5s %8s %8s', 'Variant', 'Assg', 'Sub', 'Trl', 'TrlX', 'Actv', 'Cncl', 'Rfnd', 'Net%', 'MRR', 'ARPU')];
|
||||
$currency = $report['currency'];
|
||||
$lines = [sprintf('%-8s %5s %5s %5s %5s %6s %8s %8s %8s %8s', 'Variant', 'Assg', 'Actd', 'Card', 'Paid', 'A2P%', 'MRR', 'Cost', 'Burn', 'CM')];
|
||||
|
||||
foreach (self::LABELS as $key => $label) {
|
||||
$row = $report['variants'][$key];
|
||||
$mature = $row['assignedMature'] > 0;
|
||||
$mature = $row['activatedMature'] > 0;
|
||||
|
||||
$lines[] = sprintf(
|
||||
'%-8s %5d %4d %5d %5d %5d %5d %5d %5s %8s %8s',
|
||||
'%-8s %5d %5d %5d %5d %6s %8s %8s %8s %8s',
|
||||
$label,
|
||||
$row['assigned'],
|
||||
$row['activated'],
|
||||
$row['subscribed'],
|
||||
$row['trialing'],
|
||||
$row['trialingCanceling'],
|
||||
$row['active'],
|
||||
$row['canceled'],
|
||||
$row['refunded'],
|
||||
$mature ? ((int) round($row['netActiveRate'] * 100)).'%' : 'pend',
|
||||
$revenue ? $this->money($row['mrrCents'], $report['currency']) : '—',
|
||||
$revenue && $mature ? $this->money((int) $row['arpuCents'], $report['currency']) : '—',
|
||||
$row['activeMature'],
|
||||
$mature ? ((int) round($row['activationToPaidRate'] * 100)).'%' : 'pend',
|
||||
$revenue ? $this->money($row['mrrCents'], $currency) : '—',
|
||||
$mature ? $this->money($row['costCents'], $currency) : '—',
|
||||
$mature ? $this->money($row['wastedCostCents'], $currency) : '—',
|
||||
$revenue && $mature ? $this->money($row['contributionMarginCents'], $currency) : '—',
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -100,7 +103,7 @@ class SendExperimentFunnelReportCommand extends Command
|
|||
}
|
||||
|
||||
/**
|
||||
* @param array{startedAt: ?CarbonImmutable, currency: string, revenueAvailable: bool, variants: array<string, array<string, mixed>>} $report
|
||||
* @param array{startedAt: ?CarbonImmutable, currency: string, revenueAvailable: bool, costPerConnectionCents: int, variants: array<string, array<string, mixed>>} $report
|
||||
* @return array<string, mixed>
|
||||
*/
|
||||
private function buildEmbed(array $report): array
|
||||
|
|
@ -117,12 +120,15 @@ class SendExperimentFunnelReportCommand extends Command
|
|||
],
|
||||
[
|
||||
'name' => 'Legend',
|
||||
'value' => 'Assg = assigned · Sub = started a plan · Trl = currently in trial · TrlX = of those, already scheduled to cancel at trial end (won\'t convert) · Actv/Cncl = current status · Rfnd = self-service refunds (pay_now) · Net% = live, non-refunded subs ÷ assigned (mature users only) · MRR = monthly run-rate of those subs (yearly ÷ 12) · ARPU = MRR ÷ assigned · `pend`/`—` = no mature data yet.',
|
||||
'value' => sprintf(
|
||||
'Assg = signups · Actd = activated (connected a bank or enabled AI = cost triggered) · Card = completed checkout (card on file) · Paid = live non-refunded subs (mature) · A2P%% = Paid ÷ activated (mature) · MRR = monthly run-rate of paid subs (yearly ÷ 12) · Cost = est. connection cost of the mature cohort (%s/connection) · Burn = connection cost of mature users who never converted (money lost) · CM = MRR − Cost · `pend`/`—` = no mature data yet.',
|
||||
$this->money($report['costPerConnectionCents'], $report['currency']),
|
||||
),
|
||||
'inline' => false,
|
||||
],
|
||||
[
|
||||
'name' => '⚠️ How to read it',
|
||||
'value' => 'Each variant is gated by its own decision window (control 15d, reduced 7d, pay_now 3d), so pay_now matures first — compare only once all three have mature volume, and check significance before calling a winner. **ARPU is the revenue metric to compare.** MRR is run-rate, so it does not credit pay_now\'s yearly upfront cash; true LTV also needs a churn rate the experiment is too young to have.',
|
||||
'value' => 'Each variant is gated by its own decision window (control 15d, reduced 7d, pay_now 3d), so pay_now matures first — compare only once all three have mature volume, and check significance before calling a winner. **CM (contribution margin) is the decision metric.** Assg/Actd/Card are lifetime counts; A2P%/Cost/Burn/CM cover only the mature cohort, so the raw Actd→Card→Paid funnel mixes cohorts (immature carded users can\'t be Paid yet) — read it for volume, compare variants on A2P%/CM. Burn is what the connect-and-leave leak costs. Cost is a flat per-connection estimate across all providers, not per-provider billing.',
|
||||
'inline' => false,
|
||||
],
|
||||
],
|
||||
|
|
|
|||
|
|
@ -27,19 +27,31 @@ class ExperimentFunnelCollector
|
|||
* subscription — an exact, heuristic-free metric that is comparable across
|
||||
* variants once each cohort clears its own decision window.
|
||||
*
|
||||
* The funnel is assigned → activated → carded (subscribed) → net-paying:
|
||||
* "activated" = the user connected a bank or enabled AI, i.e. triggered the
|
||||
* paid infrastructure that costs us money, whether or not they ever paid. The
|
||||
* gap activated → carded (completed Checkout with a card) is where a user
|
||||
* connects a bank and walks away without paying — the exact leak the flat
|
||||
* per-connection cost estimate quantifies.
|
||||
*
|
||||
* Revenue is the monthly-recurring run-rate (MRR) of the mature, net-active
|
||||
* subscriptions, with yearly plans normalised to a monthly equivalent, plus
|
||||
* ARPU = MRR ÷ assigned (mature) — the per-signup revenue each variant earns.
|
||||
* This is run-rate, not realised cash, so it does not credit pay_now's yearly
|
||||
* upfront payment any differently from a monthly one. Full LTV needs a churn
|
||||
* rate the experiment is too young to have; ARPU is the proxy until then.
|
||||
* ARPU = MRR ÷ assigned (mature). Cost is a flat estimate: connections of the
|
||||
* mature cohort × `$costPerConnectionCents`; "wasted" cost is the same for
|
||||
* mature users who did not convert (money burned). Contribution margin =
|
||||
* MRR − cost, the decision metric once every variant has mature volume. This
|
||||
* is run-rate, not realised cash, so it does not credit pay_now's yearly
|
||||
* upfront payment any differently from a monthly one.
|
||||
*
|
||||
* @param int $costPerConnectionCents flat estimated cost per bank connection
|
||||
* @return array{
|
||||
* startedAt: ?CarbonImmutable,
|
||||
* currency: string,
|
||||
* revenueAvailable: bool,
|
||||
* costPerConnectionCents: int,
|
||||
* variants: array<string, array{
|
||||
* assigned: int,
|
||||
* activated: int,
|
||||
* subscribed: int,
|
||||
* trialing: int,
|
||||
* trialingCanceling: int,
|
||||
|
|
@ -48,14 +60,19 @@ class ExperimentFunnelCollector
|
|||
* pastDue: int,
|
||||
* refunded: int,
|
||||
* assignedMature: int,
|
||||
* activatedMature: int,
|
||||
* activeMature: int,
|
||||
* netActiveRate: ?float,
|
||||
* activationToPaidRate: ?float,
|
||||
* mrrCents: int,
|
||||
* arpuCents: ?int,
|
||||
* costCents: int,
|
||||
* wastedCostCents: int,
|
||||
* contributionMarginCents: int,
|
||||
* }>
|
||||
* }
|
||||
*/
|
||||
public function collect(): array
|
||||
public function collect(int $costPerConnectionCents = 40): array
|
||||
{
|
||||
$startedValue = config('subscriptions.experiment.started_at');
|
||||
$startedAt = $startedValue !== null ? CarbonImmutable::parse($startedValue) : null;
|
||||
|
|
@ -68,7 +85,7 @@ class ExperimentFunnelCollector
|
|||
];
|
||||
|
||||
if ($startedAt === null) {
|
||||
return ['startedAt' => null, 'currency' => $currency, 'revenueAvailable' => false, 'variants' => $variants];
|
||||
return ['startedAt' => null, 'currency' => $currency, 'revenueAvailable' => false, 'costPerConnectionCents' => $costPerConnectionCents, 'variants' => $variants];
|
||||
}
|
||||
|
||||
$now = CarbonImmutable::now('UTC');
|
||||
|
|
@ -81,7 +98,13 @@ class ExperimentFunnelCollector
|
|||
->when($excluded !== [], fn ($query) => $query->whereNotIn('email', $excluded))
|
||||
->with(['subscriptions' => fn ($query) => $query->where('type', 'default')])
|
||||
->select(['id', 'created_at'])
|
||||
->chunkById(500, function ($users) use (&$variants, $windows, $now, $monthlyEquiv): void {
|
||||
->withCount([
|
||||
// Cost is incurred by every connection ever opened, so count
|
||||
// soft-deleted (revoked) ones too — they still cost us money.
|
||||
'bankingConnections as connection_count' => fn ($query) => $query->withTrashed(),
|
||||
'aiConsents as ai_consent_count',
|
||||
])
|
||||
->chunkById(500, function ($users) use (&$variants, $windows, $now, $monthlyEquiv, $costPerConnectionCents): void {
|
||||
Feature::for($users)->load([SubscriptionExperiment::class]);
|
||||
|
||||
foreach ($users as $user) {
|
||||
|
|
@ -95,6 +118,13 @@ class ExperimentFunnelCollector
|
|||
|
||||
$row['assigned']++;
|
||||
|
||||
$connections = (int) ($user->connection_count ?? 0);
|
||||
$activated = $connections > 0 || (int) ($user->ai_consent_count ?? 0) > 0;
|
||||
|
||||
if ($activated) {
|
||||
$row['activated']++;
|
||||
}
|
||||
|
||||
/** @var Subscription|null $subscription */
|
||||
$subscription = $user->subscriptions->sortByDesc('created_at')->first();
|
||||
$status = $subscription?->stripe_status;
|
||||
|
|
@ -117,9 +147,18 @@ class ExperimentFunnelCollector
|
|||
if ($mature) {
|
||||
$row['assignedMature']++;
|
||||
|
||||
$connectionCostCents = $connections * $costPerConnectionCents;
|
||||
$row['costCents'] += $connectionCostCents;
|
||||
|
||||
if ($activated) {
|
||||
$row['activatedMature']++;
|
||||
}
|
||||
|
||||
if ($netActive) {
|
||||
$row['activeMature']++;
|
||||
$row['mrrCents'] += (int) ($monthlyEquiv[$subscription->stripe_price] ?? 0);
|
||||
} else {
|
||||
$row['wastedCostCents'] += $connectionCostCents;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -131,26 +170,32 @@ class ExperimentFunnelCollector
|
|||
$variants[$key]['netActiveRate'] = $row['assignedMature'] > 0
|
||||
? $row['activeMature'] / $row['assignedMature']
|
||||
: null;
|
||||
$variants[$key]['activationToPaidRate'] = $row['activatedMature'] > 0
|
||||
? $row['activeMature'] / $row['activatedMature']
|
||||
: null;
|
||||
$variants[$key]['arpuCents'] = $row['assignedMature'] > 0
|
||||
? (int) round($row['mrrCents'] / $row['assignedMature'])
|
||||
: null;
|
||||
$variants[$key]['contributionMarginCents'] = $row['mrrCents'] - $row['costCents'];
|
||||
}
|
||||
|
||||
return [
|
||||
'startedAt' => $startedAt,
|
||||
'currency' => $currency,
|
||||
'revenueAvailable' => $monthlyEquiv !== [],
|
||||
'costPerConnectionCents' => $costPerConnectionCents,
|
||||
'variants' => $variants,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{assigned: int, subscribed: int, trialing: int, trialingCanceling: int, active: int, canceled: int, pastDue: int, refunded: int, assignedMature: int, activeMature: int, netActiveRate: ?float, mrrCents: int, arpuCents: ?int}
|
||||
* @return array{assigned: int, activated: int, subscribed: int, trialing: int, trialingCanceling: int, active: int, canceled: int, pastDue: int, refunded: int, assignedMature: int, activatedMature: int, activeMature: int, netActiveRate: ?float, activationToPaidRate: ?float, mrrCents: int, arpuCents: ?int, costCents: int, wastedCostCents: int, contributionMarginCents: int}
|
||||
*/
|
||||
private function emptyRow(): array
|
||||
{
|
||||
return [
|
||||
'assigned' => 0,
|
||||
'activated' => 0,
|
||||
'subscribed' => 0,
|
||||
'trialing' => 0,
|
||||
'trialingCanceling' => 0,
|
||||
|
|
@ -159,10 +204,15 @@ class ExperimentFunnelCollector
|
|||
'pastDue' => 0,
|
||||
'refunded' => 0,
|
||||
'assignedMature' => 0,
|
||||
'activatedMature' => 0,
|
||||
'activeMature' => 0,
|
||||
'netActiveRate' => null,
|
||||
'activationToPaidRate' => null,
|
||||
'mrrCents' => 0,
|
||||
'arpuCents' => null,
|
||||
'costCents' => 0,
|
||||
'wastedCostCents' => 0,
|
||||
'contributionMarginCents' => 0,
|
||||
];
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
<?php
|
||||
|
||||
use App\Features\SubscriptionExperiment;
|
||||
use App\Models\AiConsent;
|
||||
use App\Models\BankingConnection;
|
||||
use App\Models\User;
|
||||
use App\Services\Stats\ExperimentFunnelCollector;
|
||||
use Carbon\CarbonImmutable;
|
||||
|
|
@ -29,11 +31,12 @@ beforeEach(function () {
|
|||
|
||||
/**
|
||||
* Create a user whose id buckets into the wanted variant, anchored to a signup,
|
||||
* with an optional default subscription.
|
||||
* with an optional default subscription and any bank connections / AI consent
|
||||
* that mark the user as "activated" and drive the connection-cost columns.
|
||||
*
|
||||
* @param array{status: string, at: CarbonImmutable, endsAt?: CarbonImmutable, refundedAt?: CarbonImmutable}|null $subscription
|
||||
*/
|
||||
function experimentUser(string $variant, CarbonImmutable $signup, ?array $subscription = null): User
|
||||
function experimentUser(string $variant, CarbonImmutable $signup, ?array $subscription = null, int $connections = 0, bool $aiConsent = false): User
|
||||
{
|
||||
do {
|
||||
$id = (string) Str::uuid();
|
||||
|
|
@ -53,6 +56,14 @@ function experimentUser(string $variant, CarbonImmutable $signup, ?array $subscr
|
|||
]);
|
||||
}
|
||||
|
||||
if ($connections > 0) {
|
||||
BankingConnection::factory()->count($connections)->for($user)->create();
|
||||
}
|
||||
|
||||
if ($aiConsent) {
|
||||
AiConsent::factory()->for($user)->create();
|
||||
}
|
||||
|
||||
return $user;
|
||||
}
|
||||
|
||||
|
|
@ -131,6 +142,48 @@ it('computes MRR and ARPU from the net-active subscriptions', function () {
|
|||
->and($payNow['arpuCents'])->toBe(0);
|
||||
});
|
||||
|
||||
it('marks a user activated when they connect a bank or enable AI', function () {
|
||||
$signup = CarbonImmutable::parse('2026-06-05');
|
||||
|
||||
experimentUser(SubscriptionExperiment::CONTROL, $signup); // neither → not activated
|
||||
experimentUser(SubscriptionExperiment::CONTROL, $signup, connections: 1); // bank → activated
|
||||
experimentUser(SubscriptionExperiment::CONTROL, $signup, aiConsent: true); // AI → activated
|
||||
|
||||
$control = app(ExperimentFunnelCollector::class)->collect()['variants'][SubscriptionExperiment::CONTROL];
|
||||
|
||||
expect($control['assigned'])->toBe(3)
|
||||
->and($control['activated'])->toBe(2)
|
||||
->and($control['activatedMature'])->toBe(2);
|
||||
});
|
||||
|
||||
it('computes connection cost, wasted burn and contribution margin', function () {
|
||||
$signup = CarbonImmutable::parse('2026-06-05');
|
||||
|
||||
// Converted control user with 2 connections: cost 2×€0.40, no burn, MRR €3.99.
|
||||
experimentUser(SubscriptionExperiment::CONTROL, $signup, ['status' => 'active', 'at' => $signup->addDay()], connections: 2);
|
||||
// Activated non-payer with 3 connections: pure burn, no revenue.
|
||||
experimentUser(SubscriptionExperiment::CONTROL, $signup, null, connections: 3);
|
||||
|
||||
$control = app(ExperimentFunnelCollector::class)->collect()['variants'][SubscriptionExperiment::CONTROL];
|
||||
|
||||
expect($control['costCents'])->toBe(200) // (2+3) × 40
|
||||
->and($control['wastedCostCents'])->toBe(120) // 3 × 40, the non-payer
|
||||
->and($control['mrrCents'])->toBe(399)
|
||||
->and($control['contributionMarginCents'])->toBe(199) // 399 − 200
|
||||
->and($control['activationToPaidRate'])->toBe(0.5); // 1 paid ÷ 2 activated
|
||||
});
|
||||
|
||||
it('scales connection cost by the cost-per-connection argument', function () {
|
||||
$signup = CarbonImmutable::parse('2026-06-05');
|
||||
|
||||
experimentUser(SubscriptionExperiment::CONTROL, $signup, null, connections: 2);
|
||||
|
||||
$control = app(ExperimentFunnelCollector::class)->collect(100)['variants'][SubscriptionExperiment::CONTROL];
|
||||
|
||||
expect($control['costCents'])->toBe(200) // 2 × 100
|
||||
->and($control['wastedCostCents'])->toBe(200);
|
||||
});
|
||||
|
||||
it('marks revenue unavailable when Stripe prices cannot be loaded', function () {
|
||||
Cache::forget('experiment_funnel_monthly_equiv');
|
||||
Cache::put('experiment_funnel_monthly_equiv', [], now()->addHour()); // empty = unavailable
|
||||
|
|
|
|||
Loading…
Reference in New Issue