feat(stats): add MRR and ARPU per variant to the experiment funnel
Revenue dimension for the winner decision: per-variant MRR (monthly run-rate of mature net-active subs, yearly normalised to monthly) and ARPU (MRR / assigned), priced from Stripe and cached for an hour. Falls back to '—' when Stripe is unreachable. ARPU is the comparable revenue metric; full LTV still needs churn.
This commit is contained in:
Víctor Falcón
parent
051ef165f6
commit
cb5bc47905
|
|
@ -50,37 +50,49 @@ class SendExperimentFunnelReportCommand extends Command
|
|||
}
|
||||
|
||||
/**
|
||||
* @param array{startedAt: ?CarbonImmutable, variants: array<string, array<string, mixed>>} $report
|
||||
* @param array{startedAt: ?CarbonImmutable, currency: string, revenueAvailable: bool, variants: array<string, array<string, mixed>>} $report
|
||||
* @return list<string>
|
||||
*/
|
||||
private function tableLines(array $report): array
|
||||
{
|
||||
$lines = [sprintf('%-8s %5s %4s %5s %5s %5s %5s %5s %5s', 'Variant', 'Assg', 'Sub', 'Trial', 'Actv', 'Cncl', 'PstD', 'Rfnd', 'Net%')];
|
||||
$revenue = $report['revenueAvailable'];
|
||||
$lines = [sprintf('%-8s %5s %4s %5s %5s %5s %5s %8s %8s', 'Variant', 'Assg', 'Sub', 'Actv', 'Cncl', 'Rfnd', 'Net%', 'MRR', 'ARPU')];
|
||||
|
||||
foreach (self::LABELS as $key => $label) {
|
||||
$row = $report['variants'][$key];
|
||||
$mature = $row['assignedMature'] > 0;
|
||||
|
||||
$lines[] = sprintf(
|
||||
'%-8s %5d %4d %5d %5d %5d %5d %5d %5s',
|
||||
'%-8s %5d %4d %5d %5d %5d %5s %8s %8s',
|
||||
$label,
|
||||
$row['assigned'],
|
||||
$row['subscribed'],
|
||||
$row['trialing'],
|
||||
$row['active'],
|
||||
$row['canceled'],
|
||||
$row['pastDue'],
|
||||
$row['refunded'],
|
||||
$row['assignedMature'] === 0
|
||||
? 'pend'
|
||||
: ((int) round($row['netActiveRate'] * 100)).'%',
|
||||
$mature ? ((int) round($row['netActiveRate'] * 100)).'%' : 'pend',
|
||||
$revenue ? $this->money($row['mrrCents'], $report['currency']) : '—',
|
||||
$revenue && $mature ? $this->money((int) $row['arpuCents'], $report['currency']) : '—',
|
||||
);
|
||||
}
|
||||
|
||||
return $lines;
|
||||
}
|
||||
|
||||
private function money(int $cents, string $currency): string
|
||||
{
|
||||
$symbol = match (strtolower($currency)) {
|
||||
'eur' => '€',
|
||||
'gbp' => '£',
|
||||
'usd' => '$',
|
||||
default => $currency.' ',
|
||||
};
|
||||
|
||||
return $symbol.number_format($cents / 100, 2);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array{startedAt: ?CarbonImmutable, variants: array<string, array<string, mixed>>} $report
|
||||
* @param array{startedAt: ?CarbonImmutable, currency: string, revenueAvailable: bool, variants: array<string, array<string, mixed>>} $report
|
||||
* @return array<string, mixed>
|
||||
*/
|
||||
private function buildEmbed(array $report): array
|
||||
|
|
@ -97,12 +109,12 @@ class SendExperimentFunnelReportCommand extends Command
|
|||
],
|
||||
[
|
||||
'name' => 'Legend',
|
||||
'value' => 'Assg = assigned to the variant · Sub = started a plan · Trial/Actv/Cncl/PstD = current subscription status · Rfnd = self-service refunds (pay_now) · Net% = live, non-refunded subscriptions ÷ assigned, counting only users past their decision window · `pend` = no mature users yet.',
|
||||
'value' => 'Assg = assigned · Sub = started a plan · Actv/Cncl = current status · Rfnd = self-service refunds (pay_now) · Net% = live, non-refunded subs ÷ assigned (mature users only) · MRR = monthly run-rate of those subs (yearly ÷ 12) · ARPU = MRR ÷ assigned · `pend`/`—` = no mature data yet.',
|
||||
'inline' => false,
|
||||
],
|
||||
[
|
||||
'name' => '⚠️ Read at equal age',
|
||||
'value' => 'Net% gates each variant by its own decision window (control 15d, reduced 7d, pay_now 3d), so pay_now matures first. Compare Net% only once all three have meaningful mature volume.',
|
||||
'name' => '⚠️ How to read it',
|
||||
'value' => 'Each variant is gated by its own decision window (control 15d, reduced 7d, pay_now 3d), so pay_now matures first — compare only once all three have mature volume, and check significance before calling a winner. **ARPU is the revenue metric to compare.** MRR is run-rate, so it does not credit pay_now\'s yearly upfront cash; true LTV also needs a churn rate the experiment is too young to have.',
|
||||
'inline' => false,
|
||||
],
|
||||
],
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@ namespace App\Services\Stats;
|
|||
use App\Features\SubscriptionExperiment;
|
||||
use App\Models\User;
|
||||
use Carbon\CarbonImmutable;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Laravel\Cashier\Cashier;
|
||||
use Laravel\Pennant\Feature;
|
||||
|
||||
class ExperimentFunnelCollector
|
||||
|
|
@ -24,8 +26,17 @@ class ExperimentFunnelCollector
|
|||
* subscription — an exact, heuristic-free metric that is comparable across
|
||||
* variants once each cohort clears its own decision window.
|
||||
*
|
||||
* Revenue is the monthly-recurring run-rate (MRR) of the mature, net-active
|
||||
* subscriptions, with yearly plans normalised to a monthly equivalent, plus
|
||||
* ARPU = MRR ÷ assigned (mature) — the per-signup revenue each variant earns.
|
||||
* This is run-rate, not realised cash, so it does not credit pay_now's yearly
|
||||
* upfront payment any differently from a monthly one. Full LTV needs a churn
|
||||
* rate the experiment is too young to have; ARPU is the proxy until then.
|
||||
*
|
||||
* @return array{
|
||||
* startedAt: ?CarbonImmutable,
|
||||
* currency: string,
|
||||
* revenueAvailable: bool,
|
||||
* variants: array<string, array{
|
||||
* assigned: int,
|
||||
* subscribed: int,
|
||||
|
|
@ -37,6 +48,8 @@ class ExperimentFunnelCollector
|
|||
* assignedMature: int,
|
||||
* activeMature: int,
|
||||
* netActiveRate: ?float,
|
||||
* mrrCents: int,
|
||||
* arpuCents: ?int,
|
||||
* }>
|
||||
* }
|
||||
*/
|
||||
|
|
@ -44,6 +57,7 @@ class ExperimentFunnelCollector
|
|||
{
|
||||
$startedValue = config('subscriptions.experiment.started_at');
|
||||
$startedAt = $startedValue !== null ? CarbonImmutable::parse($startedValue) : null;
|
||||
$currency = strtoupper((string) config('cashier.currency', 'eur'));
|
||||
|
||||
$variants = [
|
||||
SubscriptionExperiment::CONTROL => $this->emptyRow(),
|
||||
|
|
@ -52,19 +66,20 @@ class ExperimentFunnelCollector
|
|||
];
|
||||
|
||||
if ($startedAt === null) {
|
||||
return ['startedAt' => null, 'variants' => $variants];
|
||||
return ['startedAt' => null, 'currency' => $currency, 'revenueAvailable' => false, 'variants' => $variants];
|
||||
}
|
||||
|
||||
$now = CarbonImmutable::now('UTC');
|
||||
$excluded = (array) config('ai_suggestions.report.excluded_emails', []);
|
||||
$windows = $this->decisionWindows();
|
||||
$monthlyEquiv = $this->monthlyEquivByPriceId();
|
||||
|
||||
User::query()
|
||||
->where('users.created_at', '>=', $startedAt)
|
||||
->when($excluded !== [], fn ($query) => $query->whereNotIn('email', $excluded))
|
||||
->with(['subscriptions' => fn ($query) => $query->where('type', 'default')])
|
||||
->select(['id', 'created_at'])
|
||||
->chunkById(500, function ($users) use (&$variants, $windows, $now): void {
|
||||
->chunkById(500, function ($users) use (&$variants, $windows, $now, $monthlyEquiv): void {
|
||||
Feature::for($users)->load([SubscriptionExperiment::class]);
|
||||
|
||||
foreach ($users as $user) {
|
||||
|
|
@ -97,7 +112,11 @@ class ExperimentFunnelCollector
|
|||
|
||||
if ($mature) {
|
||||
$row['assignedMature']++;
|
||||
$row['activeMature'] += $netActive ? 1 : 0;
|
||||
|
||||
if ($netActive) {
|
||||
$row['activeMature']++;
|
||||
$row['mrrCents'] += (int) ($monthlyEquiv[$subscription->stripe_price] ?? 0);
|
||||
}
|
||||
}
|
||||
|
||||
unset($row);
|
||||
|
|
@ -108,13 +127,21 @@ class ExperimentFunnelCollector
|
|||
$variants[$key]['netActiveRate'] = $row['assignedMature'] > 0
|
||||
? $row['activeMature'] / $row['assignedMature']
|
||||
: null;
|
||||
$variants[$key]['arpuCents'] = $row['assignedMature'] > 0
|
||||
? (int) round($row['mrrCents'] / $row['assignedMature'])
|
||||
: null;
|
||||
}
|
||||
|
||||
return ['startedAt' => $startedAt, 'variants' => $variants];
|
||||
return [
|
||||
'startedAt' => $startedAt,
|
||||
'currency' => $currency,
|
||||
'revenueAvailable' => $monthlyEquiv !== [],
|
||||
'variants' => $variants,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{assigned: int, subscribed: int, trialing: int, active: int, canceled: int, pastDue: int, refunded: int, assignedMature: int, activeMature: int, netActiveRate: ?float}
|
||||
* @return array{assigned: int, subscribed: int, trialing: int, active: int, canceled: int, pastDue: int, refunded: int, assignedMature: int, activeMature: int, netActiveRate: ?float, mrrCents: int, arpuCents: ?int}
|
||||
*/
|
||||
private function emptyRow(): array
|
||||
{
|
||||
|
|
@ -129,9 +156,56 @@ class ExperimentFunnelCollector
|
|||
'assignedMature' => 0,
|
||||
'activeMature' => 0,
|
||||
'netActiveRate' => null,
|
||||
'mrrCents' => 0,
|
||||
'arpuCents' => null,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Monthly-equivalent amount (in cents) for each plan price id, from Stripe.
|
||||
* Yearly prices are divided by 12. Cached for an hour; returns [] (revenue
|
||||
* unavailable) if Stripe can't be reached, without caching the failure.
|
||||
*
|
||||
* @return array<string, int>
|
||||
*/
|
||||
private function monthlyEquivByPriceId(): array
|
||||
{
|
||||
$key = 'experiment_funnel_monthly_equiv';
|
||||
|
||||
if (Cache::has($key)) {
|
||||
return Cache::get($key);
|
||||
}
|
||||
|
||||
$lookups = array_values(array_filter([
|
||||
config('subscriptions.plans.monthly.stripe_lookup_key'),
|
||||
config('subscriptions.plans.yearly.stripe_lookup_key'),
|
||||
]));
|
||||
|
||||
if ($lookups === []) {
|
||||
return [];
|
||||
}
|
||||
|
||||
try {
|
||||
$prices = Cashier::stripe()->prices->all(['lookup_keys' => $lookups, 'limit' => 10]);
|
||||
} catch (\Throwable) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$map = [];
|
||||
foreach ($prices->data as $price) {
|
||||
$amount = (int) ($price->unit_amount ?? 0);
|
||||
$map[$price->id] = ($price->recurring->interval ?? 'month') === 'year'
|
||||
? intdiv($amount, 12)
|
||||
: $amount;
|
||||
}
|
||||
|
||||
if ($map !== []) {
|
||||
Cache::put($key, $map, now()->addHour());
|
||||
}
|
||||
|
||||
return $map;
|
||||
}
|
||||
|
||||
/**
|
||||
* Days from signup until each variant's outcome can be scored: the trial
|
||||
* length (the longer of the two reduced trials) or the refund window.
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ use App\Models\User;
|
|||
use App\Services\Stats\ExperimentFunnelCollector;
|
||||
use Carbon\CarbonImmutable;
|
||||
use Illuminate\Support\Carbon;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Illuminate\Support\Facades\Http;
|
||||
use Illuminate\Support\Str;
|
||||
|
||||
|
|
@ -21,6 +22,9 @@ beforeEach(function () {
|
|||
'ai_suggestions.report.excluded_emails' => [],
|
||||
]);
|
||||
Carbon::setTestNow(CarbonImmutable::parse('2026-06-30 12:00:00'));
|
||||
|
||||
// Seed the price→monthly-equivalent map so revenue is computed without Stripe.
|
||||
Cache::put('experiment_funnel_monthly_equiv', ['price_test' => 399], now()->addHour());
|
||||
});
|
||||
|
||||
/**
|
||||
|
|
@ -89,6 +93,43 @@ it('attributes users and their subscription status to the right variant', functi
|
|||
->and($variants[SubscriptionExperiment::PAY_NOW]['activeMature'])->toBe(1);
|
||||
});
|
||||
|
||||
it('computes MRR and ARPU from the net-active subscriptions', function () {
|
||||
$signup = CarbonImmutable::parse('2026-06-05');
|
||||
|
||||
// One control assigned with no plan, one converted (€3.99/mo net-active).
|
||||
experimentUser(SubscriptionExperiment::CONTROL, $signup);
|
||||
experimentUser(SubscriptionExperiment::CONTROL, $signup, ['status' => 'active', 'at' => $signup->addDay()]);
|
||||
// A refunded pay_now contributes no revenue.
|
||||
experimentUser(SubscriptionExperiment::PAY_NOW, $signup, [
|
||||
'status' => 'canceled', 'at' => $signup, 'endsAt' => $signup->addDay(), 'refundedAt' => $signup->addDay(),
|
||||
]);
|
||||
|
||||
$report = app(ExperimentFunnelCollector::class)->collect();
|
||||
$control = $report['variants'][SubscriptionExperiment::CONTROL];
|
||||
$payNow = $report['variants'][SubscriptionExperiment::PAY_NOW];
|
||||
|
||||
expect($report['revenueAvailable'])->toBeTrue()
|
||||
->and($report['currency'])->toBe('EUR')
|
||||
->and($control['mrrCents'])->toBe(399) // one €3.99 net-active sub
|
||||
->and($control['arpuCents'])->toBe(200) // 399 / 2 assigned, rounded
|
||||
->and($payNow['mrrCents'])->toBe(0) // refunded → no revenue
|
||||
->and($payNow['arpuCents'])->toBe(0);
|
||||
});
|
||||
|
||||
it('marks revenue unavailable when Stripe prices cannot be loaded', function () {
|
||||
Cache::forget('experiment_funnel_monthly_equiv');
|
||||
Cache::put('experiment_funnel_monthly_equiv', [], now()->addHour()); // empty = unavailable
|
||||
|
||||
experimentUser(SubscriptionExperiment::CONTROL, CarbonImmutable::parse('2026-06-05'), [
|
||||
'status' => 'active', 'at' => CarbonImmutable::parse('2026-06-05'),
|
||||
]);
|
||||
|
||||
$report = app(ExperimentFunnelCollector::class)->collect();
|
||||
|
||||
expect($report['revenueAvailable'])->toBeFalse()
|
||||
->and($report['variants'][SubscriptionExperiment::CONTROL]['mrrCents'])->toBe(0);
|
||||
});
|
||||
|
||||
it('leaves young cohorts out of the mature net-active rate', function () {
|
||||
// pay_now decides in 3d (+3 buffer); a 2-day-old signup is not mature yet.
|
||||
experimentUser(SubscriptionExperiment::PAY_NOW, CarbonImmutable::now()->subDays(2), [
|
||||
|
|
|
|||
Loading…
Reference in New Issue