fix(stats): measure conversion as ever-charged, not active-now
Paid counted users net-active at report time, a live snapshot: an older cohort (control matures ~3x later than pay_now) has had longer to churn, so its conversion looked worse for reasons unrelated to the variant. Add a time-invariant convertedMature/conversionRate (ever charged, net of refund, excluding trial-only cancels) and surface it as the Conv/Conv% columns. MRR stays the current run-rate, so the gap between Conv and MRR now reads as churn.
This commit is contained in:
Víctor Falcón
parent
a6024d35ee
commit
a0a07090c3
|
|
@ -68,7 +68,7 @@ class SendExperimentFunnelReportCommand extends Command
|
|||
$currency = $report['currency'];
|
||||
$lines = [sprintf(
|
||||
'%-8s %5s %5s %5s %5s %5s %6s %7s %7s %7s %7s %7s',
|
||||
'Variant', 'Assg', 'Actd', 'Card', 'MatU', 'Paid', 'Conv%', 'ARPU', 'MRR', 'Cost', 'Burn', 'CM',
|
||||
'Variant', 'Assg', 'Actd', 'Card', 'MatU', 'Conv', 'Conv%', 'ARPU', 'MRR', 'Cost', 'Burn', 'CM',
|
||||
)];
|
||||
|
||||
foreach (self::LABELS as $key => $label) {
|
||||
|
|
@ -83,8 +83,8 @@ class SendExperimentFunnelReportCommand extends Command
|
|||
$row['activated'],
|
||||
$row['subscribed'],
|
||||
$row['assignedMature'],
|
||||
$row['activeMature'],
|
||||
$mature ? ((int) round($row['netActiveRate'] * 100)).'%' : 'pend',
|
||||
$row['convertedMature'],
|
||||
$mature ? ((int) round($row['conversionRate'] * 100)).'%' : 'pend',
|
||||
$money && $row['arpuCents'] !== null ? $this->money($row['arpuCents'], $currency) : '—',
|
||||
$money ? $this->money($row['mrrCents'], $currency) : '—',
|
||||
$mature ? $this->money($row['costCents'], $currency) : '—',
|
||||
|
|
@ -127,14 +127,14 @@ class SendExperimentFunnelReportCommand extends Command
|
|||
[
|
||||
'name' => 'Legend',
|
||||
'value' => sprintf(
|
||||
'Assg = signups · Actd = activated (connected a bank or enabled AI = cost triggered) · Card = completed checkout (card on file) · MatU = matured assigned (cohort old enough to score for this variant) · Paid = live non-refunded subs among MatU · Conv%% = Paid ÷ MatU (net conversion, always ≤100%%, comparable across variants) · ARPU = MRR ÷ MatU (revenue per matured user) · MRR = monthly run-rate of paid subs (yearly ÷ 12) · Cost = est. connection cost of MatU (%s/connection) · Burn = connection cost of matured users who never earned net revenue (connected a bank but never paid, or paid then refunded) · CM = MRR − Cost · `pend`/`—` = no matured data yet.',
|
||||
'Assg = signups · Actd = activated (connected a bank or enabled AI = cost triggered) · Card = completed checkout (card on file) · MatU = matured assigned (cohort old enough to score for this variant) · Conv = matured users who ever converted (were charged, net of refund) — time-invariant, so it does not shrink as an older cohort has longer to churn · Conv%% = Conv ÷ MatU (always ≤100%%, comparable across variants) · ARPU = MRR ÷ MatU (revenue per matured user) · MRR = monthly run-rate of *currently* paying subs (yearly ÷ 12); Conv above MRR is churn · Cost = est. connection cost of MatU (%s/connection) · Burn = connection cost of matured users who never earned net revenue (connected a bank but never paid, or paid then refunded) · CM = MRR − Cost · `pend`/`—` = no matured data yet.',
|
||||
$this->money($report['costPerConnectionCents'], $report['currency']),
|
||||
),
|
||||
'inline' => false,
|
||||
],
|
||||
[
|
||||
'name' => '⚠️ How to read it',
|
||||
'value' => 'Each variant matures on its own decision window (control 15d, reduced 7d, pay_now 3d, +3d settle), so at any moment MatU differs a lot between variants (pay_now matures first). **Compare variants on Conv% and ARPU — normalized per matured user — not on the absolute MRR/Cost/Burn/CM totals, which scale with MatU and so mechanically favour whichever variant has matured more.** Assg/Actd/Card are lifetime counts; everything from MatU rightward covers the matured cohort only, so the raw Actd→Card→Paid funnel mixes cohorts (immature carded users can\'t be Paid yet) — read it for volume. Per-user CM is sub-cent at current volume, so treat CM as directional context, not the decision. Check significance (sample size = MatU) before calling a winner. Cost is a flat per-connection estimate across all providers, not per-provider billing.',
|
||||
'value' => 'Each variant matures on its own decision window (control 15d, reduced 7d, pay_now 3d, +3d settle), so at any moment MatU differs a lot between variants (pay_now matures first). **Compare variants on Conv% and ARPU — normalized per matured user — not on the absolute MRR/Cost/Burn/CM totals, which scale with MatU and so mechanically favour whichever variant has matured more.** Assg/Actd/Card are lifetime counts; everything from MatU rightward covers the matured cohort only, so the raw Actd→Card→Conv funnel mixes cohorts (immature carded users can\'t have matured yet) — read it for volume. Conv counts anyone ever charged (net of refund), so it is not depressed for older cohorts the way a live-active snapshot would be. Per-user CM is sub-cent at current volume, so treat CM as directional context, not the decision. Check significance (sample size = MatU) before calling a winner. Cost is a flat per-connection estimate across all providers, not per-provider billing.',
|
||||
'inline' => false,
|
||||
],
|
||||
],
|
||||
|
|
|
|||
|
|
@ -62,7 +62,9 @@ class ExperimentFunnelCollector
|
|||
* refunded: int,
|
||||
* assignedMature: int,
|
||||
* activatedMature: int,
|
||||
* convertedMature: int,
|
||||
* activeMature: int,
|
||||
* conversionRate: ?float,
|
||||
* netActiveRate: ?float,
|
||||
* activationToPaidRate: ?float,
|
||||
* mrrCents: int,
|
||||
|
|
@ -141,6 +143,21 @@ class ExperimentFunnelCollector
|
|||
$status = $subscription?->stripe_status;
|
||||
$netActive = $status === 'active' && $subscription->refunded_at === null;
|
||||
|
||||
// "Converted" is time-invariant: the user was ever charged and
|
||||
// not refunded — currently active, or churned after the trial.
|
||||
// Unlike $netActive (a live snapshot), it does not shrink as an
|
||||
// older cohort has more time to cancel, so it is comparable
|
||||
// across variants that matured at different times. Excludes
|
||||
// trial-only cancels (ended on/before the trial → never charged).
|
||||
$converted = $subscription !== null
|
||||
&& $subscription->refunded_at === null
|
||||
&& $status !== 'trialing'
|
||||
&& (
|
||||
$subscription->trial_ends_at === null
|
||||
|| $subscription->ends_at === null
|
||||
|| $subscription->ends_at->greaterThan($subscription->trial_ends_at)
|
||||
);
|
||||
|
||||
if ($subscription !== null) {
|
||||
$row['subscribed']++;
|
||||
$row['trialing'] += $status === 'trialing' ? 1 : 0;
|
||||
|
|
@ -165,6 +182,10 @@ class ExperimentFunnelCollector
|
|||
$row['activatedMature']++;
|
||||
}
|
||||
|
||||
if ($converted) {
|
||||
$row['convertedMature']++;
|
||||
}
|
||||
|
||||
if ($netActive) {
|
||||
$row['activeMature']++;
|
||||
$priceId = (string) $subscription->stripe_price;
|
||||
|
|
@ -195,6 +216,9 @@ class ExperimentFunnelCollector
|
|||
}
|
||||
|
||||
foreach ($variants as $key => $row) {
|
||||
$variants[$key]['conversionRate'] = $row['assignedMature'] > 0
|
||||
? $row['convertedMature'] / $row['assignedMature']
|
||||
: null;
|
||||
$variants[$key]['netActiveRate'] = $row['assignedMature'] > 0
|
||||
? $row['activeMature'] / $row['assignedMature']
|
||||
: null;
|
||||
|
|
@ -217,7 +241,7 @@ class ExperimentFunnelCollector
|
|||
}
|
||||
|
||||
/**
|
||||
* @return array{assigned: int, activated: int, subscribed: int, trialing: int, trialingCanceling: int, active: int, canceled: int, pastDue: int, refunded: int, assignedMature: int, activatedMature: int, activeMature: int, netActiveRate: ?float, activationToPaidRate: ?float, mrrCents: int, arpuCents: ?int, costCents: int, wastedCostCents: int, contributionMarginCents: int}
|
||||
* @return array{assigned: int, activated: int, subscribed: int, trialing: int, trialingCanceling: int, active: int, canceled: int, pastDue: int, refunded: int, assignedMature: int, activatedMature: int, convertedMature: int, activeMature: int, conversionRate: ?float, netActiveRate: ?float, activationToPaidRate: ?float, mrrCents: int, arpuCents: ?int, costCents: int, wastedCostCents: int, contributionMarginCents: int}
|
||||
*/
|
||||
private function emptyRow(): array
|
||||
{
|
||||
|
|
@ -233,7 +257,9 @@ class ExperimentFunnelCollector
|
|||
'refunded' => 0,
|
||||
'assignedMature' => 0,
|
||||
'activatedMature' => 0,
|
||||
'convertedMature' => 0,
|
||||
'activeMature' => 0,
|
||||
'conversionRate' => null,
|
||||
'netActiveRate' => null,
|
||||
'activationToPaidRate' => null,
|
||||
'mrrCents' => 0,
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ beforeEach(function () {
|
|||
* with an optional default subscription and any bank connections / AI consent
|
||||
* that mark the user as "activated" and drive the connection-cost columns.
|
||||
*
|
||||
* @param array{status: string, at: CarbonImmutable, endsAt?: CarbonImmutable, refundedAt?: CarbonImmutable}|null $subscription
|
||||
* @param array{status: string, at: CarbonImmutable, endsAt?: CarbonImmutable, trialEndsAt?: CarbonImmutable, refundedAt?: CarbonImmutable}|null $subscription
|
||||
*/
|
||||
function experimentUser(string $variant, CarbonImmutable $signup, ?array $subscription = null, int $connections = 0, bool $aiConsent = false): User
|
||||
{
|
||||
|
|
@ -53,6 +53,7 @@ function experimentUser(string $variant, CarbonImmutable $signup, ?array $subscr
|
|||
'stripe_price' => 'price_test',
|
||||
'created_at' => $subscription['at'],
|
||||
'ends_at' => $subscription['endsAt'] ?? null,
|
||||
'trial_ends_at' => $subscription['trialEndsAt'] ?? null,
|
||||
'refunded_at' => $subscription['refundedAt'] ?? null,
|
||||
]);
|
||||
}
|
||||
|
|
@ -248,6 +249,32 @@ it('reports a matured-cohort conversion capped at 100% and prints the matured de
|
|||
->assertSuccessful();
|
||||
});
|
||||
|
||||
it('measures conversion as ever-charged, not active-now, so churn does not bias it', function () {
|
||||
$signup = CarbonImmutable::parse('2026-06-05');
|
||||
|
||||
// 1) Still active → converted.
|
||||
experimentUser(SubscriptionExperiment::CONTROL, $signup, ['status' => 'active', 'at' => $signup->addDay()]);
|
||||
// 2) Paid then churned after the trial (charged, not refunded) → converted.
|
||||
experimentUser(SubscriptionExperiment::CONTROL, $signup, [
|
||||
'status' => 'canceled', 'at' => $signup, 'trialEndsAt' => $signup->addDays(15), 'endsAt' => $signup->addDays(40),
|
||||
]);
|
||||
// 3) Canceled on/before the trial end (never charged) → NOT converted.
|
||||
experimentUser(SubscriptionExperiment::CONTROL, $signup, [
|
||||
'status' => 'canceled', 'at' => $signup, 'trialEndsAt' => $signup->addDays(15), 'endsAt' => $signup->addDays(10),
|
||||
]);
|
||||
// 4) Refunded → NOT converted.
|
||||
experimentUser(SubscriptionExperiment::CONTROL, $signup, [
|
||||
'status' => 'canceled', 'at' => $signup, 'endsAt' => $signup->addDay(), 'refundedAt' => $signup->addDay(),
|
||||
]);
|
||||
|
||||
$control = app(ExperimentFunnelCollector::class)->collect()['variants'][SubscriptionExperiment::CONTROL];
|
||||
|
||||
expect($control['assignedMature'])->toBe(4)
|
||||
->and($control['convertedMature'])->toBe(2) // #1 active + #2 churned-after-paying
|
||||
->and($control['activeMature'])->toBe(1) // only #1 is active now
|
||||
->and($control['conversionRate'])->toBe(0.5); // 2 ÷ 4, time-invariant
|
||||
});
|
||||
|
||||
it('excludes churned payers from burn but keeps refunds as burn', function () {
|
||||
$signup = CarbonImmutable::parse('2026-06-05');
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue