feat(stats): measure the price experiment funnel with a CM/user decision test
Add stats:price-experiment-funnel: a 2-arm funnel (control vs high) on one shared decision window. The primary metric is contribution margin per assigned user, tested with a Welch two-sample test (unequal variances, since the higher-priced arm carries far more revenue variance); conversion is a Fisher-exact guardrail against control. Cost uses currently-active connections to match the monthly MRR flow. The report is framed monitoring-only: decide at the pre-registered horizon, not on weekly peeks. Extract the shared MonthlyEquivalentPrices lookup so both funnel collectors resolve Stripe prices through one source.
This commit is contained in:
Víctor Falcón
parent
44c368120b
commit
f7cee87a2a
|
|
@ -0,0 +1,230 @@
|
|||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Features\PriceExperiment;
|
||||
use App\Services\Discord\DiscordWebhook;
|
||||
use App\Services\Stats\BinomialProportion;
|
||||
use App\Services\Stats\PriceExperimentFunnelCollector;
|
||||
use App\Services\Stats\ProportionSignificance;
|
||||
use App\Services\Stats\WelchTTest;
|
||||
use App\Support\Money;
|
||||
use Carbon\CarbonImmutable;
|
||||
use Illuminate\Console\Command;
|
||||
|
||||
class SendPriceExperimentFunnelReportCommand extends Command
|
||||
{
|
||||
protected $signature = 'stats:price-experiment-funnel
|
||||
{--no-discord : Print the report to the console only, without posting to Discord}
|
||||
{--cost-per-connection=0.4 : Estimated monthly cost (in the Cashier currency) per active bank connection, used for the Cost/CM columns}';
|
||||
|
||||
protected $description = 'Post the price experiment funnel (control vs high) to Discord — MONITORING ONLY, decide at the pre-registered horizon';
|
||||
|
||||
private const LABELS = [
|
||||
PriceExperiment::CONTROL => 'control',
|
||||
PriceExperiment::HIGH => 'high',
|
||||
];
|
||||
|
||||
public function __construct(
|
||||
private PriceExperimentFunnelCollector $collector,
|
||||
private ProportionSignificance $significance,
|
||||
private WelchTTest $welch,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
public function handle(): int
|
||||
{
|
||||
$costPerConnectionCents = (int) round(((float) $this->option('cost-per-connection')) * 100);
|
||||
$report = $this->collector->collect($costPerConnectionCents);
|
||||
|
||||
if ($report['startedAt'] === null) {
|
||||
$this->warn('Price experiment not started — set PRICE_EXPERIMENT_STARTED_AT to begin.');
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
foreach ([...$this->tableLines($report), ...$this->significanceLines($report)] as $line) {
|
||||
$this->line($line);
|
||||
}
|
||||
|
||||
if ($this->option('no-discord')) {
|
||||
$this->info('Skipped Discord (--no-discord).');
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
$webhookUrl = config('services.discord.ai_cohort_webhook_url')
|
||||
?: config('services.discord.webhook_url');
|
||||
|
||||
(new DiscordWebhook($webhookUrl))->send('', [$this->buildEmbed($report)]);
|
||||
|
||||
$this->info('Price experiment funnel report sent to Discord.');
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array{currency: string, revenueAvailable: bool, variants: array<string, array<string, mixed>>} $report
|
||||
* @return list<string>
|
||||
*/
|
||||
private function tableLines(array $report): array
|
||||
{
|
||||
$currency = $report['currency'];
|
||||
$lines = [sprintf(
|
||||
'%-8s %5s %5s %5s %5s %5s %6s %8s %8s %8s %8s %6s',
|
||||
'Variant', 'Assg', 'Actd', 'Card', 'MatU', 'Conv', 'Conv%', 'MRR', 'Cost', 'CM', 'CM/u', 'C/Pyr',
|
||||
)];
|
||||
|
||||
foreach (self::LABELS as $key => $label) {
|
||||
$row = $report['variants'][$key];
|
||||
$mature = $row['assignedMature'] > 0;
|
||||
$showMoney = $report['revenueAvailable'] && $mature;
|
||||
|
||||
$lines[] = sprintf(
|
||||
'%-8s %5d %5d %5d %5d %5d %6s %8s %8s %8s %8s %6s',
|
||||
$label,
|
||||
$row['assigned'],
|
||||
$row['activated'],
|
||||
$row['subscribed'],
|
||||
$row['assignedMature'],
|
||||
$row['convertedMature'],
|
||||
$mature ? ((int) round($row['conversionRate'] * 100)).'%' : 'pend',
|
||||
$showMoney ? Money::format($row['mrrCents'], $currency) : '—',
|
||||
$mature ? Money::format($row['costCents'], $currency) : '—',
|
||||
$showMoney ? Money::format($row['contributionMarginCents'], $currency) : '—',
|
||||
$showMoney && $row['cmPerAssignedCents'] !== null ? Money::format($row['cmPerAssignedCents'], $currency) : '—',
|
||||
$row['connectionsPerPayer'] !== null ? number_format($row['connectionsPerPayer'], 1) : '—',
|
||||
);
|
||||
}
|
||||
|
||||
return $lines;
|
||||
}
|
||||
|
||||
/**
|
||||
* Primary decision test — Welch on contribution-margin-per-user (control vs
|
||||
* high), the metric we maximise — plus the conversion guardrail (Wilson CIs +
|
||||
* Fisher exact, a single control-vs-treatment contrast, so no multiplicity
|
||||
* correction). CM/user is the number to act on; conversion protects the funnel.
|
||||
*
|
||||
* @param array{revenueAvailable: bool, currency: string, variants: array<string, array<string, mixed>>} $report
|
||||
* @return list<string>
|
||||
*/
|
||||
private function significanceLines(array $report): array
|
||||
{
|
||||
$currency = $report['currency'];
|
||||
$control = $report['variants'][PriceExperiment::CONTROL];
|
||||
$high = $report['variants'][PriceExperiment::HIGH];
|
||||
|
||||
$lines = ['', 'PRIMARY — contribution margin per assigned user (Welch, high vs control):'];
|
||||
|
||||
if (! $report['revenueAvailable'] || $control['assignedMature'] < 2 || $high['assignedMature'] < 2) {
|
||||
$lines[] = ' pend — need mature volume in both arms and Stripe revenue to compare.';
|
||||
} else {
|
||||
foreach (['control' => $control, 'high' => $high] as $label => $row) {
|
||||
$lines[] = sprintf(' %-8s %8s/user (n=%d)', $label, Money::format((int) $row['cmPerAssignedCents'], $currency), $row['assignedMature']);
|
||||
}
|
||||
|
||||
$welch = $this->welch->compare(
|
||||
$this->cmMean($high), $this->cmVariance($high), $high['assignedMature'],
|
||||
$this->cmMean($control), $this->cmVariance($control), $control['assignedMature'],
|
||||
);
|
||||
$significant = $welch['p'] < 0.05;
|
||||
$lines[] = sprintf(
|
||||
' Δ high−control: %s/user t=%.2f p=%.3f %s α=0.05 -> %s.%s',
|
||||
Money::format((int) round($welch['diff']), $currency),
|
||||
$welch['t'], $welch['p'], $significant ? '<' : '≥',
|
||||
$significant ? 'significant' : 'not significant',
|
||||
$significant ? '' : ' Keep running.',
|
||||
);
|
||||
}
|
||||
|
||||
$lines[] = '';
|
||||
$lines[] = 'GUARDRAIL — conversion (95% Wilson CI on Conv%, n = MatU):';
|
||||
$arms = [];
|
||||
|
||||
foreach (self::LABELS as $key => $label) {
|
||||
$row = $report['variants'][$key];
|
||||
$n = (int) $row['assignedMature'];
|
||||
$k = (int) $row['convertedMature'];
|
||||
|
||||
if ($n <= 0) {
|
||||
$lines[] = sprintf(' %-8s pend (n=0)', $label);
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
[$low, $high2] = $this->significance->wilsonInterval($k, $n);
|
||||
$lines[] = sprintf(' %-8s %6s [%6s – %6s] (n=%d)', $label, $this->percent($k / $n), $this->percent($low), $this->percent($high2), $n);
|
||||
$arms[$key] = new BinomialProportion($label, $k, $n);
|
||||
}
|
||||
|
||||
if (count($arms) === 2) {
|
||||
$result = $this->significance->compare($arms[PriceExperiment::HIGH], $arms[PriceExperiment::CONTROL], comparisons: 1);
|
||||
$lines[] = sprintf(
|
||||
' Fisher exact p=%.3f %s α=%.3f -> %s.',
|
||||
$result['fisherP'], $result['significant'] ? '<' : '≥', $result['alpha'],
|
||||
$result['significant'] ? 'significant' : 'not significant',
|
||||
);
|
||||
}
|
||||
|
||||
return $lines;
|
||||
}
|
||||
|
||||
private function cmMean(array $row): float
|
||||
{
|
||||
return $row['assignedMature'] > 0 ? (float) $row['contributionMarginCents'] / $row['assignedMature'] : 0.0;
|
||||
}
|
||||
|
||||
/** Sample variance (n−1) of per-user CM, from the accumulated moments. */
|
||||
private function cmVariance(array $row): float
|
||||
{
|
||||
$n = (int) $row['assignedMature'];
|
||||
|
||||
if ($n < 2) {
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
$sum = (float) $row['contributionMarginCents'];
|
||||
|
||||
return max(0.0, ((float) $row['cmSumSq'] - ($sum * $sum) / $n) / ($n - 1));
|
||||
}
|
||||
|
||||
private function percent(float $rate): string
|
||||
{
|
||||
return number_format($rate * 100, 1).'%';
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array{startedAt: CarbonImmutable, currency: string, revenueAvailable: bool, costPerConnectionCents: int, decisionWindowDays: int, variants: array<string, array<string, mixed>>} $report
|
||||
* @return array<string, mixed>
|
||||
*/
|
||||
private function buildEmbed(array $report): array
|
||||
{
|
||||
return [
|
||||
'title' => '💶 Price Experiment — Funnel (control vs high)',
|
||||
'description' => "```\n".implode("\n", [...$this->tableLines($report), ...$this->significanceLines($report)])."\n```",
|
||||
'color' => 0x57F287,
|
||||
'fields' => [
|
||||
[
|
||||
'name' => 'Started',
|
||||
'value' => $report['startedAt']->format('D, d M Y').sprintf(' · new signups split 50/50; both arms mature at %d days.', $report['decisionWindowDays']),
|
||||
'inline' => false,
|
||||
],
|
||||
[
|
||||
'name' => 'Legend',
|
||||
'value' => sprintf(
|
||||
'Assg = signups · Actd = activated (bank or AI = cost triggered) · Card = started a subscription · MatU = matured assigned (old enough to score) · Conv%% = ever-charged net of refund ÷ MatU (time-invariant) · MRR = monthly run-rate of currently-paying subs · Cost = active connections × %s · CM = MRR − Cost · CM/u = CM ÷ MatU (the decision metric) · C/Pyr = active connections per payer.',
|
||||
Money::format($report['costPerConnectionCents'], $report['currency']),
|
||||
),
|
||||
'inline' => false,
|
||||
],
|
||||
[
|
||||
'name' => '⚠️ MONITORING ONLY',
|
||||
'value' => "**Do NOT call a winner from this weekly report.** Decide at the pre-registered horizon N (from the power analysis), not the first week Fisher/Welch crosses α — weekly peeking inflates the false-positive rate. The primary metric is **CM per assigned user** (Welch); conversion is a guardrail. CM/user is sub-cent per week at current volume, so early weeks read as 'not significant' — that is expected. Don't add/remove arms or change the split mid-flight. A 1-month CM cannot see price-driven churn, so re-read retention/LTV before permanently locking the price.",
|
||||
'inline' => false,
|
||||
],
|
||||
],
|
||||
];
|
||||
}
|
||||
}
|
||||
|
|
@ -5,9 +5,7 @@ namespace App\Services\Stats;
|
|||
use App\Features\SubscriptionExperiment;
|
||||
use App\Models\User;
|
||||
use Carbon\CarbonImmutable;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Laravel\Cashier\Cashier;
|
||||
use Laravel\Cashier\Subscription;
|
||||
|
||||
class ExperimentFunnelCollector
|
||||
|
|
@ -18,6 +16,8 @@ class ExperimentFunnelCollector
|
|||
*/
|
||||
private const SETTLE_BUFFER_DAYS = 3;
|
||||
|
||||
public function __construct(private MonthlyEquivalentPrices $prices) {}
|
||||
|
||||
/**
|
||||
* Per-variant funnel for the trial/pricing experiment. Users are attributed
|
||||
* by SubscriptionExperiment::bucket() — the deterministic crc32 split that is
|
||||
|
|
@ -94,7 +94,7 @@ class ExperimentFunnelCollector
|
|||
$now = CarbonImmutable::now('UTC');
|
||||
$excluded = (array) config('ai_suggestions.report.excluded_emails', []);
|
||||
$windows = $this->decisionWindows();
|
||||
$monthlyEquiv = $this->monthlyEquivByPriceId();
|
||||
$monthlyEquiv = $this->prices->map();
|
||||
$missingPrices = [];
|
||||
|
||||
User::query()
|
||||
|
|
@ -270,70 +270,6 @@ class ExperimentFunnelCollector
|
|||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Monthly-equivalent amount (in cents) for each plan price id, from Stripe.
|
||||
* Yearly prices are divided by 12. Fetched by product so that archived,
|
||||
* rotated price ids (Stripe mints a new id and transfers the lookup key on
|
||||
* any amount change) still resolve — otherwise subscriptions on an old id
|
||||
* would silently contribute 0 to MRR. Falls back to the current lookup keys
|
||||
* when no product is configured. Foreign-currency and one-off prices are
|
||||
* skipped. Cached for an hour; returns [] (revenue unavailable) if Stripe
|
||||
* can't be reached, without caching the failure.
|
||||
*
|
||||
* @return array<string, int>
|
||||
*/
|
||||
private function monthlyEquivByPriceId(): array
|
||||
{
|
||||
$key = 'experiment_funnel_monthly_equiv';
|
||||
|
||||
if (Cache::has($key)) {
|
||||
return Cache::get($key);
|
||||
}
|
||||
|
||||
$productId = config('subscriptions.products.pro');
|
||||
$lookups = array_values(array_filter([
|
||||
config('subscriptions.plans.monthly.stripe_lookup_key'),
|
||||
config('subscriptions.plans.yearly.stripe_lookup_key'),
|
||||
]));
|
||||
|
||||
if ($productId === null && $lookups === []) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$params = $productId !== null
|
||||
? ['product' => $productId, 'limit' => 100]
|
||||
: ['lookup_keys' => $lookups, 'limit' => 10];
|
||||
|
||||
try {
|
||||
$prices = Cashier::stripe()->prices->all($params);
|
||||
} catch (\Throwable) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$currency = strtolower((string) config('cashier.currency', 'eur'));
|
||||
$map = [];
|
||||
foreach ($prices->data as $price) {
|
||||
if ($price->recurring === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (strtolower((string) ($price->currency ?? $currency)) !== $currency) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$amount = (int) ($price->unit_amount ?? 0);
|
||||
$map[$price->id] = ($price->recurring->interval ?? 'month') === 'year'
|
||||
? (int) round($amount / 12)
|
||||
: $amount;
|
||||
}
|
||||
|
||||
if ($map !== []) {
|
||||
Cache::put($key, $map, now()->addHour());
|
||||
}
|
||||
|
||||
return $map;
|
||||
}
|
||||
|
||||
/**
|
||||
* Days from signup until each variant's outcome can be scored: the trial
|
||||
* length (the longer of the two reduced trials) or the refund window.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,77 @@
|
|||
<?php
|
||||
|
||||
namespace App\Services\Stats;
|
||||
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Laravel\Cashier\Cashier;
|
||||
|
||||
/**
|
||||
* Monthly-equivalent amount (in cents) for each active recurring Stripe price id
|
||||
* under the pro product, yearly prices divided by 12. Fetched by product so that
|
||||
* archived, rotated price ids (Stripe mints a new id and transfers the lookup key
|
||||
* on any amount change) still resolve — otherwise subscriptions on an old id would
|
||||
* silently contribute 0 to MRR — and so every experiment price tier is covered by
|
||||
* one call. Falls back to the current lookup keys when no product is configured.
|
||||
* Foreign-currency and one-off prices are skipped. Cached for an hour; returns []
|
||||
* (revenue unavailable) if Stripe can't be reached, without caching the failure.
|
||||
*
|
||||
* Shared by the trial and price experiment funnel collectors so "what does each
|
||||
* price cost per month" has a single source of truth.
|
||||
*/
|
||||
class MonthlyEquivalentPrices
|
||||
{
|
||||
private const CACHE_KEY = 'experiment_funnel_monthly_equiv';
|
||||
|
||||
/**
|
||||
* @return array<string, int>
|
||||
*/
|
||||
public function map(): array
|
||||
{
|
||||
if (Cache::has(self::CACHE_KEY)) {
|
||||
return Cache::get(self::CACHE_KEY);
|
||||
}
|
||||
|
||||
$productId = config('subscriptions.products.pro');
|
||||
$lookups = array_values(array_filter([
|
||||
config('subscriptions.plans.monthly.stripe_lookup_key'),
|
||||
config('subscriptions.plans.yearly.stripe_lookup_key'),
|
||||
]));
|
||||
|
||||
if ($productId === null && $lookups === []) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$params = $productId !== null
|
||||
? ['product' => $productId, 'limit' => 100]
|
||||
: ['lookup_keys' => $lookups, 'limit' => 10];
|
||||
|
||||
try {
|
||||
$prices = Cashier::stripe()->prices->all($params);
|
||||
} catch (\Throwable) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$currency = strtolower((string) config('cashier.currency', 'eur'));
|
||||
$map = [];
|
||||
foreach ($prices->data as $price) {
|
||||
if ($price->recurring === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (strtolower((string) ($price->currency ?? $currency)) !== $currency) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$amount = (int) ($price->unit_amount ?? 0);
|
||||
$map[$price->id] = ($price->recurring->interval ?? 'month') === 'year'
|
||||
? (int) round($amount / 12)
|
||||
: $amount;
|
||||
}
|
||||
|
||||
if ($map !== []) {
|
||||
Cache::put(self::CACHE_KEY, $map, now()->addHour());
|
||||
}
|
||||
|
||||
return $map;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,239 @@
|
|||
<?php
|
||||
|
||||
namespace App\Services\Stats;
|
||||
|
||||
use App\Features\PriceExperiment;
|
||||
use App\Models\User;
|
||||
use Carbon\CarbonImmutable;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Laravel\Cashier\Subscription;
|
||||
|
||||
/**
|
||||
* Per-variant funnel for the price experiment (control vs high). Users are
|
||||
* attributed by PriceExperiment::bucket() — the deterministic, salted crc32 split
|
||||
* that is the single source of truth for assignment — so it matches the price each
|
||||
* user was served without being perturbed by the force_variant rollout hook.
|
||||
*
|
||||
* Both arms share ONE decision window (the trial length + a settle buffer), so the
|
||||
* matured cohorts are directly comparable — unlike the trial experiment, where each
|
||||
* variant matured on its own window. The funnel is
|
||||
* assigned → activated → subscribed → net-paying.
|
||||
*
|
||||
* The decision metric is contribution margin per assigned user (MRR − connection
|
||||
* cost, ÷ assigned). Cost is the *currently-active* connections × the flat
|
||||
* per-connection estimate, matching the monthly MRR flow (revoked connections cost
|
||||
* nothing going forward). Per-user CM is accumulated as moments (sum via
|
||||
* contributionMarginCents, sum-of-squares via cmSumSq) so the report can run a
|
||||
* Welch test on CM/user across arms without retaining every value. Conversion —
|
||||
* ever-charged net of refund, time-invariant — is the guardrail metric.
|
||||
*
|
||||
* ponytail: connection cost is a blended flat estimate across all providers (only
|
||||
* Enable Banking actually bills us; API-key providers are free). It is symmetric
|
||||
* across arms, so it biases absolute CM but not the between-arm comparison. Make it
|
||||
* per-provider only if absolute margin ever needs to be trusted.
|
||||
*
|
||||
* @param int $costPerConnectionCents flat estimated monthly cost per active connection
|
||||
* @return array{
|
||||
* startedAt: ?CarbonImmutable,
|
||||
* currency: string,
|
||||
* revenueAvailable: bool,
|
||||
* costPerConnectionCents: int,
|
||||
* decisionWindowDays: int,
|
||||
* variants: array<string, array{
|
||||
* assigned: int, activated: int, subscribed: int,
|
||||
* assignedMature: int, activatedMature: int, convertedMature: int, activeMature: int,
|
||||
* payerCount: int, payerConnectionSum: int,
|
||||
* conversionRate: ?float,
|
||||
* mrrCents: int, costCents: int, contributionMarginCents: int,
|
||||
* cmPerAssignedCents: ?int, cmSumSq: float, connectionsPerPayer: ?float,
|
||||
* }>
|
||||
* }
|
||||
*/
|
||||
class PriceExperimentFunnelCollector
|
||||
{
|
||||
/** Days after the trial ends before a user's outcome is settled enough to score. */
|
||||
private const SETTLE_BUFFER_DAYS = 3;
|
||||
|
||||
public function __construct(private MonthlyEquivalentPrices $prices) {}
|
||||
|
||||
public function collect(int $costPerConnectionCents = 40): array
|
||||
{
|
||||
$startedValue = config('subscriptions.price_experiment.started_at');
|
||||
$startedAt = $startedValue !== null ? CarbonImmutable::parse($startedValue) : null;
|
||||
$currency = strtoupper((string) config('cashier.currency', 'eur'));
|
||||
$window = (int) config('subscriptions.plans.monthly.trial_days', 15) + self::SETTLE_BUFFER_DAYS;
|
||||
|
||||
$variants = [
|
||||
PriceExperiment::CONTROL => $this->emptyRow(),
|
||||
PriceExperiment::HIGH => $this->emptyRow(),
|
||||
];
|
||||
|
||||
if ($startedAt === null) {
|
||||
return [
|
||||
'startedAt' => null,
|
||||
'currency' => $currency,
|
||||
'revenueAvailable' => false,
|
||||
'costPerConnectionCents' => $costPerConnectionCents,
|
||||
'decisionWindowDays' => $window,
|
||||
'variants' => $variants,
|
||||
];
|
||||
}
|
||||
|
||||
$now = CarbonImmutable::now('UTC');
|
||||
$excluded = (array) config('ai_suggestions.report.excluded_emails', []);
|
||||
$monthlyEquiv = $this->prices->map();
|
||||
$missingPrices = [];
|
||||
|
||||
User::query()
|
||||
->withTrashed()
|
||||
->where('users.created_at', '>=', $startedAt)
|
||||
->when($excluded !== [], fn ($query) => $query->whereNotIn('email', $excluded))
|
||||
->with(['subscriptions' => fn ($query) => $query->where('type', 'default')])
|
||||
->select(['id', 'created_at'])
|
||||
->withCount([
|
||||
// "Ever connected" drives activation (cost was triggered at some point).
|
||||
'bankingConnections as connection_count' => fn ($query) => $query->withTrashed(),
|
||||
// "Currently active" drives the monthly recurring cost.
|
||||
'bankingConnections as active_connection_count',
|
||||
'aiConsents as ai_consent_count',
|
||||
])
|
||||
->chunkById(500, function ($users) use (&$variants, &$missingPrices, $window, $now, $monthlyEquiv, $costPerConnectionCents): void {
|
||||
foreach ($users as $user) {
|
||||
$variant = PriceExperiment::bucket((string) $user->id);
|
||||
|
||||
if (! isset($variants[$variant])) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$row = &$variants[$variant];
|
||||
|
||||
$row['assigned']++;
|
||||
|
||||
$everConnections = (int) ($user->connection_count ?? 0);
|
||||
$activeConnections = (int) ($user->active_connection_count ?? 0);
|
||||
$activated = $everConnections > 0 || (int) ($user->ai_consent_count ?? 0) > 0;
|
||||
|
||||
if ($activated) {
|
||||
$row['activated']++;
|
||||
}
|
||||
|
||||
/** @var Subscription|null $subscription */
|
||||
$subscription = $user->subscriptions->sortByDesc('created_at')->first();
|
||||
$status = $subscription?->stripe_status;
|
||||
$netActive = $status === 'active' && $subscription->refunded_at === null;
|
||||
|
||||
// Time-invariant "ever charged, net of refund": comparable across
|
||||
// cohorts of different ages. Excludes trial-only cancels.
|
||||
$converted = $subscription !== null
|
||||
&& $subscription->refunded_at === null
|
||||
&& $status !== 'trialing'
|
||||
&& (
|
||||
$subscription->trial_ends_at === null
|
||||
|| $subscription->ends_at === null
|
||||
|| $subscription->ends_at->greaterThan($subscription->trial_ends_at)
|
||||
);
|
||||
|
||||
if ($subscription !== null) {
|
||||
$row['subscribed']++;
|
||||
}
|
||||
|
||||
$mature = CarbonImmutable::parse($user->created_at)
|
||||
->addDays($window)
|
||||
->lessThanOrEqualTo($now);
|
||||
|
||||
if (! $mature) {
|
||||
unset($row);
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
$row['assignedMature']++;
|
||||
|
||||
if ($activated) {
|
||||
$row['activatedMature']++;
|
||||
}
|
||||
|
||||
if ($converted) {
|
||||
$row['convertedMature']++;
|
||||
}
|
||||
|
||||
$costCents = $activeConnections * $costPerConnectionCents;
|
||||
$row['costCents'] += $costCents;
|
||||
|
||||
$revenueCents = 0;
|
||||
if ($netActive) {
|
||||
$row['activeMature']++;
|
||||
$priceId = (string) $subscription->stripe_price;
|
||||
|
||||
if ($monthlyEquiv !== [] && ! isset($monthlyEquiv[$priceId])) {
|
||||
$missingPrices[$priceId] = true;
|
||||
}
|
||||
|
||||
$revenueCents = (int) ($monthlyEquiv[$priceId] ?? 0);
|
||||
$row['mrrCents'] += $revenueCents;
|
||||
$row['payerCount']++;
|
||||
$row['payerConnectionSum'] += $activeConnections;
|
||||
}
|
||||
|
||||
// Per-user monthly CM; cmSumSq feeds the Welch variance. The sum
|
||||
// equals mrrCents − costCents, so it is not stored separately.
|
||||
$cmUser = $revenueCents - $costCents;
|
||||
$row['cmSumSq'] += $cmUser * $cmUser;
|
||||
|
||||
unset($row);
|
||||
}
|
||||
});
|
||||
|
||||
if ($missingPrices !== []) {
|
||||
Log::warning('Price experiment funnel: net-active subscriptions on prices absent from the monthly-equivalent map — their MRR is undercounted as 0.', [
|
||||
'price_ids' => array_keys($missingPrices),
|
||||
]);
|
||||
}
|
||||
|
||||
foreach ($variants as $key => $row) {
|
||||
$n = $row['assignedMature'];
|
||||
$variants[$key]['conversionRate'] = $n > 0 ? (float) $row['convertedMature'] / $n : null;
|
||||
$variants[$key]['contributionMarginCents'] = $row['mrrCents'] - $row['costCents'];
|
||||
$variants[$key]['cmPerAssignedCents'] = $n > 0
|
||||
? (int) round(($row['mrrCents'] - $row['costCents']) / $n)
|
||||
: null;
|
||||
$variants[$key]['connectionsPerPayer'] = $row['payerCount'] > 0
|
||||
? (float) $row['payerConnectionSum'] / $row['payerCount']
|
||||
: null;
|
||||
}
|
||||
|
||||
return [
|
||||
'startedAt' => $startedAt,
|
||||
'currency' => $currency,
|
||||
'revenueAvailable' => $monthlyEquiv !== [],
|
||||
'costPerConnectionCents' => $costPerConnectionCents,
|
||||
'decisionWindowDays' => $window,
|
||||
'variants' => $variants,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{assigned: int, activated: int, subscribed: int, assignedMature: int, activatedMature: int, convertedMature: int, activeMature: int, payerCount: int, payerConnectionSum: int, conversionRate: ?float, mrrCents: int, costCents: int, contributionMarginCents: int, cmPerAssignedCents: ?int, cmSumSq: float, connectionsPerPayer: ?float}
|
||||
*/
|
||||
private function emptyRow(): array
|
||||
{
|
||||
return [
|
||||
'assigned' => 0,
|
||||
'activated' => 0,
|
||||
'subscribed' => 0,
|
||||
'assignedMature' => 0,
|
||||
'activatedMature' => 0,
|
||||
'convertedMature' => 0,
|
||||
'activeMature' => 0,
|
||||
'payerCount' => 0,
|
||||
'payerConnectionSum' => 0,
|
||||
'conversionRate' => null,
|
||||
'mrrCents' => 0,
|
||||
'costCents' => 0,
|
||||
'contributionMarginCents' => 0,
|
||||
'cmPerAssignedCents' => null,
|
||||
'cmSumSq' => 0.0,
|
||||
'connectionsPerPayer' => null,
|
||||
];
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
<?php
|
||||
|
||||
namespace App\Services\Stats;
|
||||
|
||||
/**
|
||||
* Welch's two-sample test for a difference in means with unequal variances — the
|
||||
* correct test for comparing contribution-margin-per-user across price arms. That
|
||||
* metric is heavily zero-inflated and, because a higher-priced arm books several
|
||||
* times the revenue variance of the control at equal conversion, a pooled-variance
|
||||
* t-test is invalid here. The two-sided p-value uses the normal approximation to
|
||||
* Student's t, accurate at the per-arm sample sizes this experiment reaches
|
||||
* (n ≳ 100 by the central limit theorem); the Satterthwaite degrees of freedom are
|
||||
* returned for context.
|
||||
*
|
||||
* ponytail: normal-approx p-value, exact only for large n. If a future experiment
|
||||
* needs it at small n, swap normalCdf() for a Student-t CDF (regularised incomplete
|
||||
* beta) — the statistic and df here are already exact.
|
||||
*/
|
||||
final class WelchTTest
|
||||
{
|
||||
/**
|
||||
* @param float $varA sample variance (n−1 denominator)
|
||||
* @param float $varB sample variance (n−1 denominator)
|
||||
* @return array{diff: float, se: float, t: float, df: float, p: float}
|
||||
*/
|
||||
public function compare(float $meanA, float $varA, int $nA, float $meanB, float $varB, int $nB): array
|
||||
{
|
||||
$seSq = ($nA > 0 ? $varA / $nA : 0.0) + ($nB > 0 ? $varB / $nB : 0.0);
|
||||
$se = sqrt($seSq);
|
||||
$diff = $meanA - $meanB;
|
||||
|
||||
if ($se <= 0.0 || $nA < 2 || $nB < 2) {
|
||||
return ['diff' => $diff, 'se' => $se, 't' => 0.0, 'df' => 0.0, 'p' => 1.0];
|
||||
}
|
||||
|
||||
$t = $diff / $se;
|
||||
$df = ($seSq ** 2) / (
|
||||
(($varA / $nA) ** 2) / ($nA - 1)
|
||||
+ (($varB / $nB) ** 2) / ($nB - 1)
|
||||
);
|
||||
$p = 2.0 * (1.0 - $this->normalCdf(abs($t)));
|
||||
|
||||
return ['diff' => $diff, 'se' => $se, 't' => $t, 'df' => $df, 'p' => $p];
|
||||
}
|
||||
|
||||
private function normalCdf(float $x): float
|
||||
{
|
||||
return 0.5 * (1.0 + $this->erf($x / sqrt(2.0)));
|
||||
}
|
||||
|
||||
/** Abramowitz & Stegun 7.1.26 — |error| < 1.5e-7. */
|
||||
private function erf(float $x): float
|
||||
{
|
||||
$sign = $x < 0 ? -1.0 : 1.0;
|
||||
$x = abs($x);
|
||||
$t = 1.0 / (1.0 + 0.3275911 * $x);
|
||||
$y = 1.0 - ((((1.061405429 * $t - 1.453152027) * $t + 1.421413741) * $t - 0.284496736) * $t + 0.254829592) * $t * exp(-$x * $x);
|
||||
|
||||
return $sign * $y;
|
||||
}
|
||||
}
|
||||
|
|
@ -17,3 +17,4 @@ Schedule::command('stats:daily-report')->dailyAt('09:00')->timezone('Europe/Madr
|
|||
Schedule::command('stats:ai-cohort-report')->monthlyOn(1, '09:00')->timezone('Europe/Madrid');
|
||||
Schedule::command('stats:subscription-funnel')->weekly()->mondays()->at('09:15')->timezone('Europe/Madrid');
|
||||
Schedule::command('stats:experiment-funnel')->weekly()->mondays()->at('09:30')->timezone('Europe/Madrid');
|
||||
Schedule::command('stats:price-experiment-funnel')->weekly()->mondays()->at('09:45')->timezone('Europe/Madrid');
|
||||
|
|
|
|||
|
|
@ -0,0 +1,200 @@
|
|||
<?php
|
||||
|
||||
use App\Features\PriceExperiment;
|
||||
use App\Models\AiConsent;
|
||||
use App\Models\BankingConnection;
|
||||
use App\Models\User;
|
||||
use App\Services\Stats\PriceExperimentFunnelCollector;
|
||||
use Carbon\CarbonImmutable;
|
||||
use Illuminate\Support\Carbon;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Illuminate\Support\Facades\Http;
|
||||
use Illuminate\Support\Str;
|
||||
|
||||
use function Pest\Laravel\artisan;
|
||||
|
||||
beforeEach(function () {
|
||||
config([
|
||||
'subscriptions.enabled' => true,
|
||||
'subscriptions.price_experiment.started_at' => '2026-06-01',
|
||||
'subscriptions.price_experiment.force_variant' => null,
|
||||
'subscriptions.plans.monthly.trial_days' => 15,
|
||||
'ai_suggestions.report.excluded_emails' => [],
|
||||
]);
|
||||
Carbon::setTestNow(CarbonImmutable::parse('2026-06-30 12:00:00'));
|
||||
|
||||
// Seed the price→monthly-equivalent map so revenue is computed without Stripe:
|
||||
// control €3.99, high €8.99.
|
||||
Cache::put('experiment_funnel_monthly_equiv', ['price_control' => 399, 'price_high' => 899], now()->addHour());
|
||||
});
|
||||
|
||||
/**
|
||||
* Create a user whose id buckets into the wanted price variant, with an optional
|
||||
* default subscription, live (active) bank connections that drive the monthly cost,
|
||||
* extra soft-deleted connections that count as "ever activated" but cost nothing,
|
||||
* and optional AI consent.
|
||||
*
|
||||
* @param array{status: string, at: CarbonImmutable, price?: string, endsAt?: CarbonImmutable, trialEndsAt?: CarbonImmutable, refundedAt?: CarbonImmutable}|null $subscription
|
||||
*/
|
||||
function priceUser(string $variant, CarbonImmutable $signup, ?array $subscription = null, int $connections = 0, int $deletedConnections = 0, bool $aiConsent = false): User
|
||||
{
|
||||
do {
|
||||
$id = (string) Str::uuid();
|
||||
} while (PriceExperiment::bucket($id) !== $variant);
|
||||
|
||||
$user = User::factory()->create(['id' => $id, 'created_at' => $signup]);
|
||||
|
||||
if ($subscription !== null) {
|
||||
$user->subscriptions()->create([
|
||||
'type' => 'default',
|
||||
'stripe_id' => 'sub_'.Str::random(12),
|
||||
'stripe_status' => $subscription['status'],
|
||||
'stripe_price' => $subscription['price'] ?? 'price_control',
|
||||
'created_at' => $subscription['at'],
|
||||
'ends_at' => $subscription['endsAt'] ?? null,
|
||||
'trial_ends_at' => $subscription['trialEndsAt'] ?? null,
|
||||
'refunded_at' => $subscription['refundedAt'] ?? null,
|
||||
]);
|
||||
}
|
||||
|
||||
if ($connections > 0) {
|
||||
BankingConnection::factory()->count($connections)->for($user)->create();
|
||||
}
|
||||
|
||||
if ($deletedConnections > 0) {
|
||||
BankingConnection::factory()->count($deletedConnections)->for($user)->create()
|
||||
->each(fn (BankingConnection $connection) => $connection->delete());
|
||||
}
|
||||
|
||||
if ($aiConsent) {
|
||||
AiConsent::factory()->for($user)->create();
|
||||
}
|
||||
|
||||
return $user;
|
||||
}
|
||||
|
||||
it('returns empty variants when the experiment has not started', function () {
|
||||
config(['subscriptions.price_experiment.started_at' => null]);
|
||||
|
||||
$report = app(PriceExperimentFunnelCollector::class)->collect();
|
||||
|
||||
expect($report['startedAt'])->toBeNull()
|
||||
->and($report['variants'][PriceExperiment::CONTROL]['assigned'])->toBe(0)
|
||||
->and($report['variants'][PriceExperiment::HIGH]['assigned'])->toBe(0);
|
||||
});
|
||||
|
||||
it('attributes users to the right price variant and counts the funnel', function () {
|
||||
$signup = CarbonImmutable::parse('2026-06-05'); // mature (15d + 3d) by the test clock
|
||||
|
||||
priceUser(PriceExperiment::CONTROL, $signup);
|
||||
priceUser(PriceExperiment::CONTROL, $signup, ['status' => 'active', 'at' => $signup->addDay(), 'price' => 'price_control']);
|
||||
priceUser(PriceExperiment::HIGH, $signup, ['status' => 'active', 'at' => $signup->addDay(), 'price' => 'price_high']);
|
||||
|
||||
$variants = app(PriceExperimentFunnelCollector::class)->collect()['variants'];
|
||||
|
||||
expect($variants[PriceExperiment::CONTROL]['assigned'])->toBe(2)
|
||||
->and($variants[PriceExperiment::CONTROL]['subscribed'])->toBe(1)
|
||||
->and($variants[PriceExperiment::CONTROL]['activeMature'])->toBe(1)
|
||||
->and($variants[PriceExperiment::CONTROL]['convertedMature'])->toBe(1)
|
||||
->and($variants[PriceExperiment::CONTROL]['conversionRate'])->toBe(0.5)
|
||||
->and($variants[PriceExperiment::HIGH]['assigned'])->toBe(1)
|
||||
->and($variants[PriceExperiment::HIGH]['activeMature'])->toBe(1);
|
||||
});
|
||||
|
||||
it('books MRR at each variant price and reports contribution margin per assigned user', function () {
|
||||
$signup = CarbonImmutable::parse('2026-06-05');
|
||||
|
||||
// High arm: one payer at €8.99 with 1 active connection (cost €0.40).
|
||||
priceUser(PriceExperiment::HIGH, $signup, ['status' => 'active', 'at' => $signup, 'price' => 'price_high'], connections: 1);
|
||||
// High arm: one assigned non-payer, no connections.
|
||||
priceUser(PriceExperiment::HIGH, $signup);
|
||||
|
||||
$high = app(PriceExperimentFunnelCollector::class)->collect()['variants'][PriceExperiment::HIGH];
|
||||
|
||||
expect($high['mrrCents'])->toBe(899) // €8.99 at the high price
|
||||
->and($high['costCents'])->toBe(40) // 1 active connection
|
||||
->and($high['contributionMarginCents'])->toBe(859) // 899 − 40
|
||||
->and($high['assignedMature'])->toBe(2)
|
||||
->and($high['cmPerAssignedCents'])->toBe(430) // round(859 / 2)
|
||||
->and($high['connectionsPerPayer'])->toBe(1.0);
|
||||
});
|
||||
|
||||
it('counts revoked connections as activation but charges cost only for live ones', function () {
|
||||
$signup = CarbonImmutable::parse('2026-06-05');
|
||||
|
||||
// 1 live connection + 2 soft-deleted: activated (ever connected), but the
|
||||
// monthly cost is only the 1 live connection — the stock-vs-flow fix.
|
||||
priceUser(PriceExperiment::CONTROL, $signup, connections: 1, deletedConnections: 2);
|
||||
|
||||
$control = app(PriceExperimentFunnelCollector::class)->collect()['variants'][PriceExperiment::CONTROL];
|
||||
|
||||
expect($control['activated'])->toBe(1)
|
||||
->and($control['activatedMature'])->toBe(1)
|
||||
->and($control['costCents'])->toBe(40); // 1 live × 40, not 3 × 40
|
||||
});
|
||||
|
||||
it('leaves young cohorts out of the matured metrics', function () {
|
||||
priceUser(PriceExperiment::HIGH, CarbonImmutable::now()->subDays(5), [
|
||||
'status' => 'active', 'at' => CarbonImmutable::now()->subDays(5), 'price' => 'price_high',
|
||||
]);
|
||||
|
||||
$high = app(PriceExperimentFunnelCollector::class)->collect()['variants'][PriceExperiment::HIGH];
|
||||
|
||||
expect($high['assigned'])->toBe(1)
|
||||
->and($high['assignedMature'])->toBe(0) // 5d < 18d window
|
||||
->and($high['conversionRate'])->toBeNull();
|
||||
});
|
||||
|
||||
it('keeps the split even when a winner is forced, instead of collapsing onto one variant', function () {
|
||||
config(['subscriptions.price_experiment.force_variant' => PriceExperiment::HIGH]);
|
||||
$signup = CarbonImmutable::parse('2026-06-05');
|
||||
|
||||
priceUser(PriceExperiment::CONTROL, $signup);
|
||||
priceUser(PriceExperiment::HIGH, $signup);
|
||||
|
||||
$variants = app(PriceExperimentFunnelCollector::class)->collect()['variants'];
|
||||
|
||||
expect($variants[PriceExperiment::CONTROL]['assigned'])->toBe(1)
|
||||
->and($variants[PriceExperiment::HIGH]['assigned'])->toBe(1);
|
||||
});
|
||||
|
||||
it('prints the primary CM/user (Welch) and conversion guardrail blocks', function () {
|
||||
$signup = CarbonImmutable::parse('2026-06-05');
|
||||
|
||||
for ($i = 0; $i < 3; $i++) {
|
||||
priceUser(PriceExperiment::CONTROL, $signup, ['status' => 'active', 'at' => $signup, 'price' => 'price_control']);
|
||||
priceUser(PriceExperiment::CONTROL, $signup);
|
||||
priceUser(PriceExperiment::HIGH, $signup, ['status' => 'active', 'at' => $signup, 'price' => 'price_high']);
|
||||
priceUser(PriceExperiment::HIGH, $signup);
|
||||
}
|
||||
|
||||
artisan('stats:price-experiment-funnel', ['--no-discord' => true])
|
||||
->expectsOutputToContain('PRIMARY')
|
||||
->expectsOutputToContain('GUARDRAIL')
|
||||
->expectsOutputToContain('Fisher exact')
|
||||
->assertSuccessful();
|
||||
});
|
||||
|
||||
it('posts the price experiment embed to discord with the monitoring-only warning', function () {
|
||||
config(['services.discord.ai_cohort_webhook_url' => 'https://discord.test/hook']);
|
||||
Http::fake(['discord.test/*' => Http::response('', 204)]);
|
||||
|
||||
priceUser(PriceExperiment::CONTROL, CarbonImmutable::parse('2026-06-05'), [
|
||||
'status' => 'active', 'at' => CarbonImmutable::parse('2026-06-05'), 'price' => 'price_control',
|
||||
]);
|
||||
|
||||
artisan('stats:price-experiment-funnel')->assertSuccessful();
|
||||
|
||||
Http::assertSent(fn ($request) => $request->url() === 'https://discord.test/hook'
|
||||
&& str_contains($request['embeds'][0]['title'], 'Price Experiment')
|
||||
&& str_contains(json_encode($request['embeds'][0]), 'MONITORING ONLY'));
|
||||
});
|
||||
|
||||
it('does not post when the experiment has not started', function () {
|
||||
config(['subscriptions.price_experiment.started_at' => null]);
|
||||
Http::fake();
|
||||
|
||||
artisan('stats:price-experiment-funnel')->assertSuccessful();
|
||||
|
||||
Http::assertNothingSent();
|
||||
});
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
<?php
|
||||
|
||||
use App\Services\Stats\WelchTTest;
|
||||
|
||||
it('returns no difference and p=1 for identical arms', function () {
|
||||
$result = (new WelchTTest)->compare(10.0, 4.0, 100, 10.0, 4.0, 100);
|
||||
|
||||
expect($result['diff'])->toBe(0.0)
|
||||
->and($result['t'])->toBe(0.0)
|
||||
->and($result['p'])->toEqualWithDelta(1.0, 1e-6); // erf approximation, |error| < 1.5e-7
|
||||
});
|
||||
|
||||
it('yields a two-sided p of ~0.317 when the difference equals one standard error', function () {
|
||||
// varA/nA = varB/nB = 0.5 → seSq = 1, se = 1; diff = 1 → t = 1.
|
||||
// Two-sided normal p = 2·(1 − Φ(1)) = 0.3173, validating the erf/normalCdf path.
|
||||
$result = (new WelchTTest)->compare(1.0, 50.0, 100, 0.0, 50.0, 100);
|
||||
|
||||
expect($result['t'])->toEqualWithDelta(1.0, 1e-9)
|
||||
->and($result['se'])->toEqualWithDelta(1.0, 1e-9)
|
||||
->and($result['p'])->toEqualWithDelta(0.3173, 1e-3);
|
||||
});
|
||||
|
||||
it('finds a large separation highly significant', function () {
|
||||
$result = (new WelchTTest)->compare(10.0, 4.0, 100, 8.0, 4.0, 100);
|
||||
|
||||
expect($result['t'])->toEqualWithDelta(7.071, 1e-2)
|
||||
->and($result['p'])->toBeLessThan(0.001);
|
||||
});
|
||||
|
||||
it('defers (p=1) when an arm has too few observations to estimate variance', function () {
|
||||
$result = (new WelchTTest)->compare(5.0, 0.0, 1, 0.0, 0.0, 1);
|
||||
|
||||
expect($result['p'])->toBe(1.0);
|
||||
});
|
||||
Loading…
Reference in New Issue