diff --git a/app/Console/Commands/SendPriceExperimentFunnelReportCommand.php b/app/Console/Commands/SendPriceExperimentFunnelReportCommand.php new file mode 100644 index 00000000..c5842838 --- /dev/null +++ b/app/Console/Commands/SendPriceExperimentFunnelReportCommand.php @@ -0,0 +1,230 @@ + 'control', + PriceExperiment::HIGH => 'high', + ]; + + public function __construct( + private PriceExperimentFunnelCollector $collector, + private ProportionSignificance $significance, + private WelchTTest $welch, + ) { + parent::__construct(); + } + + public function handle(): int + { + $costPerConnectionCents = (int) round(((float) $this->option('cost-per-connection')) * 100); + $report = $this->collector->collect($costPerConnectionCents); + + if ($report['startedAt'] === null) { + $this->warn('Price experiment not started — set PRICE_EXPERIMENT_STARTED_AT to begin.'); + + return self::SUCCESS; + } + + foreach ([...$this->tableLines($report), ...$this->significanceLines($report)] as $line) { + $this->line($line); + } + + if ($this->option('no-discord')) { + $this->info('Skipped Discord (--no-discord).'); + + return self::SUCCESS; + } + + $webhookUrl = config('services.discord.ai_cohort_webhook_url') + ?: config('services.discord.webhook_url'); + + (new DiscordWebhook($webhookUrl))->send('', [$this->buildEmbed($report)]); + + $this->info('Price experiment funnel report sent to Discord.'); + + return self::SUCCESS; + } + + /** + * @param array{currency: string, revenueAvailable: bool, variants: array>} $report + * @return list + */ + private function tableLines(array $report): array + { + $currency = $report['currency']; + $lines = [sprintf( + '%-8s %5s %5s %5s %5s %5s %6s %8s %8s %8s %8s %6s', + 'Variant', 'Assg', 'Actd', 'Card', 'MatU', 'Conv', 'Conv%', 'MRR', 'Cost', 'CM', 'CM/u', 'C/Pyr', + )]; + + foreach (self::LABELS as $key => $label) { + $row = $report['variants'][$key]; + $mature = $row['assignedMature'] > 0; + $showMoney = $report['revenueAvailable'] && $mature; + + $lines[] = sprintf( + '%-8s %5d %5d %5d %5d %5d %6s %8s %8s %8s %8s %6s', + $label, + $row['assigned'], + $row['activated'], + $row['subscribed'], + $row['assignedMature'], + $row['convertedMature'], + $mature ? ((int) round($row['conversionRate'] * 100)).'%' : 'pend', + $showMoney ? Money::format($row['mrrCents'], $currency) : '—', + $mature ? Money::format($row['costCents'], $currency) : '—', + $showMoney ? Money::format($row['contributionMarginCents'], $currency) : '—', + $showMoney && $row['cmPerAssignedCents'] !== null ? Money::format($row['cmPerAssignedCents'], $currency) : '—', + $row['connectionsPerPayer'] !== null ? number_format($row['connectionsPerPayer'], 1) : '—', + ); + } + + return $lines; + } + + /** + * Primary decision test — Welch on contribution-margin-per-user (control vs + * high), the metric we maximise — plus the conversion guardrail (Wilson CIs + + * Fisher exact, a single control-vs-treatment contrast, so no multiplicity + * correction). CM/user is the number to act on; conversion protects the funnel. + * + * @param array{revenueAvailable: bool, currency: string, variants: array>} $report + * @return list + */ + private function significanceLines(array $report): array + { + $currency = $report['currency']; + $control = $report['variants'][PriceExperiment::CONTROL]; + $high = $report['variants'][PriceExperiment::HIGH]; + + $lines = ['', 'PRIMARY — contribution margin per assigned user (Welch, high vs control):']; + + if (! $report['revenueAvailable'] || $control['assignedMature'] < 2 || $high['assignedMature'] < 2) { + $lines[] = ' pend — need mature volume in both arms and Stripe revenue to compare.'; + } else { + foreach (['control' => $control, 'high' => $high] as $label => $row) { + $lines[] = sprintf(' %-8s %8s/user (n=%d)', $label, Money::format((int) $row['cmPerAssignedCents'], $currency), $row['assignedMature']); + } + + $welch = $this->welch->compare( + $this->cmMean($high), $this->cmVariance($high), $high['assignedMature'], + $this->cmMean($control), $this->cmVariance($control), $control['assignedMature'], + ); + $significant = $welch['p'] < 0.05; + $lines[] = sprintf( + ' Δ high−control: %s/user t=%.2f p=%.3f %s α=0.05 -> %s.%s', + Money::format((int) round($welch['diff']), $currency), + $welch['t'], $welch['p'], $significant ? '<' : '≥', + $significant ? 'significant' : 'not significant', + $significant ? '' : ' Keep running.', + ); + } + + $lines[] = ''; + $lines[] = 'GUARDRAIL — conversion (95% Wilson CI on Conv%, n = MatU):'; + $arms = []; + + foreach (self::LABELS as $key => $label) { + $row = $report['variants'][$key]; + $n = (int) $row['assignedMature']; + $k = (int) $row['convertedMature']; + + if ($n <= 0) { + $lines[] = sprintf(' %-8s pend (n=0)', $label); + + continue; + } + + [$low, $high2] = $this->significance->wilsonInterval($k, $n); + $lines[] = sprintf(' %-8s %6s [%6s – %6s] (n=%d)', $label, $this->percent($k / $n), $this->percent($low), $this->percent($high2), $n); + $arms[$key] = new BinomialProportion($label, $k, $n); + } + + if (count($arms) === 2) { + $result = $this->significance->compare($arms[PriceExperiment::HIGH], $arms[PriceExperiment::CONTROL], comparisons: 1); + $lines[] = sprintf( + ' Fisher exact p=%.3f %s α=%.3f -> %s.', + $result['fisherP'], $result['significant'] ? '<' : '≥', $result['alpha'], + $result['significant'] ? 'significant' : 'not significant', + ); + } + + return $lines; + } + + private function cmMean(array $row): float + { + return $row['assignedMature'] > 0 ? (float) $row['contributionMarginCents'] / $row['assignedMature'] : 0.0; + } + + /** Sample variance (n−1) of per-user CM, from the accumulated moments. */ + private function cmVariance(array $row): float + { + $n = (int) $row['assignedMature']; + + if ($n < 2) { + return 0.0; + } + + $sum = (float) $row['contributionMarginCents']; + + return max(0.0, ((float) $row['cmSumSq'] - ($sum * $sum) / $n) / ($n - 1)); + } + + private function percent(float $rate): string + { + return number_format($rate * 100, 1).'%'; + } + + /** + * @param array{startedAt: CarbonImmutable, currency: string, revenueAvailable: bool, costPerConnectionCents: int, decisionWindowDays: int, variants: array>} $report + * @return array + */ + private function buildEmbed(array $report): array + { + return [ + 'title' => '💶 Price Experiment — Funnel (control vs high)', + 'description' => "```\n".implode("\n", [...$this->tableLines($report), ...$this->significanceLines($report)])."\n```", + 'color' => 0x57F287, + 'fields' => [ + [ + 'name' => 'Started', + 'value' => $report['startedAt']->format('D, d M Y').sprintf(' · new signups split 50/50; both arms mature at %d days.', $report['decisionWindowDays']), + 'inline' => false, + ], + [ + 'name' => 'Legend', + 'value' => sprintf( + 'Assg = signups · Actd = activated (bank or AI = cost triggered) · Card = started a subscription · MatU = matured assigned (old enough to score) · Conv%% = ever-charged net of refund ÷ MatU (time-invariant) · MRR = monthly run-rate of currently-paying subs · Cost = active connections × %s · CM = MRR − Cost · CM/u = CM ÷ MatU (the decision metric) · C/Pyr = active connections per payer.', + Money::format($report['costPerConnectionCents'], $report['currency']), + ), + 'inline' => false, + ], + [ + 'name' => '⚠️ MONITORING ONLY', + 'value' => "**Do NOT call a winner from this weekly report.** Decide at the pre-registered horizon N (from the power analysis), not the first week Fisher/Welch crosses α — weekly peeking inflates the false-positive rate. The primary metric is **CM per assigned user** (Welch); conversion is a guardrail. CM/user is sub-cent per week at current volume, so early weeks read as 'not significant' — that is expected. Don't add/remove arms or change the split mid-flight. A 1-month CM cannot see price-driven churn, so re-read retention/LTV before permanently locking the price.", + 'inline' => false, + ], + ], + ]; + } +} diff --git a/app/Services/Stats/ExperimentFunnelCollector.php b/app/Services/Stats/ExperimentFunnelCollector.php index 9a0ca74d..090ef9e5 100644 --- a/app/Services/Stats/ExperimentFunnelCollector.php +++ b/app/Services/Stats/ExperimentFunnelCollector.php @@ -5,9 +5,7 @@ namespace App\Services\Stats; use App\Features\SubscriptionExperiment; use App\Models\User; use Carbon\CarbonImmutable; -use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Log; -use Laravel\Cashier\Cashier; use Laravel\Cashier\Subscription; class ExperimentFunnelCollector @@ -18,6 +16,8 @@ class ExperimentFunnelCollector */ private const SETTLE_BUFFER_DAYS = 3; + public function __construct(private MonthlyEquivalentPrices $prices) {} + /** * Per-variant funnel for the trial/pricing experiment. Users are attributed * by SubscriptionExperiment::bucket() — the deterministic crc32 split that is @@ -94,7 +94,7 @@ class ExperimentFunnelCollector $now = CarbonImmutable::now('UTC'); $excluded = (array) config('ai_suggestions.report.excluded_emails', []); $windows = $this->decisionWindows(); - $monthlyEquiv = $this->monthlyEquivByPriceId(); + $monthlyEquiv = $this->prices->map(); $missingPrices = []; User::query() @@ -270,70 +270,6 @@ class ExperimentFunnelCollector ]; } - /** - * Monthly-equivalent amount (in cents) for each plan price id, from Stripe. - * Yearly prices are divided by 12. Fetched by product so that archived, - * rotated price ids (Stripe mints a new id and transfers the lookup key on - * any amount change) still resolve — otherwise subscriptions on an old id - * would silently contribute 0 to MRR. Falls back to the current lookup keys - * when no product is configured. Foreign-currency and one-off prices are - * skipped. Cached for an hour; returns [] (revenue unavailable) if Stripe - * can't be reached, without caching the failure. - * - * @return array - */ - private function monthlyEquivByPriceId(): array - { - $key = 'experiment_funnel_monthly_equiv'; - - if (Cache::has($key)) { - return Cache::get($key); - } - - $productId = config('subscriptions.products.pro'); - $lookups = array_values(array_filter([ - config('subscriptions.plans.monthly.stripe_lookup_key'), - config('subscriptions.plans.yearly.stripe_lookup_key'), - ])); - - if ($productId === null && $lookups === []) { - return []; - } - - $params = $productId !== null - ? ['product' => $productId, 'limit' => 100] - : ['lookup_keys' => $lookups, 'limit' => 10]; - - try { - $prices = Cashier::stripe()->prices->all($params); - } catch (\Throwable) { - return []; - } - - $currency = strtolower((string) config('cashier.currency', 'eur')); - $map = []; - foreach ($prices->data as $price) { - if ($price->recurring === null) { - continue; - } - - if (strtolower((string) ($price->currency ?? $currency)) !== $currency) { - continue; - } - - $amount = (int) ($price->unit_amount ?? 0); - $map[$price->id] = ($price->recurring->interval ?? 'month') === 'year' - ? (int) round($amount / 12) - : $amount; - } - - if ($map !== []) { - Cache::put($key, $map, now()->addHour()); - } - - return $map; - } - /** * Days from signup until each variant's outcome can be scored: the trial * length (the longer of the two reduced trials) or the refund window. diff --git a/app/Services/Stats/MonthlyEquivalentPrices.php b/app/Services/Stats/MonthlyEquivalentPrices.php new file mode 100644 index 00000000..ab3132f0 --- /dev/null +++ b/app/Services/Stats/MonthlyEquivalentPrices.php @@ -0,0 +1,77 @@ + + */ + public function map(): array + { + if (Cache::has(self::CACHE_KEY)) { + return Cache::get(self::CACHE_KEY); + } + + $productId = config('subscriptions.products.pro'); + $lookups = array_values(array_filter([ + config('subscriptions.plans.monthly.stripe_lookup_key'), + config('subscriptions.plans.yearly.stripe_lookup_key'), + ])); + + if ($productId === null && $lookups === []) { + return []; + } + + $params = $productId !== null + ? ['product' => $productId, 'limit' => 100] + : ['lookup_keys' => $lookups, 'limit' => 10]; + + try { + $prices = Cashier::stripe()->prices->all($params); + } catch (\Throwable) { + return []; + } + + $currency = strtolower((string) config('cashier.currency', 'eur')); + $map = []; + foreach ($prices->data as $price) { + if ($price->recurring === null) { + continue; + } + + if (strtolower((string) ($price->currency ?? $currency)) !== $currency) { + continue; + } + + $amount = (int) ($price->unit_amount ?? 0); + $map[$price->id] = ($price->recurring->interval ?? 'month') === 'year' + ? (int) round($amount / 12) + : $amount; + } + + if ($map !== []) { + Cache::put(self::CACHE_KEY, $map, now()->addHour()); + } + + return $map; + } +} diff --git a/app/Services/Stats/PriceExperimentFunnelCollector.php b/app/Services/Stats/PriceExperimentFunnelCollector.php new file mode 100644 index 00000000..c658c9b0 --- /dev/null +++ b/app/Services/Stats/PriceExperimentFunnelCollector.php @@ -0,0 +1,239 @@ + + * } + */ +class PriceExperimentFunnelCollector +{ + /** Days after the trial ends before a user's outcome is settled enough to score. */ + private const SETTLE_BUFFER_DAYS = 3; + + public function __construct(private MonthlyEquivalentPrices $prices) {} + + public function collect(int $costPerConnectionCents = 40): array + { + $startedValue = config('subscriptions.price_experiment.started_at'); + $startedAt = $startedValue !== null ? CarbonImmutable::parse($startedValue) : null; + $currency = strtoupper((string) config('cashier.currency', 'eur')); + $window = (int) config('subscriptions.plans.monthly.trial_days', 15) + self::SETTLE_BUFFER_DAYS; + + $variants = [ + PriceExperiment::CONTROL => $this->emptyRow(), + PriceExperiment::HIGH => $this->emptyRow(), + ]; + + if ($startedAt === null) { + return [ + 'startedAt' => null, + 'currency' => $currency, + 'revenueAvailable' => false, + 'costPerConnectionCents' => $costPerConnectionCents, + 'decisionWindowDays' => $window, + 'variants' => $variants, + ]; + } + + $now = CarbonImmutable::now('UTC'); + $excluded = (array) config('ai_suggestions.report.excluded_emails', []); + $monthlyEquiv = $this->prices->map(); + $missingPrices = []; + + User::query() + ->withTrashed() + ->where('users.created_at', '>=', $startedAt) + ->when($excluded !== [], fn ($query) => $query->whereNotIn('email', $excluded)) + ->with(['subscriptions' => fn ($query) => $query->where('type', 'default')]) + ->select(['id', 'created_at']) + ->withCount([ + // "Ever connected" drives activation (cost was triggered at some point). + 'bankingConnections as connection_count' => fn ($query) => $query->withTrashed(), + // "Currently active" drives the monthly recurring cost. + 'bankingConnections as active_connection_count', + 'aiConsents as ai_consent_count', + ]) + ->chunkById(500, function ($users) use (&$variants, &$missingPrices, $window, $now, $monthlyEquiv, $costPerConnectionCents): void { + foreach ($users as $user) { + $variant = PriceExperiment::bucket((string) $user->id); + + if (! isset($variants[$variant])) { + continue; + } + + $row = &$variants[$variant]; + + $row['assigned']++; + + $everConnections = (int) ($user->connection_count ?? 0); + $activeConnections = (int) ($user->active_connection_count ?? 0); + $activated = $everConnections > 0 || (int) ($user->ai_consent_count ?? 0) > 0; + + if ($activated) { + $row['activated']++; + } + + /** @var Subscription|null $subscription */ + $subscription = $user->subscriptions->sortByDesc('created_at')->first(); + $status = $subscription?->stripe_status; + $netActive = $status === 'active' && $subscription->refunded_at === null; + + // Time-invariant "ever charged, net of refund": comparable across + // cohorts of different ages. Excludes trial-only cancels. + $converted = $subscription !== null + && $subscription->refunded_at === null + && $status !== 'trialing' + && ( + $subscription->trial_ends_at === null + || $subscription->ends_at === null + || $subscription->ends_at->greaterThan($subscription->trial_ends_at) + ); + + if ($subscription !== null) { + $row['subscribed']++; + } + + $mature = CarbonImmutable::parse($user->created_at) + ->addDays($window) + ->lessThanOrEqualTo($now); + + if (! $mature) { + unset($row); + + continue; + } + + $row['assignedMature']++; + + if ($activated) { + $row['activatedMature']++; + } + + if ($converted) { + $row['convertedMature']++; + } + + $costCents = $activeConnections * $costPerConnectionCents; + $row['costCents'] += $costCents; + + $revenueCents = 0; + if ($netActive) { + $row['activeMature']++; + $priceId = (string) $subscription->stripe_price; + + if ($monthlyEquiv !== [] && ! isset($monthlyEquiv[$priceId])) { + $missingPrices[$priceId] = true; + } + + $revenueCents = (int) ($monthlyEquiv[$priceId] ?? 0); + $row['mrrCents'] += $revenueCents; + $row['payerCount']++; + $row['payerConnectionSum'] += $activeConnections; + } + + // Per-user monthly CM; cmSumSq feeds the Welch variance. The sum + // equals mrrCents − costCents, so it is not stored separately. + $cmUser = $revenueCents - $costCents; + $row['cmSumSq'] += $cmUser * $cmUser; + + unset($row); + } + }); + + if ($missingPrices !== []) { + Log::warning('Price experiment funnel: net-active subscriptions on prices absent from the monthly-equivalent map — their MRR is undercounted as 0.', [ + 'price_ids' => array_keys($missingPrices), + ]); + } + + foreach ($variants as $key => $row) { + $n = $row['assignedMature']; + $variants[$key]['conversionRate'] = $n > 0 ? (float) $row['convertedMature'] / $n : null; + $variants[$key]['contributionMarginCents'] = $row['mrrCents'] - $row['costCents']; + $variants[$key]['cmPerAssignedCents'] = $n > 0 + ? (int) round(($row['mrrCents'] - $row['costCents']) / $n) + : null; + $variants[$key]['connectionsPerPayer'] = $row['payerCount'] > 0 + ? (float) $row['payerConnectionSum'] / $row['payerCount'] + : null; + } + + return [ + 'startedAt' => $startedAt, + 'currency' => $currency, + 'revenueAvailable' => $monthlyEquiv !== [], + 'costPerConnectionCents' => $costPerConnectionCents, + 'decisionWindowDays' => $window, + 'variants' => $variants, + ]; + } + + /** + * @return array{assigned: int, activated: int, subscribed: int, assignedMature: int, activatedMature: int, convertedMature: int, activeMature: int, payerCount: int, payerConnectionSum: int, conversionRate: ?float, mrrCents: int, costCents: int, contributionMarginCents: int, cmPerAssignedCents: ?int, cmSumSq: float, connectionsPerPayer: ?float} + */ + private function emptyRow(): array + { + return [ + 'assigned' => 0, + 'activated' => 0, + 'subscribed' => 0, + 'assignedMature' => 0, + 'activatedMature' => 0, + 'convertedMature' => 0, + 'activeMature' => 0, + 'payerCount' => 0, + 'payerConnectionSum' => 0, + 'conversionRate' => null, + 'mrrCents' => 0, + 'costCents' => 0, + 'contributionMarginCents' => 0, + 'cmPerAssignedCents' => null, + 'cmSumSq' => 0.0, + 'connectionsPerPayer' => null, + ]; + } +} diff --git a/app/Services/Stats/WelchTTest.php b/app/Services/Stats/WelchTTest.php new file mode 100644 index 00000000..5d398a3f --- /dev/null +++ b/app/Services/Stats/WelchTTest.php @@ -0,0 +1,61 @@ + 0 ? $varA / $nA : 0.0) + ($nB > 0 ? $varB / $nB : 0.0); + $se = sqrt($seSq); + $diff = $meanA - $meanB; + + if ($se <= 0.0 || $nA < 2 || $nB < 2) { + return ['diff' => $diff, 'se' => $se, 't' => 0.0, 'df' => 0.0, 'p' => 1.0]; + } + + $t = $diff / $se; + $df = ($seSq ** 2) / ( + (($varA / $nA) ** 2) / ($nA - 1) + + (($varB / $nB) ** 2) / ($nB - 1) + ); + $p = 2.0 * (1.0 - $this->normalCdf(abs($t))); + + return ['diff' => $diff, 'se' => $se, 't' => $t, 'df' => $df, 'p' => $p]; + } + + private function normalCdf(float $x): float + { + return 0.5 * (1.0 + $this->erf($x / sqrt(2.0))); + } + + /** Abramowitz & Stegun 7.1.26 — |error| < 1.5e-7. */ + private function erf(float $x): float + { + $sign = $x < 0 ? -1.0 : 1.0; + $x = abs($x); + $t = 1.0 / (1.0 + 0.3275911 * $x); + $y = 1.0 - ((((1.061405429 * $t - 1.453152027) * $t + 1.421413741) * $t - 0.284496736) * $t + 0.254829592) * $t * exp(-$x * $x); + + return $sign * $y; + } +} diff --git a/routes/console.php b/routes/console.php index 2b195663..e6e657a3 100644 --- a/routes/console.php +++ b/routes/console.php @@ -17,3 +17,4 @@ Schedule::command('stats:daily-report')->dailyAt('09:00')->timezone('Europe/Madr Schedule::command('stats:ai-cohort-report')->monthlyOn(1, '09:00')->timezone('Europe/Madrid'); Schedule::command('stats:subscription-funnel')->weekly()->mondays()->at('09:15')->timezone('Europe/Madrid'); Schedule::command('stats:experiment-funnel')->weekly()->mondays()->at('09:30')->timezone('Europe/Madrid'); +Schedule::command('stats:price-experiment-funnel')->weekly()->mondays()->at('09:45')->timezone('Europe/Madrid'); diff --git a/tests/Feature/SendPriceExperimentFunnelReportCommandTest.php b/tests/Feature/SendPriceExperimentFunnelReportCommandTest.php new file mode 100644 index 00000000..f050c0a3 --- /dev/null +++ b/tests/Feature/SendPriceExperimentFunnelReportCommandTest.php @@ -0,0 +1,200 @@ + true, + 'subscriptions.price_experiment.started_at' => '2026-06-01', + 'subscriptions.price_experiment.force_variant' => null, + 'subscriptions.plans.monthly.trial_days' => 15, + 'ai_suggestions.report.excluded_emails' => [], + ]); + Carbon::setTestNow(CarbonImmutable::parse('2026-06-30 12:00:00')); + + // Seed the price→monthly-equivalent map so revenue is computed without Stripe: + // control €3.99, high €8.99. + Cache::put('experiment_funnel_monthly_equiv', ['price_control' => 399, 'price_high' => 899], now()->addHour()); +}); + +/** + * Create a user whose id buckets into the wanted price variant, with an optional + * default subscription, live (active) bank connections that drive the monthly cost, + * extra soft-deleted connections that count as "ever activated" but cost nothing, + * and optional AI consent. + * + * @param array{status: string, at: CarbonImmutable, price?: string, endsAt?: CarbonImmutable, trialEndsAt?: CarbonImmutable, refundedAt?: CarbonImmutable}|null $subscription + */ +function priceUser(string $variant, CarbonImmutable $signup, ?array $subscription = null, int $connections = 0, int $deletedConnections = 0, bool $aiConsent = false): User +{ + do { + $id = (string) Str::uuid(); + } while (PriceExperiment::bucket($id) !== $variant); + + $user = User::factory()->create(['id' => $id, 'created_at' => $signup]); + + if ($subscription !== null) { + $user->subscriptions()->create([ + 'type' => 'default', + 'stripe_id' => 'sub_'.Str::random(12), + 'stripe_status' => $subscription['status'], + 'stripe_price' => $subscription['price'] ?? 'price_control', + 'created_at' => $subscription['at'], + 'ends_at' => $subscription['endsAt'] ?? null, + 'trial_ends_at' => $subscription['trialEndsAt'] ?? null, + 'refunded_at' => $subscription['refundedAt'] ?? null, + ]); + } + + if ($connections > 0) { + BankingConnection::factory()->count($connections)->for($user)->create(); + } + + if ($deletedConnections > 0) { + BankingConnection::factory()->count($deletedConnections)->for($user)->create() + ->each(fn (BankingConnection $connection) => $connection->delete()); + } + + if ($aiConsent) { + AiConsent::factory()->for($user)->create(); + } + + return $user; +} + +it('returns empty variants when the experiment has not started', function () { + config(['subscriptions.price_experiment.started_at' => null]); + + $report = app(PriceExperimentFunnelCollector::class)->collect(); + + expect($report['startedAt'])->toBeNull() + ->and($report['variants'][PriceExperiment::CONTROL]['assigned'])->toBe(0) + ->and($report['variants'][PriceExperiment::HIGH]['assigned'])->toBe(0); +}); + +it('attributes users to the right price variant and counts the funnel', function () { + $signup = CarbonImmutable::parse('2026-06-05'); // mature (15d + 3d) by the test clock + + priceUser(PriceExperiment::CONTROL, $signup); + priceUser(PriceExperiment::CONTROL, $signup, ['status' => 'active', 'at' => $signup->addDay(), 'price' => 'price_control']); + priceUser(PriceExperiment::HIGH, $signup, ['status' => 'active', 'at' => $signup->addDay(), 'price' => 'price_high']); + + $variants = app(PriceExperimentFunnelCollector::class)->collect()['variants']; + + expect($variants[PriceExperiment::CONTROL]['assigned'])->toBe(2) + ->and($variants[PriceExperiment::CONTROL]['subscribed'])->toBe(1) + ->and($variants[PriceExperiment::CONTROL]['activeMature'])->toBe(1) + ->and($variants[PriceExperiment::CONTROL]['convertedMature'])->toBe(1) + ->and($variants[PriceExperiment::CONTROL]['conversionRate'])->toBe(0.5) + ->and($variants[PriceExperiment::HIGH]['assigned'])->toBe(1) + ->and($variants[PriceExperiment::HIGH]['activeMature'])->toBe(1); +}); + +it('books MRR at each variant price and reports contribution margin per assigned user', function () { + $signup = CarbonImmutable::parse('2026-06-05'); + + // High arm: one payer at €8.99 with 1 active connection (cost €0.40). + priceUser(PriceExperiment::HIGH, $signup, ['status' => 'active', 'at' => $signup, 'price' => 'price_high'], connections: 1); + // High arm: one assigned non-payer, no connections. + priceUser(PriceExperiment::HIGH, $signup); + + $high = app(PriceExperimentFunnelCollector::class)->collect()['variants'][PriceExperiment::HIGH]; + + expect($high['mrrCents'])->toBe(899) // €8.99 at the high price + ->and($high['costCents'])->toBe(40) // 1 active connection + ->and($high['contributionMarginCents'])->toBe(859) // 899 − 40 + ->and($high['assignedMature'])->toBe(2) + ->and($high['cmPerAssignedCents'])->toBe(430) // round(859 / 2) + ->and($high['connectionsPerPayer'])->toBe(1.0); +}); + +it('counts revoked connections as activation but charges cost only for live ones', function () { + $signup = CarbonImmutable::parse('2026-06-05'); + + // 1 live connection + 2 soft-deleted: activated (ever connected), but the + // monthly cost is only the 1 live connection — the stock-vs-flow fix. + priceUser(PriceExperiment::CONTROL, $signup, connections: 1, deletedConnections: 2); + + $control = app(PriceExperimentFunnelCollector::class)->collect()['variants'][PriceExperiment::CONTROL]; + + expect($control['activated'])->toBe(1) + ->and($control['activatedMature'])->toBe(1) + ->and($control['costCents'])->toBe(40); // 1 live × 40, not 3 × 40 +}); + +it('leaves young cohorts out of the matured metrics', function () { + priceUser(PriceExperiment::HIGH, CarbonImmutable::now()->subDays(5), [ + 'status' => 'active', 'at' => CarbonImmutable::now()->subDays(5), 'price' => 'price_high', + ]); + + $high = app(PriceExperimentFunnelCollector::class)->collect()['variants'][PriceExperiment::HIGH]; + + expect($high['assigned'])->toBe(1) + ->and($high['assignedMature'])->toBe(0) // 5d < 18d window + ->and($high['conversionRate'])->toBeNull(); +}); + +it('keeps the split even when a winner is forced, instead of collapsing onto one variant', function () { + config(['subscriptions.price_experiment.force_variant' => PriceExperiment::HIGH]); + $signup = CarbonImmutable::parse('2026-06-05'); + + priceUser(PriceExperiment::CONTROL, $signup); + priceUser(PriceExperiment::HIGH, $signup); + + $variants = app(PriceExperimentFunnelCollector::class)->collect()['variants']; + + expect($variants[PriceExperiment::CONTROL]['assigned'])->toBe(1) + ->and($variants[PriceExperiment::HIGH]['assigned'])->toBe(1); +}); + +it('prints the primary CM/user (Welch) and conversion guardrail blocks', function () { + $signup = CarbonImmutable::parse('2026-06-05'); + + for ($i = 0; $i < 3; $i++) { + priceUser(PriceExperiment::CONTROL, $signup, ['status' => 'active', 'at' => $signup, 'price' => 'price_control']); + priceUser(PriceExperiment::CONTROL, $signup); + priceUser(PriceExperiment::HIGH, $signup, ['status' => 'active', 'at' => $signup, 'price' => 'price_high']); + priceUser(PriceExperiment::HIGH, $signup); + } + + artisan('stats:price-experiment-funnel', ['--no-discord' => true]) + ->expectsOutputToContain('PRIMARY') + ->expectsOutputToContain('GUARDRAIL') + ->expectsOutputToContain('Fisher exact') + ->assertSuccessful(); +}); + +it('posts the price experiment embed to discord with the monitoring-only warning', function () { + config(['services.discord.ai_cohort_webhook_url' => 'https://discord.test/hook']); + Http::fake(['discord.test/*' => Http::response('', 204)]); + + priceUser(PriceExperiment::CONTROL, CarbonImmutable::parse('2026-06-05'), [ + 'status' => 'active', 'at' => CarbonImmutable::parse('2026-06-05'), 'price' => 'price_control', + ]); + + artisan('stats:price-experiment-funnel')->assertSuccessful(); + + Http::assertSent(fn ($request) => $request->url() === 'https://discord.test/hook' + && str_contains($request['embeds'][0]['title'], 'Price Experiment') + && str_contains(json_encode($request['embeds'][0]), 'MONITORING ONLY')); +}); + +it('does not post when the experiment has not started', function () { + config(['subscriptions.price_experiment.started_at' => null]); + Http::fake(); + + artisan('stats:price-experiment-funnel')->assertSuccessful(); + + Http::assertNothingSent(); +}); diff --git a/tests/Unit/WelchTTestTest.php b/tests/Unit/WelchTTestTest.php new file mode 100644 index 00000000..77b7d4e5 --- /dev/null +++ b/tests/Unit/WelchTTestTest.php @@ -0,0 +1,34 @@ +compare(10.0, 4.0, 100, 10.0, 4.0, 100); + + expect($result['diff'])->toBe(0.0) + ->and($result['t'])->toBe(0.0) + ->and($result['p'])->toEqualWithDelta(1.0, 1e-6); // erf approximation, |error| < 1.5e-7 +}); + +it('yields a two-sided p of ~0.317 when the difference equals one standard error', function () { + // varA/nA = varB/nB = 0.5 → seSq = 1, se = 1; diff = 1 → t = 1. + // Two-sided normal p = 2·(1 − Φ(1)) = 0.3173, validating the erf/normalCdf path. + $result = (new WelchTTest)->compare(1.0, 50.0, 100, 0.0, 50.0, 100); + + expect($result['t'])->toEqualWithDelta(1.0, 1e-9) + ->and($result['se'])->toEqualWithDelta(1.0, 1e-9) + ->and($result['p'])->toEqualWithDelta(0.3173, 1e-3); +}); + +it('finds a large separation highly significant', function () { + $result = (new WelchTTest)->compare(10.0, 4.0, 100, 8.0, 4.0, 100); + + expect($result['t'])->toEqualWithDelta(7.071, 1e-2) + ->and($result['p'])->toBeLessThan(0.001); +}); + +it('defers (p=1) when an arm has too few observations to estimate variance', function () { + $result = (new WelchTTest)->compare(5.0, 0.0, 1, 0.0, 0.0, 1); + + expect($result['p'])->toBe(1.0); +});